Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.avro.mapred; import static org.junit.Assert.assertEquals; import java.io.IOException; import java.io.File; import java.net.URI; import java.util.Iterator; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.avro.Schema; import org.apache.avro.file.FileReader; import org.apache.avro.file.DataFileReader; import org.apache.avro.specific.SpecificDatumReader; import org.apache.avro.util.Utf8; import org.junit.BeforeClass; import org.junit.Test; public class TestSequenceFileReader { private static final int COUNT = Integer.parseInt(System.getProperty("test.count", "10")); private static final File DIR = new File(System.getProperty("test.dir", ".")); private static final File FILE = new File(DIR, "test.seq"); private static final Schema SCHEMA = Pair.getPairSchema(Schema.create(Schema.Type.LONG), Schema.create(Schema.Type.STRING)); @BeforeClass public static void testWriteSequenceFile() throws IOException { FILE.delete(); Configuration c = new Configuration(); URI uri = FILE.toURI(); SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(uri, c), c, new Path(uri.toString()), LongWritable.class, Text.class); final LongWritable key = new LongWritable(); final Text val = new Text(); for (int i = 0; i < COUNT; ++i) { key.set(i); val.set(Integer.toString(i)); writer.append(key, val); } writer.close(); } @Test public void testReadSequenceFile() throws Exception { checkFile(new SequenceFileReader<Long, CharSequence>(FILE)); } public void checkFile(FileReader<Pair<Long, CharSequence>> reader) throws Exception { long i = 0; for (Pair<Long, CharSequence> p : reader) { assertEquals((Long) i, p.key()); assertEquals(Long.toString(i), p.value().toString()); i++; } assertEquals(COUNT, i); reader.close(); } @Test public void testSequenceFileInputFormat() throws Exception { JobConf job = new JobConf(); Path output = new Path(System.getProperty("test.dir", ".") + "/seq-out"); output.getFileSystem(job).delete(output); // configure input for Avro from sequence file AvroJob.setInputSequenceFile(job); FileInputFormat.setInputPaths(job, FILE.toURI().toString()); AvroJob.setInputSchema(job, SCHEMA); // mapper is default, identity // reducer is default, identity // configure output for avro AvroJob.setOutputSchema(job, SCHEMA); FileOutputFormat.setOutputPath(job, output); JobClient.runJob(job); checkFile(new DataFileReader<Pair<Long, CharSequence>>(new File(output.toString() + "/part-00000.avro"), new SpecificDatumReader<Pair<Long, CharSequence>>())); } private static class NonAvroMapper extends MapReduceBase implements Mapper<LongWritable, Text, AvroKey<Long>, AvroValue<Utf8>> { public void map(LongWritable key, Text value, OutputCollector<AvroKey<Long>, AvroValue<Utf8>> out, Reporter reporter) throws IOException { out.collect(new AvroKey<Long>(key.get()), new AvroValue<Utf8>(new Utf8(value.toString()))); } } @Test public void testNonAvroMapper() throws Exception { JobConf job = new JobConf(); Path output = new Path(System.getProperty("test.dir", ".") + "/seq-out"); output.getFileSystem(job).delete(output); // configure input for non-Avro sequence file job.setInputFormat(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, FILE.toURI().toString()); // use a hadoop mapper that emits Avro output job.setMapperClass(NonAvroMapper.class); // reducer is default, identity // configure output for avro FileOutputFormat.setOutputPath(job, output); AvroJob.setOutputSchema(job, SCHEMA); JobClient.runJob(job); checkFile(new DataFileReader<Pair<Long, CharSequence>>(new File(output.toString() + "/part-00000.avro"), new SpecificDatumReader<Pair<Long, CharSequence>>())); } private static class NonAvroOnlyMapper extends MapReduceBase implements Mapper<LongWritable, Text, AvroWrapper<Pair<Long, Utf8>>, NullWritable> { public void map(LongWritable key, Text value, OutputCollector<AvroWrapper<Pair<Long, Utf8>>, NullWritable> out, Reporter reporter) throws IOException { out.collect( new AvroWrapper<Pair<Long, Utf8>>(new Pair<Long, Utf8>(key.get(), new Utf8(value.toString()))), NullWritable.get()); } } @Test public void testNonAvroMapOnly() throws Exception { JobConf job = new JobConf(); Path output = new Path(System.getProperty("test.dir", ".") + "/seq-out"); output.getFileSystem(job).delete(output); // configure input for non-Avro sequence file job.setInputFormat(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, FILE.toURI().toString()); // use a hadoop mapper that emits Avro output job.setMapperClass(NonAvroOnlyMapper.class); // configure output for avro job.setNumReduceTasks(0); // map-only FileOutputFormat.setOutputPath(job, output); AvroJob.setOutputSchema(job, SCHEMA); JobClient.runJob(job); checkFile(new DataFileReader<Pair<Long, CharSequence>>(new File(output.toString() + "/part-00000.avro"), new SpecificDatumReader<Pair<Long, CharSequence>>())); } private static class NonAvroReducer extends MapReduceBase implements Reducer<AvroKey<Long>, AvroValue<Utf8>, LongWritable, Text> { public void reduce(AvroKey<Long> key, Iterator<AvroValue<Utf8>> values, OutputCollector<LongWritable, Text> out, Reporter reporter) throws IOException { while (values.hasNext()) { AvroValue<Utf8> value = values.next(); out.collect(new LongWritable(key.datum()), new Text(value.datum().toString())); } } } @Test public void testNonAvroReducer() throws Exception { JobConf job = new JobConf(); Path output = new Path(System.getProperty("test.dir", ".") + "/seq-out"); output.getFileSystem(job).delete(output); // configure input for Avro from sequence file AvroJob.setInputSequenceFile(job); AvroJob.setInputSchema(job, SCHEMA); FileInputFormat.setInputPaths(job, FILE.toURI().toString()); // mapper is default, identity // use a hadoop reducer that consumes Avro input AvroJob.setMapOutputSchema(job, SCHEMA); job.setReducerClass(NonAvroReducer.class); // configure output for non-Avro SequenceFile job.setOutputFormat(SequenceFileOutputFormat.class); FileOutputFormat.setOutputPath(job, output); // output key/value classes are default, LongWritable/Text JobClient.runJob(job); checkFile(new SequenceFileReader<Long, CharSequence>(new File(output.toString() + "/part-00000"))); } }