Example usage for org.apache.hadoop.io NullWritable get

List of usage examples for org.apache.hadoop.io NullWritable get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. NullWritable
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get()

Source Link

Document

Returns the single instance of this class.

Usage

From source file:org.apache.avro.mapred.AvroMultipleOutputs.java

License:Apache License

/**
 * Gets the output collector for a multi named output.
 * <p/>//from   w  w  w  .j av  a  2  s .  co m
 *
 * @param namedOutput the named output name
 * @param multiName   the multi name part
 * @param reporter    the reporter
 * @return the output collector for the given named output
 * @throws IOException thrown if output collector could not be created
 */
@SuppressWarnings({ "unchecked" })
private AvroCollector getCollector(String namedOutput, String multiName, Reporter reporter,
        String baseOutputFileName, Schema schema) throws IOException {

    checkNamedOutputName(namedOutput);
    if (!namedOutputs.contains(namedOutput)) {
        throw new IllegalArgumentException("Undefined named output '" + namedOutput + "'");
    }
    boolean multi = isMultiNamedOutput(conf, namedOutput);

    if (!multi && multiName != null) {
        throw new IllegalArgumentException("Name output '" + namedOutput + "' has not been defined as multi");
    }
    if (multi) {
        checkTokenName(multiName);
    }

    String baseFileName = (multi) ? namedOutput + "_" + multiName : baseOutputFileName;

    final RecordWriter writer = getRecordWriter(namedOutput, baseFileName, reporter, schema);

    return new AvroCollector() {

        @SuppressWarnings({ "unchecked" })
        public void collect(Object key) throws IOException {
            AvroWrapper wrapper = new AvroWrapper(key);
            writer.write(wrapper, NullWritable.get());
        }

        public void collect(Object key, Object value) throws IOException {
            writer.write(key, value);
        }

    };
}

From source file:org.apache.avro.mapred.AvroRecordReader.java

License:Apache License

public NullWritable createValue() {
    return NullWritable.get();
}

From source file:org.apache.avro.mapred.MapCollector.java

License:Apache License

public void collect(OUT datum) throws IOException {
    if (isMapOnly) {
        wrapper.datum(datum);/*  w  ww . ja  v  a 2  s. co  m*/
        collector.collect((KO) wrapper, (VO) NullWritable.get());
    } else {
        // split a pair
        Pair<K, V> pair = (Pair<K, V>) datum;
        keyWrapper.datum(pair.key());
        valueWrapper.datum(pair.value());
        collector.collect((KO) keyWrapper, (VO) valueWrapper);
    }
}

From source file:org.apache.avro.mapred.PipesCompatibleAvroRecordReader.java

License:Apache License

private void setClassVariables(AvroRecordReader<T> avroRecordReader, KeyValueGetter<T> keyValueGetter) {
    this.avroRecordReader = avroRecordReader;
    this.keyValueGetter = keyValueGetter;

    avroWrapper = new AvroWrapper<T>(null);
    nullWritable = NullWritable.get();
}

From source file:org.apache.avro.mapred.TestAvroMultipleOutputs.java

License:Apache License

@SuppressWarnings("deprecation")
public void testProjection() throws Exception {
    JobConf job = new JobConf();

    Integer defaultRank = new Integer(-1);

    String jsonSchema = "{\"type\":\"record\"," + "\"name\":\"org.apache.avro.mapred.Pair\"," + "\"fields\": [ "
            + "{\"name\":\"rank\", \"type\":\"int\", \"default\": -1},"
            + "{\"name\":\"value\", \"type\":\"long\"}" + "]}";

    Schema readerSchema = Schema.parse(jsonSchema);

    AvroJob.setInputSchema(job, readerSchema);

    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/myavro-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);

    AvroRecordReader<Pair<Integer, Long>> recordReader = new AvroRecordReader<Pair<Integer, Long>>(job,
            fileSplit);//  ww  w  . j  a  va  2  s.  co m

    AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<Pair<Integer, Long>>(null);
    NullWritable ignore = NullWritable.get();

    long sumOfCounts = 0;
    long numOfCounts = 0;
    while (recordReader.next(inputPair, ignore)) {
        Assert.assertEquals((Integer) inputPair.datum().get(0), defaultRank);
        sumOfCounts += (Long) inputPair.datum().get(1);
        numOfCounts++;
    }

    Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size());

    long actualSumOfCounts = 0;
    for (Long count : WordCountUtil.COUNTS.values()) {
        actualSumOfCounts += count;
    }

    Assert.assertEquals(sumOfCounts, actualSumOfCounts);

}

From source file:org.apache.avro.mapred.TestAvroMultipleOutputs.java

License:Apache License

@SuppressWarnings("deprecation")
public void testProjection_newmethods() throws Exception {
    JobConf job = new JobConf();

    Integer defaultRank = new Integer(-1);

    String jsonSchema = "{\"type\":\"record\"," + "\"name\":\"org.apache.avro.mapred.Pair\"," + "\"fields\": [ "
            + "{\"name\":\"rank\", \"type\":\"int\", \"default\": -1},"
            + "{\"name\":\"value\", \"type\":\"long\"}" + "]}";

    Schema readerSchema = Schema.parse(jsonSchema);

    AvroJob.setInputSchema(job, readerSchema);

    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/testavrofile-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);

    AvroRecordReader<Pair<Integer, Long>> recordReader = new AvroRecordReader<Pair<Integer, Long>>(job,
            fileSplit);//www .j  a  v a2s. c om

    AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<Pair<Integer, Long>>(null);
    NullWritable ignore = NullWritable.get();

    long sumOfCounts = 0;
    long numOfCounts = 0;
    while (recordReader.next(inputPair, ignore)) {
        Assert.assertEquals((Integer) inputPair.datum().get(0), defaultRank);
        sumOfCounts += (Long) inputPair.datum().get(1);
        numOfCounts++;
    }

    Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size());

    long actualSumOfCounts = 0;
    for (Long count : WordCountUtil.COUNTS.values()) {
        actualSumOfCounts += count;
    }

    Assert.assertEquals(sumOfCounts, actualSumOfCounts);

}

From source file:org.apache.avro.mapred.TestAvroMultipleOutputs.java

License:Apache License

@SuppressWarnings("deprecation")
// Test for a differnt schema output
public void testProjection1() throws Exception {
    JobConf job = new JobConf();
    Schema readerSchema = Schema.create(Schema.Type.STRING);
    AvroJob.setInputSchema(job, readerSchema);

    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/myavro1-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroWrapper<Utf8> inputPair = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    AvroRecordReader<Utf8> recordReader = new AvroRecordReader<Utf8>(job, fileSplit);
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while (recordReader.next(inputPair, ignore)) {
        sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim());
        numOfCounts++;/*from  ww  w  . j a v a 2  s  . c om*/
    }
    Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size());
    long actualSumOfCounts = 0;
    for (Long count : WordCountUtil.COUNTS.values()) {
        actualSumOfCounts += count;
    }
    Assert.assertEquals(sumOfCounts, actualSumOfCounts);
}

From source file:org.apache.avro.mapred.TestAvroMultipleOutputs.java

License:Apache License

@SuppressWarnings("deprecation")
// Test for a differnt schema output
public void testProjection_newmethods_1() throws Exception {
    JobConf job = new JobConf();
    Schema readerSchema = Schema.create(Schema.Type.STRING);
    AvroJob.setInputSchema(job, readerSchema);

    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/testavrofile1-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroWrapper<Utf8> inputPair = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    AvroRecordReader<Utf8> recordReader = new AvroRecordReader<Utf8>(job, fileSplit);
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while (recordReader.next(inputPair, ignore)) {
        sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim());
        numOfCounts++;/*from www.  j  ava 2  s  .  c o m*/
    }
    Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size());
    long actualSumOfCounts = 0;
    for (Long count : WordCountUtil.COUNTS.values()) {
        actualSumOfCounts += count;
    }
    Assert.assertEquals(sumOfCounts, actualSumOfCounts);
}

From source file:org.apache.avro.mapred.TestAvroMultipleOutputs.java

License:Apache License

public void testProjection_noreducer() throws Exception {
    JobConf job = new JobConf();
    long onel = 1;
    Schema readerSchema = Schema.create(Schema.Type.STRING);
    AvroJob.setInputSchema(job, readerSchema);
    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/myavro2-m-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroRecordReader<Utf8> recordReader_new = new AvroRecordReader<Utf8>(job, fileSplit);
    AvroWrapper<Utf8> inputPair_new = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    long testl = 0;
    while (recordReader_new.next(inputPair_new, ignore)) {
        testl = Long.parseLong(inputPair_new.datum().toString().split(":")[2].replace("}", "").trim());
        Assert.assertEquals(onel, testl);
    }//from  w  w w .  j  a  va  2 s .  c o  m
}

From source file:org.apache.avro.mapred.TestAvroTextOutputFormat.java

License:Apache License

@Test
public void testAvroTextRecordWriter() throws Exception {
    File file = new File(System.getProperty("test.dir", "."), "writer");
    Schema schema = Schema.create(Schema.Type.BYTES);
    DatumWriter<ByteBuffer> datumWriter = new GenericDatumWriter<ByteBuffer>(schema);
    DataFileWriter<ByteBuffer> fileWriter = new DataFileWriter<ByteBuffer>(datumWriter);
    fileWriter.create(schema, file);//from  w  w  w  .  jav a  2  s  . c  o m
    RecordWriter<Object, Object> rw = new AvroTextOutputFormat<Object, Object>().new AvroTextRecordWriter(
            fileWriter, "\t".getBytes(UTF8));

    rw.write(null, null);
    rw.write(null, NullWritable.get());
    rw.write(NullWritable.get(), null);
    rw.write(NullWritable.get(), NullWritable.get());

    rw.write("k1", null);
    rw.write("k2", NullWritable.get());

    rw.write(null, "v1");
    rw.write(NullWritable.get(), "v2");

    rw.write("k3", "v3");
    rw.write(new Text("k4"), new Text("v4"));

    rw.close(null);

    DatumReader<ByteBuffer> reader = new GenericDatumReader<ByteBuffer>();
    DataFileReader<ByteBuffer> fileReader = new DataFileReader<ByteBuffer>(file, reader);
    assertEquals("k1", asString(fileReader.next()));
    assertEquals("k2", asString(fileReader.next()));
    assertEquals("v1", asString(fileReader.next()));
    assertEquals("v2", asString(fileReader.next()));
    assertEquals("k3\tv3", asString(fileReader.next()));
    assertEquals("k4\tv4", asString(fileReader.next()));
    assertFalse("End", fileReader.hasNext());
}