Example usage for org.apache.hadoop.io NullWritable get

List of usage examples for org.apache.hadoop.io NullWritable get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. NullWritable
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get()

Source Link

Document

Returns the single instance of this class.

Usage

From source file:com.cloudera.sqoop.mapreduce.TextExportMapper.java

License:Apache License

public void map(LongWritable key, Text val, Context context) throws IOException, InterruptedException {
    try {/*from  w  w  w.  jav a  2 s .  c  o m*/
        recordImpl.parse(val);
        context.write(recordImpl, NullWritable.get());
    } catch (RecordParser.ParseError pe) {
        throw new IOException("Could not parse record: " + val, pe);
    }
}

From source file:com.cloudera.sqoop.mapreduce.TextImportMapper.java

License:Apache License

@Override
public void map(LongWritable key, SqoopRecord val, Context context) throws IOException, InterruptedException {

    try {//from ww  w  .  jav  a  2  s  . co m
        // Loading of LOBs was delayed until we have a Context.
        val.loadLargeObjects(lobLoader);
    } catch (SQLException sqlE) {
        throw new IOException(sqlE);
    }

    outkey.set(val.toString());
    context.write(outkey, NullWritable.get());
}

From source file:com.cloudera.sqoop.testutil.ReparseMapper.java

License:Apache License

public void map(LongWritable key, Text val, OutputCollector<Text, NullWritable> out, Reporter r)
        throws IOException {

    LOG.info("Mapper input line: " + val.toString());

    try {/*from  w  w  w  .  j a  v a  2  s  .c  o  m*/
        // Use the user's record class to parse the line back in.
        userRecord.parse(val);
    } catch (RecordParser.ParseError pe) {
        LOG.error("Got parse error: " + pe.toString());
        throw new IOException(pe);
    }

    LOG.info("Mapper output line: " + userRecord.toString());

    out.collect(new Text(userRecord.toString()), NullWritable.get());

    if (!userRecord.toString(false).equals(val.toString())) {
        // Could not format record w/o end-of-record delimiter.
        throw new IOException("Returned string w/o EOR has value [" + userRecord.toString(false) + "] when ["
                + val.toString() + "] was expected.");
    }

    if (!userRecord.toString().equals(val.toString() + "\n")) {
        // misparsed.
        throw new IOException("Returned string has value [" + userRecord.toString() + "] when ["
                + val.toString() + "\n] was expected.");
    }
}

From source file:com.conductor.hadoop.WritableValueInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(final JobContext context) throws IOException, InterruptedException {
    final Configuration conf = context.getConfiguration();

    // init the reader
    final String filePath = conf.get(INPUT_FILE_LOCATION_CONF);
    checkArgument(!Strings.isNullOrEmpty(filePath), "Missing property: " + INPUT_FILE_LOCATION_CONF);

    final FileSystem fs = getFileSystem(conf);
    final Path path = fs.makeQualified(new Path(filePath));
    final SequenceFile.Reader reader = getReader(conf, path);

    // create the splits by looping through the values of the input file
    int totalInputs = 0;
    int maxInputsPerSplit = conf.getInt(INPUTS_PER_SPLIT_CONF, DEFAULT_INPUTS_PER_SPLIT);
    long pos = 0L;
    long last = 0L;
    long lengthRemaining = fs.getFileStatus(path).getLen();
    final List<InputSplit> splits = Lists.newArrayList();
    final V value = getV(conf);
    for (final NullWritable key = NullWritable.get(); reader.next(key, value); last = reader.getPosition()) {
        if (++totalInputs % maxInputsPerSplit == 0) {
            long splitSize = last - pos;
            splits.add(new FileSplit(path, pos, splitSize, null));
            lengthRemaining -= splitSize;
            pos = last;//  w w w  .  j  ava  2s .c  om
        }
    }
    // create the last split if there is data remaining
    if (lengthRemaining != 0) {
        splits.add(new FileSplit(path, pos, lengthRemaining, null));
    }
    return splits;
}

From source file:com.conductor.hadoop.WritableValueInputFormat.java

License:Apache License

@VisibleForTesting
static <V extends Writable> void doSetupInput(final List<V> values, final Class<V> clazz,
        final int inputsPerSplit, final Job job, final Path inputPath, final Writer writer) throws IOException {
    job.getConfiguration().setClass(VALUE_TYPE_CONF, clazz, Writable.class);
    job.getConfiguration().setInt(INPUTS_PER_SPLIT_CONF, inputsPerSplit);
    job.getConfiguration().set(INPUT_FILE_LOCATION_CONF, inputPath.toString());

    // write each value to the sequence file
    int syncCounter = 0;
    for (final V input : values) {
        // each entry in the sequence file is a map input
        writer.append(NullWritable.get(), input);
        // syncing indicates an input split boundary
        if (++syncCounter % inputsPerSplit == 0) {
            writer.sync();/*from   ww w.java  2 s.  com*/
        }
    }
    // close the input file
    writer.hflush();
    writer.close();

    // delete file when JVM exits
    inputPath.getFileSystem(job.getConfiguration()).deleteOnExit(inputPath);
}

From source file:com.conversantmedia.mapreduce.example.avro.AvroInputOutputMapperTest.java

License:Apache License

@Test
public void testShakespeareThe() {
    MapDriver<AvroKey<AvroExample>, NullWritable, AvroKey<AvroExample>, NullWritable> driver = this
            .getTestDriver();/* w w w .j  a v  a 2s .c  o  m*/

    AvroUnitTestHelper helper = AvroUnitTestHelper.getInstance();
    Iterable<AvroExample> records;
    try {
        records = helper.getRecords("src/test/resources/shakespeare_counts.avro");
        for (AvroExample record : records) {
            driver.addInput(new Pair<>(new AvroKey<>(record), NullWritable.get()));
        }

        AvroExample.Builder builder = AvroExample.newBuilder();
        builder.setWord("the");
        builder.setFrequency(23437l);
        AvroKey<AvroExample> avroKey = new AvroKey<>(builder.build());
        driver.addOutput(new Pair<>(avroKey, NullWritable.get()));

        driver.runTest();
    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:com.conversantmedia.mapreduce.example.avro.AvroMultipleOutputExampleTest.java

License:Apache License

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testAvroMultipleOut() {

    ReduceDriver<Text, LongWritable, AvroKey<AvroExample>, NullWritable> driver = this.getTestDriver();
    LongWritable ONE = new LongWritable(1);
    List<String> words = new ArrayList<>();
    words.add("10");
    words.add("20");
    words.add("30");
    words.add("40");

    for (String word : words) {
        List<LongWritable> counts = new ArrayList<>();
        for (int i = 0; i < Long.parseLong(word); i++) {
            counts.add(ONE);/*from   w  ww. jav a 2 s.c om*/
        }
        driver.addInput(new Text(word), counts);
    }

    try {
        List<Pair<AvroKey<AvroExample>, NullWritable>> results = driver.run();
        assertThat(results.size(), equalTo(words.size()));
        for (Pair<AvroKey<AvroExample>, NullWritable> e : results) {
            validateRecords(e.getFirst().datum());
        }

        AvroMultipleOutputs avroOut = this.getAvroNamedOutput();
        ArgumentCaptor<AvroKey> recordArg = ArgumentCaptor.forClass(AvroKey.class);
        try {
            verify(avroOut, times(words.size())).write(recordArg.capture(), eq(NullWritable.get()),
                    eq("avroMulti"));
            for (AvroKey<AvroExample> record : recordArg.getAllValues()) {
                validateRecords(record.datum());
            }
        } catch (InterruptedException e) {
            fail(e.getMessage());
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:com.conversantmedia.mapreduce.example.avro.AvroWordCountReducer.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    int sum = 0;// www .  j a v a  2  s . c om
    for (LongWritable value : values) {
        sum += value.get();
    }
    AvroExample.Builder builder = AvroExample.newBuilder();
    builder.setWord(key.toString());
    builder.setFrequency(sum);
    AvroExample datum = builder.build();
    aKey.datum(datum);

    context.write(aKey, NullWritable.get());
    avroMultiOut.write(aKey, NullWritable.get(), "avroMulti");
}

From source file:com.conversantmedia.mapreduce.io.CompositeSortKeyTest.java

License:Apache License

@Test
public void testNaturalSortByValue() {
    // Add in a non-sorted order
    int offset = 50;
    setupInputs(offset);/*from w  w  w .j  a va 2s . c  om*/

    // Outputs should be properly sorted...
    // First 'A'
    for (int i = 0; i < offset * 3; i++) {
        driver.addOutput(new Text("A" + i), NullWritable.get());
    }
    // And 'B'
    for (int i = 0; i < offset * 3; i++) {
        driver.addOutput(new Text("B" + i), NullWritable.get());
    }

    try {
        driver.runTest();
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:com.conversantmedia.mapreduce.io.CompositeSortKeyTest.java

License:Apache License

@Test
public void testReverseSortByValue() {
    // Add in a non-sorted order
    int offset = 50;
    setupInputs(offset);/*from   w  ww.ja va2  s. c  o  m*/

    // Outputs should be properly sorted...
    // First 'A'
    for (int i = offset * 3 - 1; i > -1; i--) {
        driver.addOutput(new Text("A" + i), NullWritable.get());
    }
    // And 'B'
    for (int i = offset * 3 - 1; i > -1; i--) {
        driver.addOutput(new Text("B" + i), NullWritable.get());
    }

    try {
        driver.setKeyOrderComparator(new CompositeSortKey.ReverseSortComparator<Text, IntWritable>());
        driver.runTest();
    } catch (IOException e) {
        fail(e.getMessage());
    }
}