Example usage for org.apache.hadoop.io NullWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get()

Source Link

Document

Returns the single instance of this class.

Usage

From source file:com.google.appengine.tools.mapreduce.AppEngineMapperTest.java

License:Apache License

public void testPartialMutationPool() throws Exception {
    mapper.shouldOutputEntity(true);//w w w.jav a  2  s  .  co m
    mapper.setup(context);
    mapper.taskSetup(context);
    mapper.map(NullWritable.get(), NullWritable.get(), context);
    assertEquals(0,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
    mapper.taskCleanup(context);
    mapper.cleanup(context);
    assertEquals(1,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
}

From source file:com.google.appengine.tools.mapreduce.AppEngineMapperTest.java

License:Apache License

public void testMutationPool() throws Exception {
    mapper.shouldOutputEntity(true);//from   w w w.  j  av a2 s  .c  o  m
    mapper.setup(context);
    mapper.taskSetup(context);
    for (int i = 0; i < DatastoreMutationPool.DEFAULT_COUNT_LIMIT - 1; i++) {
        mapper.map(NullWritable.get(), NullWritable.get(), context);
    }
    assertEquals(0,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
    mapper.map(NullWritable.get(), NullWritable.get(), context);
    assertEquals(DatastoreMutationPool.DEFAULT_COUNT_LIMIT,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
    mapper.taskCleanup(context);
    mapper.cleanup(context);
    assertEquals(DatastoreMutationPool.DEFAULT_COUNT_LIMIT,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
}

From source file:com.google.appengine.tools.mapreduce.AppEngineMapperTest.java

License:Apache License

public void testMutationPoolFlushParameters() throws Exception {
    mapper.shouldOutputEntity(true);/*from w ww . jav a2 s.c  o  m*/
    mapper.setup(context);
    mapper.taskSetup(context);
    context.setMutationPoolFlushParameters(10, 100000);
    for (int i = 0; i < 10 - 1; i++) {
        mapper.map(NullWritable.get(), NullWritable.get(), context);
    }
    assertEquals(0,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
    mapper.map(NullWritable.get(), NullWritable.get(), context);
    assertEquals(10,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
    mapper.taskCleanup(context);
    mapper.cleanup(context);
    assertEquals(10,
            datastoreService.prepare(new Query("foo")).countEntities(FetchOptions.Builder.withDefaults()));
}

From source file:com.google.cloud.dataflow.sdk.io.hdfs.HDFSFileSink.java

License:Apache License

public static <T> HDFSFileSink<T, NullWritable, Text> toText(String path) {
    SerializableFunction<T, KV<NullWritable, Text>> outputConverter = new SerializableFunction<T, KV<NullWritable, Text>>() {
        @Override/*from   w w  w  .  j  a va 2s. com*/
        public KV<NullWritable, Text> apply(T input) {
            return KV.of(NullWritable.get(), new Text(input.toString()));
        }
    };
    return to(path, TextOutputFormat.class, outputConverter);
}

From source file:com.google.cloud.dataflow.sdk.io.hdfs.HDFSFileSink.java

License:Apache License

public static <T> HDFSFileSink<T, AvroKey<T>, NullWritable> toAvro(String path) {
    SerializableFunction<T, KV<AvroKey<T>, NullWritable>> outputConverter = new SerializableFunction<T, KV<AvroKey<T>, NullWritable>>() {
        @Override/* w w  w . j  a  v  a2  s  .  c o  m*/
        public KV<AvroKey<T>, NullWritable> apply(T input) {
            return KV.of(new AvroKey<>(input), NullWritable.get());
        }
    };
    return to(path, AvroKeyOutputFormat.class, outputConverter);
}

From source file:com.google.cloud.dataflow.sdk.io.hdfs.WritableCoder.java

License:Apache License

@SuppressWarnings("unchecked")
@Override//  w  w w  .ja v a  2  s.  co m
public T decode(InputStream inStream, Context context) throws IOException {
    try {
        if (type == NullWritable.class) {
            // NullWritable has no default constructor
            return (T) NullWritable.get();
        }
        T t = type.newInstance();
        t.readFields(new DataInputStream(inStream));
        return t;
    } catch (InstantiationException | IllegalAccessException e) {
        throw new CoderException("unable to deserialize record", e);
    }
}

From source file:com.google.cloud.dataflow.sdk.io.hdfs.WritableCoderTest.java

License:Apache License

@Test
public void testNullWritableEncoding() throws Exception {
    NullWritable value = NullWritable.get();
    WritableCoder<NullWritable> coder = WritableCoder.of(NullWritable.class);

    CoderProperties.coderDecodeEncodeEqual(coder, value);
}

From source file:com.hortonworks.pso.data.generator.mapreduce.DataGenMapper.java

License:Apache License

public void map(LongWritable key, NullWritable value, Context context)
        throws IOException, InterruptedException {
    Text record = new Text();
    record.set(recordGenerator.next());/*from  w  ww  .  j a  v a  2s  . co m*/
    context.write(NullWritable.get(), record);
}

From source file:com.hotels.corc.cascading.OrcFile.java

License:Apache License

/**
 * Populates the {@link Corc} with the next value from the {@link RecordReader}. Then copies the values into the
 * incoming {@link TupleEntry}./*from w  w w  .  j a v  a2  s .com*/
 */
@Override
public boolean source(FlowProcess<JobConf> flowProcess, SourceCall<Corc, RecordReader> sourceCall)
        throws IOException {
    Corc corc = sourceCall.getContext();
    @SuppressWarnings("unchecked")
    boolean next = sourceCall.getInput().next(NullWritable.get(), corc);
    if (!next) {
        return false;
    }
    TupleEntry tupleEntry = sourceCall.getIncomingEntry();
    for (Comparable<?> fieldName : tupleEntry.getFields()) {
        if (ROW_ID_NAME.equals(fieldName)) {
            tupleEntry.setObject(ROW_ID_NAME, corc.getRecordIdentifier());
        } else {
            tupleEntry.setObject(fieldName, corc.get(fieldName.toString()));
        }
    }
    return true;
}

From source file:com.hotels.corc.mapred.CorcInputFormatTest.java

License:Apache License

@Test
public void readColumnProjection() throws IOException {
    StructTypeInfo typeInfo = new StructTypeInfoBuilder().add("a", TypeInfoFactory.stringTypeInfo).build();
    CorcInputFormat.setTypeInfo(conf, typeInfo);
    CorcInputFormat.setConverterFactoryClass(conf, DefaultConverterFactory.class);

    RecordReader<NullWritable, Corc> reader = inputFormat.getRecordReader(split, conf, reporter);

    Corc corc = reader.createValue();/* w  ww . j  a v a 2  s . c  o m*/

    reader.next(NullWritable.get(), corc);
    assertThat(corc.get("a"), is((Object) "A1"));
    assertThat(corc.get("b"), is(nullValue()));
    reader.close();
}