List of usage examples for org.apache.hadoop.io NullWritable get
public static NullWritable get()
From source file:com.ebay.jetstream.event.processor.hdfs.writer.SequenceEventWriter.java
License:MIT License
@Override public void afterPropertiesSet() throws Exception { if (keyClassName == null) { keyClassName = NullWritable.class.getName(); }//from ww w .j ava 2s . com if (valueClassName == null) { valueClassName = Text.class.getName(); } if (keyTransformer == null) { keyTransformer = new EventTransformer<Object>() { @Override public Object transform(JetstreamEvent event) { return NullWritable.get(); } }; } if (valueTransformer == null) { valueTransformer = new EventTransformer<Object>() { JsonEventTransformer jsonTran = new JsonEventTransformer(); @Override public Object transform(JetstreamEvent event) { String json = jsonTran.transform(event); return new Text(json); } }; } if (compressionType == null) { compressionType = CompressionType.NONE.name(); } }
From source file:com.ebay.nest.io.sede.NullStructSerDe.java
License:Apache License
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { return NullWritable.get(); }
From source file:com.ebay.nest.io.sede.objectinspector.primitive.JavaVoidObjectInspector.java
License:Apache License
@Override public Object getPrimitiveWritableObject(Object o) { return NullWritable.get(); }
From source file:com.facebook.hive.orc.TestInputOutputFormat.java
License:Apache License
@Test public void testMROutput() throws Exception { JobConf job = new JobConf(conf); Properties properties = new Properties(); StructObjectInspector inspector;//from w w w . java 2 s.com synchronized (TestOrcFile.class) { inspector = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } SerDe serde = new OrcSerde(); OutputFormat<?, ?> outFormat = new OrcOutputFormat(); RecordWriter writer = outFormat.getRecordWriter(fs, conf, testFilePath.toString(), Reporter.NULL); writer.write(NullWritable.get(), serde.serialize(new NestedRow(1, 2, 3), inspector)); writer.write(NullWritable.get(), serde.serialize(new NestedRow(4, 5, 6), inspector)); writer.write(NullWritable.get(), serde.serialize(new NestedRow(7, 8, 9), inspector)); writer.close(Reporter.NULL); serde = new OrcSerde(); properties.setProperty("columns", "z,r"); properties.setProperty("columns.types", "int:struct<x:int,y:int>"); serde.initialize(conf, properties); inspector = (StructObjectInspector) serde.getObjectInspector(); InputFormat<?, ?> in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); InputSplit[] splits = in.getSplits(conf, 1); assertEquals(1, splits.length); conf.set("hive.io.file.readcolumn.ids", "1"); org.apache.hadoop.mapred.RecordReader reader = in.getRecordReader(splits[0], conf, Reporter.NULL); Object key = reader.createKey(); Object value = reader.createValue(); int rowNum = 0; List<? extends StructField> fields = inspector.getAllStructFieldRefs(); StructObjectInspector inner = (StructObjectInspector) fields.get(1).getFieldObjectInspector(); List<? extends StructField> inFields = inner.getAllStructFieldRefs(); IntObjectInspector intInspector = (IntObjectInspector) inFields.get(0).getFieldObjectInspector(); while (reader.next(key, value)) { assertEquals(null, inspector.getStructFieldData(value, fields.get(0))); Object sub = inspector.getStructFieldData(value, fields.get(1)); assertEquals(3 * rowNum + 1, intInspector.get(inner.getStructFieldData(sub, inFields.get(0)))); assertEquals(3 * rowNum + 2, intInspector.get(inner.getStructFieldData(sub, inFields.get(1)))); rowNum += 1; } assertEquals(3, rowNum); reader.close(); }
From source file:com.facebook.hive.orc.TestInputOutputFormat.java
License:Apache License
@Test public void testMROutput2() throws Exception { JobConf job = new JobConf(conf); // Test that you can set the output directory using this config job.set("mapred.work.output.dir", testFilePath.getParent().toString()); Properties properties = new Properties(); StructObjectInspector inspector;//ww w .j a va 2 s .c o m synchronized (TestOrcFile.class) { inspector = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(StringRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } SerDe serde = new OrcSerde(); OutputFormat<?, ?> outFormat = new OrcOutputFormat(); RecordWriter writer = outFormat.getRecordWriter(fs, job, testFilePath.getName(), Reporter.NULL); writer.write(NullWritable.get(), serde.serialize(new StringRow("a"), inspector)); writer.close(Reporter.NULL); serde = new OrcSerde(); properties.setProperty("columns", "col"); properties.setProperty("columns.types", "string"); serde.initialize(conf, properties); inspector = (StructObjectInspector) serde.getObjectInspector(); InputFormat<?, ?> in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); InputSplit[] splits = in.getSplits(conf, 1); assertEquals(1, splits.length); org.apache.hadoop.mapred.RecordReader reader = in.getRecordReader(splits[0], conf, Reporter.NULL); Object key = reader.createKey(); Object value = reader.createValue(); int rowNum = 0; List<? extends StructField> fields = inspector.getAllStructFieldRefs(); reader.next(key, value); assertEquals("a", ((StringObjectInspector) fields.get(0).getFieldObjectInspector()) .getPrimitiveJavaObject(inspector.getStructFieldData(value, fields.get(0)))); reader.close(); }
From source file:com.facebook.hive.orc.TestInputOutputFormat.java
License:Apache License
/** * Tests that passing null as the file system to getRecordWriter works, this is * to be compatible with the way Sequence and RC file tolerate nulls. * @throws Exception/* www.j av a2 s.com*/ */ @Test public void testNullFileSystem() throws Exception { conf.set("mapred.work.output.dir", testFilePath.getParent().toString()); JobConf job = new JobConf(conf); Properties properties = new Properties(); StructObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(StringRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } OrcSerde serde = new OrcSerde(); OrcOutputFormat outFormat = new OrcOutputFormat(); RecordWriter<NullWritable, OrcSerdeRow> writer = outFormat.getRecordWriter(null, conf, testFilePath.getName(), Reporter.NULL); writer.write(NullWritable.get(), (OrcSerdeRow) serde.serialize(new StringRow("a"), inspector)); writer.write(NullWritable.get(), (OrcSerdeRow) serde.serialize(new StringRow("b"), inspector)); writer.write(NullWritable.get(), (OrcSerdeRow) serde.serialize(new StringRow("c"), inspector)); writer.close(Reporter.NULL); serde = new OrcSerde(); properties.setProperty("columns", "str,str2"); serde.initialize(conf, properties); inspector = (StructObjectInspector) serde.getObjectInspector(); OrcInputFormat in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); InputSplit[] splits = in.getSplits(conf, 1); assertEquals(1, splits.length); // read the whole file org.apache.hadoop.mapred.RecordReader<NullWritable, OrcLazyRow> reader = in.getRecordReader(splits[0], conf, Reporter.NULL); NullWritable key = reader.createKey(); OrcLazyRow value = (OrcLazyRow) reader.createValue(); List<? extends StructField> fields = inspector.getAllStructFieldRefs(); StringObjectInspector strInspector = (StringObjectInspector) fields.get(0).getFieldObjectInspector(); assertEquals(true, reader.next(key, value)); assertEquals("a", strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value, fields.get(0)))); assertEquals(true, reader.next(key, value)); assertEquals("b", strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value, fields.get(0)))); assertEquals(true, reader.next(key, value)); assertEquals("c", strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value, fields.get(0)))); assertEquals(false, reader.next(key, value)); reader.close(); }
From source file:com.facebook.hiveio.output.HiveOutput.java
License:Apache License
/** * Write records to a Hive table//from ww w . j ava 2s . com * * @param outputDesc description of Hive table * @param records the records to write * @throws TException * @throws IOException * @throws InterruptedException */ public static void writeTable(HiveOutputDescription outputDesc, Iterable<HiveWritableRecord> records) throws TException, IOException, InterruptedException { long uniqueId = System.nanoTime(); String taskAttemptIdStr = "attempt_200707121733_" + (int) uniqueId + "_m_000005_0"; String profile = Long.toString(uniqueId); HiveConf conf = new HiveConf(HiveOutput.class); conf.setInt("mapred.task.partition", 1); conf.set("mapred.task.id", taskAttemptIdStr); HiveApiOutputFormat.initProfile(conf, outputDesc, profile); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(profile); TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptIdStr); TaskAttemptContext taskContext = new HackTaskAttemptContext(new JobConf(conf), taskAttemptID); JobContext jobContext = new HackJobContext(new JobConf(conf), taskAttemptID.getJobID()); RecordWriterImpl recordWriter = outputFormat.getRecordWriter(taskContext); HiveApiOutputCommitter committer = outputFormat.getOutputCommitter(taskContext); committer.setupJob(jobContext); committer.setupTask(taskContext); for (HiveWritableRecord record : records) { recordWriter.write(NullWritable.get(), record); } recordWriter.close(taskContext); committer.commitTask(taskContext); committer.commitJob(jobContext); }
From source file:com.facebook.hiveio.output.OutputCmd.java
License:Apache License
/** * Write output/*w ww . j av a 2s . c o m*/ * * @param context Context * @throws Exception */ public void write(Context context) throws Exception { PerThread threadLocal = context.perThread.get(); HiveApiOutputCommitter outputCommitter = context.outputFormat.getOutputCommitter(threadLocal.taskContext()); outputCommitter.setupTask(threadLocal.taskContext()); RecordWriter<WritableComparable, HiveWritableRecord> recordWriter = context.outputFormat .getRecordWriter(threadLocal.taskContext()); HiveWritableRecord record = HiveRecordFactory.newWritableRecord(context.schema); // TODO: allow type promotions: see https://github.com/facebook/hive-io-experimental/issues/15 record.set(0, 11L); record.set(1, 22.22); record.set(2, true); record.set(3, "foo"); recordWriter.write(NullWritable.get(), record); record.set(0, 33L); record.set(1, 44.44); record.set(2, false); record.set(3, "bar"); recordWriter.write(NullWritable.get(), record); recordWriter.close(threadLocal.taskContext()); if (outputCommitter.needsTaskCommit(threadLocal.taskContext())) { outputCommitter.commitTask(threadLocal.taskContext()); } }
From source file:com.facebook.hiveio.output.RecordWriterImpl.java
License:Apache License
/** * Write serialized data to Hive/* w w w . j av a2s . c o m*/ * * @param key Key * @param value Value * @param serializedValue Serialized value * @throws IOException I/O errors */ private void write(WritableComparable key, HiveWritableRecord value, Writable serializedValue) throws IOException { observer.beginWrite(key, value); baseWriter.write(NullWritable.get(), serializedValue); observer.endWrite(key, value); }
From source file:com.google.appengine.tools.mapreduce.AppEngineMapperTest.java
License:Apache License
/** * Just makes sure that things don't explode in the no mutation pool * case./*from w w w .j a va2s . c o m*/ */ public void testNoMutationPool() throws Exception { mapper.setup(context); mapper.taskSetup(context); mapper.map(NullWritable.get(), NullWritable.get(), context); mapper.taskCleanup(context); mapper.cleanup(context); }