List of usage examples for org.apache.hadoop.io ByteWritable ByteWritable
public ByteWritable(byte value)
From source file:org.apache.giraph.examples.StarMain.java
License:Apache License
/** * Apply the rule "ExtendStar" to a given match. * @param vertex The base vertex.//from w ww. j a v a2s.c o m * @param match The match object. * @param matchIndex Match index. * @return true if the rule was applied. * @throws IOException On I/O errors. */ protected boolean applyExtendStar(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match, long matchIndex) throws IOException { VertexId cur1 = match.getVertexId(1); LOG.info("Vertex " + vertex.getId() + " applying rule ExtendStar with match " + match); VertexId new0 = deriveVertexId(vertex.getId(), (int) matchIndex, 0); addVertexRequest(new0, new ByteWritable(TYPE_VERTEX)); VertexId src0 = cur1; VertexId trg0 = new0; Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_VERTEX_LEFT)); addEdgeRequest(src0, edge0); return true; }
From source file:org.apache.giraph.examples.TwoTimesTwo.java
License:Apache License
/** * Apply the rule "TwoTimesTwo" to a given match. * @param vertex The base vertex./*from w w w . ja va2 s .co m*/ * @param match The match object. * @param matchIndex Match index. * @return true if the rule was applied. * @throws IOException On I/O errors. */ protected boolean applyTwoTimesTwo(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match, long matchIndex) throws IOException { VertexId cur0 = match.getVertexId(0); VertexId cur1 = match.getVertexId(1); LOG.info("Vertex " + vertex.getId() + " applying rule TwoTimesTwo with match " + match); VertexId new0 = VertexId.randomVertexId(); addVertexRequest(new0, new ByteWritable(TYPE_VERTEX_CONTAINER)); VertexId src0 = new0; VertexId trg0 = cur0; Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_VERTEX_CONTAINER_VERTICES)); addEdgeRequest(src0, edge0); VertexId src1 = new0; VertexId trg1 = cur1; Edge<VertexId, ByteWritable> edge1 = EdgeFactory.create(trg1, new ByteWritable(TYPE_VERTEX_CONTAINER_VERTICES)); addEdgeRequest(src1, edge1); return true; }
From source file:org.apache.giraph.examples.WheelMain.java
License:Apache License
/** * Apply the rule "Wheel" to a given match. * @param vertex The base vertex.// ww w .ja v a 2s. c o m * @param match The match object. * @param matchIndex Match index. * @return true if the rule was applied. * @throws IOException On I/O errors. */ protected boolean applyWheel(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match, long matchIndex) throws IOException { VertexId cur0 = match.getVertexId(0); VertexId cur1 = match.getVertexId(1); VertexId cur2 = match.getVertexId(2); VertexId cur3 = match.getVertexId(3); LOG.info("Vertex " + vertex.getId() + " applying rule Wheel with match " + match); removeEdgesRequest(cur0, cur1); removeEdgesRequest(cur3, cur1); removeEdgesRequest(cur1, cur2); removeVertexRequest(cur1); VertexId src0 = cur0; VertexId trg0 = cur2; Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_VERTEX_CONTAINER_VERTICES)); addEdgeRequest(src0, edge0); VertexId src1 = cur3; VertexId trg1 = cur2; Edge<VertexId, ByteWritable> edge1 = EdgeFactory.create(trg1, new ByteWritable(TYPE_VERTEX_RIGHT)); addEdgeRequest(src1, edge1); return true; }
From source file:org.apache.giraph.types.ops.ByteTypeOps.java
License:Apache License
@Override public ByteWritable createCopy(ByteWritable from) { return new ByteWritable(from.get()); }
From source file:org.apache.hawq.pxf.plugins.hdfs.utilities.RecordkeyAdapter.java
License:Apache License
private ValConverter initializeConverter(Object key) { if (key instanceof Integer) { return new ValConverter() { @Override/*from w w w . jav a2 s. c o m*/ public Writable get(Object key) { return (new IntWritable((Integer) key)); } }; } else if (key instanceof Byte) { return new ValConverter() { @Override public Writable get(Object key) { return (new ByteWritable((Byte) key)); } }; } else if (key instanceof Boolean) { return new ValConverter() { @Override public Writable get(Object key) { return (new BooleanWritable((Boolean) key)); } }; } else if (key instanceof Double) { return new ValConverter() { @Override public Writable get(Object key) { return (new DoubleWritable((Double) key)); } }; } else if (key instanceof Float) { return new ValConverter() { @Override public Writable get(Object key) { return (new FloatWritable((Float) key)); } }; } else if (key instanceof Long) { return new ValConverter() { @Override public Writable get(Object key) { return (new LongWritable((Long) key)); } }; } else if (key instanceof String) { return new ValConverter() { @Override public Writable get(Object key) { return (new Text((String) key)); } }; } else { return new ValConverter() { @Override public Writable get(Object key) { throw new UnsupportedOperationException( "Unsupported recordkey data type " + key.getClass().getName()); } }; } }
From source file:org.apache.hawq.pxf.plugins.hdfs.utilities.RecordkeyAdapterTest.java
License:Apache License
/** * Test convertKeyValue for Byte type//from ww w .ja v a 2 s.co m */ @Test public void convertKeyValueByte() { byte key = 1; initRecordkeyAdapter(); runConvertKeyValue(key, new ByteWritable(key)); }
From source file:org.apache.orc.mapred.TestOrcOutputFormat.java
License:Apache License
@Test public void testAllTypes() throws Exception { conf.set("mapreduce.task.attempt.id", "attempt_20160101_0001_m_000001_0"); conf.setOutputCommitter(NullOutputCommitter.class); final String typeStr = "struct<b1:binary,b2:boolean,b3:tinyint," + "c:char(10),d1:date,d2:decimal(20,5),d3:double,fff:float,int:int," + "l:array<bigint>,map:map<smallint,string>," + "str:struct<u:uniontype<timestamp,varchar(100)>>,ts:timestamp>"; OrcConf.MAPRED_OUTPUT_SCHEMA.setString(conf, typeStr); FileOutputFormat.setOutputPath(conf, workDir); TypeDescription type = TypeDescription.fromString(typeStr); // build a row object OrcStruct row = (OrcStruct) OrcStruct.createValue(type); ((BytesWritable) row.getFieldValue(0)).set(new byte[] { 1, 2, 3, 4 }, 0, 4); ((BooleanWritable) row.getFieldValue(1)).set(true); ((ByteWritable) row.getFieldValue(2)).set((byte) 23); ((Text) row.getFieldValue(3)).set("aaabbbcccddd"); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); ((DateWritable) row.getFieldValue(4)).set(DateWritable.millisToDays(format.parse("2016-04-01").getTime())); ((HiveDecimalWritable) row.getFieldValue(5)).set(new HiveDecimalWritable("1.23")); ((DoubleWritable) row.getFieldValue(6)).set(1.5); ((FloatWritable) row.getFieldValue(7)).set(4.5f); ((IntWritable) row.getFieldValue(8)).set(31415); OrcList<LongWritable> longList = (OrcList<LongWritable>) row.getFieldValue(9); longList.add(new LongWritable(123)); longList.add(new LongWritable(456)); OrcMap<ShortWritable, Text> map = (OrcMap<ShortWritable, Text>) row.getFieldValue(10); map.put(new ShortWritable((short) 1000), new Text("aaaa")); map.put(new ShortWritable((short) 123), new Text("bbbb")); OrcStruct struct = (OrcStruct) row.getFieldValue(11); OrcUnion union = (OrcUnion) struct.getFieldValue(0); union.set((byte) 1, new Text("abcde")); ((OrcTimestamp) row.getFieldValue(12)).set("1996-12-11 15:00:00"); NullWritable nada = NullWritable.get(); RecordWriter<NullWritable, OrcStruct> writer = new OrcOutputFormat<OrcStruct>().getRecordWriter(fs, conf, "all.orc", Reporter.NULL); for (int r = 0; r < 10; ++r) { row.setFieldValue(8, new IntWritable(r * 10)); writer.write(nada, row);/*w w w .ja v a 2 s. c o m*/ } union.set((byte) 0, new OrcTimestamp("2011-12-25 12:34:56")); for (int r = 0; r < 10; ++r) { row.setFieldValue(8, new IntWritable(r * 10 + 100)); writer.write(nada, row); } OrcStruct row2 = new OrcStruct(type); writer.write(nada, row2); row.setFieldValue(8, new IntWritable(210)); writer.write(nada, row); writer.close(Reporter.NULL); FileSplit split = new FileSplit(new Path(workDir, "all.orc"), 0, 100000, new String[0]); RecordReader<NullWritable, OrcStruct> reader = new OrcInputFormat<OrcStruct>().getRecordReader(split, conf, Reporter.NULL); nada = reader.createKey(); row = reader.createValue(); for (int r = 0; r < 22; ++r) { assertEquals(true, reader.next(nada, row)); if (r == 20) { for (int c = 0; c < 12; ++c) { assertEquals(null, row.getFieldValue(c)); } } else { assertEquals(new BytesWritable(new byte[] { 1, 2, 3, 4 }), row.getFieldValue(0)); assertEquals(new BooleanWritable(true), row.getFieldValue(1)); assertEquals(new ByteWritable((byte) 23), row.getFieldValue(2)); assertEquals(new Text("aaabbbcccd"), row.getFieldValue(3)); assertEquals(new DateWritable(DateWritable.millisToDays(format.parse("2016-04-01").getTime())), row.getFieldValue(4)); assertEquals(new HiveDecimalWritable("1.23"), row.getFieldValue(5)); assertEquals(new DoubleWritable(1.5), row.getFieldValue(6)); assertEquals(new FloatWritable(4.5f), row.getFieldValue(7)); assertEquals(new IntWritable(r * 10), row.getFieldValue(8)); assertEquals(longList, row.getFieldValue(9)); assertEquals(map, row.getFieldValue(10)); if (r < 10) { union.set((byte) 1, new Text("abcde")); } else { union.set((byte) 0, new OrcTimestamp("2011-12-25 12:34:56")); } assertEquals("row " + r, struct, row.getFieldValue(11)); assertEquals("row " + r, new OrcTimestamp("1996-12-11 15:00:00"), row.getFieldValue(12)); } } assertEquals(false, reader.next(nada, row)); }
From source file:org.apache.phoenix.hive.objectinspector.PhoenixByteObjectInspector.java
License:Apache License
@Override public ByteWritable getPrimitiveWritableObject(Object o) { return new ByteWritable(get(o)); }
From source file:org.apache.sysml.runtime.transform.GTFMTDReducer.java
License:Apache License
@SuppressWarnings({ "unchecked", "deprecation" })
private long generateOffsetsFile(ArrayList<OffsetCount> list) throws IllegalArgumentException, IOException {
Collections.sort(list);/*from ww w . j a v a 2 s .c o m*/
SequenceFile.Writer writer = null;
long lineOffset = 0;
try {
writer = new SequenceFile.Writer(FileSystem.get(_rJob), _rJob,
new Path(_agents.getOffsetFile() + "/part-00000"), ByteWritable.class, OffsetCount.class);
for (OffsetCount oc : list) {
long count = oc.count;
oc.count = lineOffset;
writer.append(new ByteWritable((byte) 0), oc);
lineOffset += count;
}
} finally {
IOUtilFunctions.closeSilently(writer);
}
list.clear();
return lineOffset;
}
From source file:org.apache.tajo.plan.util.WritableTypeConverter.java
License:Apache License
public static Writable convertDatum2Writable(Datum value) { switch (value.kind()) { case INT1:/*w w w. j a v a 2s .c o m*/ return new ByteWritable(value.asByte()); case INT2: return new ShortWritable(value.asInt2()); case INT4: return new IntWritable(value.asInt4()); case INT8: return new LongWritable(value.asInt8()); case FLOAT4: return new FloatWritable(value.asFloat4()); case FLOAT8: return new DoubleWritable(value.asFloat8()); // NOTE: value should be DateDatum case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE); // NOTE: value should be TimestampDatum case TIMESTAMP: TimestampWritable result = new TimestampWritable(); result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8())); return result; case CHAR: { String str = value.asChars(); return new HiveCharWritable(new HiveChar(str, str.length())); } case TEXT: return new Text(value.asChars()); case VARBINARY: return new BytesWritable(value.asByteArray()); case NULL_TYPE: return null; } throw new TajoRuntimeException(new NotImplementedException(TypeStringEncoder.encode(value.type()))); }