List of usage examples for org.apache.hadoop.io Text getBytes
@Override public byte[] getBytes()
From source file:edu.jhuapl.accumulo.proxy.ProxyTableOperations.java
License:Apache License
public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { Set<ByteBuffer> tsplits = new TreeSet<ByteBuffer>(); for (Text key : partitionKeys) { tsplits.add(ByteBuffer.wrap(key.getBytes())); }//from www.j a v a 2 s . c o m try { client.addSplits(token, tableName, tsplits); } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { throw ExceptionFactory.tableNotFoundException(tableName, tnfe); } catch (TException te) { throw ExceptionFactory.accumuloException(te); } }
From source file:edu.jhuapl.accumulo.proxy.ThiftHelperTest.java
License:Apache License
@Test public void testToByteBuffer() { ByteBuffer buf = ThriftHelper.toByteBuffer(null); Assert.assertNull(buf);//from w w w .j a va 2 s. c o m Text text = new Text("test test"); buf = ThriftHelper.toByteBuffer(text); Assert.assertArrayEquals(text.getBytes(), buf.array()); }
From source file:edu.jhuapl.accumulo.proxy.ThriftHelper.java
License:Apache License
public static ByteBuffer toByteBuffer(Text val) { if (val == null) { return null; } else {// w ww . j a va2 s. c o m return ByteBuffer.wrap(val.getBytes()); } }
From source file:edu.mit.ll.graphulo.pig.backend.GraphuloOneTableStorage.java
License:Apache License
@Override protected Tuple getTuple(Key key, Value value) throws IOException { SortedMap<Key, Value> rowKVs = WholeRowIterator.decodeRow(key, value); Tuple tuple = TupleFactory.getInstance().newTuple(columns.size() + 1); final Text cfHolder = new Text(); final Text cqHolder = new Text(); final Text row = key.getRow(); int tupleOffset = 0; tuple.set(tupleOffset, new DataByteArray(Text.decode(row.getBytes(), 0, row.getLength()))); for (Column column : this.columns) { tupleOffset++;// w w w . j a v a 2s . com switch (column.getType()) { case LITERAL: cfHolder.set(column.getColumnFamily()); if (null != column.getColumnQualifier()) { cqHolder.set(column.getColumnQualifier()); } else { cqHolder.set(EMPTY_TEXT); } // Get the key where our literal would exist (accounting for // "colf:colq" or "colf:" empty colq) Key literalStartKey = new Key(row, cfHolder, cqHolder); SortedMap<Key, Value> tailMap = rowKVs.tailMap(literalStartKey); // Find the element if (tailMap.isEmpty()) { tuple.set(tupleOffset, EMPTY_DATA_BYTE_ARRAY); } else { Key actualKey = tailMap.firstKey(); // Only place it in the tuple if it matches the user // request, avoid using a value from a // key with the wrong colqual if (0 == literalStartKey.compareTo(actualKey, PartialKey.ROW_COLFAM_COLQUAL)) { tuple.set(tupleOffset, new DataByteArray(tailMap.get(actualKey).get())); } else { // This row doesn't have the column we were looking for tuple.set(tupleOffset, EMPTY_DATA_BYTE_ARRAY); } } break; case COLFAM_PREFIX: cfHolder.set(column.getColumnFamily()); Range colfamPrefixRange = Range.prefix(row, cfHolder); Key colfamPrefixStartKey = new Key(row, cfHolder); SortedMap<Key, Value> cfTailMap = rowKVs.tailMap(colfamPrefixStartKey); // Find the element if (cfTailMap.isEmpty()) { tuple.set(tupleOffset, EMPTY_DATA_BYTE_ARRAY); } else { HashMap<String, DataByteArray> tupleMap = new HashMap<String, DataByteArray>(); // Build up a map for all the entries in this row that match // the colfam prefix for (Entry<Key, Value> entry : cfTailMap.entrySet()) { if (colfamPrefixRange.contains(entry.getKey())) { entry.getKey().getColumnFamily(cfHolder); entry.getKey().getColumnQualifier(cqHolder); DataByteArray val = new DataByteArray(entry.getValue().get()); // Avoid adding an extra ':' when colqual is empty if (0 == cqHolder.getLength()) { tupleMap.put(cfHolder.toString(), val); } else { tupleMap.put(cfHolder.toString() + COLON + cqHolder.toString(), val); } } else { break; } } if (!tupleMap.isEmpty()) { tuple.set(tupleOffset, tupleMap); } } break; case COLQUAL_PREFIX: cfHolder.set(column.getColumnFamily()); cqHolder.set(column.getColumnQualifier()); Range colqualPrefixRange = Range.prefix(row, cfHolder, cqHolder); Key colqualPrefixStartKey = new Key(row, cfHolder, cqHolder); SortedMap<Key, Value> cqTailMap = rowKVs.tailMap(colqualPrefixStartKey); if (cqTailMap.isEmpty()) { tuple.set(tupleOffset, EMPTY_DATA_BYTE_ARRAY); } else { HashMap<String, DataByteArray> tupleMap = new HashMap<String, DataByteArray>(); // Build up a map for all the entries in this row that match // the colqual prefix for (Entry<Key, Value> entry : cqTailMap.entrySet()) { if (colqualPrefixRange.contains(entry.getKey())) { entry.getKey().getColumnFamily(cfHolder); entry.getKey().getColumnQualifier(cqHolder); DataByteArray val = new DataByteArray(entry.getValue().get()); // Avoid the extra ':' on empty colqual if (0 == cqHolder.getLength()) { tupleMap.put(cfHolder.toString(), val); } else { tupleMap.put(cfHolder.toString() + COLON + cqHolder.toString(), val); } } else { break; } } if (!tupleMap.isEmpty()) { tuple.set(tupleOffset, tupleMap); } } break; default: break; } } return tuple; }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazySerDe.java
License:Apache License
/** * Deserialize a table record to a Lazy struct. *//* w w w .ja v a 2s . co m*/ @SuppressWarnings("deprecation") @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; if (b.getSize() == 0) { return null; } // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.get()); cachedLazyStruct.init(byteArrayRef.getData(), 0, b.getSize()); } else if (field instanceof Text) { Text t = (Text) field; if (t.getLength() == 0) { return null; } byteArrayRef.setData(t.getBytes()); cachedLazyStruct.init(byteArrayRef.getData(), 0, t.getLength()); } else { throw new SerDeException(getClass().toString() + ": expects either BytesWritable or Text object!"); } return cachedLazyStruct; }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazySerDe.java
License:Apache License
/** * A recursive function that serialize an object to a byte buffer based on * its object inspector./*from ww w . j a v a 2 s .co m*/ * * @param byteStream * the byte stream storing the serialization data * @param obj * the object to serialize * @param objInspector * the object inspector */ private void serialize(Output byteStream, Object obj, ObjectInspector objInspector) { // do nothing for null object if (null == obj) { return; } switch (objInspector.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector; switch (poi.getPrimitiveCategory()) { case VOID: { return; } case BOOLEAN: { boolean v = ((BooleanObjectInspector) poi).get(obj); byteStream.write((byte) (v ? 1 : 0)); return; } case BYTE: { ByteObjectInspector boi = (ByteObjectInspector) poi; byte v = boi.get(obj); byteStream.write(v); return; } case SHORT: { ShortObjectInspector spoi = (ShortObjectInspector) poi; short v = spoi.get(obj); byteStream.write((byte) (v >> 8)); byteStream.write((byte) (v)); return; } case INT: { IntObjectInspector ioi = (IntObjectInspector) poi; int v = ioi.get(obj); LazyUtils.writeVInt(byteStream, v); return; } case LONG: { LongObjectInspector loi = (LongObjectInspector) poi; long v = loi.get(obj); LazyUtils.writeVLong(byteStream, v); return; } case FLOAT: { FloatObjectInspector foi = (FloatObjectInspector) poi; int v = Float.floatToIntBits(foi.get(obj)); byteStream.write((byte) (v >> 24)); byteStream.write((byte) (v >> 16)); byteStream.write((byte) (v >> 8)); byteStream.write((byte) (v)); return; } case DOUBLE: { DoubleObjectInspector doi = (DoubleObjectInspector) poi; long v = Double.doubleToLongBits(doi.get(obj)); byteStream.write((byte) (v >> 56)); byteStream.write((byte) (v >> 48)); byteStream.write((byte) (v >> 40)); byteStream.write((byte) (v >> 32)); byteStream.write((byte) (v >> 24)); byteStream.write((byte) (v >> 16)); byteStream.write((byte) (v >> 8)); byteStream.write((byte) (v)); return; } case STRING: { StringObjectInspector soi = (StringObjectInspector) poi; Text t = soi.getPrimitiveWritableObject(obj); /* write byte size of the string which is a vint */ int length = t.getLength(); LazyUtils.writeVInt(byteStream, length); /* write string itself */ byte[] data = t.getBytes(); byteStream.write(data, 0, length); return; } default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); } } } case LIST: { ListObjectInspector loi = (ListObjectInspector) objInspector; ObjectInspector eoi = loi.getListElementObjectInspector(); // 1/ reserve spaces for the byte size of the list // which is a integer and takes four bytes int byteSizeStart = byteStream.getCount(); byteStream.write((byte) 0); byteStream.write((byte) 0); byteStream.write((byte) 0); byteStream.write((byte) 0); int listStart = byteStream.getCount(); // 2/ write the size of the list as a VInt int size = loi.getListLength(obj); LazyUtils.writeVInt(byteStream, size); // 3/ write the null bytes byte nullByte = 0; for (int eid = 0; eid < size; eid++) { // set the bit to 1 if an element is not null if (null != loi.getListElement(obj, eid)) { nullByte |= 1 << (eid % 8); } // store the byte every eight elements or // if this is the last element if (7 == eid % 8 || eid == size - 1) { byteStream.write(nullByte); nullByte = 0; } } // 4/ write element by element from the list for (int eid = 0; eid < size; eid++) { serialize(byteStream, loi.getListElement(obj, eid), eoi); } // 5/ update the list byte size int listEnd = byteStream.getCount(); int listSize = listEnd - listStart; byte[] bytes = byteStream.getData(); bytes[byteSizeStart] = (byte) (listSize >> 24); bytes[byteSizeStart + 1] = (byte) (listSize >> 16); bytes[byteSizeStart + 2] = (byte) (listSize >> 8); bytes[byteSizeStart + 3] = (byte) (listSize); return; } case MAP: { MapObjectInspector moi = (MapObjectInspector) objInspector; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); // 1/ reserve spaces for the byte size of the map // which is a integer and takes four bytes int byteSizeStart = byteStream.getCount(); byteStream.write((byte) 0); byteStream.write((byte) 0); byteStream.write((byte) 0); byteStream.write((byte) 0); int mapStart = byteStream.getCount(); // 2/ write the size of the map which is a VInt int size = map.size(); LazyUtils.writeVInt(byteStream, size); // 3/ write the null bytes int b = 0; byte nullByte = 0; for (Map.Entry<?, ?> entry : map.entrySet()) { // set the bit to 1 if a key is not null if (null != entry.getKey()) { nullByte |= 1 << (b % 8); } else if (!nullMapKey) { nullMapKey = true; LOG.warn("Null map key encountered! Ignoring similar problems."); } b++; // set the bit to 1 if a value is not null if (null != entry.getValue()) { nullByte |= 1 << (b % 8); } b++; // write the byte to stream every 4 key-value pairs // or if this is the last key-value pair if (0 == b % 8 || b == size * 2) { byteStream.write(nullByte); nullByte = 0; } } // 4/ write key-value pairs one by one for (Map.Entry<?, ?> entry : map.entrySet()) { serialize(byteStream, entry.getKey(), koi); serialize(byteStream, entry.getValue(), voi); } // 5/ update the byte size of the map int mapEnd = byteStream.getCount(); int mapSize = mapEnd - mapStart; byte[] bytes = byteStream.getData(); bytes[byteSizeStart] = (byte) (mapSize >> 24); bytes[byteSizeStart + 1] = (byte) (mapSize >> 16); bytes[byteSizeStart + 2] = (byte) (mapSize >> 8); bytes[byteSizeStart + 3] = (byte) (mapSize); return; } case STRUCT: { // 1/ reserve spaces for the byte size of the struct // which is a integer and takes four bytes int byteSizeStart = byteStream.getCount(); byteStream.write((byte) 0); byteStream.write((byte) 0); byteStream.write((byte) 0); byteStream.write((byte) 0); int structStart = byteStream.getCount(); // 2/ serialize the struct serializeStruct(byteStream, obj, (StructObjectInspector) objInspector); // 3/ update the byte size of the struct int structEnd = byteStream.getCount(); int structSize = structEnd - structStart; byte[] bytes = byteStream.getData(); bytes[byteSizeStart] = (byte) (structSize >> 24); bytes[byteSizeStart + 1] = (byte) (structSize >> 16); bytes[byteSizeStart + 2] = (byte) (structSize >> 8); bytes[byteSizeStart + 3] = (byte) (structSize); return; } default: { throw new RuntimeException("Unrecognized type: " + objInspector.getCategory()); } } }
From source file:edu.uci.ics.hyracks.hdfs.lib.TextKeyValueParserFactory.java
License:Apache License
@Override public IKeyValueParser<LongWritable, Text> createKeyValueParser(final IHyracksTaskContext ctx) throws HyracksDataException { final ArrayTupleBuilder tb = new ArrayTupleBuilder(1); final ByteBuffer buffer = ctx.allocateFrame(); final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize()); appender.reset(buffer, true);/*from w ww . jav a 2 s.c om*/ return new IKeyValueParser<LongWritable, Text>() { @Override public void open(IFrameWriter writer) { } @Override public void parse(LongWritable key, Text value, IFrameWriter writer, String fileString) throws HyracksDataException { tb.reset(); tb.addField(value.getBytes(), 0, value.getLength()); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { FrameUtils.flushFrame(buffer, writer); appender.reset(buffer, true); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new HyracksDataException("tuple cannot be appended into the frame"); } } } @Override public void close(IFrameWriter writer) throws HyracksDataException { FrameUtils.flushFrame(buffer, writer); } }; }
From source file:edu.uci.ics.hyracks.imru.dataflow.Hdtest.java
License:Apache License
public static JobSpecification createJob() throws Exception { JobSpecification spec = new JobSpecification(); spec.setFrameSize(4096);//from ww w. j a v a2 s. c o m String PATH_TO_HADOOP_CONF = "/home/wangrui/a/imru/hadoop-0.20.2/conf"; String HDFS_INPUT_PATH = "/customer/customer.tbl,/customer_result/part-0"; JobConf conf = new JobConf(); conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); FileInputFormat.setInputPaths(conf, HDFS_INPUT_PATH); conf.setInputFormat(TextInputFormat.class); RecordDescriptor recordDesc = new RecordDescriptor( new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() }); InputSplit[] splits = conf.getInputFormat().getSplits(conf, 1); HDFSReadOperatorDescriptor readOperator = new HDFSReadOperatorDescriptor(spec, recordDesc, conf, splits, new String[] { "NC0", "NC1" }, new IKeyValueParserFactory<LongWritable, Text>() { @Override public IKeyValueParser<LongWritable, Text> createKeyValueParser(final IHyracksTaskContext ctx) { return new IKeyValueParser<LongWritable, Text>() { TupleWriter tupleWriter; @Override public void open(IFrameWriter writer) throws HyracksDataException { tupleWriter = new TupleWriter(ctx, writer, 1); } @Override public void parse(LongWritable key, Text value, IFrameWriter writer, String fileString) throws HyracksDataException { try { tupleWriter.write(value.getBytes(), 0, value.getLength()); tupleWriter.finishField(); tupleWriter.finishTuple(); } catch (IOException e) { throw new HyracksDataException(e); } } @Override public void close(IFrameWriter writer) throws HyracksDataException { tupleWriter.close(); } }; } }); // createPartitionConstraint(spec, readOperator, new String[] {"NC0"}); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, readOperator, new String[] { "NC0", "NC1" }); IOperatorDescriptor writer = new HDFSOD(spec, null, null, null); // createPartitionConstraint(spec, writer, outSplits); spec.connect(new OneToOneConnectorDescriptor(spec), readOperator, 0, writer, 0); spec.addRoot(writer); return spec; }
From source file:edu.umn.cs.spatialHadoop.core.CSVOGC.java
License:Open Source License
@Override public void fromText(Text text) { byte[] bytes = text.getBytes(); int separatorsEncountered = 0; int i1 = 0;/*from w w w.j av a 2s. co m*/ // Locate the required column while (separatorsEncountered < column && i1 < text.getLength()) { if (bytes[i1++] == separator) separatorsEncountered++; } if (i1 == text.getLength()) { this.prefix = new byte[i1]; System.arraycopy(bytes, 0, prefix, 0, i1); super.geom = null; this.suffix = null; return; } int i2 = i1 + 1; while (i2 < text.getLength() && bytes[i2] != separator) i2++; // Copy prefix and suffix if (i1 == 0) { prefix = null; } else { prefix = new byte[i1]; System.arraycopy(bytes, 0, prefix, 0, i1); } if (i2 == text.getLength()) { suffix = null; } else { suffix = new byte[text.getLength() - i2]; System.arraycopy(bytes, i2, suffix, 0, text.getLength() - i2); } // Chop prefix and suffix and leave only the selected column text.set(bytes, i1, i2 - i1); super.fromText(text); }
From source file:edu.umn.cs.spatialHadoop.core.GridInfo.java
License:Open Source License
@Override public void fromText(Text text) { super.fromText(text); if (text.getLength() > 0) { // Remove the first comma text.set(text.getBytes(), 1, text.getLength() - 1); columns = (int) TextSerializerHelper.consumeInt(text, ','); rows = (int) TextSerializerHelper.consumeInt(text, '\0'); }/*w ww . ja va 2 s.co m*/ }