List of usage examples for org.apache.hadoop.io BytesWritable getLength
@Override public int getLength()
From source file:com.cloudera.seismic.segy.SegyUnloader.java
License:Open Source License
private void write(Path path, DataOutputStream out, Configuration conf) throws Exception { System.out.println("Reading: " + path); SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), path, conf); BytesWritable value = new BytesWritable(); while (reader.next(NullWritable.get(), value)) { out.write(value.getBytes(), 0, value.getLength()); }// w ww .j a v a2s. co m reader.close(); }
From source file:com.cloudera.sqoop.lib.BlobRef.java
License:Apache License
@Override protected InputStream getInternalSource(BytesWritable data) { return new ByteArrayInputStream(data.getBytes(), 0, data.getLength()); }
From source file:com.cloudera.sqoop.lib.BlobRef.java
License:Apache License
@Override protected byte[] getInternalData(BytesWritable data) { return Arrays.copyOf(data.getBytes(), data.getLength()); }
From source file:com.cloudera.sqoop.lib.BlobRef.java
License:Apache License
@Override protected BytesWritable deepCopyData(BytesWritable data) { return new BytesWritable(Arrays.copyOf(data.getBytes(), data.getLength())); }
From source file:com.dasasian.chok.lucene.integration.LuceneClientTest.java
License:Apache License
@Test public void testGetBinaryDetails() throws Exception { File index = temporaryFolder.newFolder("indexWithBinaryData"); File indexShard = new File(index, "binaryShard"); if (!indexShard.mkdirs()) { throw new RuntimeException("Unable to create directory " + indexShard.getAbsolutePath()); }/* w ww .ja va2 s. c o m*/ String textFieldName = "textField"; String binaryFieldName = "binaryField"; String textFieldContent = "sample text"; byte[] bytesFieldContent = new byte[] { 1, 2, 3 }; IndexWriter indexWriter = new IndexWriter(FSDirectory.open(indexShard), new StandardAnalyzer(Version.LUCENE_30), true, MaxFieldLength.UNLIMITED); Document document = new Document(); document.add(new Field(binaryFieldName, bytesFieldContent, Store.YES)); document.add(new Field(textFieldName, textFieldContent, Store.NO, Index.ANALYZED)); indexWriter.addDocument(document); indexWriter.close(true); DeployClient deployClient = new DeployClient(miniCluster.getProtocol()); IndexState indexState = deployClient.addIndex(index.getName(), index.getAbsolutePath(), 1).joinDeployment(); assertEquals(IndexState.DEPLOYED, indexState); LuceneClient client = new LuceneClient(miniCluster.createInteractionProtocol()); final Query query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()) .parse(textFieldName + ": " + textFieldContent); final Hits hits = client.search(query, new String[] { index.getName() }, 10); assertNotNull(hits); assertEquals(1, hits.getHits().size()); final Hit hit = hits.getHits().get(0); final MapWritable details = client.getDetails(hit); final Set<Writable> keySet = details.keySet(); assertEquals(1, keySet.size()); final Writable writable = details.get(new Text(binaryFieldName)); assertNotNull(writable); assertThat(writable, instanceOf(BytesWritable.class)); BytesWritable bytesWritable = (BytesWritable) writable; bytesWritable.setCapacity(bytesWritable.getLength());// getBytes() returns // the full array assertArrayEquals(bytesFieldContent, bytesWritable.getBytes()); client.close(); }
From source file:com.datasalt.utils.io.Serialization.java
License:Apache License
public <T> T deser(Object obj, BytesWritable writable) throws IOException { return (T) deser(obj, writable.getBytes(), 0, writable.getLength()); }
From source file:com.ebay.nest.io.sede.binarysortable.BinarySortableSerDe.java
License:Apache License
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inputByteBuffer.reset(data.getBytes(), 0, data.getLength()); try {//from w w w . j a v a 2 s .c o m for (int i = 0; i < columnNames.size(); i++) { row.set(i, deserialize(inputByteBuffer, columnTypes.get(i), columnSortOrderIsDesc[i], row.get(i))); } } catch (IOException e) { throw new SerDeException(e); } return row; }
From source file:com.ebay.nest.io.sede.binarysortable.BinarySortableSerDe.java
License:Apache License
static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) throws SerDeException { // Is this field a null? if (o == null) { buffer.write((byte) 0, invert); return;/* w ww. jav a 2 s . c om*/ } // This field is not a null. buffer.write((byte) 1, invert); switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; switch (poi.getPrimitiveCategory()) { case VOID: { return; } case BOOLEAN: { boolean v = ((BooleanObjectInspector) poi).get(o); buffer.write((byte) (v ? 2 : 1), invert); return; } case BYTE: { ByteObjectInspector boi = (ByteObjectInspector) poi; byte v = boi.get(o); buffer.write((byte) (v ^ 0x80), invert); return; } case SHORT: { ShortObjectInspector spoi = (ShortObjectInspector) poi; short v = spoi.get(o); buffer.write((byte) ((v >> 8) ^ 0x80), invert); buffer.write((byte) v, invert); return; } case INT: { IntObjectInspector ioi = (IntObjectInspector) poi; int v = ioi.get(o); serializeInt(buffer, v, invert); return; } case LONG: { LongObjectInspector loi = (LongObjectInspector) poi; long v = loi.get(o); buffer.write((byte) ((v >> 56) ^ 0x80), invert); buffer.write((byte) (v >> 48), invert); buffer.write((byte) (v >> 40), invert); buffer.write((byte) (v >> 32), invert); buffer.write((byte) (v >> 24), invert); buffer.write((byte) (v >> 16), invert); buffer.write((byte) (v >> 8), invert); buffer.write((byte) v, invert); return; } case FLOAT: { FloatObjectInspector foi = (FloatObjectInspector) poi; int v = Float.floatToIntBits(foi.get(o)); if ((v & (1 << 31)) != 0) { // negative number, flip all bits v = ~v; } else { // positive number, flip the first bit v = v ^ (1 << 31); } buffer.write((byte) (v >> 24), invert); buffer.write((byte) (v >> 16), invert); buffer.write((byte) (v >> 8), invert); buffer.write((byte) v, invert); return; } case DOUBLE: { DoubleObjectInspector doi = (DoubleObjectInspector) poi; long v = Double.doubleToLongBits(doi.get(o)); if ((v & (1L << 63)) != 0) { // negative number, flip all bits v = ~v; } else { // positive number, flip the first bit v = v ^ (1L << 63); } buffer.write((byte) (v >> 56), invert); buffer.write((byte) (v >> 48), invert); buffer.write((byte) (v >> 40), invert); buffer.write((byte) (v >> 32), invert); buffer.write((byte) (v >> 24), invert); buffer.write((byte) (v >> 16), invert); buffer.write((byte) (v >> 8), invert); buffer.write((byte) v, invert); return; } case STRING: { StringObjectInspector soi = (StringObjectInspector) poi; Text t = soi.getPrimitiveWritableObject(o); serializeBytes(buffer, t.getBytes(), t.getLength(), invert); return; } case VARCHAR: { HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) poi; HiveVarcharWritable hc = hcoi.getPrimitiveWritableObject(o); // use varchar's text field directly Text t = hc.getTextValue(); serializeBytes(buffer, t.getBytes(), t.getLength(), invert); return; } case BINARY: { BinaryObjectInspector baoi = (BinaryObjectInspector) poi; BytesWritable ba = baoi.getPrimitiveWritableObject(o); byte[] toSer = new byte[ba.getLength()]; System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength()); serializeBytes(buffer, toSer, ba.getLength(), invert); return; } case DATE: { DateObjectInspector doi = (DateObjectInspector) poi; int v = doi.getPrimitiveWritableObject(o).getDays(); serializeInt(buffer, v, invert); return; } case TIMESTAMP: { TimestampObjectInspector toi = (TimestampObjectInspector) poi; TimestampWritable t = toi.getPrimitiveWritableObject(o); byte[] data = t.getBinarySortable(); for (int i = 0; i < data.length; i++) { buffer.write(data[i], invert); } return; } case DECIMAL: { // decimals are encoded in three pieces: // sign: 1, 2 or 3 for smaller, equal or larger than 0 respectively // factor: Number that indicates the amount of digits you have to move // the decimal point left or right until the resulting number is smaller // than zero but has something other than 0 as the first digit. // digits: which is a string of all the digits in the decimal. If the number // is negative the binary string will be inverted to get the correct ordering. // Example: 0.00123 // Sign is 3 (bigger than 0) // Factor is -2 (move decimal point 2 positions right) // Digits are: 123 HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi; HiveDecimal dec = boi.getPrimitiveJavaObject(o); // get the sign of the big decimal int sign = dec.compareTo(HiveDecimal.ZERO); // we'll encode the absolute value (sign is separate) dec = dec.abs(); // get the scale factor to turn big decimal into a decimal < 1 int factor = dec.precision() - dec.scale(); factor = sign == 1 ? factor : -factor; // convert the absolute big decimal to string dec.scaleByPowerOfTen(Math.abs(dec.scale())); String digits = dec.unscaledValue().toString(); // finally write out the pieces (sign, scale, digits) buffer.write((byte) (sign + 1), invert); buffer.write((byte) ((factor >> 24) ^ 0x80), invert); buffer.write((byte) (factor >> 16), invert); buffer.write((byte) (factor >> 8), invert); buffer.write((byte) factor, invert); serializeBytes(buffer, digits.getBytes(decimalCharSet), digits.length(), sign == -1 ? !invert : invert); return; } default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); } } } case LIST: { ListObjectInspector loi = (ListObjectInspector) oi; ObjectInspector eoi = loi.getListElementObjectInspector(); // \1 followed by each element int size = loi.getListLength(o); for (int eid = 0; eid < size; eid++) { buffer.write((byte) 1, invert); serialize(buffer, loi.getListElement(o, eid), eoi, invert); } // and \0 to terminate buffer.write((byte) 0, invert); return; } case MAP: { MapObjectInspector moi = (MapObjectInspector) oi; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); // \1 followed by each key and then each value Map<?, ?> map = moi.getMap(o); for (Map.Entry<?, ?> entry : map.entrySet()) { buffer.write((byte) 1, invert); serialize(buffer, entry.getKey(), koi, invert); serialize(buffer, entry.getValue(), voi, invert); } // and \0 to terminate buffer.write((byte) 0, invert); return; } case STRUCT: { StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { serialize(buffer, soi.getStructFieldData(o, fields.get(i)), fields.get(i).getFieldObjectInspector(), invert); } return; } case UNION: { UnionObjectInspector uoi = (UnionObjectInspector) oi; byte tag = uoi.getTag(o); buffer.write(tag, invert); serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert); return; } default: { throw new RuntimeException("Unrecognized type: " + oi.getCategory()); } } }
From source file:com.ebay.nest.io.sede.ByteStreamTypedSerDe.java
License:Apache License
@Override public Object deserialize(Writable field) throws SerDeException { Object retObj = super.deserialize(field); BytesWritable b = (BytesWritable) field; bis.reset(b.getBytes(), b.getLength()); return (retObj); }
From source file:com.ebay.nest.io.sede.lazy.LazyBinary.java
License:Apache License
public LazyBinary(LazyBinary other) { super(other); BytesWritable incoming = other.getWritableObject(); byte[] bytes = new byte[incoming.getLength()]; System.arraycopy(incoming.getBytes(), 0, bytes, 0, incoming.getLength()); data = new BytesWritable(bytes); }