List of usage examples for org.apache.hadoop.io BytesWritable getSize
@Deprecated public int getSize()
From source file:Relevance.java
License:Apache License
public static byte[] getBytes(BytesWritable bw) { byte[] ret = new byte[bw.getSize()]; System.arraycopy(bw.get(), 0, ret, 0, bw.getSize()); return ret;//from w w w . ja va2 s .c o m }
From source file:com.m6d.hive.protobuf.TestKVAsSeq.java
License:Apache License
@Ignore public void testRank() throws Exception { Path p = new Path(this.ROOT_DIR, "kv"); SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p, BytesWritable.class, BytesWritable.class); Person.Builder bbuild = Person.newBuilder(); Person ed = bbuild.setEmail("ed@email.com").setName("ed").setId(1).build(); Assert.assertEquals("ed", ed.getName()); BytesWritable key = new BytesWritable(); key.set("ed".getBytes(), 0, 2); BytesWritable value = new BytesWritable(); ByteArrayOutputStream s = new ByteArrayOutputStream(); ed.writeTo(s);//from w w w . jav a 2 s .c om Person g = Person.parseFrom(s.toByteArray()); int bsize = s.toByteArray().length; Assert.assertEquals(g.getName(), ed.getName()); value.set(s.toByteArray(), 0, s.size()); w.append(key, value); w.close(); //DataOutputBuffer itkey = new DataOutputBuffer(); //SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf()); //SequenceFile.ValueBytes v = r.createValueBytes(); //while (r.nextRaw(itkey, v) != -1){ // v.writeUncompressedBytes(itkey); // Person other = Person.parseFrom(itkey.getData()); // Assert.assertEquals(ed.getName(), other.getName()); //} SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf()); BytesWritable itKey = new BytesWritable(); BytesWritable itValue = new BytesWritable(); r.next(itKey, itValue); System.out.println(itValue); Assert.assertEquals(bsize, itValue.getLength()); Assert.assertEquals(bsize, itValue.getSize()); //Assert.assertEquals(bsize, itValue.getCapacity()); for (int i = 0; i < bsize; i++) { Assert.assertEquals(itValue.getBytes()[i], s.toByteArray()[i]); } //Assert.assertEquals(itValue.getBytes(), s.toByteArray()); byte[] whatIWant = new byte[itValue.getLength()]; System.arraycopy(itValue.getBytes(), 0, whatIWant, 0, itValue.getLength()); Person other = Person.parseFrom(whatIWant); Assert.assertEquals(ed.getName(), other.getName()); }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazySerDe.java
License:Apache License
/** * Deserialize a table record to a Lazy struct. */// w ww.j a v a 2 s. c o m @SuppressWarnings("deprecation") @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; if (b.getSize() == 0) { return null; } // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.get()); cachedLazyStruct.init(byteArrayRef.getData(), 0, b.getSize()); } else if (field instanceof Text) { Text t = (Text) field; if (t.getLength() == 0) { return null; } byteArrayRef.setData(t.getBytes()); cachedLazyStruct.init(byteArrayRef.getData(), 0, t.getLength()); } else { throw new SerDeException(getClass().toString() + ": expects either BytesWritable or Text object!"); } return cachedLazyStruct; }
From source file:edu.uci.ics.hivesterix.test.serde.SerDeTest.java
License:Apache License
/** * Test the LazySimpleSerDe class./*from www . ja va 2 s . co m*/ */ public void testLazySimpleSerDe() throws Throwable { try { // Create the SerDe LazySimpleSerDe serDe = new LazySimpleSerDe(); Configuration conf = new Configuration(); Properties tbl = createProperties(); serDe.initialize(conf, tbl); LazySerDe outputSerde = new LazySerDe(); outputSerde.initialize(conf, tbl); // Data String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\tqf"; byte[] bytes = s.getBytes(); Writable bytesWritable = new BytesWritable(bytes); // Test // deserializeAndSerialize(serDe, t, s, expectedFieldsData); Object row = serDe.deserialize(bytesWritable); // test my serde StructObjectInspector simpleInspector = (StructObjectInspector) serDe.getObjectInspector(); List<Object> fields = simpleInspector.getStructFieldsDataAsList(row); List<? extends StructField> fieldRefs = simpleInspector.getAllStructFieldRefs(); int i = 0; for (Object field : fields) { BytesWritable fieldWritable = (BytesWritable) outputSerde.serialize(field, fieldRefs.get(i).getFieldObjectInspector()); System.out.print(fieldWritable.getSize() + "|"); i++; } // Writable output = outputSerde.serialize(row, serDe // .getObjectInspector()); // System.out.println(output); // // Object row2 = outputSerde.deserialize(output); // Writable output2 = serDe.serialize(row2, outputSerde // .getObjectInspector()); // System.out.println(output2); // System.out.println(output); // deserializeAndSerialize(outputSerde, t, s, expectedFieldsData); } catch (Throwable e) { e.printStackTrace(); throw e; } }
From source file:es.pic.astro.hadoop.io.BinaryOutputFormat.java
License:Apache License
/** * create the final out file, and output row by row. After one row is * appended, a configured row separator is appended * * @param jc/*w ww.ja v a 2 s .co m*/ * the job configuration file * @param outPath * the final output file to be created * @param valueClass * the value class used for create * @param isCompressed * whether the content is compressed or not * @param tableProperties * the tableProperties of this file's corresponding table * @param progress * progress used for status report * @return the RecordWriter */ @Override public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { FileSystem fs = outPath.getFileSystem(jc); final OutputStream outStream = Utilities.createCompressedStream(jc, fs.create(outPath, progress), isCompressed); return new RecordWriter() { @Override public void write(Writable r) throws IOException { if (r instanceof Text) { Text tr = (Text) r; outStream.write(tr.getBytes(), 0, tr.getLength()); } else { // DynamicSerDe always writes out BytesWritable BytesWritable bw = (BytesWritable) r; outStream.write(bw.get(), 0, bw.getSize()); } } @Override public void close(boolean abort) throws IOException { outStream.close(); } }; }
From source file:org.altlaw.util.hadoop.SeqFilePrinter.java
License:Apache License
/** Runs the process. Keys are printed to standard output; * information about the sequence file is printed to standard * error. *//* ww w .j a v a2 s.c om*/ public void execute() throws Exception { Path path = new Path(inputFile); SequenceFile.Reader reader = new SequenceFile.Reader(setup.getLocalFileSystem(), path, setup.getConf()); try { System.out.println("Key type is " + reader.getKeyClassName()); System.out.println("Value type is " + reader.getValueClassName()); if (reader.isCompressed()) { System.err.println("Values are compressed."); if (reader.isBlockCompressed()) { System.err.println("Records are block-compressed."); } System.err.println("Compression type is " + reader.getCompressionCodec().getClass().getName()); } System.out.println(""); Writable key = (Writable) (reader.getKeyClass().newInstance()); Writable val = (Writable) (reader.getValueClass().newInstance()); String value; while (reader.next(key, val)) { System.out.println("============================================================"); System.out.println("KEY:\t" + key.toString()); if (val instanceof BytesWritable) { BytesWritable v = (BytesWritable) val; value = new String(v.get(), 0, v.getSize()); } else { value = val.toString(); } System.out.println("VALUE:\n" + value); } } finally { reader.close(); } }
From source file:org.altlaw.util.hadoop.SeqValueSizes.java
License:Apache License
/** Runs the process. Keys are printed to standard output; * information about the sequence file is printed to standard * error. *//*w w w. j a va 2s . co m*/ public void execute() throws Exception { Path path = new Path(inputFile); SequenceFile.Reader reader = new SequenceFile.Reader(setup.getLocalFileSystem(), path, setup.getConf()); try { System.err.println("Key type is " + reader.getKeyClassName()); System.err.println("Value type is " + reader.getValueClassName()); if (reader.isCompressed()) { System.err.println("Values are compressed."); if (reader.isBlockCompressed()) { System.err.println("Records are block-compressed."); } System.err.println("Compression type is " + reader.getCompressionCodec().getClass().getName()); } System.err.println(""); Writable key = (Writable) (reader.getKeyClass().newInstance()); BytesWritable value = new BytesWritable(); while (reader.next(key, value)) { System.out.print(key.toString()); System.out.print("\t"); System.out.println(value.getSize()); } } finally { reader.close(); } }
From source file:org.cloudata.core.tablet.backup.RestoreBinaryMap.java
License:Apache License
@Override public void map(BytesWritable key, BytesWritable value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { if (confErr != null) { throw confErr; }//from ww w. j a v a2 s . com ByteArrayInputStream bin = new ByteArrayInputStream(key.get(), 0, key.getSize()); DataInputStream in = new DataInputStream(bin); Row row = new Row(); row.readFields(in); uploader.put(row); count++; if (count % 10000 == 0) { LOG.info(count + " cells restored"); } in.close(); }
From source file:org.cloudata.core.tablet.backup.RestoreBinaryPartitionMap.java
License:Apache License
@Override public void map(BytesWritable key, BytesWritable value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { ByteArrayInputStream bin = new ByteArrayInputStream(key.get(), 0, key.getSize()); DataInputStream in = new DataInputStream(bin); Row row = new Row(); row.readFields(in);// w w w.j a v a 2 s . co m lastRow = row; in.close(); }
From source file:org.platform.modules.hadoop.format.output.CustomOutputFormat.java
License:Apache License
/** * create the final out file, and output row by row. After one row is * appended, a configured row separator is appended * //from w w w. ja v a2s . co m * @param jc * the job configuration file * @param outPath * the final output file to be created * @param valueClass * the value class used for create * @param isCompressed * whether the content is compressed or not * @param tableProperties * the tableProperties of this file's corresponding table * @param progress * progress used for status report * @return the RecordWriter */ @Override public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { int rowSeparator = 0; String rowSeparatorString = tableProperties.getProperty(serdeConstants.LINE_DELIM, "\n"); try { rowSeparator = Byte.parseByte(rowSeparatorString); } catch (NumberFormatException e) { rowSeparator = rowSeparatorString.charAt(0); } final int finalRowSeparator = rowSeparator; FileSystem fs = outPath.getFileSystem(jc); final OutputStream outStream = Utilities.createCompressedStream(jc, fs.create(outPath), isCompressed); return new RecordWriter() { @SuppressWarnings("deprecation") public void write(Writable r) throws IOException { if (r instanceof Text) { Text tr = (Text) r; String strReplace = tr.toString().toLowerCase().replace(":", "::"); Text txtReplace = new Text(); txtReplace.set(strReplace); outStream.write(txtReplace.getBytes(), 0, txtReplace.getLength()); // outStream.write(tr.getBytes(), 0, tr.getLength()); outStream.write(finalRowSeparator); } else { // DynamicSerDe always writes out BytesWritable BytesWritable bw = (BytesWritable) r; outStream.write(bw.get(), 0, bw.getSize()); outStream.write(finalRowSeparator); } } public void close(boolean abort) throws IOException { outStream.close(); } }; }