List of usage examples for org.apache.hadoop.io BytesWritable getLength
@Override public int getLength()
From source file:com.m6d.hive.protobuf.LongTest.java
License:Apache License
public void testWriteReadProto() throws Exception { Path p = new Path(this.ROOT_DIR, "reallybigfile2"); SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK); long startLoad = System.currentTimeMillis(); int toLoad = load; for (int i = 0; i < toLoad; i++) { Person.Builder bbuild = Person.newBuilder(); Person ed = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt()) .setHobby(Hobby.newBuilder().setName(randomString())).build(); Person bo = bbuild.setEmail(randomString()).setName(randomString()).setId(randomInt()) .setHobby(Hobby.newBuilder().setName(randomString())).build(); BytesWritable key = new BytesWritable(); BytesWritable value = new BytesWritable(); ByteArrayOutputStream s = new ByteArrayOutputStream(); ed.writeTo(s);/*w w w. j a v a 2 s .c o m*/ ByteArrayOutputStream t = new ByteArrayOutputStream(); bo.writeTo(t); key.set(s.toByteArray(), 0, s.size()); value.set(t.toByteArray(), 0, t.size()); w.append(key, value); } w.close(); long start = System.currentTimeMillis(); SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf()); BytesWritable readkey = new BytesWritable(); BytesWritable readval = new BytesWritable(); while (r.next(readkey, readval)) { byte[] c = new byte[readkey.getLength()]; System.arraycopy(readkey.getBytes(), 0, c, 0, readkey.getLength()); Person.parseFrom(c); byte[] d = new byte[readval.getLength()]; System.arraycopy(readval.getBytes(), 0, d, 0, readval.getLength()); Person.parseFrom(d); } long end = System.currentTimeMillis(); System.out.println("reading proto took" + (end - start)); r.close(); }
From source file:com.m6d.hive.protobuf.ProtobufDeserializer.java
License:Apache License
@Override public Object deserialize(Writable field) throws SerDeException { //if (!(field instanceof Pair)) { // throw new SerDeException("Writable was not an instance of Pair. It was " + field.getClass()); //}/* w w w. j av a 2s. c om*/ BytesWritable key = null; BytesWritable value = null; if (field instanceof Pair) { Pair p = (Pair) field; key = (BytesWritable) p.getKey(); value = (BytesWritable) p.getValue(); } else if (field instanceof BytesWritable) { parseFrom = null; value = (BytesWritable) field; } Message parsedResult = null; Message vparsedResult = null; try { if (parseFrom != null) { byte[] b = new byte[key.getLength()]; System.arraycopy(key.getBytes(), 0, b, 0, key.getLength()); parsedResult = (Message) parseFrom.invoke(null, b); } if (vparseFrom != null) { byte[] c = new byte[value.getLength()]; System.arraycopy(value.getBytes(), 0, c, 0, value.getLength()); vparsedResult = (Message) vparseFrom.invoke(null, c); } } catch (IllegalAccessException ex) { throw new SerDeException(ex.getMessage(), ex); } catch (IllegalArgumentException ex) { throw new SerDeException(ex.getMessage(), ex); } catch (InvocationTargetException ex) { throw new SerDeException(ex.getMessage(), ex); } row.clear(); keyRow.clear(); if (parseFrom != null) { try { this.matchProtoToRow(parsedResult, keyRow, keyOIs, keyColumnNames); } catch (Exception ex) { throw new SerDeException(ex); } row.add(keyRow); } else { row.add(null); } valueRow.clear(); if (vparseFrom != null) { try { this.matchProtoToRow(vparsedResult, valueRow, valueOIs, valueColumnNames); } catch (Exception ex) { throw new SerDeException(ex); } row.add(valueRow); } else { row.add(null); } return row; }
From source file:com.m6d.hive.protobuf.TestKVAsSeq.java
License:Apache License
@Ignore public void testRank() throws Exception { Path p = new Path(this.ROOT_DIR, "kv"); SequenceFile.Writer w = SequenceFile.createWriter(this.getFileSystem(), new Configuration(), p, BytesWritable.class, BytesWritable.class); Person.Builder bbuild = Person.newBuilder(); Person ed = bbuild.setEmail("ed@email.com").setName("ed").setId(1).build(); Assert.assertEquals("ed", ed.getName()); BytesWritable key = new BytesWritable(); key.set("ed".getBytes(), 0, 2); BytesWritable value = new BytesWritable(); ByteArrayOutputStream s = new ByteArrayOutputStream(); ed.writeTo(s);//from w ww.j av a 2s. c o m Person g = Person.parseFrom(s.toByteArray()); int bsize = s.toByteArray().length; Assert.assertEquals(g.getName(), ed.getName()); value.set(s.toByteArray(), 0, s.size()); w.append(key, value); w.close(); //DataOutputBuffer itkey = new DataOutputBuffer(); //SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf()); //SequenceFile.ValueBytes v = r.createValueBytes(); //while (r.nextRaw(itkey, v) != -1){ // v.writeUncompressedBytes(itkey); // Person other = Person.parseFrom(itkey.getData()); // Assert.assertEquals(ed.getName(), other.getName()); //} SequenceFile.Reader r = new SequenceFile.Reader(this.getFileSystem(), p, this.createJobConf()); BytesWritable itKey = new BytesWritable(); BytesWritable itValue = new BytesWritable(); r.next(itKey, itValue); System.out.println(itValue); Assert.assertEquals(bsize, itValue.getLength()); Assert.assertEquals(bsize, itValue.getSize()); //Assert.assertEquals(bsize, itValue.getCapacity()); for (int i = 0; i < bsize; i++) { Assert.assertEquals(itValue.getBytes()[i], s.toByteArray()[i]); } //Assert.assertEquals(itValue.getBytes(), s.toByteArray()); byte[] whatIWant = new byte[itValue.getLength()]; System.arraycopy(itValue.getBytes(), 0, whatIWant, 0, itValue.getLength()); Person other = Person.parseFrom(whatIWant); Assert.assertEquals(ed.getName(), other.getName()); }
From source file:com.marklogic.mapreduce.examples.BinaryReader.java
License:Apache License
@Override public void write(DocumentURI uri, BytesWritable content) throws IOException, InterruptedException { String pathStr = dir.getName() + uri.getUri(); Path path = new Path(pathStr); FileSystem fs = path.getFileSystem(conf); FSDataOutputStream out = fs.create(path, false); System.out.println("writing to: " + path); out.write(content.getBytes(), 0, content.getLength()); out.flush();//from ww w . j av a2s .co m out.close(); }
From source file:com.metamx.druid.indexer.SortableBytes.java
License:Open Source License
public static SortableBytes fromBytesWritable(BytesWritable bytes) { return fromBytes(bytes.getBytes(), 0, bytes.getLength()); }
From source file:com.pinterest.terrapin.hadoop.HFileRecordWriter.java
License:Apache License
@Override public void write(BytesWritable key, BytesWritable value) throws IOException, InterruptedException { // Mapreduce reuses the same Text objects and hence sometimes they have some // additional left over garbage at the end from previous keys. So we need to // retrieve the String objects which do not have the additional garbage. getBytes() // call does not work well. byte[] row = new byte[key.getLength()]; byte[] val = new byte[value.getLength()]; for (int i = 0; i < row.length; i++) { row[i] = key.getBytes()[i];//from www. j a va 2 s.c o m } for (int i = 0; i < val.length; i++) { val[i] = value.getBytes()[i]; } writer.append(new KeyValue(row, Bytes.toBytes("cf"), Bytes.toBytes(""), val)); }
From source file:com.rapleaf.hank.hadoop.KeyAndPartitionWritable.java
License:Apache License
public KeyAndPartitionWritable(Domain domain, BytesWritable key) { this.key = key; int partition = domain.getPartitioner().partition(ByteBuffer.wrap(key.getBytes(), 0, key.getLength()), domain.getNumParts());//w ww. ja v a 2 s . c om this.partition = new IntWritable(partition); }
From source file:com.rramos.bigdata.utils.GenericUDFSha2.java
License:Apache License
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { if (digest == null) { return null; }// w w w . j a v a 2 s . c o m digest.reset(); if (isStr) { Text n = GenericUDFParamUtils.getTextValue(arguments, 0, converters); if (n == null) { return null; } digest.update(n.getBytes(), 0, n.getLength()); } else { BytesWritable bWr = GenericUDFParamUtils.getBinaryValue(arguments, 0, converters); if (bWr == null) { return null; } digest.update(bWr.getBytes(), 0, bWr.getLength()); } byte[] resBin = digest.digest(); String resStr = Hex.encodeHexString(resBin); output.set(resStr); return output; }
From source file:com.scaleunlimited.classify.datum.ModelDatum.java
License:Apache License
public BaseModel getModel() throws Exception { String className = _tupleEntry.getString(MODEL_FN); BytesWritable modelData = (BytesWritable) (_tupleEntry.getObject(MODEL_DATA_FN)); DataInputBuffer dib = new DataInputBuffer(); dib.reset(modelData.getBytes(), modelData.getLength()); BaseModel model = (BaseModel) Class.forName(className).newInstance(); model.readFields(dib);//from ww w . j a v a 2 s . c om return model; }
From source file:com.shmsoft.dmass.main.Reduce.java
License:Apache License
protected void processMap(MapWritable value) throws IOException { Metadata allMetadata = getAllMetadata(value); Metadata standardMetadata = getStandardMetadata(allMetadata, outputFileCount); columnMetadata.addMetadata(standardMetadata); columnMetadata.addMetadata(allMetadata); if (!isMaster) { columnMetadata.addMetadataValue(DocumentMetadataKeys.MASTER_DUPLICATE, UPIFormat.format(outputFileCount)); }//w ww . j a v a 2 s .c om String originalFileName = new File(allMetadata.get(DocumentMetadataKeys.DOCUMENT_ORIGINAL_PATH)).getName(); // add the text to the text folder String documentText = allMetadata.get(DocumentMetadataKeys.DOCUMENT_TEXT); String textEntryName = ParameterProcessing.TEXT + "/" + UPIFormat.format(outputFileCount) + "_" + originalFileName + ".txt"; if (textEntryName != null) { zipFileWriter.addTextFile(textEntryName, documentText); } columnMetadata.addMetadataValue(DocumentMetadataKeys.LINK_TEXT, textEntryName); // add the native file to the native folder String nativeEntryName = ParameterProcessing.NATIVE + "/" + UPIFormat.format(outputFileCount) + "_" + originalFileName; BytesWritable bytesWritable = (BytesWritable) value.get(new Text(ParameterProcessing.NATIVE)); if (bytesWritable != null) { // some large exception files are not passed zipFileWriter.addBinaryFile(nativeEntryName, bytesWritable.getBytes(), bytesWritable.getLength()); History.appendToHistory(nativeEntryName); } columnMetadata.addMetadataValue(DocumentMetadataKeys.LINK_NATIVE, nativeEntryName); // add the pdf made from native to the PDF folder String pdfNativeEntryName = ParameterProcessing.PDF_FOLDER + "/" + UPIFormat.format(outputFileCount) + "_" + new File(allMetadata.get(DocumentMetadataKeys.DOCUMENT_ORIGINAL_PATH)).getName() + ".pdf"; BytesWritable pdfBytesWritable = (BytesWritable) value.get(new Text(ParameterProcessing.NATIVE_AS_PDF)); if (pdfBytesWritable != null) { zipFileWriter.addBinaryFile(pdfNativeEntryName, pdfBytesWritable.getBytes(), pdfBytesWritable.getLength()); History.appendToHistory(pdfNativeEntryName); } // add exception to the exception folder String exception = allMetadata.get(DocumentMetadataKeys.PROCESSING_EXCEPTION); if (exception != null) { String exceptionEntryName = "exception/" + UPIFormat.format(outputFileCount) + "_" + new File(allMetadata.get(DocumentMetadataKeys.DOCUMENT_ORIGINAL_PATH)).getName(); if (bytesWritable != null) { zipFileWriter.addBinaryFile(exceptionEntryName, bytesWritable.getBytes(), bytesWritable.getLength()); } columnMetadata.addMetadataValue(DocumentMetadataKeys.LINK_EXCEPTION, exceptionEntryName); } }