List of usage examples for org.apache.hadoop.io WritableUtils writeVLong
public static void writeVLong(DataOutput stream, long i) throws IOException
From source file:io.gzet.community.accumulo.EdgeWritable.java
License:Apache License
public void write(DataOutput dataOutput) throws IOException { WritableUtils.writeString(dataOutput, sourceVertex); WritableUtils.writeString(dataOutput, destVertex); WritableUtils.writeVLong(dataOutput, count); }
From source file:io.prestosql.rcfile.TestRcFileDecoderUtils.java
License:Apache License
private static Slice writeVintOld(SliceOutput output, long value) throws IOException { output.reset();// w w w . j av a2 s. c om WritableUtils.writeVLong(output, value); Slice vLongOld = Slices.copyOf(output.slice()); output.reset(); RcFileDecoderUtils.writeVLong(output, value); Slice vLongNew = Slices.copyOf(output.slice()); assertEquals(vLongNew, vLongOld); if (value == (int) value) { output.reset(); WritableUtils.writeVInt(output, (int) value); Slice vIntOld = Slices.copyOf(output.slice()); assertEquals(vIntOld, vLongOld); output.reset(); RcFileDecoderUtils.writeVInt(output, (int) value); Slice vIntNew = Slices.copyOf(output.slice()); assertEquals(vIntNew, vLongOld); } return vLongOld; }
From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java
License:Apache License
protected void writeObject(Writable obj, DataOutputStream stream) throws IOException { // For Text and BytesWritable, encode them directly, so that they end up // in C++ as the natural translations. System.err.println("obj: " + obj); DataOutputBuffer buffer = new DataOutputBuffer(); if (obj instanceof Text) { Text t = (Text) obj; int len = t.getLength(); WritableUtils.writeVLong(stream, len); stream.flush();//from w ww. java 2 s . c o m stream.write(t.getBytes(), 0, len); stream.flush(); System.err.println("len: " + len); } else if (obj instanceof BytesWritable) { BytesWritable b = (BytesWritable) obj; int len = b.getLength(); WritableUtils.writeVLong(stream, len); stream.write(b.getBytes(), 0, len); System.err.println("len: " + len); } else { buffer.reset(); obj.write(buffer); int length = buffer.getLength(); WritableUtils.writeVInt(stream, length); stream.write(buffer.getData(), 0, length); System.err.println("len: " + length); } stream.flush(); }
From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationStub.java
License:Apache License
public void binaryProtocolStub() { try {/* w ww . ja v a 2 s . co m*/ System.err.println("binaryProtocolStub()"); initSoket(); System.err.println("done with socket()"); // output code WritableUtils.writeVInt(dataOut, 50); IntWritable wt = new IntWritable(); wt.set(123); writeObject(wt, dataOut); writeObject(new Text("value"), dataOut); // PARTITIONED_OUTPUT WritableUtils.writeVInt(dataOut, 51); WritableUtils.writeVInt(dataOut, 0); writeObject(wt, dataOut); writeObject(new Text("value"), dataOut); // STATUS WritableUtils.writeVInt(dataOut, 52); Text.writeString(dataOut, "PROGRESS"); dataOut.flush(); // progress WritableUtils.writeVInt(dataOut, 53); dataOut.writeFloat(0.55f); // register counter WritableUtils.writeVInt(dataOut, 55); // id WritableUtils.writeVInt(dataOut, 0); Text.writeString(dataOut, "group"); Text.writeString(dataOut, "name"); // increment counter WritableUtils.writeVInt(dataOut, 56); WritableUtils.writeVInt(dataOut, 0); WritableUtils.writeVLong(dataOut, 2); // map item int intValue = WritableUtils.readVInt(dataInput); System.out.println("intValue:" + intValue); IntWritable iw = new IntWritable(); readObject(iw, dataInput); System.out.println("key:" + iw.get()); Text txt = new Text(); readObject(txt, dataInput); System.out.println("value:" + txt.toString()); // done // end of session WritableUtils.writeVInt(dataOut, 54); System.out.println("finish"); dataOut.flush(); dataOut.close(); } catch (Exception x) { x.printStackTrace(); } finally { closeSoket(); } }
From source file:it.crs4.seal.recab.ObservationCount.java
License:Open Source License
public void write(DataOutput out) throws IOException { WritableUtils.writeVLong(out, observations); WritableUtils.writeVLong(out, mismatches); }
From source file:ivory.core.data.index.PostingsListDocSortedNonPositional.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { if (rawBytes != null) { // this would happen if we're reading in an already-encoded // postings; if that's the case, simply write out the byte array WritableUtils.writeVInt(out, postingsAdded); WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); // df WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); // cf WritableUtils.writeVInt(out, rawBytes.length); out.write(rawBytes);/*from w w w . j a v a 2s.c o m*/ } else { try { bitOut.padAndFlush(); bitOut.close(); if (numPostings != postingsAdded) { throw new RuntimeException( "Error, number of postings added doesn't match number of expected postings. Expected " + numPostings + ", got " + postingsAdded); } WritableUtils.writeVInt(out, postingsAdded); WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); // df WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); // cf byte[] bytes = bytesOut.toByteArray(); WritableUtils.writeVInt(out, bytes.length); out.write(bytes); } catch (ArithmeticException e) { throw new RuntimeException("ArithmeticException caught \"" + e.getMessage() + "\": check to see if collection size or df is set properly."); } } }
From source file:ivory.core.data.index.PostingsListDocSortedPositional.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { if (rawBytes != null) { // This would happen if we're reading in an already-encoded postings; if that's the case, // simply write out the byte array. WritableUtils.writeVInt(out, postingsAdded); WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); WritableUtils.writeVInt(out, rawBytes.length); out.write(rawBytes);/*from w ww . ja v a 2 s . c om*/ } else { try { bitsOut.padAndFlush(); bitsOut.close(); if (numPostings != postingsAdded) { throw new RuntimeException( "Error: number of postings added doesn't match number of expected postings. Expected " + numPostings + ", got " + postingsAdded); } WritableUtils.writeVInt(out, postingsAdded); WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); byte[] bytes = bytesOut.toByteArray(); WritableUtils.writeVInt(out, bytes.length); out.write(bytes); } catch (ArithmeticException e) { throw new RuntimeException("ArithmeticException caught \"" + e.getMessage() + "\": check to see if collection size or df is set properly."); } } }
From source file:ivory.core.data.index.PostingsListDocSortedPositionalPForDelta.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, postingsAdded); WritableUtils.writeVInt(out, df == 0 ? postingsAdded : df); WritableUtils.writeVLong(out, cf == 0 ? sumOfPostingsScore : cf); out.writeInt(lastBlockSize);/*from w w w . j ava2s . co m*/ out.writeInt(docidCompressed.length); for (int i = 0; i < docidCompressed.length; i++) { out.writeInt(docidCompressed[i].length); for (int j = 0; j < docidCompressed[i].length; j++) { out.writeInt(docidCompressed[i][j]); } } for (int i = 0; i < tfCompressed.length; i++) { out.writeInt(tfCompressed[i].length); for (int j = 0; j < tfCompressed[i].length; j++) { out.writeInt(tfCompressed[i][j]); } } for (int i = 0; i < offsetCompressed.length; i++) { out.writeInt(offsetCompressed[i].length); for (int j = 0; j < offsetCompressed[i].length; j++) { out.writeInt(offsetCompressed[i][j]); } } out.writeInt(positionsLastBlockSize); out.writeInt(positionsCompressed.length); for (int i = 0; i < positionsCompressed.length; i++) { out.writeInt(positionsCompressed[i].length); for (int j = 0; j < positionsCompressed[i].length; j++) { out.writeInt(positionsCompressed[i][j]); } } }
From source file:ivory.data.PostingsListDocSortedPositional.java
License:Apache License
public void write(DataOutput out) throws IOException { if (mRawBytes != null) { // this would happen if we're reading in an already-encoded // postings; if that's the case, simply write out the byte array WritableUtils.writeVInt(out, mPostingsAdded); WritableUtils.writeVInt(out, mDf == 0 ? mPostingsAdded : mDf); // df WritableUtils.writeVLong(out, mCf == 0 ? mSumOfPostingsScore : mCf); // cf WritableUtils.writeVInt(out, mRawBytes.length); out.write(mRawBytes);/*from w w w .j a v a 2s . c o m*/ } else { try { mBitsOut.padAndFlush(); mBitsOut.close(); if (mNumPostings != mPostingsAdded) { throw new RuntimeException( "Error, number of postings added doesn't match number of expected postings. Expected " + mNumPostings + ", got " + mPostingsAdded); } WritableUtils.writeVInt(out, mPostingsAdded); WritableUtils.writeVInt(out, mDf == 0 ? mPostingsAdded : mDf); // df WritableUtils.writeVLong(out, mCf == 0 ? mSumOfPostingsScore : mCf); // cf byte[] bytes = mBytesOut.toByteArray(); WritableUtils.writeVInt(out, bytes.length); out.write(bytes); } catch (ArithmeticException e) { throw new RuntimeException("ArithmeticException caught \"" + e.getMessage() + "\": check to see if collection size or df is set properly."); } sLogger.info("writing postings: cf=" + mSumOfPostingsScore + ", df=" + mNumPostings); } }
From source file:org.apache.accumulo.core.data.Key.java
License:Apache License
@Override public void write(DataOutput out) throws IOException { int colFamilyOffset = row.length; int colQualifierOffset = colFamilyOffset + colFamily.length; int colVisibilityOffset = colQualifierOffset + colQualifier.length; int totalLen = colVisibilityOffset + colVisibility.length; WritableUtils.writeVInt(out, colFamilyOffset); WritableUtils.writeVInt(out, colQualifierOffset); WritableUtils.writeVInt(out, colVisibilityOffset); WritableUtils.writeVInt(out, totalLen); out.write(row);//from w w w.j a v a 2 s. co m out.write(colFamily); out.write(colQualifier); out.write(colVisibility); WritableUtils.writeVLong(out, timestamp); out.writeBoolean(deleted); }