List of usage examples for org.apache.hadoop.io WritableUtils getVIntSize
public static int getVIntSize(long i)
From source file:cn.iie.haiep.hbase.value.Bytes.java
License:Apache License
/** * @param vint Integer to make a vint of. * @return Vint as bytes array./* w w w. j a v a2 s. com*/ */ public static byte[] vintToBytes(final long vint) { long i = vint; int size = WritableUtils.getVIntSize(i); byte[] result = new byte[size]; int offset = 0; if (i >= -112 && i <= 127) { result[offset] = (byte) i; return result; } int len = -112; if (i < 0) { i ^= -1L; // take one's complement' len = -120; } long tmp = i; while (tmp != 0) { tmp = tmp >> 8; len--; } result[offset++] = (byte) len; len = (len < -120) ? -(len + 120) : -(len + 112); for (int idx = len; idx != 0; idx--) { int shiftbits = (idx - 1) * 8; long mask = 0xFFL << shiftbits; result[offset++] = (byte) ((i & mask) >> shiftbits); } return result; }
From source file:com.chinamobile.bcbsp.util.Bytes.java
License:Apache License
/** * Converts a long to a byte array./* w ww . ja v a2 s. co m*/ * * @param vint * Integer to make a vint of. * @return Vint as bytes array. */ public static byte[] vintToBytes(final long vint) { long i = vint; int size = WritableUtils.getVIntSize(i); byte[] result = new byte[size]; int offset = 0; if (i >= -112 && i <= 127) { result[offset] = (byte) i; return result; } int len = -112; if (i < 0) { i ^= -1L; // take one's complement' len = -120; } long tmp = i; while (tmp != 0) { tmp = tmp >> 8; len--; } result[offset++] = (byte) len; len = (len < -120) ? -(len + 120) : -(len + 112); for (int idx = len; idx != 0; idx--) { int shiftbits = (idx - 1) * 8; long mask = 0xFFL << shiftbits; result[offset++] = (byte) ((i & mask) >> shiftbits); } return result; }
From source file:com.koda.integ.hbase.test.TestUtils.java
License:Open Source License
public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv, final boolean includeMvccVersion) { // keep pushing the limit out. assume enough capacity bb.limit(bb.position() + kv.getLength()); bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength()); // TODO tags/*from w ww .j a v a 2 s . c om*/ if (includeMvccVersion) { // TODO: 0.98 - comatibility int numMvccVersionBytes = WritableUtils.getVIntSize(0/*kv.getMemstoreTS()*/); bb.limit(bb.limit() + numMvccVersionBytes); ByteBufferUtils.writeVLong(bb, 0/*kv.getMemstoreTS()*/); } }
From source file:org.apache.gora.util.ByteUtils.java
License:Apache License
/** * @param vint Integer to make a vint of. * @return Vint as bytes array./*from w w w . j a va 2s . c o m*/ */ public static byte[] vintToBytes(final long vint) { long i = vint; int size = WritableUtils.getVIntSize(i); byte[] result = new byte[size]; int offset = 0; if (i >= -112 && i <= 127) { result[offset] = ((byte) i); return result; } int len = -112; if (i < 0) { i ^= -1L; // take one's complement' len = -120; } long tmp = i; while (tmp != 0) { tmp = tmp >> 8; len--; } result[offset++] = (byte) len; len = (len < -120) ? -(len + 120) : -(len + 112); for (int idx = len; idx != 0; idx--) { int shiftbits = (idx - 1) * 8; long mask = 0xFFL << shiftbits; result[offset++] = (byte) ((i & mask) >> shiftbits); } return result; }
From source file:org.apache.phoenix.cache.aggcache.SpillManager.java
License:Apache License
/** * Helper method to deserialize the key part from a serialized byte array * @param data//from w w w .java 2 s. co m * @return * @throws IOException */ static ImmutableBytesPtr getKey(byte[] data) throws IOException { DataInputStream input = null; try { input = new DataInputStream(new ByteArrayInputStream(data)); // key length int keyLength = WritableUtils.readVInt(input); int offset = WritableUtils.getVIntSize(keyLength); // key return new ImmutableBytesPtr(data, offset, keyLength); } finally { if (input != null) { input.close(); input = null; } } }
From source file:org.apache.phoenix.cache.aggcache.SpillManager.java
License:Apache License
private Aggregator[] getAggregators(byte[] data) throws IOException { DataInputStream input = null; try {// ww w .j a v a2 s .co m input = new DataInputStream(new ByteArrayInputStream(data)); // key length int keyLength = WritableUtils.readVInt(input); int vIntKeyLength = WritableUtils.getVIntSize(keyLength); ImmutableBytesPtr ptr = new ImmutableBytesPtr(data, vIntKeyLength, keyLength); // value length input.skip(keyLength); int valueLength = WritableUtils.readVInt(input); int vIntValLength = WritableUtils.getVIntSize(keyLength); KeyValue keyValue = KeyValueUtil.newKeyValue(ptr.get(), ptr.getOffset(), ptr.getLength(), QueryConstants.SINGLE_COLUMN_FAMILY, QueryConstants.SINGLE_COLUMN, QueryConstants.AGG_TIMESTAMP, data, vIntKeyLength + keyLength + vIntValLength, valueLength); Tuple result = new SingleKeyValueTuple(keyValue); TupleUtil.getAggregateValue(result, ptr); KeyValueSchema schema = aggregators.getValueSchema(); ValueBitSet tempValueSet = ValueBitSet.newInstance(schema); tempValueSet.clear(); tempValueSet.or(ptr); int i = 0, maxOffset = ptr.getOffset() + ptr.getLength(); SingleAggregateFunction[] funcArray = aggregators.getFunctions(); Aggregator[] sAggs = new Aggregator[funcArray.length]; Boolean hasValue; schema.iterator(ptr); while ((hasValue = schema.next(ptr, i, maxOffset, tempValueSet)) != null) { SingleAggregateFunction func = funcArray[i]; sAggs[i++] = hasValue ? func.newServerAggregator(conf, ptr) : func.newServerAggregator(conf); } return sAggs; } finally { Closeables.closeQuietly(input); } }
From source file:org.apache.phoenix.index.IndexMaintainer.java
License:Apache License
/** * For client-side to append serialized IndexMaintainers of keyValueIndexes * @param dataTable data table/*w w w . ja v a 2s. co m*/ * @param indexMetaDataPtr bytes pointer to hold returned serialized value * @param keyValueIndexes indexes to serialize */ public static void serializeAdditional(PTable table, ImmutableBytesWritable indexMetaDataPtr, List<PTable> keyValueIndexes, PhoenixConnection connection) { int nMutableIndexes = indexMetaDataPtr.getLength() == 0 ? 0 : ByteUtil.vintFromBytes(indexMetaDataPtr); int nIndexes = nMutableIndexes + keyValueIndexes.size(); int estimatedSize = indexMetaDataPtr.getLength() + 1; // Just in case new size increases buffer if (indexMetaDataPtr.getLength() == 0) { estimatedSize += table.getRowKeySchema().getEstimatedByteSize(); } for (PTable index : keyValueIndexes) { estimatedSize += index.getIndexMaintainer(table, connection).getEstimatedByteSize(); } TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedSize + 1); DataOutput output = new DataOutputStream(stream); try { // Encode data table salting in sign of number of indexes WritableUtils.writeVInt(output, nIndexes * (table.getBucketNum() == null ? 1 : -1)); // Serialize current mutable indexes, subtracting the vint size from the length // as its still included if (indexMetaDataPtr.getLength() > 0) { output.write(indexMetaDataPtr.get(), indexMetaDataPtr.getOffset(), indexMetaDataPtr.getLength() - WritableUtils.getVIntSize(nMutableIndexes)); } else { table.getRowKeySchema().write(output); } // Serialize mutable indexes afterwards for (PTable index : keyValueIndexes) { IndexMaintainer maintainer = index.getIndexMaintainer(table, connection); byte[] protoBytes = IndexMaintainer.toProto(maintainer).toByteArray(); WritableUtils.writeVInt(output, protoBytes.length); output.write(protoBytes); } } catch (IOException e) { throw new RuntimeException(e); // Impossible } indexMetaDataPtr.set(stream.getBuffer(), 0, stream.size()); }
From source file:org.apache.phoenix.index.IndexMaintainer.java
License:Apache License
public int getEstimatedByteSize() { int size = WritableUtils.getVIntSize(nIndexSaltBuckets); size += WritableUtils.getVIntSize(estimatedIndexRowKeyBytes); size += WritableUtils.getVIntSize(indexedColumns.size()); size += viewIndexId == null ? 0 : viewIndexId.length; for (ColumnReference ref : indexedColumns) { size += WritableUtils.getVIntSize(ref.getFamily().length); size += ref.getFamily().length;/*from w w w .j a va2 s .co m*/ size += WritableUtils.getVIntSize(ref.getQualifier().length); size += ref.getQualifier().length; } for (int i = 0; i < indexedColumnTypes.size(); i++) { PDataType type = indexedColumnTypes.get(i); size += WritableUtils.getVIntSize(type.ordinal()); } Set<ColumnReference> dataTableColRefs = coveredColumnsMap.keySet(); size += WritableUtils.getVIntSize(dataTableColRefs.size()); for (ColumnReference ref : dataTableColRefs) { size += WritableUtils.getVIntSize(ref.getFamilyWritable().getSize()); size += ref.getFamily().length; size += WritableUtils.getVIntSize(ref.getQualifierWritable().getSize()); size += ref.getQualifier().length; } size += indexTableName.length + WritableUtils.getVIntSize(indexTableName.length); size += rowKeyMetaData.getByteSize(); size += dataEmptyKeyValueCF.length + WritableUtils.getVIntSize(dataEmptyKeyValueCF.length); size += emptyKeyValueCFPtr.getLength() + WritableUtils.getVIntSize(emptyKeyValueCFPtr.getLength()); size += WritableUtils.getVIntSize(nDataCFs + 1); size += WritableUtils.getVIntSize(indexedExpressions.size()); for (Expression expression : indexedExpressions) { size += WritableUtils.getVIntSize(ExpressionType.valueOf(expression).ordinal()); } size += estimatedExpressionSize; return size; }
From source file:org.apache.phoenix.schema.KeyValueSchema.java
License:Apache License
private int getVarLengthBytes(int length) { return length + WritableUtils.getVIntSize(length); }
From source file:org.apache.phoenix.schema.ValueSchema.java
License:Apache License
private void init(int minNullable, List<Field> fields, boolean rowKeyOrderOptimizable) { this.rowKeyOrderOptimizable = rowKeyOrderOptimizable; this.minNullable = minNullable; this.fields = ImmutableList.copyOf(fields); int estimatedLength = 0; boolean isMaxLength = true, isFixedLength = true; int positions = 0; for (Field field : fields) { int fieldEstLength = 0; PDataType type = field.getDataType(); if (type != null) { Integer byteSize = type.getByteSize(); if (type.isFixedWidth()) { fieldEstLength += field.getByteSize(); } else { isFixedLength = false;/* w w w. ja v a 2 s.c om*/ // Account for vint for length if not fixed if (byteSize == null) { isMaxLength = false; fieldEstLength += ESTIMATED_VARIABLE_LENGTH_SIZE; } else { fieldEstLength += WritableUtils.getVIntSize(byteSize); fieldEstLength = byteSize; } } } positions += field.getCount(); estimatedLength += fieldEstLength * field.getCount(); } fieldIndexByPosition = new int[positions]; for (int i = 0, j = 0; i < fields.size(); i++) { Field field = fields.get(i); Arrays.fill(fieldIndexByPosition, j, j + field.getCount(), i); j += field.getCount(); } this.isFixedLength = isFixedLength; this.isMaxLength = isMaxLength; this.estimatedLength = estimatedLength; }