Example usage for org.apache.hadoop.io DataInputBuffer getData

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataInputBuffer getData.

Prototype

public byte[] getData() 

Source Link

Usage

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEByteReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 0, inBuf.getLength());
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//from www .jav  a2 s  . c  o  m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.BINARY);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        byte tmp = Byte.parseByte(cr.readBytes().toStringUsingUTF8());
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

---

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEByteReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//  ww  w  .j a va 2 s .  c  o  m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.BINARY);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        byte tmp = Byte.parseByte(cr.readBytes().toStringUsingUTF8());
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();

}

---

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEIntReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 0, inBuf.getLength());
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();/* w w w  .j  a  va  2s  .  c om*/
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.INT32);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = cr.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();

    inBuf.close();
    return decoding.getData();

}

---

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEIntReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//from www. j a v  a  2 s .co m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.INT32);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = cr.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

---

From source file:cn.ac.ncic.mastiff.io.coding.RedBlackTreeStringReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    DataOutputBuffer transfer = new DataOutputBuffer();
    transfer.write(inBuf.getData(), 12, inBuf.getLength() - 12);
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(transfer.getData(), 0, transfer.getLength());
    int dictionarySize = dib.readInt();
    int length1 = dib.readInt();
    byte[] data = transfer.getData();
    transfer.close();/*from   w  w  w .j  a v a2 s  .  co  m*/
    dib.reset(data, Integer.SIZE + Integer.SIZE, length1);
    FlexibleEncoding.ORC.StreamName name = new FlexibleEncoding.ORC.StreamName(0,
            OrcProto.Stream.Kind.DICTIONARY_DATA);
    ByteBuffer inBuf1 = ByteBuffer.allocate(length1);
    inBuf1.put(dib.getData(), 0, dib.getLength());
    inBuf1.flip();
    InStream in = InStream.create("test1", inBuf1, null, dictionarySize);
    if (in.available() > 0) {
        dictionaryBuffer = new DynamicByteArray(64, in.available());
        dictionaryBuffer.readAll(in);
        in.close();
        // read the lengths    google  proto buffer
        name = new StreamName(1, OrcProto.Stream.Kind.LENGTH);
        dib.reset(data, 4 + 4 + length1, 4);
        int length2 = dib.readInt();
        dib.reset(data, 4 + 4 + length1 + 4, length2);
        //  in = streams.get(name);
        ByteBuffer inBuf2 = ByteBuffer.allocate(length2);
        inBuf2.put(dib.getData(), 0, length2);
        inBuf2.flip();
        in = InStream.create("test2", inBuf2, null, dictionarySize);
        //    IntegerReader lenReader = createIntegerReader(encodings.get(columnId)
        //        .getKind(), in, false);
        IntegerReader lenReader = createIntegerReader(OrcProto.ColumnEncoding.Kind.DIRECT_V2, in, false);
        int offset = 0;
        dictionaryOffsets = new int[dictionarySize + 1];
        for (int i = 0; i < dictionarySize; ++i) {
            dictionaryOffsets[i] = offset;
            offset += (int) lenReader.next();
        }
        dictionaryOffsets[dictionarySize] = offset;
        in.close();
        name = new FlexibleEncoding.ORC.StreamName(2, OrcProto.Stream.Kind.DATA);
        dib.reset(data, 4 + 4 + length1 + 4 + length2, 4);
        int length3 = dib.readInt();
        dib.reset(data, 4 + 4 + length1 + 4 + length2 + 4, length3);
        ByteBuffer inBuf3 = ByteBuffer.allocate(length3);
        inBuf3.put(dib.getData(), 0, length3);
        inBuf3.flip();
        in = InStream.create("test3", inBuf3, null, dictionarySize);
        reader = createIntegerReader(OrcProto.ColumnEncoding.Kind.DIRECT_V2, in, false);
    }
    inBuf.close();
    DataOutputBuffer decoding = new DataOutputBuffer();
    DataOutputBuffer offsets = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    int dataoffset = 12;
    String str;
    for (int i = 0; i < numPairs; i++) {
        str = readEachValue(null);
        decoding.writeUTF(str);
        //      if(i<5){
        //        System.out.println("304  bin[i]  "+str+"  decoding    "+ decoding.size());
        //      }
        dataoffset = decoding.size();
        offsets.writeInt(dataoffset);
    }
    System.out.println("315  offset.size() " + offsets.size() + "  decoding.szie   " + decoding.size());
    System.out.println("316  dataoffet   " + dataoffset);
    decoding.write(offsets.getData(), 0, offsets.size());
    inBuf.close();
    offsets.close();
    dib.close();
    System.out.println("316   decoding   " + decoding.size() + decoding.getLength() + " decoding.getData()   "
            + decoding.getData().length);
    inBuf1.clear();
    return decoding.getData();
}

---

From source file:org.apache.tez.engine.common.combine.CombineInput.java

License:Apache License

private boolean nextKeyValue() throws IOException, InterruptedException {
    if (!hasMore) {
        key = null;/*from ww w. ja  v  a2 s  .  c  o  m*/
        value = null;
        return false;
    }
    firstValue = !nextKeyIsSame;
    DataInputBuffer nextKey = input.getKey();
    currentRawKey.set(nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition());
    buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength());
    key = keyDeserializer.deserialize(key);
    DataInputBuffer nextVal = input.getValue();
    buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength());
    value = valueDeserializer.deserialize(value);

    hasMore = input.next();
    if (hasMore) {
        nextKey = input.getKey();
        nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, currentRawKey.getLength(),
                nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition()) == 0;
    } else {
        nextKeyIsSame = false;
    }
    inputValueCounter.increment(1);
    return true;
}

---

From source file:org.apache.tez.engine.common.shuffle.impl.InMemoryWriter.java

License:Apache License

public void append(DataInputBuffer key, DataInputBuffer value) throws IOException {
    int keyLength = key.getLength() - key.getPosition();
    if (keyLength < 0) {
        throw new IOException("Negative key-length not allowed: " + keyLength + " for " + key);
    }/*w w w.  j  a v  a2  s  .c o m*/

    boolean sameKey = (key == IFile.REPEAT_KEY);

    int valueLength = value.getLength() - value.getPosition();
    if (valueLength < 0) {
        throw new IOException("Negative value-length not allowed: " + valueLength + " for " + value);
    }

    if (sameKey) {
        WritableUtils.writeVInt(out, IFile.RLE_MARKER);
        WritableUtils.writeVInt(out, valueLength);
        out.write(value.getData(), value.getPosition(), valueLength);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("InMemWriter.append" + " key.data=" + key.getData() + " key.pos=" + key.getPosition()
                    + " key.len=" + key.getLength() + " val.data=" + value.getData() + " val.pos="
                    + value.getPosition() + " val.len=" + value.getLength());
        }
        WritableUtils.writeVInt(out, keyLength);
        WritableUtils.writeVInt(out, valueLength);
        out.write(key.getData(), key.getPosition(), keyLength);
        out.write(value.getData(), value.getPosition(), valueLength);
    }

}

---

From source file:org.apache.tez.engine.common.task.impl.ValuesIterator.java

License:Apache License

/** 
 * read the next key /*w w  w  . j a  va2  s .c om*/
 */
private void readNextKey() throws IOException {
    more = in.next();
    if (more) {
        DataInputBuffer nextKeyBytes = in.getKey();
        keyIn.reset(nextKeyBytes.getData(), nextKeyBytes.getPosition(), nextKeyBytes.getLength());
        nextKey = keyDeserializer.deserialize(nextKey);
        hasNext = key != null && (comparator.compare(key, nextKey) == 0);
    } else {
        hasNext = false;
    }
}

---

From source file:org.apache.tez.engine.common.task.impl.ValuesIterator.java

License:Apache License

/**
 * Read the next value/*from   w  w w .  jav  a2  s .  com*/
 * @throws IOException
 */
private void readNextValue() throws IOException {
    DataInputBuffer nextValueBytes = in.getValue();
    valueIn.reset(nextValueBytes.getData(), nextValueBytes.getPosition(), nextValueBytes.getLength());
    value = valDeserializer.deserialize(value);
}

---

From source file:org.apache.tez.mapreduce.hadoop.mapreduce.ReduceContextImpl.java

License:Apache License

/**
 * Advance to the next key/value pair.//  w w  w . ja va2 s.c  om
 */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (!hasMore) {
        key = null;
        value = null;
        return false;
    }
    firstValue = !nextKeyIsSame;
    DataInputBuffer nextKey = input.getKey();
    currentRawKey.set(nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition());
    buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength());
    key = keyDeserializer.deserialize(key);
    DataInputBuffer nextVal = input.getValue();
    buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength());
    value = valueDeserializer.deserialize(value);

    currentKeyLength = nextKey.getLength() - nextKey.getPosition();
    currentValueLength = nextVal.getLength() - nextVal.getPosition();

    if (isMarked) {
        backupStore.write(nextKey, nextVal);
    }

    hasMore = input.next();
    if (hasMore) {
        nextKey = input.getKey();
        nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, currentRawKey.getLength(),
                nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition()) == 0;
    } else {
        nextKeyIsSame = false;
    }
    inputValueCounter.increment(1);
    return true;
}

---

• «
• 1
• 2
• »