Example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer

List of usage examples for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. DataOutputBuffer
  6. DataOutputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer.

Prototype

public DataOutputBuffer()

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryBitPackingZigZarIntReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 0, inBuf.getLength());
    FlexibleEncoding.Parquet.DeltaBinaryPackingValuesReader reader = new FlexibleEncoding.Parquet.DeltaBinaryPackingValuesReader();
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();/*from w  w w .  ja  va  2s.  co  m*/
    reader.initFromPage(numPairs, byteBuf.array(), 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {

        int tmp = reader.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryBitPackingZigZarIntReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    FlexibleEncoding.Parquet.DeltaBinaryPackingValuesReader reader = new FlexibleEncoding.Parquet.DeltaBinaryPackingValuesReader();
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();/* ww  w  . j a v  a2s  .c o m*/
    reader.initFromPage(numPairs, byteBuf.array(), 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = reader.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryPackingStringReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    System.out.println("280    inBuf.length   " + inBuf.getLength());
    FlexibleEncoding.Parquet.DeltaByteArrayReader reader = new FlexibleEncoding.Parquet.DeltaByteArrayReader();
    DataOutputBuffer transfer = new DataOutputBuffer();
    //   transfer.write(inBuf.getData(), 12, inBuf.getLength()-12);
    transfer.write(inBuf.getData(), 0, inBuf.getLength());
    byte[] data = transfer.getData();
    System.out.println("286   byte [] data  " + data.length + "  numPairs  " + numPairs);
    inBuf.close();/*w w w. ja  va2s.c  o  m*/
    Binary[] bin = new Utils().readData(reader, data, numPairs);
    System.out.println("2998   Binary[] bin   " + bin.length);
    DataOutputBuffer decoding = new DataOutputBuffer();
    DataOutputBuffer offset = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    int dataoffset = 12;
    // int dataoffset=0 ;
    String str;
    for (int i = 0; i < numPairs; i++) {
        str = bin[i].toStringUsingUTF8();
        decoding.writeUTF(str);
        //      if(i<5){
        //        System.out.println("304  bin[i]  "+str+"  decoding    "+ decoding.size());
        //      }
        dataoffset = decoding.size();
        //  decoding.writeBytes(str);
        offset.writeInt(dataoffset);
    }

    //    System.out.println("315  offset.size() "+offset.getData().length+"  decoding.szie   "+decoding.size());
    //    System.out.println("316  dataoffet   "+dataoffset);
    //  System.out.println("number  of Pairs =  "+ceshi);
    decoding.write(offset.getData(), 0, offset.size());
    inBuf.close();
    offset.close();
    System.out.println("316   decoding   " + decoding.size() + "   " + decoding.getLength()
            + " decoding.getData()   " + decoding.getData().length);
    //   inBuf.reset(decoding.getData(), 0, decoding.size());
    System.out.println(" 280  " + inBuf.getLength() + "   " + inBuf.getData().length);
    return inBuf.getData();
}

From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryPackingStringReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    System.out.println("280    inBuf.length   " + inBuf.getLength());
    FlexibleEncoding.Parquet.DeltaByteArrayReader reader = new FlexibleEncoding.Parquet.DeltaByteArrayReader();
    DataOutputBuffer transfer = new DataOutputBuffer();
    transfer.write(inBuf.getData(), 12, inBuf.getLength() - 12);
    byte[] data = transfer.getData();
    System.out.println("286   byte [] data  " + data.length + "  numPairs  " + numPairs);
    inBuf.close();//from  w w w .  j  a va  2  s  . co  m
    Binary[] bin = new Utils().readData(reader, data, numPairs);
    System.out.println("2998   Binary[] bin   " + bin.length);
    ByteArrayOutputStream bos1 = new ByteArrayOutputStream();
    DataOutputStream dos1 = new DataOutputStream(bos1);
    ByteArrayOutputStream bos2 = new ByteArrayOutputStream();
    DataOutputStream dos2 = new DataOutputStream(bos2);
    //    DataOutputBuffer   decoding = new DataOutputBuffer();
    //    DataOutputBuffer   offset = new DataOutputBuffer();
    dos1.writeInt(decompressedSize);
    dos1.writeInt(numPairs);
    dos1.writeInt(startPos);
    int dataoffset = 12;
    String str;
    for (int i = 0; i < numPairs; i++) {
        str = bin[i].toStringUsingUTF8();
        dos1.writeUTF(str);
        dataoffset = dos1.size();
        dos2.writeInt(dataoffset);
    }
    System.out.println("315  offset.size() " + bos2.size() + "  decoding.szie   " + bos2.toByteArray().length);
    System.out.println("316  dataoffet   " + dataoffset);
    dos1.write(bos2.toByteArray(), 0, bos2.size());
    inBuf.close();
    System.out.println("316   bos1  " + bos1.toByteArray().length + "    " + bos1.size());
    byte[] bytes = bos1.toByteArray();
    dos2.close();
    bos2.close();
    bos1.close();
    dos1.close();
    return bytes;
}

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEByteReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 0, inBuf.getLength());
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//from w  ww  .j a va  2 s  . c o m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.BINARY);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        byte tmp = Byte.parseByte(cr.readBytes().toStringUsingUTF8());
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEByteReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//  w ww. j a v a2s.  c  o m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.BINARY);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        byte tmp = Byte.parseByte(cr.readBytes().toStringUsingUTF8());
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();

}

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEIntReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 0, inBuf.getLength());
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();/*  w w  w  . j a v a 2s .  c o m*/
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.INT32);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = cr.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();

    inBuf.close();
    return decoding.getData();

}

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEIntReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();/* w w  w.j a va 2s .c o m*/
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.INT32);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = cr.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

From source file:cn.ac.ncic.mastiff.io.coding.RedBlackTreeStringReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.Parquet.DeltaByteArrayReader reader = new FlexibleEncoding.Parquet.DeltaByteArrayReader();
    DataOutputBuffer transfer = new DataOutputBuffer();
    //   transfer.write(inBuf.getData(), 12, inBuf.getLength()-12);
    transfer.write(inBuf.getData(), 0, inBuf.getLength());
    byte[] data = transfer.getData();
    System.out.println("286   byte [] data  " + data.length + "  numPairs  " + numPairs);
    inBuf.close();/* ww w.j a  va2s .c  om*/
    Binary[] bin = new Utils().readData(reader, data, numPairs);
    System.out.println("2998   Binary[] bin   " + bin.length);
    // bb = ByteBuffer.wrap(page, 0, page.length);
    //  int  count=0 ;
    DataOutputBuffer decoding = new DataOutputBuffer();
    DataOutputBuffer offset = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    int dataoffset = 12;
    // int dataoffset=0 ;
    String str;
    for (int i = 0; i < numPairs; i++) {
        str = bin[i].toStringUsingUTF8();
        decoding.writeUTF(str);
        //      if(i<5){
        //        System.out.println("304  bin[i]  "+str+"  decoding    "+ decoding.size());
        //      }
        dataoffset = decoding.size();
        offset.writeInt(dataoffset);
    }
    System.out.println("315  offset.size() " + offset.size() + "  decoding.szie   " + decoding.size());
    System.out.println("316  dataoffet   " + dataoffset);
    //  System.out.println("number  of Pairs =  "+ceshi);
    decoding.write(offset.getData(), 0, offset.size());
    inBuf.close();
    offset.close();
    System.out.println("316   decoding   " + decoding.size() + "   " + decoding.getLength()
            + " decoding.getData()   " + decoding.getData().length);
    return decoding.getData();
}

From source file:cn.ac.ncic.mastiff.io.coding.RedBlackTreeStringReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    DataOutputBuffer transfer = new DataOutputBuffer();
    transfer.write(inBuf.getData(), 12, inBuf.getLength() - 12);
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(transfer.getData(), 0, transfer.getLength());
    int dictionarySize = dib.readInt();
    int length1 = dib.readInt();
    byte[] data = transfer.getData();
    transfer.close();//from   w w w.  ja  v  a2s  . c o m
    dib.reset(data, Integer.SIZE + Integer.SIZE, length1);
    FlexibleEncoding.ORC.StreamName name = new FlexibleEncoding.ORC.StreamName(0,
            OrcProto.Stream.Kind.DICTIONARY_DATA);
    ByteBuffer inBuf1 = ByteBuffer.allocate(length1);
    inBuf1.put(dib.getData(), 0, dib.getLength());
    inBuf1.flip();
    InStream in = InStream.create("test1", inBuf1, null, dictionarySize);
    if (in.available() > 0) {
        dictionaryBuffer = new DynamicByteArray(64, in.available());
        dictionaryBuffer.readAll(in);
        in.close();
        // read the lengths    google  proto buffer
        name = new StreamName(1, OrcProto.Stream.Kind.LENGTH);
        dib.reset(data, 4 + 4 + length1, 4);
        int length2 = dib.readInt();
        dib.reset(data, 4 + 4 + length1 + 4, length2);
        //  in = streams.get(name);
        ByteBuffer inBuf2 = ByteBuffer.allocate(length2);
        inBuf2.put(dib.getData(), 0, length2);
        inBuf2.flip();
        in = InStream.create("test2", inBuf2, null, dictionarySize);
        //    IntegerReader lenReader = createIntegerReader(encodings.get(columnId)
        //        .getKind(), in, false);
        IntegerReader lenReader = createIntegerReader(OrcProto.ColumnEncoding.Kind.DIRECT_V2, in, false);
        int offset = 0;
        dictionaryOffsets = new int[dictionarySize + 1];
        for (int i = 0; i < dictionarySize; ++i) {
            dictionaryOffsets[i] = offset;
            offset += (int) lenReader.next();
        }
        dictionaryOffsets[dictionarySize] = offset;
        in.close();
        name = new FlexibleEncoding.ORC.StreamName(2, OrcProto.Stream.Kind.DATA);
        dib.reset(data, 4 + 4 + length1 + 4 + length2, 4);
        int length3 = dib.readInt();
        dib.reset(data, 4 + 4 + length1 + 4 + length2 + 4, length3);
        ByteBuffer inBuf3 = ByteBuffer.allocate(length3);
        inBuf3.put(dib.getData(), 0, length3);
        inBuf3.flip();
        in = InStream.create("test3", inBuf3, null, dictionarySize);
        reader = createIntegerReader(OrcProto.ColumnEncoding.Kind.DIRECT_V2, in, false);
    }
    inBuf.close();
    DataOutputBuffer decoding = new DataOutputBuffer();
    DataOutputBuffer offsets = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    int dataoffset = 12;
    String str;
    for (int i = 0; i < numPairs; i++) {
        str = readEachValue(null);
        decoding.writeUTF(str);
        //      if(i<5){
        //        System.out.println("304  bin[i]  "+str+"  decoding    "+ decoding.size());
        //      }
        dataoffset = decoding.size();
        offsets.writeInt(dataoffset);
    }
    System.out.println("315  offset.size() " + offsets.size() + "  decoding.szie   " + decoding.size());
    System.out.println("316  dataoffet   " + dataoffset);
    decoding.write(offsets.getData(), 0, offsets.size());
    inBuf.close();
    offsets.close();
    dib.close();
    System.out.println("316   decoding   " + decoding.size() + decoding.getLength() + " decoding.getData()   "
            + decoding.getData().length);
    inBuf1.clear();
    return decoding.getData();
}