Example usage for org.apache.hadoop.io DataInputBuffer DataInputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataInputBuffer DataInputBuffer.

Prototype

public DataInputBuffer() 

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEByteReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//www .j  a v  a 2  s. c o m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.BINARY);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        byte tmp = Byte.parseByte(cr.readBytes().toStringUsingUTF8());
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();

}

---

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEIntReader.java

License:Apache License

public byte[] CompressensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 0, inBuf.getLength());
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//from ww  w. ja  v  a  2s. c  o  m
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.INT32);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = cr.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();

    inBuf.close();
    return decoding.getData();

}

---

From source file:cn.ac.ncic.mastiff.io.coding.DictionaryBitPackingRLEIntReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    FlexibleEncoding.ORC.DynamicByteArray dynamicBuffer = new FlexibleEncoding.ORC.DynamicByteArray();
    dynamicBuffer.add(inBuf.getData(), 12, inBuf.getLength() - 12);
    ByteBuffer byteBuf = ByteBuffer.allocate(dynamicBuffer.size());
    dynamicBuffer.setByteBuffer(byteBuf, 0, dynamicBuffer.size());
    byteBuf.flip();//w w  w  . j  av  a 2  s  . c  om
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(byteBuf.array(), 0, byteBuf.array().length);
    int dictionarySize = dib.readInt();
    int OnlydictionarySize = dib.readInt();
    dib.reset(byteBuf.array(), 8, dictionarySize - 4);
    byte[] dictionaryBuffer = dib.getData();
    dib.reset(byteBuf.array(), 4 + dictionarySize, (byteBuf.array().length - dictionarySize - 4));
    byte[] dictionaryId = dib.getData();
    dib.close();
    DictionaryValuesReader cr = initDicReader(OnlydictionarySize, dictionaryBuffer,
            PrimitiveType.PrimitiveTypeName.INT32);
    cr.initFromPage(numPairs, dictionaryId, 0);
    DataOutputBuffer decoding = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    for (int i = 0; i < numPairs; i++) {
        int tmp = cr.readInteger();
        decoding.writeInt(tmp);
    }
    byteBuf.clear();
    inBuf.close();
    return decoding.getData();
}

---

From source file:cn.ac.ncic.mastiff.io.coding.RedBlackTreeStringReader.java

License:Apache License

@Override
public byte[] ensureDecompressed() throws IOException {
    DataOutputBuffer transfer = new DataOutputBuffer();
    transfer.write(inBuf.getData(), 12, inBuf.getLength() - 12);
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(transfer.getData(), 0, transfer.getLength());
    int dictionarySize = dib.readInt();
    int length1 = dib.readInt();
    byte[] data = transfer.getData();
    transfer.close();//from w w  w .  ja va2  s. com
    dib.reset(data, Integer.SIZE + Integer.SIZE, length1);
    FlexibleEncoding.ORC.StreamName name = new FlexibleEncoding.ORC.StreamName(0,
            OrcProto.Stream.Kind.DICTIONARY_DATA);
    ByteBuffer inBuf1 = ByteBuffer.allocate(length1);
    inBuf1.put(dib.getData(), 0, dib.getLength());
    inBuf1.flip();
    InStream in = InStream.create("test1", inBuf1, null, dictionarySize);
    if (in.available() > 0) {
        dictionaryBuffer = new DynamicByteArray(64, in.available());
        dictionaryBuffer.readAll(in);
        in.close();
        // read the lengths    google  proto buffer
        name = new StreamName(1, OrcProto.Stream.Kind.LENGTH);
        dib.reset(data, 4 + 4 + length1, 4);
        int length2 = dib.readInt();
        dib.reset(data, 4 + 4 + length1 + 4, length2);
        //  in = streams.get(name);
        ByteBuffer inBuf2 = ByteBuffer.allocate(length2);
        inBuf2.put(dib.getData(), 0, length2);
        inBuf2.flip();
        in = InStream.create("test2", inBuf2, null, dictionarySize);
        //    IntegerReader lenReader = createIntegerReader(encodings.get(columnId)
        //        .getKind(), in, false);
        IntegerReader lenReader = createIntegerReader(OrcProto.ColumnEncoding.Kind.DIRECT_V2, in, false);
        int offset = 0;
        dictionaryOffsets = new int[dictionarySize + 1];
        for (int i = 0; i < dictionarySize; ++i) {
            dictionaryOffsets[i] = offset;
            offset += (int) lenReader.next();
        }
        dictionaryOffsets[dictionarySize] = offset;
        in.close();
        name = new FlexibleEncoding.ORC.StreamName(2, OrcProto.Stream.Kind.DATA);
        dib.reset(data, 4 + 4 + length1 + 4 + length2, 4);
        int length3 = dib.readInt();
        dib.reset(data, 4 + 4 + length1 + 4 + length2 + 4, length3);
        ByteBuffer inBuf3 = ByteBuffer.allocate(length3);
        inBuf3.put(dib.getData(), 0, length3);
        inBuf3.flip();
        in = InStream.create("test3", inBuf3, null, dictionarySize);
        reader = createIntegerReader(OrcProto.ColumnEncoding.Kind.DIRECT_V2, in, false);
    }
    inBuf.close();
    DataOutputBuffer decoding = new DataOutputBuffer();
    DataOutputBuffer offsets = new DataOutputBuffer();
    decoding.writeInt(decompressedSize);
    decoding.writeInt(numPairs);
    decoding.writeInt(startPos);
    int dataoffset = 12;
    String str;
    for (int i = 0; i < numPairs; i++) {
        str = readEachValue(null);
        decoding.writeUTF(str);
        //      if(i<5){
        //        System.out.println("304  bin[i]  "+str+"  decoding    "+ decoding.size());
        //      }
        dataoffset = decoding.size();
        offsets.writeInt(dataoffset);
    }
    System.out.println("315  offset.size() " + offsets.size() + "  decoding.szie   " + decoding.size());
    System.out.println("316  dataoffet   " + dataoffset);
    decoding.write(offsets.getData(), 0, offsets.size());
    inBuf.close();
    offsets.close();
    dib.close();
    System.out.println("316   decoding   " + decoding.size() + decoding.getLength() + " decoding.getData()   "
            + decoding.getData().length);
    inBuf1.clear();
    return decoding.getData();
}

---

From source file:com.alibaba.wasp.EntityGroupInfo.java

License:Apache License

/**
 * Parses all the EntityGroupInfo instances from the passed in stream until
 * EOF. Presumes the EntityGroupInfo's were serialized to the stream with
 * {@link #toDelimitedByteArray()}/*from www  .ja  va 2 s  . c  o  m*/
 * 
 * @param bytes
 *          serialized bytes
 * @param offset
 *          the start offset into the byte[] buffer
 * @param length
 *          how far we should read into the byte[] buffer
 * @return All the entityGroupInfos that are in the byte array. Keeps reading
 *         till we hit the end.
 */
public static List<EntityGroupInfo> parseDelimitedFrom(final byte[] bytes, final int offset, final int length)
        throws IOException {
    if (bytes == null) {
        throw new IllegalArgumentException("Can't build an object with empty bytes array");
    }
    DataInputBuffer in = new DataInputBuffer();
    List<EntityGroupInfo> egis = new ArrayList<EntityGroupInfo>();
    try {
        in.reset(bytes, offset, length);
        while (in.available() > 0) {
            EntityGroupInfo egi = parseFrom(in);
            egis.add(egi);
        }
    } finally {
        in.close();
    }
    return egis;
}

---

From source file:com.asakusafw.dmdl.java.emitter.driver.WritableDriverTest.java

License:Apache License

/**
 * primitives./* w ww .j a  v  a  2 s  .c  o m*/
 * @throws Exception if test was failed
 */
@Test
public void primitives() throws Exception {
    ModelLoader loader = generate();
    ModelWrapper object = loader.newModel("Primitives");
    assertThat(object.unwrap(), instanceOf(Writable.class));

    object.set("type_boolean", true);
    object.set("type_byte", (byte) 64);
    object.set("type_short", (short) 256);
    object.set("type_int", 100);
    object.set("type_long", 200L);
    object.set("type_float", 300.f);
    object.set("type_double", 400.d);
    object.set("type_decimal", new BigDecimal("1234.567"));
    object.set("type_text", new Text("Hello, world!"));
    object.set("type_date", new Date(2011, 3, 31));
    object.set("type_datetime", new DateTime(2011, 3, 31, 23, 30, 1));

    Writable writable = (Writable) object.unwrap();

    DataOutputBuffer output = new DataOutputBuffer();
    writable.write(output);

    Writable copy = (Writable) loader.newModel("Primitives").unwrap();
    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), output.getLength());
    copy.readFields(input);

    assertThat(input.read(), is(-1));
    assertThat(writable, equalTo(copy));
}

---

From source file:com.asakusafw.dmdl.thundergate.driver.ModelInputDriverTest.java

License:Apache License

/**
 * input/output simple records./*from  w w  w .j ava 2 s  . c  om*/
 * @throws Exception if test was failed
 */
@SuppressWarnings("unchecked")
@Test
public void simple_record() throws Exception {
    ModelLoader loader = generateJava("simple_record");

    Class<?> type = loader.modelType("Simple");
    assertThat(type.isAnnotationPresent(ModelInputLocation.class), is(true));
    assertThat(type.isAnnotationPresent(ModelOutputLocation.class), is(true));

    ModelWrapper object = loader.newModel("Simple");
    DataOutputBuffer output = new DataOutputBuffer();
    try (ModelOutput<Object> modelOut = (ModelOutput<Object>) type.getAnnotation(ModelOutputLocation.class)
            .value().getDeclaredConstructor(RecordEmitter.class)
            .newInstance(new TsvEmitter(new OutputStreamWriter(output, "UTF-8")))) {
        object.set("sid", 1L);
        object.set("value", new Text("hello"));
        modelOut.write(object.unwrap());

        object.set("sid", 2L);
        object.set("value", new Text("world"));
        modelOut.write(object.unwrap());

        object.set("sid", 3L);
        object.set("value", null);
        modelOut.write(object.unwrap());
    }

    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), output.getLength());
    try (ModelInput<Object> modelIn = (ModelInput<Object>) type.getAnnotation(ModelInputLocation.class).value()
            .getDeclaredConstructor(RecordParser.class)
            .newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")))) {
        ModelWrapper copy = loader.newModel("Simple");

        modelIn.readTo(copy.unwrap());
        assertThat(copy.get("sid"), is((Object) 1L));
        assertThat(copy.get("value"), is((Object) new Text("hello")));

        modelIn.readTo(copy.unwrap());
        assertThat(copy.get("sid"), is((Object) 2L));
        assertThat(copy.get("value"), is((Object) new Text("world")));

        modelIn.readTo(copy.unwrap());
        assertThat(copy.get("sid"), is((Object) 3L));
        assertThat(copy.getOption("value").isNull(), is(true));

        assertThat(input.read(), is(-1));
    }
}

---

From source file:com.asakusafw.dmdl.thundergate.driver.ModelInputDriverTest.java

License:Apache License

/**
 * all primitive types.//  w w  w.j av a 2  s .c  o  m
 * @throws Exception if test was failed
 */
@SuppressWarnings("unchecked")
@Test
public void primitives() throws Exception {
    ModelLoader loader = generateJava("primitives");

    Class<?> type = loader.modelType("Primitives");
    assertThat(type.isAnnotationPresent(ModelInputLocation.class), is(true));
    assertThat(type.isAnnotationPresent(ModelOutputLocation.class), is(true));

    ModelWrapper object = loader.newModel("Primitives");

    object.set("type_boolean", true);
    object.set("type_byte", (byte) 64);
    object.set("type_short", (short) 256);
    object.set("type_int", 100);
    object.set("type_long", 200L);
    object.set("type_float", 300.f);
    object.set("type_double", 400.d);
    object.set("type_decimal", new BigDecimal("1234.567"));
    object.set("type_text", new Text("Hello, world!"));
    object.set("type_date", new Date(2011, 3, 31));
    object.set("type_datetime", new DateTime(2011, 3, 31, 23, 30, 1));

    DataOutputBuffer output = new DataOutputBuffer();
    try (ModelOutput<Object> modelOut = (ModelOutput<Object>) type.getAnnotation(ModelOutputLocation.class)
            .value().getDeclaredConstructor(RecordEmitter.class)
            .newInstance(new TsvEmitter(new OutputStreamWriter(output, "UTF-8")))) {
        modelOut.write(object.unwrap());
        modelOut.write(object.unwrap());
        modelOut.write(object.unwrap());
    }

    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), output.getLength());
    try (ModelInput<Object> modelIn = (ModelInput<Object>) type.getAnnotation(ModelInputLocation.class).value()
            .getDeclaredConstructor(RecordParser.class)
            .newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")))) {
        ModelWrapper copy = loader.newModel("Primitives");
        modelIn.readTo(copy.unwrap());
        assertThat(object.unwrap(), equalTo(copy.unwrap()));
        assertThat(input.read(), is(-1));
    }
}

---

From source file:com.asakusafw.runtime.io.util.WritableTestRoot.java

License:Apache License

static <T extends Writable> T des(T writable, byte[] serialized) throws IOException {
    DataInputBuffer buf = new DataInputBuffer();
    buf.reset(serialized, serialized.length);
    writable.readFields(buf);/*from w  w w . jav a2 s.c o  m*/
    return writable;
}

---

From source file:com.asakusafw.runtime.stage.collector.SortableSlotTest.java

License:Apache License

static <T extends Writable> T read(T writable, byte[] bytes) {
    DataInputBuffer buffer = new DataInputBuffer();
    buffer.reset(bytes, bytes.length);//from   ww w .j av a 2  s.c o  m
    try {
        writable.readFields(buffer);
    } catch (IOException e) {
        throw new AssertionError(e);
    }
    return writable;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• »