Example usage for org.apache.hadoop.io DataInputBuffer DataInputBuffer

List of usage examples for org.apache.hadoop.io DataInputBuffer DataInputBuffer

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. DataInputBuffer
  6. DataInputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataInputBuffer DataInputBuffer.

Prototype

public DataInputBuffer()

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:TestILineIndex.java

License:Open Source License

public void testPersistable() throws IOException {
    ILineIndex ili = new ILineIndex(1, 1);

    DataOutputBuffer dob = new DataOutputBuffer();
    ili.persistent(dob);//from w  ww  .  ja va 2 s. c  o  m
    byte[] data = dob.getData();

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    ILineIndex ili2 = new ILineIndex();
    ili2.unpersistent(dib);

    assertEquals(ili.beginline(), ili2.beginline());
    assertEquals(ili.endline(), ili2.endline());
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecord() {
    try {//from ww w.j  av a2s .  co  m
        String fileName = prefix + "testChunkToRecord";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        byte[] fb = new byte[ConstVar.Sizeof_Float];
        float f = (float) 5.5;
        Util.float2bytes(fb, f);
        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, fb,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());
        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        value = new byte[4];
        in.readFully(value);
        float fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;
        if (bitSet.length() != (fieldNum / 8 + 1) * 8) {
            fail("bitSet.len:" + bitSet.length());
        }

        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("bitSet should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }

        {
        }
        lv = Util.bytes2long(value, 0, len);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        fail("should not exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecordNull() {
    try {//from ww  w.j  ava 2 s. co m
        String fileName = prefix + "testChunkToRecord2";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, null,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        in.readFloat();

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value, 0, strLen);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value != null) {
            fail("error value not null");
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        e.printStackTrace();
        fail("should not exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistenUnitVar() {
    try {/*ww  w  . j a  v  a2  s .  com*/
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Read);
        fd.head = head;

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        info.len = 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        String file = prefix + "testPersistentUnitVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        byte[] buffer = unit.loadUnitBuffer(in);

        unit.loadDataMeta(buffer, true);

        if (unit.recordNum() != 100) {
            fail("error recordNum:" + unit.recordNum());
        }

        if (unit.offsetArray() == null) {
            fail("error offsetArray, null");
        }

        if (unit.offsetArray().length != 100) {
            fail("error offsetArray len:" + unit.offsetArray().length);
        }

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        ByteArrayInputStream stream1 = new ByteArrayInputStream(buffer);
        DataInputStream stream = new DataInputStream(stream1);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(buffer, 0, buffer.length);
        for (int i = 0; i < unit.offsetArray().length; i++) {
            if (unit.offsetArray()[i] != full7chunkLen * i) {
                fail("error meta offset:" + unit.offsetArray()[i] + "i:" + i);
            }

            DataChunk chunk = new DataChunk((short) 7);
            chunk.unpersistent(unit.offsetArray()[i], full7chunkLen, inputBuffer);

            Record record = chunk.toRecord(fieldMap, true, arrayList);
            judgeFixedRecord(record);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistenUnitNotVar() {
    try {//from  w w w  . j  ava2  s . com
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Read);
        fd.head = head;

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        info.len = 100 * full6chunkLen + ConstVar.DataChunkMetaOffset;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        String file = prefix + "testPersistentUnitNotVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        byte[] buffer = unit.loadUnitBuffer(in);

        unit.loadDataMeta(buffer, false);

        if (unit.recordNum() != 100) {
            fail("error recordNum:" + unit.recordNum());
        }

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        ByteArrayInputStream stream1 = new ByteArrayInputStream(buffer);
        DataInputStream stream = new DataInputStream(stream1);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(buffer, 0, buffer.length);
        for (int i = 0; i < 100; i++) {
            DataChunk chunk = new DataChunk((short) 6);
            chunk.unpersistent(i * 29, full6chunkLen, inputBuffer);

            Record record = chunk.toRecord(fieldMap, true, arrayList);
            judgeFixedRecord(record);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testTransferUnit() {
    try {/*from   w  w w .j  a  v a  2 s  .c o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testTransferUnitOneRecord_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        for (int i = 0; i < 100; i++) {
            unit.addRecord(record);
        }

        if (unit.offset() != 123) {
            fail("error offset1:" + unit.offset());
        }

        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(((DataOutputBuffer) unit.metasBuffer).getData(), 0,
                ((DataOutputBuffer) unit.metasBuffer).getLength());
        for (int i = 0; i < 100; i++) {
            long value = inputBuffer.readLong();
            if (value != 123 + i * full7chunkLen) {
                fail("error data offset1:" + value + "i:" + i);
            }
        }

        if (unit.metaOffset() != 123 + full7chunkLen * 100) {
            fail("error metaOffset1:" + unit.metaOffset());
        }

        unit.transfer(2000);

        if (unit.offset() != 2000) {
            fail("error offset2:" + unit.offset());
        }

        inputBuffer.reset(((DataOutputBuffer) unit.metasBuffer).getData(), 0,
                ((DataOutputBuffer) unit.metasBuffer).getLength());
        for (int i = 0; i < 100; i++) {
            long value = inputBuffer.readLong();
            if (value != 2000 + i * full7chunkLen) {
                fail("error data offset2:" + value + "i:" + i);
            }
        }
        if (unit.metaOffset() != 2000 + full7chunkLen * 100) {
            fail("error metaOffset2:" + unit.metaOffset());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:TestISegmentIndex.java

License:Open Source License

public void testPersistable() throws IOException {
    IFileInfo fileInfo = TestUtil.genfileinfo(true, 2);
    ISegmentIndex index = new ISegmentIndex(fileInfo);

    index.update(TestUtil.genseginfo(fileInfo, 0, 2));
    index.update(TestUtil.genseginfo(fileInfo, 1, 2));
    index.update(TestUtil.genseginfo(fileInfo, 2, 2));
    index.update(TestUtil.genseginfo(fileInfo, 3, 2));
    index.update(TestUtil.genseginfo(fileInfo, 4, 2));

    DataOutputBuffer dob = new DataOutputBuffer();
    index.persistent(dob);//  w ww  .j a  v a2  s .c o  m
    byte[] data = dob.getData();

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    ISegmentIndex index2 = new ISegmentIndex(fileInfo);
    index2.unpersistent(dib);

    assertEquals(5, index2.getSegnum());

    assertEquals(0, index2.getSegid(0));
    assertEquals(0, index2.getSegid(3));
    assertEquals(1, index2.getSegid(4));
    assertEquals(1, index2.getSegid(7));
    assertEquals(4, index2.getSegid(19));
    assertEquals(5, index2.getSegid(20));

    assertEquals(0, index2.getSegid(new IFieldValue(0)));
    assertEquals(0, index2.getSegid(new IFieldValue(3)));
    assertEquals(1, index2.getSegid(new IFieldValue(4)));
    assertEquals(1, index2.getSegid(new IFieldValue(7)));
    assertEquals(4, index2.getSegid(new IFieldValue(19)));
    assertEquals(5, index2.getSegid(new IFieldValue(20)));

}

From source file:TestIndexValue.java

License:Open Source License

public void testIndexValue() throws IOException {

    IndexValue value = new IndexValue((short) 1, 2);

    DataOutputBuffer dob = new DataOutputBuffer();
    value.write(dob);//  w w w.j av  a  2 s  . c  om
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IndexValue value2 = new IndexValue((short) 1, 2);
    value2.readFields(dib);

    assertEquals(value.getFileindex(), value2.getFileindex());
    assertEquals(value.getRowid(), value2.getRowid());

}

From source file:TestIFieldType.java

License:Open Source License

public void testWritable() throws IOException {
    IFieldType ift = new IFieldType(ConstVar.FieldType_Int, (short) 1);

    DataOutputBuffer dob = new DataOutputBuffer();
    ift.write(dob);// w  w  w  . ja v  a 2 s  . c  o m
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IFieldType ift2 = new IFieldType();
    ift2.readFields(dib);

    assertEquals(ift.getIndex(), ift2.getIndex());
    assertEquals(ift.getLen(), ift2.getLen());
    assertEquals(ift.getType(), ift2.getType());
}

From source file:TestIFieldType.java

License:Open Source License

public void testPersistable() throws IOException {
    IFieldType ift = new IFieldType(ConstVar.FieldType_Int, (short) 1);

    DataOutputBuffer dob = new DataOutputBuffer();
    ift.persistent(dob);//from   w  w w.j  av  a2  s.c o  m
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IFieldType ift2 = new IFieldType();
    ift2.unpersistent(dib);

    assertEquals(ift.getIndex(), ift2.getIndex());
    assertEquals(ift.getLen(), ift2.getLen());
    assertEquals(ift.getType(), ift2.getType());
}