Example usage for org.apache.hadoop.fs FSDataOutputStream getPos

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FSDataOutputStream getPos.

Prototype

public long getPos() 

Source Link

Document

Get the current position in the output stream.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentField() throws IOException {
    {// ww  w. j  av a  2s  .c o  m
        String file = prefix + "testPersistentField";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Field field = new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1);
        field.persistent(out);
        if (out.getPos() != 7) {
            fail("error out.pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);
        byte type = in.readByte();
        int len = in.readInt();
        short idx = in.readShort();

        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

    }
    {
    }
    {
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentFieldMap() {
    try {/*from  ww w . j a va  2  s.com*/
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataOutputStream out1 = fs.create(path);

        FieldMap fm1 = new FieldMap();
        fm1.persistent(out1);
        if (out1.getPos() != 2) {
            fail("persisitent null fieldmap fail, pos:" + out1.getPos());
        }
        out1.close();

        FSDataInputStream in1 = fs.open(path);
        short fieldNum = in1.readShort();
        if (fieldNum != 0) {
            fail("persistent null fieldmap fail, fieldNum:" + fieldNum);
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataOutputStream out2 = fs.create(path);

        FieldMap fm2 = new FieldMap();
        fm2.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1));
        fm2.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 3));
        fm2.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 5));
        fm2.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 7));
        fm2.persistent(out2);
        if (out2.getPos() != (2 + 4 * 7)) {
            fail("persistent 4 field fail, pos:" + out2.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);
        fieldNum = in2.readShort();
        if (fieldNum != 4) {
            fail("persistent 4 field fail, fieldNum:" + fieldNum);
        }

        byte type = in2.readByte();
        int len = in2.readInt();
        short idx = in2.readShort();
        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Short) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Short) {
            fail("fail len:" + len);
        }
        if (idx != 3) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Int) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Int) {
            fail("fail len:" + len);
        }
        if (idx != 5) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Long) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Long) {
            fail("fail len:" + len);
        }
        if (idx != 7) {
            fail("fail idx:" + idx);
        }
    } catch (IOException e) {
        fail("testPersistentField fail1:" + e.getMessage());
    } catch (Exception e) {
        fail("testPersistentField fail2:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentHead() {

    try {/*from  w ww  . jav a 2 s. c  o  m*/
        String fileName = prefix + "testPersistentHead";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Head head = new Head();
        head.persistent(out);
        if (out.getPos() != 17) {
            fail("persistent error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int magic = in.readInt();
        short ver = in.readShort();
        byte var = in.readByte();
        byte compress = in.readByte();
        byte compressStyle = in.readByte();
        short primaryIndex = in.readShort();
        byte encode = in.readByte();
        byte encodeStyle = in.readByte();
        short keyLen = in.readShort();
        short fieldNum = in.readShort();

        if (magic != head.magic()) {
            fail("error magic:" + magic);
        }
        if (ver != head.ver()) {
            fail("error ver:" + ver);
        }
        if (var != 0) {
            fail("error var:" + var);
        }
        if (compress != head.compress()) {
            fail("error compress:" + compress);
        }
        if (compressStyle != head.compressStyle()) {
            fail("error compressStyle:" + compressStyle);
        }
        if (primaryIndex != head.primaryIndex()) {
            fail("error primaryIndex:" + primaryIndex);
        }
        if (encode != head.encode()) {
            fail("error encode:" + encode);
        }
        if (encodeStyle != head.encodeStyle()) {
            fail("error encodeStyle:" + encodeStyle);
        }
        if (keyLen != 0) {
            fail("error keyLen:" + keyLen);
        }
        if (fieldNum != 0) {
            fail("error fieldNum:" + fieldNum);
        }

        fileName = prefix + "testPersistentHead2";
        path = new Path(fileName);
        FSDataOutputStream out2 = fs.create(path);

        Head head2 = new Head();
        String key = "hello konten";

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        head2.setFieldMap(fieldMap);
        head2.setKey(key);

        head2.persistent(out2);
        if (out2.getPos() != 13 + 2 + key.length() + fieldMap.len()) {
            fail("persistent error pos:" + out.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);

        magic = in2.readInt();
        ver = in2.readShort();
        var = in2.readByte();
        compress = in2.readByte();
        compressStyle = in2.readByte();
        primaryIndex = in2.readShort();
        encode = in2.readByte();
        encodeStyle = in2.readByte();

        keyLen = in2.readShort();
        if (keyLen == 0) {
            fail("error keyLen:" + keyLen);
        }
        byte[] buf = new byte[keyLen];
        in2.readFully(buf);
        String keykey = new String(buf);
        if (!key.equals(keykey)) {
            fail("error key:" + keykey);
        }

        FieldMap fieldMap22 = new FieldMap();
        fieldMap22.unpersistent(in2);
        if (fieldMap22.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap22.fieldNum());
        }
        Field field = fieldMap22.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap22.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecord() {
    try {//from   w ww. j  a  v a  2s  .c o m
        String fileName = prefix + "testChunkToRecord";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        byte[] fb = new byte[ConstVar.Sizeof_Float];
        float f = (float) 5.5;
        Util.float2bytes(fb, f);
        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, fb,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());
        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        value = new byte[4];
        in.readFully(value);
        float fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;
        if (bitSet.length() != (fieldNum / 8 + 1) * 8) {
            fail("bitSet.len:" + bitSet.length());
        }

        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("bitSet should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }

        {
        }
        lv = Util.bytes2long(value, 0, len);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        fail("should not exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecordNull() {
    try {/*from w w w .  ja v  a2  s  .  c  o m*/
        String fileName = prefix + "testChunkToRecord2";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, null,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        in.readFloat();

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value, 0, strLen);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value != null) {
            fail("error value not null");
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        e.printStackTrace();
        fail("should not exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitVar() {
    try {//from www  .  j a v  a  2s .  c  om
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitVar_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        int count = 100;
        for (int i = 0; i < count; i++) {
            unit.addRecord(record);
        }

        String file = prefix + "testPersistentUnitVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        unit.persistent(out);
        long pos = out.getPos();
        if (pos != full7chunkLen * count + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error pos:" + pos);
        }
        out.close();

        long len = unit.len();
        if (len != count * full7chunkLen + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error unit.len" + len);
        }

        FSDataInputStream in = fs.open(path);
        in.seek(len - 8 - 4);
        long metaOffset = in.readLong();
        if (metaOffset != full7chunkLen * count) {
            fail("error metaOffset:" + metaOffset);
        }

        in.seek(len - 8 - 4 - 4);
        int recordNum = in.readInt();
        if (recordNum != count) {
            fail("error recordNum:" + recordNum);
        }

        in.seek(metaOffset);
        for (int i = 0; i < recordNum; i++) {
            long offset = in.readLong();
            if (offset != full7chunkLen * i) {
                fail("error offset:" + offset + "i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitNotVar() {
    try {/* w  ww .j  a  va2  s  .  c  o  m*/
        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitNotVar_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(6);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));

        int count = 100;
        for (int i = 0; i < count; i++) {
            unit.addRecord(record);
        }

        String file = prefix + "testPersistentUnitNotVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        unit.persistent(out);
        long pos = out.getPos();
        if (pos != full6chunkLen * count + ConstVar.DataChunkMetaOffset) {
            fail("error pos:" + pos);
        }
        out.close();

        long len = unit.len();
        if (len != count * full6chunkLen + ConstVar.DataChunkMetaOffset) {
            fail("error unit.len" + len);
        }

        FSDataInputStream in = fs.open(path);
        in.seek(len - 8 - 4);
        long metaOffset = in.readLong();
        if (metaOffset != full6chunkLen * count) {
            fail("error metaOffset:" + metaOffset);
        }

        in.seek(len - 8 - 4 - 4);
        int recordNum = in.readInt();
        if (recordNum != count) {
            fail("error recordNum:" + recordNum);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineUnitIndex() {
    try {// www  .  j  av  a  2s.  c  o  m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);

            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;

            segment.addUnit(unit);
        }

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);
        if (out.getPos() != unitSize * ConstVar.LineIndexRecordLen) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        if (segment.lineIndexOffset() != 0) {
            fail("error line index offset:" + segment.lineIndexOffset());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment.keyIndexOffset());
        }

        FSDataInputStream in = fs.open(path);

        for (int i = 0; i < unitSize; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != (i + 1) * 100) {
                fail("error begin line:" + beginLine + " i:" + i);
            }
            if (endLine != (i + 2) * 100) {
                fail("error end line:" + endLine + " i:" + i);
            }
            if (offset != i * 100) {
                fail("error offset:" + offset + " i:" + i);
            }
            if (len != unitlen) {
                fail("error len:" + len + " i:" + i);
            }
            if (idx != i) {
                fail("error idx:" + idx + " i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitIndexMeta() {
    try {/*  w  w w . j a va2  s .  c  o  m*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistentUnitIndexMeta(out);
        if (out.getPos() != ConstVar.IndexMetaOffset) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int recordNum = in.readInt();
        int unitNum = in.readInt();
        long keyIndexOffset = in.readLong();
        long lineIndexOffset = in.readLong();

        if (recordNum != 234) {
            fail("error recordnum:" + recordNum);
        }
        if (unitNum != unitSize) {
            fail("error unitNum:" + unitNum);
        }
        if (keyIndexOffset != -1) {
            fail("error key index offset:" + keyIndexOffset);
        }
        if (lineIndexOffset != -1) {
            fail("error line inded offset:" + lineIndexOffset);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentDummy() {
    try {/*from  w  w w  .j  a v  a 2s  .  c  om*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentDummy_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentDummy";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);
        segment.persistentUnitIndexMeta(out);

        segment.persistentDummy(out);
        if (out.getPos() != segment.unitIndex().len() + ConstVar.IndexMetaOffset + segment.remain()) {
            fail("error pos:" + out.getPos());
        }
        out.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »