Example usage for org.apache.hadoop.fs Path Path

List of usage examples for org.apache.hadoop.fs Path Path

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecordNull() {
    try {/*  w  w  w  . ja v a  2 s .  c  o  m*/
        String fileName = prefix + "testChunkToRecord2";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, null,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        in.readFloat();

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value, 0, strLen);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value != null) {
            fail("error value not null");
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        e.printStackTrace();
        fail("should not exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitVar() {
    try {/*  w w w.jav a2  s .  c  o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitVar_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        int count = 100;
        for (int i = 0; i < count; i++) {
            unit.addRecord(record);
        }

        String file = prefix + "testPersistentUnitVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        unit.persistent(out);
        long pos = out.getPos();
        if (pos != full7chunkLen * count + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error pos:" + pos);
        }
        out.close();

        long len = unit.len();
        if (len != count * full7chunkLen + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error unit.len" + len);
        }

        FSDataInputStream in = fs.open(path);
        in.seek(len - 8 - 4);
        long metaOffset = in.readLong();
        if (metaOffset != full7chunkLen * count) {
            fail("error metaOffset:" + metaOffset);
        }

        in.seek(len - 8 - 4 - 4);
        int recordNum = in.readInt();
        if (recordNum != count) {
            fail("error recordNum:" + recordNum);
        }

        in.seek(metaOffset);
        for (int i = 0; i < recordNum; i++) {
            long offset = in.readLong();
            if (offset != full7chunkLen * i) {
                fail("error offset:" + offset + "i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitNotVar() {
    try {//from w  w  w  . jav  a  2 s  . c o  m
        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitNotVar_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(6);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));

        int count = 100;
        for (int i = 0; i < count; i++) {
            unit.addRecord(record);
        }

        String file = prefix + "testPersistentUnitNotVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        unit.persistent(out);
        long pos = out.getPos();
        if (pos != full6chunkLen * count + ConstVar.DataChunkMetaOffset) {
            fail("error pos:" + pos);
        }
        out.close();

        long len = unit.len();
        if (len != count * full6chunkLen + ConstVar.DataChunkMetaOffset) {
            fail("error unit.len" + len);
        }

        FSDataInputStream in = fs.open(path);
        in.seek(len - 8 - 4);
        long metaOffset = in.readLong();
        if (metaOffset != full6chunkLen * count) {
            fail("error metaOffset:" + metaOffset);
        }

        in.seek(len - 8 - 4 - 4);
        int recordNum = in.readInt();
        if (recordNum != count) {
            fail("error recordNum:" + recordNum);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistenUnitVar() {
    try {/*from www.j  a v  a2  s.  c o  m*/
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Read);
        fd.head = head;

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        info.len = 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        String file = prefix + "testPersistentUnitVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        byte[] buffer = unit.loadUnitBuffer(in);

        unit.loadDataMeta(buffer, true);

        if (unit.recordNum() != 100) {
            fail("error recordNum:" + unit.recordNum());
        }

        if (unit.offsetArray() == null) {
            fail("error offsetArray, null");
        }

        if (unit.offsetArray().length != 100) {
            fail("error offsetArray len:" + unit.offsetArray().length);
        }

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        ByteArrayInputStream stream1 = new ByteArrayInputStream(buffer);
        DataInputStream stream = new DataInputStream(stream1);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(buffer, 0, buffer.length);
        for (int i = 0; i < unit.offsetArray().length; i++) {
            if (unit.offsetArray()[i] != full7chunkLen * i) {
                fail("error meta offset:" + unit.offsetArray()[i] + "i:" + i);
            }

            DataChunk chunk = new DataChunk((short) 7);
            chunk.unpersistent(unit.offsetArray()[i], full7chunkLen, inputBuffer);

            Record record = chunk.toRecord(fieldMap, true, arrayList);
            judgeFixedRecord(record);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistenUnitNotVar() {
    try {//from   ww w.  j a va2  s. c om
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Read);
        fd.head = head;

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        info.len = 100 * full6chunkLen + ConstVar.DataChunkMetaOffset;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        String file = prefix + "testPersistentUnitNotVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        byte[] buffer = unit.loadUnitBuffer(in);

        unit.loadDataMeta(buffer, false);

        if (unit.recordNum() != 100) {
            fail("error recordNum:" + unit.recordNum());
        }

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        ByteArrayInputStream stream1 = new ByteArrayInputStream(buffer);
        DataInputStream stream = new DataInputStream(stream1);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(buffer, 0, buffer.length);
        for (int i = 0; i < 100; i++) {
            DataChunk chunk = new DataChunk((short) 6);
            chunk.unpersistent(i * 29, full6chunkLen, inputBuffer);

            Record record = chunk.toRecord(fieldMap, true, arrayList);
            judgeFixedRecord(record);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineUnit() {
    try {/*from  w  ww.j a  va2 s. c o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByLine_tmp", head);

        String fileName = prefix + "testUnitGetRecordByLine";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        try {
            Record record = unit2.getRecordByLine(-1);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }
        try {
            Record record = unit2.getRecordByLine(120);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }

        for (int i = 0; i < 100; i++) {
            try {
                Record record = unit2.getRecordByLine(i);

                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != i + index) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + lv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value, 0, len);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderUnit() {
    try {/*from ww  w. j a v a  2 s.co  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByOrder_tmp", head);

        String fileName = prefix + "testUnitGetRecordByOrder";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        Record[] records = unit2.getRecordByValue(null, 0, ConstVar.OP_GetAll);
        if (records.length != 100) {
            fail("error record.len:" + records.length);
        }

        for (int i = 0; i < 100; i++) {
            Record record = records[i];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByValueUnit() {
    try {/*  w  w w  . j  av a 2  s . c o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByValue_tmp", head);

        String fileName = prefix + "testUnitGetRecordByValue";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        FieldValue[] values1 = new FieldValue[2];
        values1[0] = new FieldValue((short) (3), (short) 3);
        values1[1] = new FieldValue((int) (3), (short) 5);
        Record[] records1 = unit2.getRecordByValue(values1, values1.length, ConstVar.OP_GetSpecial);
        if (records1 != null) {
            fail("should return null");
        }

        seg2.units().add(unit2);

        for (int i = 0; i < 100; i++) {
            int base = 0;
            FieldValue[] values = new FieldValue[2];
            values[0] = new FieldValue((short) (1 + i), (short) 1);
            values[1] = new FieldValue((int) (2 + i), (short) 2);
            Record[] records = unit2.getRecordByValue(values, values.length, ConstVar.OP_GetSpecial);
            if (i < 100) {
                if (records == null) {
                    fail("records null:" + i);
                }

                if (records.length != 1) {
                    fail("error record.len:" + records.length + "i:" + i);
                }
            } else {
                if (records != null) {
                    fail("should return null:" + i);
                }
            }

            if (records == null) {
                continue;
            }

            Record record = records[0];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index + base)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index + base) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i + base)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i + base)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + (i + base);
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineUnitIndex() {
    try {/*from  ww w  .j  a  v  a 2s . co m*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);

            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;

            segment.addUnit(unit);
        }

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);
        if (out.getPos() != unitSize * ConstVar.LineIndexRecordLen) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        if (segment.lineIndexOffset() != 0) {
            fail("error line index offset:" + segment.lineIndexOffset());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment.keyIndexOffset());
        }

        FSDataInputStream in = fs.open(path);

        for (int i = 0; i < unitSize; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != (i + 1) * 100) {
                fail("error begin line:" + beginLine + " i:" + i);
            }
            if (endLine != (i + 2) * 100) {
                fail("error end line:" + endLine + " i:" + i);
            }
            if (offset != i * 100) {
                fail("error offset:" + offset + " i:" + i);
            }
            if (len != unitlen) {
                fail("error len:" + len + " i:" + i);
            }
            if (idx != i) {
                fail("error idx:" + idx + " i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitIndexMeta() {
    try {//from w  ww.j  av  a  2  s .c o  m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistentUnitIndexMeta(out);
        if (out.getPos() != ConstVar.IndexMetaOffset) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int recordNum = in.readInt();
        int unitNum = in.readInt();
        long keyIndexOffset = in.readLong();
        long lineIndexOffset = in.readLong();

        if (recordNum != 234) {
            fail("error recordnum:" + recordNum);
        }
        if (unitNum != unitSize) {
            fail("error unitNum:" + unitNum);
        }
        if (keyIndexOffset != -1) {
            fail("error key index offset:" + keyIndexOffset);
        }
        if (lineIndexOffset != -1) {
            fail("error line inded offset:" + lineIndexOffset);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}