Example usage for org.apache.hadoop.fs FileSystem open

List of usage examples for org.apache.hadoop.fs FileSystem open

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineUnit() {
    try {//from ww w . j  av  a  2  s. c o  m
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByLine_tmp", head);

        String fileName = prefix + "testUnitGetRecordByLine";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        try {
            Record record = unit2.getRecordByLine(-1);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }
        try {
            Record record = unit2.getRecordByLine(120);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }

        for (int i = 0; i < 100; i++) {
            try {
                Record record = unit2.getRecordByLine(i);

                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != i + index) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + lv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value, 0, len);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderUnit() {
    try {//w w w.ja v  a  2 s.  co m
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByOrder_tmp", head);

        String fileName = prefix + "testUnitGetRecordByOrder";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        Record[] records = unit2.getRecordByValue(null, 0, ConstVar.OP_GetAll);
        if (records.length != 100) {
            fail("error record.len:" + records.length);
        }

        for (int i = 0; i < 100; i++) {
            Record record = records[i];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByValueUnit() {
    try {//  w  ww . j  av  a  2 s  . com
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByValue_tmp", head);

        String fileName = prefix + "testUnitGetRecordByValue";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        FieldValue[] values1 = new FieldValue[2];
        values1[0] = new FieldValue((short) (3), (short) 3);
        values1[1] = new FieldValue((int) (3), (short) 5);
        Record[] records1 = unit2.getRecordByValue(values1, values1.length, ConstVar.OP_GetSpecial);
        if (records1 != null) {
            fail("should return null");
        }

        seg2.units().add(unit2);

        for (int i = 0; i < 100; i++) {
            int base = 0;
            FieldValue[] values = new FieldValue[2];
            values[0] = new FieldValue((short) (1 + i), (short) 1);
            values[1] = new FieldValue((int) (2 + i), (short) 2);
            Record[] records = unit2.getRecordByValue(values, values.length, ConstVar.OP_GetSpecial);
            if (i < 100) {
                if (records == null) {
                    fail("records null:" + i);
                }

                if (records.length != 1) {
                    fail("error record.len:" + records.length + "i:" + i);
                }
            } else {
                if (records != null) {
                    fail("should return null:" + i);
                }
            }

            if (records == null) {
                continue;
            }

            Record record = records[0];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index + base)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index + base) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i + base)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i + base)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + (i + base);
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineUnitIndex() {
    try {// w w  w . ja  v  a  2 s . c  om
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);

            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;

            segment.addUnit(unit);
        }

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);
        if (out.getPos() != unitSize * ConstVar.LineIndexRecordLen) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        if (segment.lineIndexOffset() != 0) {
            fail("error line index offset:" + segment.lineIndexOffset());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment.keyIndexOffset());
        }

        FSDataInputStream in = fs.open(path);

        for (int i = 0; i < unitSize; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != (i + 1) * 100) {
                fail("error begin line:" + beginLine + " i:" + i);
            }
            if (endLine != (i + 2) * 100) {
                fail("error end line:" + endLine + " i:" + i);
            }
            if (offset != i * 100) {
                fail("error offset:" + offset + " i:" + i);
            }
            if (len != unitlen) {
                fail("error len:" + len + " i:" + i);
            }
            if (idx != i) {
                fail("error idx:" + idx + " i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitIndexMeta() {
    try {//from   www . java  2 s. co  m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistentUnitIndexMeta(out);
        if (out.getPos() != ConstVar.IndexMetaOffset) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int recordNum = in.readInt();
        int unitNum = in.readInt();
        long keyIndexOffset = in.readLong();
        long lineIndexOffset = in.readLong();

        if (recordNum != 234) {
            fail("error recordnum:" + recordNum);
        }
        if (unitNum != unitSize) {
            fail("error unitNum:" + unitNum);
        }
        if (keyIndexOffset != -1) {
            fail("error key index offset:" + keyIndexOffset);
        }
        if (lineIndexOffset != -1) {
            fail("error line inded offset:" + lineIndexOffset);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentUnitIndexMeta() {
    try {/*  w w w  .  jav a2  s.  c  o m*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnpersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        int unitSize = 100;

        segment.unpersistentIndexMeta(in);

        if (segment.recordNum() != 234) {
            fail("error record num:" + segment.recordNum());
        }
        if (segment.unitNum() != unitSize) {
            fail("error unit num:" + segment.unitNum());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error keyIndex offset:" + segment.keyIndexOffset());
        }
        if (segment.lineIndexOffset() != -1) {
            fail("error lineIndexOffset:" + segment.lineIndexOffset());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentLineUnitIndex() {
    try {//from   w w w .  j  a  v  a 2s  . c o m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnpersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        int unitSize = 100;

        segment.setUnitIndex(new UnitIndex());
        segment.setUnitNum(unitSize);
        segment.unpersistentLineUnitIndex(in);

        if (segment.unitIndex().len() != ConstVar.LineIndexRecordLen * unitSize) {
            fail("error unitIndex len:" + segment.unitIndex().len());
        }

        if (segment.unitIndex().lineIndexInfos().size() != unitSize) {
            fail("error line index size:" + segment.unitIndex().lineIndexInfos().size());
        }
        if (segment.unitIndex().keyIndexInfos().size() != 0) {
            fail("error key index size:" + segment.unitIndex().keyIndexInfos().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentSegment() {
    try {//from w  w w  .jav  a 2s .c  o  m
        IndexInfo info = new IndexInfo();
        info.offset = 0;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentSegment_tmp", head);

        String fileName = prefix + "testPersistentSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        fd.setOut(out);
        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            segment.addUnit(unit);
            if (unit.len() != 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset) {
                fail("error unit.len:" + unit.len());
            }
        }

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistent(out);

        if (out.getPos() != fd.confSegmentSize()) {
            System.out.println("seg.len:" + segment.len() + "seg.remain:" + segment.remain() + "index.len"
                    + segment.unitIndex().len());
            fail("error pos:" + out.getPos());
        }
        out.close();

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        FSDataInputStream in = fs.open(path);

        in.seek(segment.lineIndexOffset());

        info.offset = 0;
        info.len = segment.len();
        fd.setWorkStatus(ConstVar.WS_Read);
        Segment segment2 = new Segment(info, fd);
        segment2.unpersistentUnitIndex(in);
        if (segment2.recordNum() != 234) {
            fail("error recordnum:" + segment2.recordNum());
        }
        if (segment2.unitNum() != unitSize) {
            fail("error unitNum:" + segment2.unitNum());
        }
        if (segment2.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment2.keyIndexOffset());
        }
        if (segment2.lineIndexOffset() != unitlen * unitSize) {
            fail("error line index offset:" + segment2.lineIndexOffset());
        }
        if (segment2.units().size() != unitSize) {
            fail("error units.size:" + segment2.units().size());
        }

        UnitIndex index = segment2.unitIndex();
        if (index.lineIndexInfos().size() != unitSize) {
            fail("error line unit index size:" + index.lineIndexInfos().size());
        }
        if (index.keyIndexInfos().size() != 0) {
            fail("error key unit index size:" + index.keyIndexInfos().size());
        }

        for (int i = 0; i < unitSize; i++) {
            IndexInfo ii = index.lineIndexInfos().get(i);
            if (ii.beginLine() != (1 + i) * 100) {
                fail("error beginline:" + ii.beginLine() + "i:" + i);
            }
            if (ii.endLine() != (2 + i) * 100) {
                fail("error end line:" + ii.endLine() + "i:" + i);
            }
            if (ii.offset() != i * 100) {
                fail("error offset:" + ii.offset() + "i:" + i);
            }
            if (ii.len != unitlen) {
                fail("error len:" + ii.len() + "i:" + i);
            }
            if (ii.idx() != i) {
                fail("error idx:" + ii.idx() + "i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineSegment() {
    Segment segment = null;/*from  ww  w  . j  a va  2 s  .c om*/
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Write);
        fd.head = head;
        fd.setOut(out);

        IndexInfo info = new IndexInfo();
        info.offset = 0;

        segment = new Segment(info, fd);

        int recordNum = 150000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
            record = null;
        }

        segment.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        fd.setIn(in);
        fd.setWorkStatus(ConstVar.WS_Read);

        info.offset = 0;
        info.len = segment.len();
        info.beginLine = 0;
        info.endLine = 1500000;
        Segment segment2 = new Segment(info, fd);
        Record record = segment2.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }

        record = segment2.getRecordByLine(150000);
        if (record != null) {
            fail("should get null");
        }
        record = segment2.getRecordByLine(150001);
        if (record != null) {
            fail("should get null");
        }

        record = segment2.getRecordByLine(0);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

        int line = 150000 - 1;
        record = segment2.getRecordByLine(line);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderSegment() {
    Segment segment = null;/*from  w w  w . java  2 s  .  co  m*/
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByValueSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Write);
        fd.head = head;
        fd.setOut(out);

        IndexInfo info = new IndexInfo();
        info.offset = 0;

        segment = new Segment(info, fd);

        int recordNum = 150000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
            record = null;
        }

        segment.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        fd.setIn(in);
        fd.setWorkStatus(ConstVar.WS_Read);

        info.offset = 0;
        info.len = segment.len();

        Segment segment2 = new Segment(info, fd);

        FieldValue[] values = new FieldValue[2];
        values[0] = new FieldValue((byte) 1, (short) 2);
        values[1] = new FieldValue((short) 2, (short) 3);
        Record[] records = segment2.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null, index error, records.len:" + records.length);
        }

        values[0] = new FieldValue((byte) 1, (short) 0);
        values[1] = new FieldValue((short) 3, (short) 1);
        records = segment2.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null, value error");
        }

        values[0] = new FieldValue((byte) 1, (short) 0);
        values[1] = new FieldValue((short) 2, (short) 1);
        records = segment2.getRecordByOrder(values, values.length);
        if (records == null) {
            fail("should not get null");
        }
        if (records.length != 150000) {
            fail("error result size:" + records.length);
        }

        for (int i = 0; i < 150000; i++) {
            judgeFixedRecord(records[i]);
        }

        records = segment2.getRecordByOrder(null, 0);
        if (records == null) {
            fail("should not get null");
        }
        if (records.length != 150000) {
            fail("error result size:" + records.length);
        }

        for (int i = 0; i < 150000; i++) {
            judgeFixedRecord(records[i]);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}