Example usage for org.apache.hadoop.fs FileSystem get

List of usage examples for org.apache.hadoop.fs FileSystem get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem get.

Prototype

public static FileSystem get(Configuration conf) throws IOException

Source Link

Document

Returns the configured FileSystem implementation.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineUnit() {
    try {/*from  w  w w  . j  av  a  2  s.c  o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByLine_tmp", head);

        String fileName = prefix + "testUnitGetRecordByLine";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        try {
            Record record = unit2.getRecordByLine(-1);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }
        try {
            Record record = unit2.getRecordByLine(120);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }

        for (int i = 0; i < 100; i++) {
            try {
                Record record = unit2.getRecordByLine(i);

                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != i + index) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + lv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value, 0, len);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderUnit() {
    try {/*from  ww w.  j a  v a 2s .c  om*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByOrder_tmp", head);

        String fileName = prefix + "testUnitGetRecordByOrder";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        Record[] records = unit2.getRecordByValue(null, 0, ConstVar.OP_GetAll);
        if (records.length != 100) {
            fail("error record.len:" + records.length);
        }

        for (int i = 0; i < 100; i++) {
            Record record = records[i];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByValueUnit() {
    try {/* w w  w. j  av  a  2s . c  o m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByValue_tmp", head);

        String fileName = prefix + "testUnitGetRecordByValue";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        FieldValue[] values1 = new FieldValue[2];
        values1[0] = new FieldValue((short) (3), (short) 3);
        values1[1] = new FieldValue((int) (3), (short) 5);
        Record[] records1 = unit2.getRecordByValue(values1, values1.length, ConstVar.OP_GetSpecial);
        if (records1 != null) {
            fail("should return null");
        }

        seg2.units().add(unit2);

        for (int i = 0; i < 100; i++) {
            int base = 0;
            FieldValue[] values = new FieldValue[2];
            values[0] = new FieldValue((short) (1 + i), (short) 1);
            values[1] = new FieldValue((int) (2 + i), (short) 2);
            Record[] records = unit2.getRecordByValue(values, values.length, ConstVar.OP_GetSpecial);
            if (i < 100) {
                if (records == null) {
                    fail("records null:" + i);
                }

                if (records.length != 1) {
                    fail("error record.len:" + records.length + "i:" + i);
                }
            } else {
                if (records != null) {
                    fail("should return null:" + i);
                }
            }

            if (records == null) {
                continue;
            }

            Record record = records[0];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index + base)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index + base) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i + base)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i + base)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + (i + base);
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineUnitIndex() {
    try {//from  w w  w .  j  a  va  2 s. c  om
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);

            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;

            segment.addUnit(unit);
        }

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);
        if (out.getPos() != unitSize * ConstVar.LineIndexRecordLen) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        if (segment.lineIndexOffset() != 0) {
            fail("error line index offset:" + segment.lineIndexOffset());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment.keyIndexOffset());
        }

        FSDataInputStream in = fs.open(path);

        for (int i = 0; i < unitSize; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != (i + 1) * 100) {
                fail("error begin line:" + beginLine + " i:" + i);
            }
            if (endLine != (i + 2) * 100) {
                fail("error end line:" + endLine + " i:" + i);
            }
            if (offset != i * 100) {
                fail("error offset:" + offset + " i:" + i);
            }
            if (len != unitlen) {
                fail("error len:" + len + " i:" + i);
            }
            if (idx != i) {
                fail("error idx:" + idx + " i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitIndexMeta() {
    try {//from w w  w  .  j  a  va2 s. c  o m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistentUnitIndexMeta(out);
        if (out.getPos() != ConstVar.IndexMetaOffset) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int recordNum = in.readInt();
        int unitNum = in.readInt();
        long keyIndexOffset = in.readLong();
        long lineIndexOffset = in.readLong();

        if (recordNum != 234) {
            fail("error recordnum:" + recordNum);
        }
        if (unitNum != unitSize) {
            fail("error unitNum:" + unitNum);
        }
        if (keyIndexOffset != -1) {
            fail("error key index offset:" + keyIndexOffset);
        }
        if (lineIndexOffset != -1) {
            fail("error line inded offset:" + lineIndexOffset);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentDummy() {
    try {/*from w ww  .ja  v a2 s.  com*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentDummy_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentDummy";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);
        segment.persistentUnitIndexMeta(out);

        segment.persistentDummy(out);
        if (out.getPos() != segment.unitIndex().len() + ConstVar.IndexMetaOffset + segment.remain()) {
            fail("error pos:" + out.getPos());
        }
        out.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentUnitIndexMeta() {
    try {// ww  w .  j a  va2  s .co  m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnpersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        int unitSize = 100;

        segment.unpersistentIndexMeta(in);

        if (segment.recordNum() != 234) {
            fail("error record num:" + segment.recordNum());
        }
        if (segment.unitNum() != unitSize) {
            fail("error unit num:" + segment.unitNum());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error keyIndex offset:" + segment.keyIndexOffset());
        }
        if (segment.lineIndexOffset() != -1) {
            fail("error lineIndexOffset:" + segment.lineIndexOffset());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentLineUnitIndex() {
    try {/*from   w  w w  .j av a 2  s  .c  om*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnpersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        int unitSize = 100;

        segment.setUnitIndex(new UnitIndex());
        segment.setUnitNum(unitSize);
        segment.unpersistentLineUnitIndex(in);

        if (segment.unitIndex().len() != ConstVar.LineIndexRecordLen * unitSize) {
            fail("error unitIndex len:" + segment.unitIndex().len());
        }

        if (segment.unitIndex().lineIndexInfos().size() != unitSize) {
            fail("error line index size:" + segment.unitIndex().lineIndexInfos().size());
        }
        if (segment.unitIndex().keyIndexInfos().size() != 0) {
            fail("error key index size:" + segment.unitIndex().keyIndexInfos().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentSegment() {
    try {/* w  ww. ja  v a 2  s.c o m*/
        IndexInfo info = new IndexInfo();
        info.offset = 0;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentSegment_tmp", head);

        String fileName = prefix + "testPersistentSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        fd.setOut(out);
        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            segment.addUnit(unit);
            if (unit.len() != 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset) {
                fail("error unit.len:" + unit.len());
            }
        }

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistent(out);

        if (out.getPos() != fd.confSegmentSize()) {
            System.out.println("seg.len:" + segment.len() + "seg.remain:" + segment.remain() + "index.len"
                    + segment.unitIndex().len());
            fail("error pos:" + out.getPos());
        }
        out.close();

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        FSDataInputStream in = fs.open(path);

        in.seek(segment.lineIndexOffset());

        info.offset = 0;
        info.len = segment.len();
        fd.setWorkStatus(ConstVar.WS_Read);
        Segment segment2 = new Segment(info, fd);
        segment2.unpersistentUnitIndex(in);
        if (segment2.recordNum() != 234) {
            fail("error recordnum:" + segment2.recordNum());
        }
        if (segment2.unitNum() != unitSize) {
            fail("error unitNum:" + segment2.unitNum());
        }
        if (segment2.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment2.keyIndexOffset());
        }
        if (segment2.lineIndexOffset() != unitlen * unitSize) {
            fail("error line index offset:" + segment2.lineIndexOffset());
        }
        if (segment2.units().size() != unitSize) {
            fail("error units.size:" + segment2.units().size());
        }

        UnitIndex index = segment2.unitIndex();
        if (index.lineIndexInfos().size() != unitSize) {
            fail("error line unit index size:" + index.lineIndexInfos().size());
        }
        if (index.keyIndexInfos().size() != 0) {
            fail("error key unit index size:" + index.keyIndexInfos().size());
        }

        for (int i = 0; i < unitSize; i++) {
            IndexInfo ii = index.lineIndexInfos().get(i);
            if (ii.beginLine() != (1 + i) * 100) {
                fail("error beginline:" + ii.beginLine() + "i:" + i);
            }
            if (ii.endLine() != (2 + i) * 100) {
                fail("error end line:" + ii.endLine() + "i:" + i);
            }
            if (ii.offset() != i * 100) {
                fail("error offset:" + ii.offset() + "i:" + i);
            }
            if (ii.len != unitlen) {
                fail("error len:" + ii.len() + "i:" + i);
            }
            if (ii.idx() != i) {
                fail("error idx:" + ii.idx() + "i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineSegment() {
    Segment segment = null;//from  w  ww. j a v a 2  s  .  c o m
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Write);
        fd.head = head;
        fd.setOut(out);

        IndexInfo info = new IndexInfo();
        info.offset = 0;

        segment = new Segment(info, fd);

        int recordNum = 150000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
            record = null;
        }

        segment.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        fd.setIn(in);
        fd.setWorkStatus(ConstVar.WS_Read);

        info.offset = 0;
        info.len = segment.len();
        info.beginLine = 0;
        info.endLine = 1500000;
        Segment segment2 = new Segment(info, fd);
        Record record = segment2.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }

        record = segment2.getRecordByLine(150000);
        if (record != null) {
            fail("should get null");
        }
        record = segment2.getRecordByLine(150001);
        if (record != null) {
            fail("should get null");
        }

        record = segment2.getRecordByLine(0);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

        int line = 150000 - 1;
        record = segment2.getRecordByLine(line);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}