Example usage for org.apache.hadoop.fs Path Path

List of usage examples for org.apache.hadoop.fs Path Path

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentLineIndexInfo() {
    try {/*  w  w w . ja  va 2 s  . c  o m*/
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        IndexInfo info = new IndexInfo();
        info.unpersistentLineIndexInfo(in);
        in.close();

        if (info.beginLine != 11) {
            fail("beginLine fail:" + info.beginLine);
        }
        if (info.endLine != 22) {
            fail("endLine fail:" + info.endLine);
        }
        if (info.offset != 33) {
            fail("offset fail:" + info.offset);
        }
        if (info.len != 44) {
            fail("len fail:" + info.len);
        }
        if (info.idx != 55) {
            fail("idx fail:" + info.idx);
        }

        fs.close();
    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentKeyIndexInfo() {
    try {// w ww  .j  a v a2  s .  com
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        IndexInfo info = new IndexInfo();
        info.unpersistentKeyIndexInfo(in);
        in.close();

        if (info.beginKey != 111) {
            fail("beginKey fail:" + info.beginKey);
        }
        if (info.endKey != 222) {
            fail("beginKey fail:" + info.beginKey);
        }

        fs.close();
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndex() {
    try {/*  w  w w .j  a  v  a  2  s .  c o  m*/
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginLine = i;
            info.endLine = i + 10;
            info.offset = i;
            info.len = i + 20;
            info.idx = i;

            index.addIndexInfo(info, ConstVar.LineMode);
        }

        String fileName = prefix + "testPersistentLineIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != i) {
                fail("beginLine fail:" + beginLine);
            }
            if (endLine != i + 10) {
                fail("endLine fail:" + endLine);
            }
            if (offset != i) {
                fail("offset fail:" + offset);
            }
            if (len != i + 20) {
                fail("len fail:" + len);
            }
            if (idx != i) {
                fail("idx fail:" + idx);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndex() {
    try {/*from w  w  w. j  a  v  a  2  s. com*/
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginKey = i;
            info.endKey = i + 10;

            index.addIndexInfo(info, ConstVar.KeyMode);
        }

        String fileName = prefix + "testPersistentKeyIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginKey = in.readInt();
            int endKey = in.readInt();

            if (beginKey != i) {
                fail("beginKey fail:" + beginKey);
            }
            if (endKey != i + 10) {
                fail("endKey fail:" + endKey);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentField() throws IOException {
    {/*w  w w .j ava  2 s . co m*/
        String file = prefix + "testPersistentField";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Field field = new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1);
        field.persistent(out);
        if (out.getPos() != 7) {
            fail("error out.pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);
        byte type = in.readByte();
        int len = in.readInt();
        short idx = in.readShort();

        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

    }
    {
    }
    {
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentFieldMap() {
    try {//from   ww  w .  jav a2  s.  c  om
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataOutputStream out1 = fs.create(path);

        FieldMap fm1 = new FieldMap();
        fm1.persistent(out1);
        if (out1.getPos() != 2) {
            fail("persisitent null fieldmap fail, pos:" + out1.getPos());
        }
        out1.close();

        FSDataInputStream in1 = fs.open(path);
        short fieldNum = in1.readShort();
        if (fieldNum != 0) {
            fail("persistent null fieldmap fail, fieldNum:" + fieldNum);
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataOutputStream out2 = fs.create(path);

        FieldMap fm2 = new FieldMap();
        fm2.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1));
        fm2.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 3));
        fm2.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 5));
        fm2.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 7));
        fm2.persistent(out2);
        if (out2.getPos() != (2 + 4 * 7)) {
            fail("persistent 4 field fail, pos:" + out2.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);
        fieldNum = in2.readShort();
        if (fieldNum != 4) {
            fail("persistent 4 field fail, fieldNum:" + fieldNum);
        }

        byte type = in2.readByte();
        int len = in2.readInt();
        short idx = in2.readShort();
        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Short) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Short) {
            fail("fail len:" + len);
        }
        if (idx != 3) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Int) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Int) {
            fail("fail len:" + len);
        }
        if (idx != 5) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Long) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Long) {
            fail("fail len:" + len);
        }
        if (idx != 7) {
            fail("fail idx:" + idx);
        }
    } catch (IOException e) {
        fail("testPersistentField fail1:" + e.getMessage());
    } catch (Exception e) {
        fail("testPersistentField fail2:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentFieldMap() {
    try {/* w w w . j a  v  a 2 s  . c  om*/
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataInputStream in1 = fs.open(path);

        FieldMap fm1 = new FieldMap();
        fm1.unpersistent(in1);
        in1.close();

        if (fm1.len() != 2) {
            fail("unpersistent fieldMap fail, len:" + fm1.len());
        }
        if (fm1.fieldNum() != 0) {
            fail("unpersistent fieldMap fail, fieldNum:" + fm1.fieldNum());
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataInputStream in2 = fs.open(path);

        FieldMap fm2 = new FieldMap();
        fm2.unpersistent(in2);
        if (fm2.len() != 2 + 4 * 7) {
            fail("unpersistent fieldMap fail, len:" + fm2.len());
        }
        if (fm2.fieldNum() != 4) {
            fail("unpersistent fieldMap fail, fieldNum:" + fm2.fieldNum());
        }

        Field field = fm2.fields.get((short) 1);
        if (field.type != ConstVar.FieldType_Byte) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Byte) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 1) {
            fail("unpersistent fail idx:" + field.index);
        }

        field = fm2.fields.get((short) 3);
        if (field.type != ConstVar.FieldType_Short) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Short) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 3) {
            fail("unpersistent fail idx:" + field.index);
        }

        field = fm2.fields.get((short) 5);
        if (field.type != ConstVar.FieldType_Int) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Int) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 5) {
            fail("unpersistent fail idx:" + field.index);
        }

        field = fm2.fields.get((short) 7);
        if (field.type != ConstVar.FieldType_Long) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Long) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 7) {
            fail("unpersistent fail idx:" + field.index);
        }
    } catch (IOException e) {
        fail("unpersistent fieldMap fail:" + e.getMessage());
    } catch (Exception e) {
        fail("unpersistent failMap fail:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentHead() {

    try {/*from w w  w.  j a  va  2  s . c  o m*/
        String fileName = prefix + "testPersistentHead";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Head head = new Head();
        head.persistent(out);
        if (out.getPos() != 17) {
            fail("persistent error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int magic = in.readInt();
        short ver = in.readShort();
        byte var = in.readByte();
        byte compress = in.readByte();
        byte compressStyle = in.readByte();
        short primaryIndex = in.readShort();
        byte encode = in.readByte();
        byte encodeStyle = in.readByte();
        short keyLen = in.readShort();
        short fieldNum = in.readShort();

        if (magic != head.magic()) {
            fail("error magic:" + magic);
        }
        if (ver != head.ver()) {
            fail("error ver:" + ver);
        }
        if (var != 0) {
            fail("error var:" + var);
        }
        if (compress != head.compress()) {
            fail("error compress:" + compress);
        }
        if (compressStyle != head.compressStyle()) {
            fail("error compressStyle:" + compressStyle);
        }
        if (primaryIndex != head.primaryIndex()) {
            fail("error primaryIndex:" + primaryIndex);
        }
        if (encode != head.encode()) {
            fail("error encode:" + encode);
        }
        if (encodeStyle != head.encodeStyle()) {
            fail("error encodeStyle:" + encodeStyle);
        }
        if (keyLen != 0) {
            fail("error keyLen:" + keyLen);
        }
        if (fieldNum != 0) {
            fail("error fieldNum:" + fieldNum);
        }

        fileName = prefix + "testPersistentHead2";
        path = new Path(fileName);
        FSDataOutputStream out2 = fs.create(path);

        Head head2 = new Head();
        String key = "hello konten";

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        head2.setFieldMap(fieldMap);
        head2.setKey(key);

        head2.persistent(out2);
        if (out2.getPos() != 13 + 2 + key.length() + fieldMap.len()) {
            fail("persistent error pos:" + out.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);

        magic = in2.readInt();
        ver = in2.readShort();
        var = in2.readByte();
        compress = in2.readByte();
        compressStyle = in2.readByte();
        primaryIndex = in2.readShort();
        encode = in2.readByte();
        encodeStyle = in2.readByte();

        keyLen = in2.readShort();
        if (keyLen == 0) {
            fail("error keyLen:" + keyLen);
        }
        byte[] buf = new byte[keyLen];
        in2.readFully(buf);
        String keykey = new String(buf);
        if (!key.equals(keykey)) {
            fail("error key:" + keykey);
        }

        FieldMap fieldMap22 = new FieldMap();
        fieldMap22.unpersistent(in2);
        if (fieldMap22.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap22.fieldNum());
        }
        Field field = fieldMap22.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap22.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentHead() {
    try {//w  w w.ja  va 2  s.  c  o m
        String fileName = prefix + "testPersistentHead2";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        Head head = new Head();

        head.setMagic(in.readInt());
        head.unpersistent(in);

        if (head.compress() != 0) {
            fail("compress not 0");
        }
        if (head.encode() != 0) {
            fail("encode != 0");
        }
        if (head.var() != 0) {
            fail("var != 0");
        }
        if (head.primaryIndex() != -1) {
            fail("primary index != -1");
        }
        if (head.magic() != ConstVar.DataMagic) {
            fail("invalid init magic");
        }
        if (head.ver() != ConstVar.Ver) {
            fail("invalid init ver");
        }

        String key = "hello konten";
        if (!head.key.equals(key)) {
            fail("error key:" + head.key);
        }

        FieldMap fieldMap = head.fieldMap();
        if (fieldMap.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap.fieldNum());
        }
        Field field = fieldMap.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecord() {
    try {/* ww w.  j ava  2  s . com*/
        String fileName = prefix + "testChunkToRecord";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        byte[] fb = new byte[ConstVar.Sizeof_Float];
        float f = (float) 5.5;
        Util.float2bytes(fb, f);
        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, fb,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());
        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        value = new byte[4];
        in.readFully(value);
        float fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;
        if (bitSet.length() != (fieldNum / 8 + 1) * 8) {
            fail("bitSet.len:" + bitSet.length());
        }

        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("bitSet should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }

        {
        }
        lv = Util.bytes2long(value, 0, len);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        fail("should not exception:" + e.getMessage());
    }
}