List of usage examples for org.apache.hadoop.fs Path Path
public Path(URI aUri)
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentDummy() { try {/*from www.j ava2s . c o m*/ IndexInfo info = new IndexInfo(); info.offset = 12; Head head = new Head(); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentDummy_tmp", head); Segment segment = new Segment(info, fd); int unitSize = 100; for (int i = 0; i < unitSize; i++) { IndexInfo indexInfo = new IndexInfo(); indexInfo.offset = i * 100; indexInfo.len = 77; indexInfo.beginLine = (i + 1) * 100; indexInfo.endLine = (i + 2) * 100; indexInfo.idx = i; Unit unit = new Unit(indexInfo, segment); unit.beginLine = (i + 1) * 100; unit.endLine = (i + 2) * 100; unit.setFull(); unit.setMetaOffset(indexInfo.offset); segment.addUnit(unit); } String fileName = prefix + "testPersistentDummy"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); segment.persistentUnitIndex(out); segment.recordNum = 234; segment.setBeginLine(1); segment.setEndLine(235); segment.persistentUnitIndexMeta(out); segment.persistentDummy(out); if (out.getPos() != segment.unitIndex().len() + ConstVar.IndexMetaOffset + segment.remain()) { fail("error pos:" + out.getPos()); } out.close(); } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testUnpersistentUnitIndexMeta() { try {//from w ww . ja v a 2s . c o m IndexInfo info = new IndexInfo(); info.offset = 12; Head head = new Head(); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testUnpersistentUnitIndexMeta_tmp", head); Segment segment = new Segment(info, fd); String fileName = prefix + "testPersistentUnitIndexMeta"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream in = fs.open(path); int unitSize = 100; segment.unpersistentIndexMeta(in); if (segment.recordNum() != 234) { fail("error record num:" + segment.recordNum()); } if (segment.unitNum() != unitSize) { fail("error unit num:" + segment.unitNum()); } if (segment.keyIndexOffset() != -1) { fail("error keyIndex offset:" + segment.keyIndexOffset()); } if (segment.lineIndexOffset() != -1) { fail("error lineIndexOffset:" + segment.lineIndexOffset()); } } catch (IOException e) { e.printStackTrace(); fail("get ioexception:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testUnpersistentLineUnitIndex() { try {/*from w ww .j a v a 2 s . c o m*/ IndexInfo info = new IndexInfo(); info.offset = 12; Head head = new Head(); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testUnpersistentLineUnitIndex_tmp", head); Segment segment = new Segment(info, fd); String fileName = prefix + "testPersistentLineUnitIndex"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream in = fs.open(path); int unitSize = 100; segment.setUnitIndex(new UnitIndex()); segment.setUnitNum(unitSize); segment.unpersistentLineUnitIndex(in); if (segment.unitIndex().len() != ConstVar.LineIndexRecordLen * unitSize) { fail("error unitIndex len:" + segment.unitIndex().len()); } if (segment.unitIndex().lineIndexInfos().size() != unitSize) { fail("error line index size:" + segment.unitIndex().lineIndexInfos().size()); } if (segment.unitIndex().keyIndexInfos().size() != 0) { fail("error key index size:" + segment.unitIndex().keyIndexInfos().size()); } } catch (IOException e) { e.printStackTrace(); fail("get ioexception:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentSegment() { try {// ww w . j av a 2s. co m IndexInfo info = new IndexInfo(); info.offset = 0; Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentSegment_tmp", head); String fileName = prefix + "testPersistentSegment"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); fd.setOut(out); Segment segment = new Segment(info, fd); int unitSize = 100; for (int i = 0; i < unitSize; i++) { IndexInfo indexInfo = new IndexInfo(); indexInfo.offset = i * 100; indexInfo.len = 77; indexInfo.beginLine = (i + 1) * 100; indexInfo.endLine = (i + 2) * 100; indexInfo.idx = i; Unit unit = new Unit(indexInfo, segment); addRecord2Unit(unit, 100); unit.beginLine = (i + 1) * 100; unit.endLine = (i + 2) * 100; segment.addUnit(unit); if (unit.len() != 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset) { fail("error unit.len:" + unit.len()); } } segment.recordNum = 234; segment.setBeginLine(1); segment.setEndLine(235); segment.persistent(out); if (out.getPos() != fd.confSegmentSize()) { System.out.println("seg.len:" + segment.len() + "seg.remain:" + segment.remain() + "index.len" + segment.unitIndex().len()); fail("error pos:" + out.getPos()); } out.close(); int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset; FSDataInputStream in = fs.open(path); in.seek(segment.lineIndexOffset()); info.offset = 0; info.len = segment.len(); fd.setWorkStatus(ConstVar.WS_Read); Segment segment2 = new Segment(info, fd); segment2.unpersistentUnitIndex(in); if (segment2.recordNum() != 234) { fail("error recordnum:" + segment2.recordNum()); } if (segment2.unitNum() != unitSize) { fail("error unitNum:" + segment2.unitNum()); } if (segment2.keyIndexOffset() != -1) { fail("error key index offset:" + segment2.keyIndexOffset()); } if (segment2.lineIndexOffset() != unitlen * unitSize) { fail("error line index offset:" + segment2.lineIndexOffset()); } if (segment2.units().size() != unitSize) { fail("error units.size:" + segment2.units().size()); } UnitIndex index = segment2.unitIndex(); if (index.lineIndexInfos().size() != unitSize) { fail("error line unit index size:" + index.lineIndexInfos().size()); } if (index.keyIndexInfos().size() != 0) { fail("error key unit index size:" + index.keyIndexInfos().size()); } for (int i = 0; i < unitSize; i++) { IndexInfo ii = index.lineIndexInfos().get(i); if (ii.beginLine() != (1 + i) * 100) { fail("error beginline:" + ii.beginLine() + "i:" + i); } if (ii.endLine() != (2 + i) * 100) { fail("error end line:" + ii.endLine() + "i:" + i); } if (ii.offset() != i * 100) { fail("error offset:" + ii.offset() + "i:" + i); } if (ii.len != unitlen) { fail("error len:" + ii.len() + "i:" + i); } if (ii.idx() != i) { fail("error idx:" + ii.idx() + "i:" + i); } } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testGetRecordByLineSegment() { Segment segment = null;//from w w w . ja v a2s .c o m try { FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0)); fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1)); fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2)); fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3)); fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4)); fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5)); fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6)); Head head = new Head(); head.setFieldMap(fieldMap); String fileName = prefix + "testGetRecordByLineSegment"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.setWorkStatus(ConstVar.WS_Write); fd.head = head; fd.setOut(out); IndexInfo info = new IndexInfo(); info.offset = 0; segment = new Segment(info, fd); int recordNum = 150000; for (int i = 0; i < recordNum; i++) { Record record = new Record(7); record.addValue(new FieldValue((byte) 1, (short) 0)); record.addValue(new FieldValue((short) 2, (short) 1)); record.addValue(new FieldValue((int) 3, (short) 2)); record.addValue(new FieldValue((long) 4, (short) 3)); record.addValue(new FieldValue((float) 5.5, (short) 4)); record.addValue(new FieldValue((double) 6.6, (short) 5)); record.addValue(new FieldValue("hello konten", (short) 6)); segment.addRecord(record); record = null; } segment.persistent(out); out.close(); FSDataInputStream in = fs.open(path); fd.setIn(in); fd.setWorkStatus(ConstVar.WS_Read); info.offset = 0; info.len = segment.len(); info.beginLine = 0; info.endLine = 1500000; Segment segment2 = new Segment(info, fd); Record record = segment2.getRecordByLine(-1); if (record != null) { fail("should get null"); } record = segment2.getRecordByLine(150000); if (record != null) { fail("should get null"); } record = segment2.getRecordByLine(150001); if (record != null) { fail("should get null"); } record = segment2.getRecordByLine(0); if (record == null) { fail("should not get null"); } judgeFixedRecord(record); int line = 150000 - 1; record = segment2.getRecordByLine(line); if (record == null) { fail("should not get null"); } judgeFixedRecord(record); } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testGetRecordByOrderSegment() { Segment segment = null;/*from w w w .j a v a2 s .co m*/ try { FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0)); fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1)); fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2)); fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3)); fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4)); fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5)); fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6)); Head head = new Head(); head.setFieldMap(fieldMap); String fileName = prefix + "testGetRecordByValueSegment"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.setWorkStatus(ConstVar.WS_Write); fd.head = head; fd.setOut(out); IndexInfo info = new IndexInfo(); info.offset = 0; segment = new Segment(info, fd); int recordNum = 150000; for (int i = 0; i < recordNum; i++) { Record record = new Record(7); record.addValue(new FieldValue((byte) 1, (short) 0)); record.addValue(new FieldValue((short) 2, (short) 1)); record.addValue(new FieldValue((int) 3, (short) 2)); record.addValue(new FieldValue((long) 4, (short) 3)); record.addValue(new FieldValue((float) 5.5, (short) 4)); record.addValue(new FieldValue((double) 6.6, (short) 5)); record.addValue(new FieldValue("hello konten", (short) 6)); segment.addRecord(record); record = null; } segment.persistent(out); out.close(); FSDataInputStream in = fs.open(path); fd.setIn(in); fd.setWorkStatus(ConstVar.WS_Read); info.offset = 0; info.len = segment.len(); Segment segment2 = new Segment(info, fd); FieldValue[] values = new FieldValue[2]; values[0] = new FieldValue((byte) 1, (short) 2); values[1] = new FieldValue((short) 2, (short) 3); Record[] records = segment2.getRecordByOrder(values, values.length); if (records != null) { fail("should get null, index error, records.len:" + records.length); } values[0] = new FieldValue((byte) 1, (short) 0); values[1] = new FieldValue((short) 3, (short) 1); records = segment2.getRecordByOrder(values, values.length); if (records != null) { fail("should get null, value error"); } values[0] = new FieldValue((byte) 1, (short) 0); values[1] = new FieldValue((short) 2, (short) 1); records = segment2.getRecordByOrder(values, values.length); if (records == null) { fail("should not get null"); } if (records.length != 150000) { fail("error result size:" + records.length); } for (int i = 0; i < 150000; i++) { judgeFixedRecord(records[i]); } records = segment2.getRecordByOrder(null, 0); if (records == null) { fail("should not get null"); } if (records.length != 150000) { fail("error result size:" + records.length); } for (int i = 0; i < 150000; i++) { judgeFixedRecord(records[i]); } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testSetNoPreFileName() { try {/*from w ww . j a v a 2 s. c o m*/ Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); Head head = new Head(); String fileName = prefix + "testSetNoPreFileName"; fd.create(fileName, head); Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); if (!fs.isFile(path)) { fail("create file fail"); } } catch (IOException e) { e.printStackTrace(); fail("get ioexception:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testSetPreFileName() { try {//www . j av a 2 s . c o m Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); Head head = new Head(); //String fileName = "hdfs://tdw-172-25-38-246:54310/user/tdwadmin/testSetPreFileName"; String fileName = "/user/tdwadmin/se_test/fs/basic/testSetPreFileName"; fd.create(fileName, head); fd.close(); Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); if (!fs.isFile(path)) { fail("create file fail"); } } catch (IOException e) { fail(e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testClose() { try {//w w w .j av a 2 s . c o m Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testClose", head); int size = 100 * 10000; for (int i = 0; i < size; i++) { Record record = new Record(7); record.addValue(new FieldValue((byte) 1, (short) 0)); record.addValue(new FieldValue((short) 2, (short) 1)); record.addValue(new FieldValue((int) 3, (short) 2)); record.addValue(new FieldValue((long) 4, (short) 3)); record.addValue(new FieldValue((float) 5.5, (short) 4)); record.addValue(new FieldValue((double) 6.6, (short) 5)); record.addValue(new FieldValue("hello konten", (short) 6)); fd.addRecord(record); } if (fd.recordNum() != size) { fail("error record num:" + fd.recordNum()); } if (fd.currentSegment().currentUnit() == null) { fail("null current unit"); } if (fd.currentSegment() == null) { fail("null current seg"); } if (fd.segmentNum() != 0) { fail("error segment num:" + fd.segmentNum()); } int headLen = head.len(); long currentUnitLen = fd.currentSegment().currentUnit().len(); long segmentLen = fd.currentSegment().len() + currentUnitLen + ConstVar.LineIndexRecordLen; long remain = fd.currentSegment().remain(); int unitNum = fd.currentSegment().unitNum(); fd.close(); int indexLen = ConstVar.LineIndexRecordLen * fd.segmentNum(); int metaLen = ConstVar.IndexMetaOffset; long fileLen = fd.getFileLen(); if (fileLen != headLen + segmentLen + indexLen + metaLen) { fail("error file len:" + fileLen); } if (fd.in() != null) { fail("in should set null"); } if (fd.out() != null) { fail("out should set null"); } if (fd.recordNum() != 0) { fail("record num should set 0"); } if (fd.keyIndexOffset != -1) { fail("key index offset not -1"); } if (fd.lineIndexOffset != -1) { fail("line index offset not -1"); } if (fd.currentOffset != -1) { fail("current offset not -1"); } if (fd.hasLoadAllSegmentDone) { fail("has load all segment Done not false"); } String fileName = prefix + "testClose"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream in = fs.open(path); long metaOffset = fileLen - ConstVar.IndexMetaOffset; in.seek(metaOffset); int recordNum = in.readInt(); int segNum = in.readInt(); long keyIndexOffset = in.readLong(); long lineIndexOffset = in.readLong(); if (recordNum != size) { fail("error record num:" + recordNum); } if (segNum != 1) { fail("error segNum:" + segNum); } if (keyIndexOffset != -1) { fail("error key index offset:" + keyIndexOffset); } if (lineIndexOffset != (headLen + segmentLen)) { fail("error line index offset:" + lineIndexOffset); } in.seek(lineIndexOffset); for (int i = 0; i < segNum; i++) { int beginLine = in.readInt(); int endLine = in.readInt(); long offset = in.readLong(); long len = in.readLong(); int idx = in.readInt(); if (beginLine != 0) { fail("error beginLine:" + beginLine); } if (endLine != size) { fail("error end line:" + endLine); } if (offset != head.len()) { fail("error offset:" + offset); } long tlen = size * full7chunkLen + size * 8 + ConstVar.DataChunkMetaOffset * (unitNum + 1) + 28 * (unitNum + 1) + 24; if (len != tlen) { fail("error len:" + len); } } } catch (IOException e) { e.printStackTrace(); fail("get ioexception:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testNoFillLastSegment() { try {/*from www . j a va2s.c o m*/ String fileName = prefix + "testNoFillLastSegment"; Head head = new Head(); FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0)); fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1)); fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2)); fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3)); fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4)); fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5)); fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6)); head.setFieldMap(fieldMap); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(fileName, head); Record record = new Record(7); record.addValue(new FieldValue((byte) 1, (short) 0)); record.addValue(new FieldValue((short) 2, (short) 1)); record.addValue(new FieldValue((int) 3, (short) 2)); record.addValue(new FieldValue((long) 4, (short) 3)); record.addValue(new FieldValue((float) 5.5, (short) 4)); record.addValue(new FieldValue((double) 6.6, (short) 5)); record.addValue(new FieldValue("hello konten", (short) 6)); fd.addRecord(record); fd.close(); FileSystem fs = FileSystem.get(conf); long fileLen = fs.getFileStatus(new Path(fileName)).getLen(); int tlen = head.len() + full7chunkLen + 8 + ConstVar.DataChunkMetaOffset + ConstVar.LineIndexRecordLen + ConstVar.IndexMetaOffset + ConstVar.LineIndexRecordLen + ConstVar.IndexMetaOffset; if (fileLen != tlen) { fail("error file len:" + fileLen); } FormatDataFile fd2 = new FormatDataFile(new Configuration()); fd2.open(fileName); if (fd2.recordNum() != 1) { fail("error record num:" + fd2.recordNum()); } if (fd2.segmentNum() != 1) { fail("error segment num:" + fd2.segmentNum()); } } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } }