List of usage examples for org.apache.hadoop.fs FileStatus isFile
public boolean isFile()
From source file:org.apache.rya.reasoning.mr.MRReasoningUtils.java
License:Apache License
/** * Load serialized schema information from a file. *///from w w w . j ava 2 s.com public static Schema loadSchema(Configuration conf) { SchemaWritable schema = new SchemaWritable(); try { FileSystem fs = FileSystem.get(conf); Path schemaPath = getSchemaPath(conf); if (fs.isDirectory(schemaPath)) { for (FileStatus status : fs.listStatus(schemaPath)) { schemaPath = status.getPath(); if (status.isFile() && status.getLen() > 0 && !schemaPath.getName().startsWith(DEBUG_OUT)) { break; } } } SequenceFile.Reader in = new SequenceFile.Reader(conf, SequenceFile.Reader.file(schemaPath)); NullWritable key = NullWritable.get(); in.next(key, schema); in.close(); } catch (IOException e) { e.printStackTrace(); } return schema; }
From source file:org.apache.slider.server.avro.RoleHistoryWriter.java
License:Apache License
/** * Find all history entries in a dir. The dir is created if it is * not already defined.//from w ww .j a v a 2 s. c om * * The scan uses the match pattern {@link SliderKeys#HISTORY_FILENAME_MATCH_PATTERN} * while dropping empty files and directories which match the pattern. * The list is then sorted with a comparator that sorts on filename, * relying on the filename of newer created files being later than the old ones. * * * * @param fs filesystem * @param dir dir to scan * @param includeEmptyFiles should empty files be included in the result? * @return a possibly empty list * @throws IOException IO problems * @throws FileNotFoundException if the target dir is actually a path */ public List<Path> findAllHistoryEntries(FileSystem fs, Path dir, boolean includeEmptyFiles) throws IOException { assert fs != null; assert dir != null; if (!fs.exists(dir)) { fs.mkdirs(dir); } else if (!fs.isDirectory(dir)) { throw new FileNotFoundException("Not a directory " + dir.toString()); } PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN); FileStatus[] stats = fs.listStatus(dir, filter); List<Path> paths = new ArrayList<Path>(stats.length); for (FileStatus stat : stats) { log.debug("Possible entry: {}", stat.toString()); if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) { paths.add(stat.getPath()); } } sortHistoryPaths(paths); return paths; }
From source file:org.apache.slider.test.ContractTestUtils.java
License:Apache License
/** * Read a file, verify its length and contents match the expected array * @param fs filesystem// w ww.j a v a 2s . c om * @param path path to file * @param original original dataset * @throws IOException IO Problems */ public static void verifyFileContents(FileSystem fs, Path path, byte[] original) throws IOException { FileStatus stat = fs.getFileStatus(path); String statText = stat.toString(); assertTrue("not a file " + statText, stat.isFile()); assertEquals("wrong length " + statText, original.length, stat.getLen()); byte[] bytes = readDataset(fs, path, original.length); compareByteArrays(original, bytes, original.length); }
From source file:org.apache.streams.hdfs.WebHdfsPersistReaderTask.java
License:Apache License
@Override public void run() { for (FileStatus fileStatus : reader.status) { BufferedReader bufferedReader; LOGGER.info("Found " + fileStatus.getPath().getName()); if (fileStatus.isFile() && !fileStatus.getPath().getName().startsWith("_")) { LOGGER.info("Started Processing " + fileStatus.getPath().getName()); try { bufferedReader = new BufferedReader( new InputStreamReader(reader.client.open(fileStatus.getPath()))); } catch (Exception e) { e.printStackTrace();//from ww w . ja va 2 s .c o m LOGGER.error(e.getMessage()); return; } String line = ""; do { try { line = bufferedReader.readLine(); if (!Strings.isNullOrEmpty(line)) { reader.countersCurrent.incrementAttempt(); String[] fields = line.split(Character.toString(reader.DELIMITER)); // Temporarily disabling timestamp reads to make reader and writer compatible // This capability will be restore in PR for STREAMS-169 //StreamsDatum entry = new StreamsDatum(fields[3], fields[0], new DateTime(Long.parseLong(fields[2]))); StreamsDatum entry = new StreamsDatum(fields[3], fields[0]); write(entry); reader.countersCurrent.incrementStatus(DatumStatus.SUCCESS); } } catch (Exception e) { e.printStackTrace(); LOGGER.warn(e.getMessage()); reader.countersCurrent.incrementStatus(DatumStatus.FAIL); } } while (!Strings.isNullOrEmpty(line)); LOGGER.info("Finished Processing " + fileStatus.getPath().getName()); try { bufferedReader.close(); } catch (Exception e) { e.printStackTrace(); LOGGER.error(e.getMessage()); } } } }
From source file:org.apache.tajo.engine.planner.LogicalPlanner.java
License:Apache License
private void updatePhysicalInfo(TableDesc desc) { if (desc.getPath() != null) { try {/*from www . j a va 2 s.c om*/ FileSystem fs = desc.getPath().getFileSystem(new Configuration()); FileStatus status = fs.getFileStatus(desc.getPath()); if (desc.getStats() != null && (status.isDirectory() || status.isFile())) { ContentSummary summary = fs.getContentSummary(desc.getPath()); if (summary != null) { long volume = summary.getLength(); desc.getStats().setNumBytes(volume); } } } catch (Throwable t) { LOG.warn(t); } } }
From source file:org.apache.tajo.engine.planner.physical.PhysicalPlanUtil.java
License:Apache License
/** * * @param fs/* w ww . j a va2 s. co m*/ * @param path The table path * @param result The final result files to be used * @param startFileIndex * @param numResultFiles * @param currentFileIndex * @param partitioned A flag to indicate if this table is partitioned * @param currentDepth Current visiting depth of partition directories * @param maxDepth The partition depth of this table * @throws IOException */ private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result, int startFileIndex, int numResultFiles, AtomicInteger currentFileIndex, boolean partitioned, int currentDepth, int maxDepth) throws IOException { // Intermediate directory if (fs.isDirectory(path)) { FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter); if (files != null && files.length > 0) { for (FileStatus eachFile : files) { // checking if the enough number of files are found if (result.size() >= numResultFiles) { return; } if (eachFile.isDirectory()) { getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, currentFileIndex, partitioned, currentDepth + 1, // increment a visiting depth maxDepth); // if partitioned table, we should ignore files located in the intermediate directory. // we can ensure that this file is in leaf directory if currentDepth == maxDepth. } else if (eachFile.isFile() && eachFile.getLen() > 0 && (!partitioned || currentDepth == maxDepth)) { if (currentFileIndex.get() >= startFileIndex) { result.add(eachFile); } currentFileIndex.incrementAndGet(); } } } // Files located in leaf directory } else { FileStatus fileStatus = fs.getFileStatus(path); if (fileStatus != null && fileStatus.getLen() > 0) { if (currentFileIndex.get() >= startFileIndex) { result.add(fileStatus); } currentFileIndex.incrementAndGet(); if (result.size() >= numResultFiles) { return; } } } }
From source file:org.apache.tajo.engine.planner.PlannerUtil.java
License:Apache License
private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result, int startFileIndex, int numResultFiles, AtomicInteger currentFileIndex) throws IOException { if (fs.isDirectory(path)) { FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter); if (files != null && files.length > 0) { for (FileStatus eachFile : files) { if (result.size() >= numResultFiles) { return; }//from w w w. j a v a2 s . com if (eachFile.isDirectory()) { getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, currentFileIndex); } else if (eachFile.isFile() && eachFile.getLen() > 0) { if (currentFileIndex.get() >= startFileIndex) { result.add(eachFile); } currentFileIndex.incrementAndGet(); } } } } else { FileStatus fileStatus = fs.getFileStatus(path); if (fileStatus != null && fileStatus.getLen() > 0) { if (currentFileIndex.get() >= startFileIndex) { result.add(fileStatus); } currentFileIndex.incrementAndGet(); if (result.size() >= numResultFiles) { return; } } } }
From source file:org.apache.tajo.engine.query.TestHBaseTable.java
License:Apache License
@Test public void testInsertIntoLocation() throws Exception { executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text) " + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:', " + "'hbase.split.rowkeys'='010,040,060,080', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); assertTableExists("hbase_mapped_table"); try {/*from ww w .j a v a 2s . com*/ // create test table KeyValueSet tableOptions = new KeyValueSet(); tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); schema.addColumn("name", Type.TEXT); schema.addColumn("comment", Type.TEXT); List<String> datas = new ArrayList<String>(); DecimalFormat df = new DecimalFormat("000"); for (int i = 99; i >= 0; i--) { datas.add(df.format(i) + "|value" + i + "|comment-" + i); } TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", schema, tableOptions, datas.toArray(new String[] {}), 2); executeString("insert into location '/tmp/hfile_test' " + "select id, name, comment from base_table ") .close(); FileSystem fs = testingCluster.getDefaultFileSystem(); Path path = new Path("/tmp/hfile_test"); assertTrue(fs.exists(path)); FileStatus[] files = fs.listStatus(path); assertNotNull(files); assertEquals(2, files.length); int index = 0; for (FileStatus eachFile : files) { assertEquals("/tmp/hfile_test/part-01-00000" + index + "-00" + index, eachFile.getPath().toUri().getPath()); for (FileStatus subFile : fs.listStatus(eachFile.getPath())) { assertTrue(subFile.isFile()); assertTrue(subFile.getLen() > 0); } index++; } } finally { executeString("DROP TABLE base_table PURGE").close(); executeString("DROP TABLE hbase_mapped_table PURGE").close(); } }
From source file:org.apache.tajo.engine.query.TestJoinBroadcast.java
License:Apache License
private List<Path> getPartitionPathList(FileSystem fs, Path path) throws Exception { FileStatus[] files = fs.listStatus(path); List<Path> paths = new ArrayList<Path>(); if (files != null) { for (FileStatus eachFile : files) { if (eachFile.isFile()) { paths.add(path);/*w w w .jav a 2 s .c om*/ return paths; } else { paths.addAll(getPartitionPathList(fs, eachFile.getPath())); } } } return paths; }
From source file:org.apache.tajo.engine.query.TestJoinQuery.java
License:Apache License
protected static List<Path> getPartitionPathList(FileSystem fs, Path path) throws Exception { FileStatus[] files = fs.listStatus(path); List<Path> paths = new ArrayList<Path>(); if (files != null) { for (FileStatus eachFile : files) { if (eachFile.isFile()) { paths.add(path);//from ww w. j a va2s. c om return paths; } else { paths.addAll(getPartitionPathList(fs, eachFile.getPath())); } } } return paths; }