List of usage examples for org.apache.hadoop.fs FileSystem isFile
@Deprecated public boolean isFile(Path f) throws IOException
From source file:org.bgi.flexlab.gaea.tools.mapreduce.vcfqualitycontrol.VCFQualityControlOptions.java
License:Open Source License
public void traversalInputPath(Path path) { Configuration conf = new Configuration(); FileSystem fs = null; try {//w ww . ja v a 2 s. c om fs = path.getFileSystem(conf); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { if (!fs.exists(path)) { System.err.println("Input File Path is not exist! Please check -I var."); System.exit(-1); } if (fs.isFile(path)) { inputList.add(path); } else { FileStatus stats[] = fs.listStatus(path); for (FileStatus file : stats) { Path filePath = file.getPath(); if (!fs.isFile(filePath)) { traversalInputPath(filePath); } else { inputList.add(filePath); } } } } catch (IOException ioe) { throw new RuntimeException(ioe); } }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
private void moveTaskOutputs(TaskAttemptContext context, FileSystem fs, Path jobOutputDir, Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context)); LOG.info("Renaming:" + taskOutput + " to:" + finalOutputPath); if (!fs.rename(taskOutput, finalOutputPath)) { LOG.info("Rename Failed for:" + taskOutput + " to:" + finalOutputPath + " Trying Delete and then Rename"); if (!fs.delete(finalOutputPath, true)) { throw new IOException("Failed to delete earlier output of task: " + attemptId); }//from w w w. j ava 2s . c om LOG.info("Renaming:" + taskOutput + " to: " + finalOutputPath); if (!fs.rename(taskOutput, finalOutputPath)) { throw new IOException("Failed to save output of task: " + attemptId); } } LOG.info("Moved " + taskOutput + " to " + finalOutputPath); } else if (fs.getFileStatus(taskOutput).isDir()) { FileStatus[] paths = fs.listStatus(taskOutput); Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context)); LOG.info("Moving " + taskOutput + " to " + finalOutputPath); fs.mkdirs(finalOutputPath); if (paths != null) { for (FileStatus path : paths) { LOG.info("Moving " + path.getPath()); moveTaskOutputs(context, fs, jobOutputDir, path.getPath()); } } } }
From source file:org.exem.flamingo.shared.util.HdfsUtils.java
License:Apache License
/** * ? ?.//from ww w . j ava2 s .c o m * * @param fs FileSystem * @param path ? Path * @return ? <tt>true</tt> */ public static boolean isDirectory(FileSystem fs, String path) { try { return !fs.isFile(new Path(path)); } catch (Exception ex) { throw new ServiceException(ExceptionUtils.getMessage("Cannot access '{}'", path), ex); } }
From source file:org.exem.flamingo.shared.util.HdfsUtils.java
License:Apache License
/** * ?? ?./*from w w w. j a va 2 s. c o m*/ * * @param fs FileSystem * @param path ? Path * @return ?? <tt>true</tt> */ public static boolean isFile(FileSystem fs, String path) { try { return fs.isFile(new Path(path)); } catch (Exception ex) { throw new ServiceException(ExceptionUtils.getMessage("Cannot access '{}'", path), ex); } }
From source file:org.kitesdk.data.spi.filesystem.FileSystemUtil.java
License:Apache License
private static <T> T visit(PathVisitor<T> visitor, FileSystem fs, Path path, List<Path> followedLinks) throws IOException { if (fs.isFile(path)) { return visitor.file(fs, path); } else if (IS_SYMLINK != null && IS_SYMLINK.<Boolean>invoke(fs.getFileStatus(path))) { Preconditions.checkArgument(!followedLinks.contains(path), "Encountered recursive path structure at link: " + path); followedLinks.add(path); // no need to remove return visit(visitor, fs, fs.getLinkTarget(path), followedLinks); }/*w ww . j a v a 2 s. co m*/ List<T> children = Lists.newArrayList(); FileStatus[] statuses = fs.listStatus(path, PathFilters.notHidden()); for (FileStatus stat : statuses) { children.add(visit(visitor, fs, stat.getPath())); } return visitor.directory(fs, path, children); }
From source file:org.mrgeo.hdfs.ingest.HdfsImageIngestDataProvider.java
License:Apache License
public static boolean canOpen(final Configuration conf, final String name) throws IOException { final Path imagePath = new Path(name); final FileSystem fs = HadoopFileUtils.getFileSystem(conf, imagePath); return fs.exists(imagePath) && fs.isFile(imagePath); }
From source file:org.mrgeo.mapalgebra.SplitVectorMapOp.java
License:Apache License
@Override public void build(Progress p) throws IOException, JobFailedException, JobCancelledException { ProgressHierarchy ph = new ProgressHierarchy(p); ph.createChild(1.0f);//from w w w . j a v a2s. co m ph.createChild(1.0f); MapOp inputMapOp = _inputs.get(0); Path inputPath = null; // TODO: // The following code is an ugly hack until we have time to re-factor the direct // reading of vector data. Right now, there is generic code for doing this (see // AutoFeatureInputFormat), but it's tightly coupled to the map/reduce InputFormat // and splits. We need to re-factor it so that the core part for reading the // vector data is independent of InputFormat. This represents a first step in that // direction where InlineCsvInputFormat itself was re-factored. That's why there's // a special case below - because the other vector formats have not been re-factored // and there is no generic interface in place for reading any vector data. if (inputMapOp instanceof InlineCsvMapOp) { InlineCsvInputFormatDescriptor ifd = (InlineCsvInputFormatDescriptor) ((VectorMapOp) inputMapOp) .getVectorOutput(); determineOutputForInlineCsvInput(ifd, ','); return; } else if (inputMapOp instanceof VectorMapOp) { inputPath = new Path(((VectorMapOp) inputMapOp).getOutputName()); } else { // defensive code since input should be VectorMapOp - see addInput() throw new IllegalArgumentException("Invalid value for vector argument to SplitVector"); } // SplitVectorDriver svd = new SplitVectorDriver(); // svd.run(getConf(), inputPath, splitCount, currentSplit, splitType, _outputName, p, jobListener); FileSystem dfs = HadoopFileUtils.getFileSystem(inputPath); if (!dfs.exists(inputPath)) { throw new IOException( String.format("Cannot read contrast measures, %s does not exist", inputPath.toString())); } FileStatus[] outputFiles = dfs.listStatus(inputPath); List<Path> tsvFiles = new ArrayList<Path>(); if (dfs.isFile(inputPath)) { tsvFiles.add(inputPath); } else { for (FileStatus fileStatus : outputFiles) { if (fileStatus.isDir() == false) { Path fp = fileStatus.getPath(); String name = fp.getName(); if (name.startsWith("part-")) { tsvFiles.add(fp); } } } } FSDataOutputStream os = dfs.create(new Path(_outputName), true); java.io.PrintWriter pw = new java.io.PrintWriter(new java.io.OutputStreamWriter(os)); try { long lineNumber = 0; boolean isTestSplit = splitType.equalsIgnoreCase("test"); for (Path tsvFile : tsvFiles) { InputStream is = HadoopFileUtils.open(tsvFile); // dfs.open(tsvFile); java.io.BufferedReader r = new java.io.BufferedReader(new java.io.InputStreamReader(is)); try { String line; while ((line = r.readLine()) != null) { if ((lineNumber % splitCount) == (currentSplit - 1)) { if (isTestSplit) { pw.println(line); } } else { if (!isTestSplit) { pw.println(line); } } lineNumber++; } } finally { r.close(); if (is != null) { is.close(); } } } } finally { pw.close(); if (os != null) { os.close(); } } // Copy the input columns to the output columns, excluding the stats // because we've filtered the actual data, so the stats will be wrong. Path inputColumnsPath = new Path(inputPath.toString() + ".columns"); Path outputColumnsPath = new Path(_outputName + ".columns"); ColumnDefinitionFile inputCDF = new ColumnDefinitionFile(inputColumnsPath); ColumnDefinitionFile outputCDF = new ColumnDefinitionFile(); Vector<Column> columns = inputCDF.getColumns(); Vector<Column> newColumns = new Vector<Column>(); for (Column column : columns) { newColumns.add(new Column(column.getName(), column.getType())); } outputCDF.setColumns(newColumns); outputCDF.store(outputColumnsPath); _output = new BasicInputFormatDescriptor(_outputName); }
From source file:org.mrgeo.test.MapOpTestVectorUtils.java
License:Apache License
public List readVectorOutputAsText(final Configuration conf, final Path vectorPath) throws IOException { // read in the output file final FileSystem fs = HadoopFileUtils.getFileSystem(conf, vectorPath); ArrayList results = new ArrayList(); if (fs.isFile(vectorPath)) { final FSDataInputStream fdis = fs.open(vectorPath); final BufferedReader br = new BufferedReader(new InputStreamReader(fdis)); try {//from w ww .j a va2 s.co m String line = br.readLine(); while (line != null) { results.add(line); line = br.readLine(); } } finally { br.close(); if (fdis != null) { fdis.close(); } } } else { Path srcVector = new Path(vectorPath, "part*"); FileStatus files[] = fs.globStatus(srcVector); for (FileStatus fileStat : files) { final FSDataInputStream fdis = fs.open(fileStat.getPath()); final BufferedReader br = new BufferedReader(new InputStreamReader(fdis)); try { String line = br.readLine(); while (line != null) { results.add(line); line = br.readLine(); } } finally { br.close(); if (fdis != null) { fdis.close(); } } } } return results; }
From source file:org.openflamingo.engine.util.HdfsUtils.java
License:Apache License
/** * ? ?.//ww w .j a va2 s.c o m * * @param fs FileSystem * @param path ? Path * @return ? <tt>true</tt> */ public static boolean isDir(FileSystem fs, String path) { try { return !fs.isFile(new Path(path)); } catch (Exception ex) { String message = MessageFormatter.format(" '{}'? .", path) .getMessage(); throw new FileSystemException(message, ex); } }
From source file:org.openflamingo.engine.util.HdfsUtils.java
License:Apache License
/** * ?? ?./*from ww w . j a v a2 s.com*/ * * @param fs FileSystem * @param path ? Path * @return ?? <tt>true</tt> */ public static boolean isFile(FileSystem fs, String path) { try { return fs.isFile(new Path(path)); } catch (Exception ex) { String message = MessageFormatter.format(" '{}'? .", path) .getMessage(); throw new FileSystemException(message, ex); } }