List of usage examples for org.apache.hadoop.fs FileUtil stat2Paths
public static Path[] stat2Paths(FileStatus[] stats, Path path)
From source file:com.cloudera.crunch.impl.mr.exec.CrunchJob.java
License:Open Source License
private void handleMultiPaths() throws IOException { if (!multiPaths.isEmpty()) { // Need to handle moving the data from the output directory of the // job to the output locations specified in the paths. FileSystem fs = FileSystem.get(job.getConfiguration()); for (int i = 0; i < multiPaths.size(); i++) { Path src = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + i + "-*"); Path[] srcs = FileUtil.stat2Paths(fs.globStatus(src), src); Path dst = multiPaths.get(i); if (!fs.exists(dst)) { fs.mkdirs(dst);/*from w w w . j a va 2s .co m*/ } int minPartIndex = getMinPartIndex(dst, fs); for (Path s : srcs) { fs.rename(s, getDestFile(s, dst, minPartIndex++)); } } } }
From source file:com.gruter.hadoop.customShell.CustomShell.java
License:Apache License
int runCmdHandler(CmdHandler handler, String[] args, int startIndex, boolean recursive) throws IOException { int errors = 0; for (int i = startIndex; i < args.length; i++) { Path srcPath = new Path(args[i]); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path[] paths = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); // if nothing matches to given glob pattern then increment error count if (paths.length == 0) { System.err.println(handler.getName() + ": could not get status for '" + args[i] + "'"); errors++;//w w w . j a va 2 s .co m } for (Path path : paths) { try { FileStatus file = srcFs.getFileStatus(path); if (file == null) { System.err.println(handler.getName() + ": could not get status for '" + path + "'"); errors++; } else { errors += runCmdHandler(handler, file, srcFs, recursive); } } catch (IOException e) { String msg = (e.getMessage() != null ? e.getLocalizedMessage() : (e.getCause().getMessage() != null ? e.getCause().getLocalizedMessage() : "null")); System.err.println( handler.getName() + ": could not get status for '" + path + "': " + msg.split("\n")[0]); errors++; } } } return (errors > 0 || handler.getErrorCode() != 0) ? 1 : 0; }
From source file:com.revolutionanalytics.hadoop.hdfs.DelayedExceptionThrowing.java
License:Apache License
final void globAndProcess(Path srcPattern, FileSystem srcFs) throws IOException { ArrayList<IOException> exceptions = new ArrayList<IOException>(); for (Path p : FileUtil.stat2Paths(srcFs.globStatus(srcPattern), srcPattern)) try {// w w w . ja va 2s . c o m process(p, srcFs); } catch (IOException ioe) { exceptions.add(ioe); } if (!exceptions.isEmpty()) if (exceptions.size() == 1) throw exceptions.get(0); else throw new IOException("Multiple IOExceptions: " + exceptions); }
From source file:nur.aini.hadoop.CopyMergeRegexToLocal.java
License:GNU General Public License
public void run(String srcf, String dst) { final Path srcPath = new Path("./" + srcf); final Path desPath = new Path(dst); try {//from w w w . j a v a2 s.co m Path[] srcs = FileUtil.stat2Paths(hdfs.globStatus(srcPath), srcPath); OutputStream out = FileSystem.getLocal(conf).create(desPath); for (int i = 0; i < srcs.length; i++) { System.out.println(srcs[i]); InputStream in = hdfs.open(srcs[i]); IOUtils.copyBytes(in, out, conf, false); in.close(); } out.close(); } catch (IOException ex) { System.err.print(ex.getMessage()); } }
From source file:org.apache.crunch.impl.mr.exec.CrunchJob.java
License:Apache License
private synchronized void handleMultiPaths() throws IOException { if (!multiPaths.isEmpty()) { // Need to handle moving the data from the output directory of the // job to the output locations specified in the paths. FileSystem fs = FileSystem.get(job.getConfiguration()); for (int i = 0; i < multiPaths.size(); i++) { Path src = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + i + "-*"); Path[] srcs = FileUtil.stat2Paths(fs.globStatus(src), src); Path dst = multiPaths.get(i); if (!fs.exists(dst)) { fs.mkdirs(dst);/*from w ww .ja v a2 s . co m*/ } int minPartIndex = getMinPartIndex(dst, fs); for (Path s : srcs) { fs.rename(s, getDestFile(s, dst, minPartIndex++)); } } } }
From source file:org.apache.crunch.io.avro.AvroPathPerKeyTarget.java
License:Apache License
@Override public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException { FileSystem srcFs = workingPath.getFileSystem(conf); Path base = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + index); Path[] keys = FileUtil.stat2Paths(srcFs.listStatus(base), base); FileSystem dstFs = path.getFileSystem(conf); if (!dstFs.exists(path)) { dstFs.mkdirs(path);/* w ww. j ava2 s . c o m*/ } boolean sameFs = isCompatible(srcFs, path); for (Path key : keys) { Path[] srcs = FileUtil.stat2Paths(srcFs.listStatus(key), key); Path targetPath = new Path(path, key.getName()); dstFs.mkdirs(targetPath); for (Path s : srcs) { Path d = getDestFile(conf, s, targetPath, s.getName().contains("-m-")); if (sameFs) { srcFs.rename(s, d); } else { FileUtil.copy(srcFs, s, dstFs, d, true, true, conf); } } } dstFs.create(getSuccessIndicator(), true).close(); }
From source file:org.apache.crunch.io.impl.FileTargetImpl.java
License:Apache License
@Override public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException { FileSystem srcFs = workingPath.getFileSystem(conf); Path src = getSourcePattern(workingPath, index); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(src), src); FileSystem dstFs = path.getFileSystem(conf); if (!dstFs.exists(path)) { dstFs.mkdirs(path);/*from w ww . j a v a 2 s . co m*/ } boolean sameFs = isCompatible(srcFs, path); for (Path s : srcs) { Path d = getDestFile(conf, s, path, s.getName().contains("-m-")); if (sameFs) { srcFs.rename(s, d); } else { FileUtil.copy(srcFs, s, dstFs, d, true, true, conf); } } dstFs.create(getSuccessIndicator(), true).close(); }
From source file:org.exem.flamingo.shared.util.HdfsUtils.java
License:Apache License
/** * ? ? ??./*www .j a va 2 s . c o m*/ * * @param source ?? * @param target ?? * @param fs Hadoop FileSystem */ public static void move(String source, String target, FileSystem fs) throws Exception { Path srcPath = new Path(source); Path[] srcs = FileUtil.stat2Paths(fs.globStatus(srcPath), srcPath); Path dst = new Path(target); if (srcs.length > 1 && !fs.getFileStatus(dst).isDirectory()) { throw new ServiceException("When moving multiple files, destination should be a directory."); } for (int i = 0; i < srcs.length; i++) { if (!fs.rename(srcs[i], dst)) { FileStatus srcFstatus = null; FileStatus dstFstatus = null; try { srcFstatus = fs.getFileStatus(srcs[i]); } catch (FileNotFoundException e) { throw new FileNotFoundException(srcs[i] + ": No such file or directory"); } try { dstFstatus = fs.getFileStatus(dst); } catch (IOException e) { // Nothing } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDirectory() && !dstFstatus.isDirectory()) { throw new ServiceException( "cannot overwrite non directory " + dst + " with directory " + srcs[i]); } } throw new ServiceException("Failed to rename " + srcs[i] + " to " + dst); } } }
From source file:org.openflamingo.engine.util.HdfsUtils.java
License:Apache License
/** * ? ? ??.// w w w.j a v a2s . c o m * * @param source ?? * @param target ?? * @param fs Hadoop FileSystem */ public static void move(String source, String target, FileSystem fs) throws Exception { Path srcPath = new Path(source); Path[] srcs = FileUtil.stat2Paths(fs.globStatus(srcPath), srcPath); Path dst = new Path(target); if (srcs.length > 1 && !fs.getFileStatus(dst).isDir()) { throw new FileSystemException("When moving multiple files, destination should be a directory."); } for (int i = 0; i < srcs.length; i++) { if (!fs.rename(srcs[i], dst)) { FileStatus srcFstatus = null; FileStatus dstFstatus = null; try { srcFstatus = fs.getFileStatus(srcs[i]); } catch (FileNotFoundException e) { throw new FileNotFoundException(srcs[i] + ": No such file or directory"); } try { dstFstatus = fs.getFileStatus(dst); } catch (IOException e) { // Nothing } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDir() && !dstFstatus.isDir()) { throw new FileSystemException( "cannot overwrite non directory " + dst + " with directory " + srcs[i]); } } throw new FileSystemException("Failed to rename " + srcs[i] + " to " + dst); } } }
From source file:org.smartfrog.services.hadoop.operations.utils.DfsUtils.java
License:Open Source License
/** * Move files that match the file pattern <i>srcPath</i> * to a destination file.//w w w .j ava 2s . c o m * When moving mutiple files, the destination must be a directory. * Otherwise, IOException is thrown. * Based on {@link org.apache.hadoop.fs.FsShell#rename(String, String)} * * @param fileSystem filesystem to work with * @param srcPath a file pattern specifying source files * @param dstPath a destination file/directory * @throws IOException for any problem * @see org.apache.hadoop.fs.FileSystem#globStatus(Path) */ public static void rename(FileSystem fileSystem, Path srcPath, Path dstPath) throws IOException { Path[] srcs = FileUtil.stat2Paths(fileSystem.globStatus(srcPath), srcPath); FileStatus destStatus = fileSystem.getFileStatus(dstPath); if (srcs.length > 1 && !destStatus.isDir()) { throw new IOException("When moving multiple files, " + "destination should be a directory."); } for (Path src : srcs) { if (!fileSystem.rename(src, dstPath)) { FileStatus srcFstatus; FileStatus dstFstatus; try { srcFstatus = fileSystem.getFileStatus(src); } catch (FileNotFoundException e) { FileNotFoundException fnf = new FileNotFoundException( src + ": No such file or directory in " + fileSystem.getUri()); fnf.initCause(e); throw fnf; } try { dstFstatus = fileSystem.getFileStatus(dstPath); } catch (IOException ignored) { dstFstatus = null; } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDir() && !dstFstatus.isDir()) { throw new IOException("cannot overwrite non directory " + dstPath + " with directory " + srcPath + " in " + fileSystem.getUri()); } } throw new IOException( "Failed to rename '" + srcPath + "' to '" + dstPath + "'" + " in " + fileSystem.getUri()); } } }