List of usage examples for org.apache.hadoop.fs FileSystem rename
public abstract boolean rename(Path src, Path dst) throws IOException;
From source file:org.openflamingo.collector.handler.LocalToHdfsHandler.java
License:Apache License
/** * ?? ? ??./* w w w .j ava2 s .c o m*/ * * @param fs ? * @return ?? ? <tt>true</tt> * @throws IOException ?? ?? */ public boolean copyToErrorDirectory(FileStatus fs) throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); String errorDirectory = correctPath(jobContext.getValue(local.getErrorDirectory().trim())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); if (fs.getPath().getName().endsWith(PROCESSING_FILE_QUALIFIER)) { Path errorPath = new Path(errorDirectory, fs.getPath().getName().replaceAll(PROCESSING_FILE_QUALIFIER, "")); logger.info( " ? ?? . '{}' ?? '{}' ??.", fs.getPath(), errorPath); return workingDirectoryFS.rename(fs.getPath(), errorPath); } return false; }
From source file:org.openflamingo.engine.util.HdfsUtils.java
License:Apache License
/** * ? ? ??./*from w ww .j a va 2 s . c om*/ * * @param source ?? * @param target ?? * @param fs Hadoop FileSystem */ public static void move(String source, String target, FileSystem fs) throws Exception { Path srcPath = new Path(source); Path[] srcs = FileUtil.stat2Paths(fs.globStatus(srcPath), srcPath); Path dst = new Path(target); if (srcs.length > 1 && !fs.getFileStatus(dst).isDir()) { throw new FileSystemException("When moving multiple files, destination should be a directory."); } for (int i = 0; i < srcs.length; i++) { if (!fs.rename(srcs[i], dst)) { FileStatus srcFstatus = null; FileStatus dstFstatus = null; try { srcFstatus = fs.getFileStatus(srcs[i]); } catch (FileNotFoundException e) { throw new FileNotFoundException(srcs[i] + ": No such file or directory"); } try { dstFstatus = fs.getFileStatus(dst); } catch (IOException e) { // Nothing } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDir() && !dstFstatus.isDir()) { throw new FileSystemException( "cannot overwrite non directory " + dst + " with directory " + srcs[i]); } } throw new FileSystemException("Failed to rename " + srcs[i] + " to " + dst); } } }
From source file:org.openflamingo.uploader.handler.LocalToHdfsHandler.java
License:Open Source License
@Override public void execute() throws Exception { // ? ?? ? ??. copyToWorkingDirectory();/*www . j a v a 2s. c om*/ // ?? ? ? ?? ??. List<FileStatus> files = getFilesFromWorkingDirectory(); if (files.size() < 1) { jobLogger.info( " ?? ? ? ."); return; } // ? ?? HDFS . Iterator<FileStatus> iterator = files.iterator(); while (iterator.hasNext()) { // ? ?? ? ? ??. FileStatus workingFile = iterator.next(); FileSystem workingFS = getFileSystem(workingFile.getPath()); // ? ?? . String processingFileName = workingFile.getPath().getName() + PROCESSING_FILE_QUALIFIER; String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory())); Path processingFile = new Path(workingDirectory, processingFileName); boolean renamed = workingFS.rename(workingFile.getPath(), processingFile); jobLogger.debug( " ? ? '{}'? '{}' ?? .", workingFile.getPath(), processingFile); if (renamed) { // Outgress? HDFS ??. Hdfs hdfs = job.getPolicy().getOutgress().getHdfs(); // ?? HDFS? FileSystem ??. String cluster = jobContext.getValue(hdfs.getCluster()); Configuration configuration = getConfiguration(jobContext.getModel(), cluster); FileSystem targetFS = FileSystem.get(configuration); jobLogger.info( "HDFS? Hadoop Cluster '{}'? Hadoop Cluster? ? ? .", cluster); // HDFS? target, staging . String targetDirectory = jobContext.getValue(hdfs.getTargetPath()); String stagingDirectory = jobContext.getValue(hdfs.getStagingPath()); jobLogger.info( "HDFS? ? '{}'? ? '{}'.", targetDirectory, stagingDirectory); // ? ? ?? . int hash = Math.abs((workingFile.getPath().toString() + processingFile.toString()).hashCode()) + Integer.parseInt(JVMIDUtils.generateUUID()); if (hash < 0) hash = -hash; jobLogger.debug( "? '{}'? ? '{}'? '{}'? ?.", new Object[] { stagingDirectory, processingFile.getName(), hash }); // ? ? . // FIXME Path stagingFile = new Path(stagingDirectory, DateUtils.parseDate(jobContext.getStartDate(), "yyyyMMddHHmmss") + "_" + String.valueOf(hash)); try { targetFS.copyFromLocalFile(false, false, processingFile, stagingFile); } catch (Exception ex) { jobLogger.warn( " ? ? '{}'? ? ? '{}' ? ??.", processingFile, stagingFile); copyToErrorDirectory(workingFile); continue; } jobLogger.info( " ? ? '{}'? ? ? '{}' .", processingFile, stagingFile); // ? ?? ? ??. Path targetFile = new Path(targetDirectory, workingFile.getPath().getName()); targetFS.rename(stagingFile, targetFile); jobLogger.info("? ? '{}' ?? '{}' ??.", stagingFile, targetFile); // ?? ??. copyToCompleteDirectory(workingFS.getFileStatus(processingFile)); } } }
From source file:org.openflamingo.uploader.handler.LocalToHdfsHandler.java
License:Open Source License
/** * ?? ?? ? ?? .//from w w w . j a v a 2 s . c o m * * @return ? ?(?? null) * @throws IOException ? ? , ?? ?? */ public List<FileStatus> copyToWorkingDirectory() throws IOException { // ?? ? ? ? Selector Pattern? . SelectorPattern selectorPattern = SelectorPatternFactory.getSelectorPattern( this.local.getSourceDirectory().getConditionType(), jobContext.getValue(this.local.getSourceDirectory().getCondition()), jobContext); String sourceDirectory = correctPath(jobContext.getValue(local.getSourceDirectory().getPath())); String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory())); FileSystem sourceDirectoryFS = getFileSystem(sourceDirectory); List<FileStatus> files = new LinkedList<FileStatus>(); for (FileStatus sourceFile : sourceDirectoryFS.listStatus(new Path(sourceDirectory))) { if (!sourceFile.isDir()) { if (sourceFile.getPath().getName().startsWith(".") || sourceFile.getPath().getName().startsWith("_") || sourceFile.getPath().getName().endsWith(".work")) { jobLogger.info(" ? '{}'? .", sourceFile.getPath()); continue; } if (selectorPattern.accept(sourceFile.getPath())) { // ?? ??. Path workPath = new Path(workingDirectory, sourceFile.getPath().getName()); sourceDirectoryFS.rename(sourceFile.getPath(), workPath); jobLogger.info("? ? '{}'? '{}' ??.", sourceFile.getPath(), workPath); files.add(sourceDirectoryFS.getFileStatus(workPath)); } } } return files; }
From source file:org.openflamingo.uploader.handler.LocalToHdfsHandler.java
License:Open Source License
/** * ?? ??./*from www . j a va 2 s . co m*/ * * @param fileToMove ? * @return ?? ? <tt>true</tt> * @throws IOException ?? ?? */ public boolean copyToCompleteDirectory(FileStatus fileToMove) throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory())); String completeDirectory = correctPath(jobContext.getValue(local.getCompleteDirectory())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); boolean success = false; if (local.isRemoveAfterCopy()) { jobLogger.info("? . ? ? '{}'? ." + fileToMove.getPath()); success = workingDirectoryFS.delete(fileToMove.getPath(), false); if (!success) { jobLogger.info("? ? '{}'? .", fileToMove.getPath()); } } else { Path completedPath = new Path(completeDirectory, fileToMove.getPath().getName().replaceAll(PROCESSING_FILE_QUALIFIER, "")); jobLogger.info( "? . ? ? '{}'? '{}' ??.", fileToMove.getPath(), completedPath); success = workingDirectoryFS.rename(fileToMove.getPath(), completedPath); if (!success) { jobLogger.warn("? ??? ? ."); } } return success; }
From source file:org.openflamingo.uploader.handler.LocalToHdfsHandler.java
License:Open Source License
/** * ?? ? ??./* w w w. ja v a 2s. co m*/ * * @param fs ? * @return ?? ? <tt>true</tt> * @throws IOException ?? ?? */ public boolean copyToErrorDirectory(FileStatus fs) throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory())); String errorDirectory = correctPath(jobContext.getValue(local.getErrorDirectory())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); if (fs.getPath().getName().endsWith(PROCESSING_FILE_QUALIFIER)) { Path errorPath = new Path(errorDirectory, fs.getPath().getName().replaceAll(PROCESSING_FILE_QUALIFIER, "")); jobLogger.info( " ? ?? . '{}' ?? '{}' ??.", fs.getPath(), errorPath); return workingDirectoryFS.rename(fs.getPath(), errorPath); } return false; }
From source file:org.sf.xrime.algorithms.kcore.undirected.AllKCoreAlgorithm.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { try {/* w w w . j a va 2s. co m*/ if (getSource().getPaths() == null || getSource().getPaths().size() == 0 || getDestination().getPaths() == null || getDestination().getPaths().size() == 0) { throw new ProcessorExecutionException("No input and/or output paths specified."); } // The prefix used by temp directories which store intermediate results of each steps. String temp_dir_prefix = getDestination().getPath().toString() + "/kcore_"; // Create the temporary directory manager. SequenceTempDirMgr dirMgr = new SequenceTempDirMgr(temp_dir_prefix, context); // Sequence number begins with zero. dirMgr.setSeqNum(0); Path tmpDir; // 1. Transform input from outgoing adjacency vertexes lists to AdjSetVertex. System.out.println("-------->" + dirMgr.getSeqNum() + ": Transform input to AdjSetVertex"); Transformer transformer = new OutAdjVertex2AdjSetVertexTransformer(); // Inherit settings from this algorithm. transformer.setConf(context); transformer.setSrcPath(getSource().getPath()); // Generate temporary directory. tmpDir = dirMgr.getTempDir(); // And use it as the destination directory. transformer.setDestPath(tmpDir); transformer.setMapperNum(getMapperNum()); transformer.setReducerNum(getReducerNum()); transformer.execute(); // 2. Recursively invoke ElementRemoval to remove vertexes, and lines incident with // them, of degree less than k. And, loop to find all possible k. Graph src; Graph dest; // The current k value to generate k-core for. int the_k_value = 1; // The number of vertexes in current k-core. long vertexes_num = -1; // Get the file system client object. FileSystem fs_client = FileSystem.get(context); while (vertexes_num != 0) { // If we still have k-core with larger k. while (true) { // 2.1. Invoke ElementRemoval. System.out .println("-------->" + dirMgr.getSeqNum() + ": Recursively delete vertexes and lines"); GraphAlgorithm element_rm = new ElementRemoval(); // Inherit settings from this algorithm. element_rm.setConf(context); src = new Graph(Graph.defaultGraph()); // Use the output directory of last step as the input directory of this step. src.setPath(tmpDir); dest = new Graph(Graph.defaultGraph()); // Generate a new temporary directory. tmpDir = dirMgr.getTempDir(); dest.setPath(tmpDir); element_rm.setSource(src); element_rm.setDestination(dest); element_rm.setMapperNum(getMapperNum()); element_rm.setReducerNum(getReducerNum()); // Specify the K we are interested in. element_rm.setParameter(ElementRemoval.K_OF_CORE, "" + the_k_value); element_rm.execute(); // 2.2. Check for convergence. RunningJob conv_result = element_rm.getFinalStatus(); long found_vertexes_num = MRConsoleReader.getReduceOutputRecordNum(conv_result); // Check whether the number of remaining vertexes changes. if (found_vertexes_num != vertexes_num) { // Changed! We need another iteration. vertexes_num = found_vertexes_num; } else { // Converged! Record it by rename the directory. System.out.println("--------> Determined " + the_k_value + "-core"); // Determine the destination path. Path dest_path = new Path( getDestination().getPath().toString() + "/" + the_k_value + "-core"); // Rename it. fs_client.rename(tmpDir, dest_path); // Remember this. tmpDir = dest_path; // Textify this core. System.out.println("--------> Textifying " + the_k_value + "-core"); transformer = new SequenceFileToTextFileTransformer(); // Inherit settings from this algorithm. transformer.setConf(context); transformer.setSrcPath(dest_path); // Generate a directory for textify purpose. Path dest_txt_path = new Path( getDestination().getPath().toString() + "/" + the_k_value + "-core-txt"); // And use it as the destination directory. transformer.setDestPath(dest_txt_path); transformer.setMapperNum(getMapperNum()); transformer.setReducerNum(getReducerNum()); transformer.execute(); // Increase the k value. the_k_value++; break; } } } // Delete all useless temporary directories. dirMgr.deleteAll(); } catch (IllegalAccessException e) { throw new ProcessorExecutionException(e); } catch (IOException e) { throw new ProcessorExecutionException(e); } catch (NumberFormatException e) { throw new ProcessorExecutionException(e); } }
From source file:org.smartfrog.services.hadoop.benchmark.citerank.CiteRank.java
License:Open Source License
private static void overwrite(FileSystem fs, Path src, Path dst) throws IOException { fs.delete(dst, true);/* w w w . ja v a 2 s . c o m*/ fs.rename(src, dst); }
From source file:org.smartfrog.services.hadoop.operations.utils.DfsUtils.java
License:Open Source License
/** * Move files that match the file pattern <i>srcPath</i> * to a destination file.//from w w w . j a va 2 s . c om * When moving mutiple files, the destination must be a directory. * Otherwise, IOException is thrown. * Based on {@link org.apache.hadoop.fs.FsShell#rename(String, String)} * * @param fileSystem filesystem to work with * @param srcPath a file pattern specifying source files * @param dstPath a destination file/directory * @throws IOException for any problem * @see org.apache.hadoop.fs.FileSystem#globStatus(Path) */ public static void rename(FileSystem fileSystem, Path srcPath, Path dstPath) throws IOException { Path[] srcs = FileUtil.stat2Paths(fileSystem.globStatus(srcPath), srcPath); FileStatus destStatus = fileSystem.getFileStatus(dstPath); if (srcs.length > 1 && !destStatus.isDir()) { throw new IOException("When moving multiple files, " + "destination should be a directory."); } for (Path src : srcs) { if (!fileSystem.rename(src, dstPath)) { FileStatus srcFstatus; FileStatus dstFstatus; try { srcFstatus = fileSystem.getFileStatus(src); } catch (FileNotFoundException e) { FileNotFoundException fnf = new FileNotFoundException( src + ": No such file or directory in " + fileSystem.getUri()); fnf.initCause(e); throw fnf; } try { dstFstatus = fileSystem.getFileStatus(dstPath); } catch (IOException ignored) { dstFstatus = null; } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDir() && !dstFstatus.isDir()) { throw new IOException("cannot overwrite non directory " + dstPath + " with directory " + srcPath + " in " + fileSystem.getUri()); } } throw new IOException( "Failed to rename '" + srcPath + "' to '" + dstPath + "'" + " in " + fileSystem.getUri()); } } }
From source file:org.springframework.data.hadoop.fs.FsShell.java
License:Apache License
public void mv(String src, String src2, String... dst) { Object[] va = parseVarargs(src, src2, dst); @SuppressWarnings({ "unchecked" }) List<Path> sources = (List<Path>) va[0]; Path dstPath = (Path) va[1]; try {/* w ww. j a v a 2 s.c o m*/ FileSystem dstFs = getFS(dstPath); boolean isDstDir = !dstFs.isFile(dstPath); if (sources.size() > 1 && !isDstDir) { throw new IllegalArgumentException("Destination must be a dir when moving multiple files"); } for (Path srcPath : sources) { FileSystem srcFs = getFS(srcPath); URI srcURI = srcFs.getUri(); URI dstURI = dstFs.getUri(); if (srcURI.compareTo(dstURI) != 0) { throw new IllegalArgumentException("src and destination filesystems do not match."); } Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); if (srcs.length > 1 && !isDstDir) { throw new IllegalArgumentException( "When moving multiple files, destination should be a directory."); } for (Path s : srcs) { if (!srcFs.rename(s, dstPath)) { FileStatus srcFstatus = null; FileStatus dstFstatus = null; try { srcFstatus = srcFs.getFileStatus(s); } catch (FileNotFoundException e) { // ignore } try { dstFstatus = dstFs.getFileStatus(dstPath); } catch (IOException e) { } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDir() && !dstFstatus.isDir()) { throw new IllegalArgumentException( "cannot overwrite non directory " + dstPath + " with directory " + s); } } throw new HadoopException("Failed to rename " + s + " to " + dstPath); } } } } catch (IOException ex) { throw new HadoopException("Cannot rename resources " + ex.getMessage(), ex); } }