Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:org.mrgeo.hdfs.utils.HadoopFileUtils.java

License:Apache License

/**
 * Moves a file in DFS//w  w  w  .jav a 2s  .  c  o m
 *
 * @param inputPath  input path
 * @param outputPath output path
 * @throws IOException if unable to move file
 */
public static void move(final Configuration conf, final Path inputPath, final Path outputPath)
        throws IOException {
    final FileSystem fs = getFileSystem(conf, inputPath);

    if (!fs.rename(inputPath, outputPath)) {
        throw new IOException("Cannot rename " + inputPath.toString() + " to " + outputPath.toString());
    }
    if (!fs.exists(outputPath)) {
        throw new IOException("Output path does not exist: " + outputPath.toString());
    }
}

---

From source file:org.mrgeo.hdfs.utils.HadoopFileUtils.java

License:Apache License

/**
 * For each subdirectory below srcParent, move that subdirectory below targetParent. If a
 * sub-directory already exists under the targetParent, delete it and then perform the move. This
 * function does not moves files below the parent, just directories.
 *
 * @param conf//from   w w  w  .  ja  va2s.c o m
 * @param targetParent
 * @throws IOException
 */
public static void moveChildDirectories(final Configuration conf, final Path srcParent, final Path targetParent)
        throws IOException {
    // We want to move each of the child directories of the output path
    // to the actual target directory (toPath). If any of the subdirs
    // already exists under the target path, we need to delete it first,
    // then perform the move.

    final FileSystem fs = getFileSystem(conf, srcParent);
    if (!fs.exists(targetParent)) {
        fs.mkdirs(targetParent);
    }
    final FileStatus[] children = fs.listStatus(srcParent);
    for (final FileStatus stat : children) {
        if (stat.isDir()) {
            final Path srcPath = stat.getPath();
            final String name = srcPath.getName();
            final Path target = new Path(targetParent, name);
            if (fs.exists(target)) {
                fs.delete(target, true);
            }
            if (fs.rename(srcPath, targetParent) == false) {
                final String msg = MessageFormat.format("Error moving temporary file {0} to output path {1}.",
                        srcPath.toString(), target.toString());
                throw new IOException(msg);
            }
        }
    }
}

---

From source file:org.mrgeo.mapalgebra.VectorMapOp.java

License:Apache License

/**
 * After a map op chain is executed, moveOutput will be called on the root map op. By default, the
 * map op's output is stored in a location other than where its final resting place will be, and
 * this method is responsible for moving the content to that location (e.g. toPath).
 * /*from   w w  w. j a  va2s  . co  m*/
 * @param toName
 * @throws IOException
 */
@Override
public void moveOutput(final String toName) throws IOException {
    HadoopFileUtils.delete(toName);
    final Path tmpOut = new Path(getOutputName());
    FileSystem fs = HadoopFileUtils.getFileSystem(tmpOut);
    // just move the file to the output path -- much faster.
    if (fs.rename(tmpOut, new Path(toName)) == false) {
        throw new IOException("Error moving temporary file to output path.");
    }
    _outputName = toName;
    final Path columns = new Path(tmpOut.toString() + ".columns");
    if (fs.exists(columns)) {
        String toColumns = toName + ".columns";
        HadoopFileUtils.delete(toColumns);
        if (fs.rename(columns, new Path(toColumns)) == false) {
            throw new IOException("Error moving temporary columns file to output path.");
        }
    }
}

---

From source file:org.opencloudengine.flamingo.mapreduce.util.HdfsUtils.java

License:Apache License

/**
 * HDFS ??  ??   ?? ??./*from   ww  w.j a  va2s  . c  o m*/
 *
 * @param conf            Hadoop Configuration
 * @param path            ?? ?
 * @param prefixToAppend  ?? ??  ?? prefix?  ?
 * @param targetDirectory ? 
 * @throws java.io.IOException ?? ??   
 */
public static void moveFileToDirectory(Configuration conf, String path, String prefixToAppend,
        String targetDirectory) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);
    FileStatus[] statuses = fileSystem.listStatus(new Path(path));
    for (FileStatus fileStatus : statuses) {
        String filename = prefixToAppend + "_" + fileStatus.getPath().getName();
        if (!isExist(conf, targetDirectory + "/" + filename)) {
            fileSystem.rename(fileStatus.getPath(), new Path(targetDirectory + "/" + filename));
        } else {
            throw new RuntimeException(
                    "\t  Warn: '" + fileStatus.getPath() + "' cannot moved. Already exists.");
        }
    }
}

---

From source file:org.opencloudengine.flamingo.mapreduce.util.HdfsUtils.java

License:Apache License

/**
 * HDFS ??  ??   ?? ??.//from   ww w.ja  va  2s  .  com
 *
 * @param conf            Hadoop Configuration
 * @param delayFiles      ?? ? ?
 * @param targetDirectory ? 
 * @throws java.io.IOException ?? ??   
 */
public static void moveFilesToDirectory(Configuration conf, List<String> delayFiles, String targetDirectory)
        throws IOException {
    for (String path : delayFiles) {
        String filename = FileUtils.getFilename(path);
        String delayedFilePrefix = filename.split("-")[0];
        String outputHead = delayedFilePrefix.replaceAll("delay", "");
        String outputMiddle = delayedFilePrefix.substring(0, 5); // todo
        String outputTail = filename.replaceAll(delayedFilePrefix, "");

        System.out.println(
                "Acceleration Dir " + targetDirectory + "/" + outputHead + "_" + outputMiddle + outputTail);
        makeDirectoryIfNotExists(targetDirectory, conf);

        FileSystem fileSystem = FileSystem.get(conf);
        fileSystem.rename(new Path(path),
                new Path(targetDirectory + "/" + outputHead + "_" + outputMiddle + outputTail));

        System.out.println("\t Moved: '" + path + "' --> '" + targetDirectory + "'");
    }
}

---

From source file:org.opencloudengine.flamingo.mapreduce.util.HdfsUtils.java

License:Apache License

/**
 *  ?  ?? MERGE./*from   w  ww .j  a  v  a2  s. c o  m*/
 *
 * @param hdfsUrl HDFS URL
 * @param path    HDFS Path
 * @throws java.io.IOException Get Merge   
 */
public static void merge(String hdfsUrl, String path) throws IOException {
    //  ?  ?? Get Merge  ?? ?.
    Configuration conf = new Configuration();
    conf.set("fs.default.name", hdfsUrl);
    FileSystem fileSystem = FileSystem.get(conf);
    Path source = new Path(path);
    if (!fileSystem.getFileStatus(source).isDir()) {
        // ? ??? ??? Get Merge .
        return;
    }
    Path target = new Path(path + "_temporary");
    FileUtil.copyMerge(fileSystem, source, fileSystem, target, true, conf, null);

    // ?  ?? .
    fileSystem.delete(source, true);

    //  ?? ?  ? .
    Path in = new Path(path + "_temporary");
    Path out = new Path(path);
    fileSystem.rename(in, out);

    //   .
    fileSystem.delete(new Path(path + "_temporary"), true);
}

---

From source file:org.opencloudengine.garuda.backend.hdfs.HdfsServiceImpl.java

License:Open Source License

private Path _rename(String path, String rename) throws Exception {
    FileSystem fs = fileSystemFactory.getFileSystem();
    Path fsPath = new Path(path);
    FileStatus fileStatus = fs.getFileStatus(fsPath);
    HdfsFileInfo hdfsFileInfo = new HdfsFileInfo(fileStatus, fs.getContentSummary(fsPath));
    String parentPath = hdfsFileInfo.getPath();

    String newPath = parentPath + "/" + rename;
    Path path1 = new Path(newPath);
    if (StringUtils.isEmpty(rename)) {
        logger.warn("Failed rename HDFS file, Rename is empty : {}", newPath);
        throw new ServiceException(" ? ??  .");
    }//from   ww w .ja  v  a 2s.c o m

    fs.rename(fsPath, path1);
    fs.close();
    return path1;
}

---

From source file:org.openflamingo.collector.handler.LocalToHdfsHandler.java

License:Apache License

@Override
public void execute() throws Exception {
    // ? ?? ?   ??.
    copyToWorkingDirectory();//from   w ww  . ja va2  s.  c om

    // ??  ? ? ?? ??.
    List<FileStatus> files = getFilesFromWorkingDirectory();

    if (files.size() < 1) {
        logger.info(
                " ??  ?   ? .");
        return;
    }

    //  ? ??   HDFS .
    Iterator<FileStatus> iterator = files.iterator();
    while (iterator.hasNext()) {
        //  ? ??  ? ? ??.
        FileStatus workingFile = iterator.next();
        FileSystem workingFS = getFileSystem(workingFile.getPath());

        // ? ??  .
        String processingFileName = workingFile.getPath().getName() + PROCESSING_FILE_QUALIFIER;
        String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim()));
        Path processingFile = new Path(workingDirectory, processingFileName);
        boolean renamed = workingFS.rename(workingFile.getPath(), processingFile);
        logger.debug(
                " ? ? '{}'? '{}' ??   .",
                workingFile.getPath(), processingFile);

        if (renamed) {
            // Target HDFS  ??.
            ToHdfs hdfs = job.getPolicy().getEgress().getToHdfs();

            // ??  HDFS? FileSystem  ??.
            String cluster = jobContext.getValue(hdfs.getCluster());
            Configuration configuration = getConfiguration(jobContext.getModel(), cluster);
            FileSystem targetFS = FileSystem.get(configuration);
            logger.info(
                    "HDFS?    Hadoop Cluster '{}'? Hadoop Cluster? ? ? .",
                    cluster);

            // HDFS? target, staging  .
            String targetDirectory = jobContext.getValue(hdfs.getTargetPath());
            String stagingDirectory = jobContext.getValue(hdfs.getStagingPath());
            logger.info(
                    "HDFS?     ?  '{}'? ?  '{}'.",
                    targetDirectory, stagingDirectory);

            // ? ?  ??  .
            int hash = Math.abs((workingFile.getPath().toString() + processingFile.toString()).hashCode())
                    + Integer.parseInt(JVMIDUtils.generateUUID());
            if (hash < 0)
                hash = -hash;
            logger.debug(
                    "?  '{}'?  ? '{}'?   '{}'? ?.",
                    new Object[] { stagingDirectory, processingFile.getName(), hash });

            // ? ? .
            // FIXME
            Path stagingFile = new Path(stagingDirectory,
                    DateUtils.parseDate(jobContext.getStartDate(), "yyyyMMddHHmmss") + "_"
                            + String.valueOf(hash));
            try {
                targetFS.copyFromLocalFile(false, false, processingFile, stagingFile);
            } catch (Exception ex) {
                logger.warn(
                        " ? ? '{}'? ? ? '{}'    ?  ??.",
                        processingFile, stagingFile);
                copyToErrorDirectory(workingFile);
                continue;
            }
            logger.info(
                    " ? ? '{}'? ? ? '{}' .",
                    processingFile, stagingFile);

            // ? ??  ?  ??.
            Path targetFile = new Path(targetDirectory, workingFile.getPath().getName());
            targetFS.rename(stagingFile, targetFile);
            logger.info("? ? '{}' ?? '{}' ??.",
                    stagingFile, targetFile);

            //  ??   ??.
            copyToCompleteDirectory(workingFS.getFileStatus(processingFile));
        }
    }
}

---

From source file:org.openflamingo.collector.handler.LocalToHdfsHandler.java

License:Apache License

/**
 *  ??    ??  ? ?? ./*  w  w  w. j  a v  a 2s  .  c om*/
 *
 * @return  ? ?(??    null)
 * @throws IOException ? ?   , ?? ??   
 */
public List<FileStatus> copyToWorkingDirectory() throws IOException {
    // ?? ?  ? ?   Selector Pattern? .
    SelectorPattern selectorPattern = SelectorPatternFactory.getSelectorPattern(
            this.local.getSourceDirectory().getConditionType().trim(),
            jobContext.getValue(this.local.getSourceDirectory().getCondition().trim()), jobContext);
    String sourceDirectory = correctPath(jobContext.getValue(local.getSourceDirectory().getPath().trim()));
    String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim()));

    FileSystem sourceDirectoryFS = getFileSystem(sourceDirectory);
    List<FileStatus> files = new LinkedList<FileStatus>();
    for (FileStatus sourceFile : sourceDirectoryFS.listStatus(new Path(sourceDirectory))) {
        if (!sourceFile.isDir()) {
            if (sourceFile.getPath().getName().startsWith(".") || sourceFile.getPath().getName().startsWith("_")
                    || sourceFile.getPath().getName().endsWith(".work")) {
                logger.info(" ? '{}'?   .", sourceFile.getPath());
                continue;
            }
            if (selectorPattern.accept(sourceFile.getPath())) {
                // ??   ??.
                Path workPath = new Path(workingDirectory, sourceFile.getPath().getName());
                sourceDirectoryFS.rename(sourceFile.getPath(), workPath);
                logger.info("? ? '{}'?   '{}' ??.",
                        sourceFile.getPath(), workPath);
                files.add(sourceDirectoryFS.getFileStatus(workPath));
            }
        }
    }
    return files;
}

---

From source file:org.openflamingo.collector.handler.LocalToHdfsHandler.java

License:Apache License

/**
 * ??   ??.//ww  w. j a  va 2 s .c  o m
 *
 * @param fileToMove ?
 * @return ?? ? <tt>true</tt>
 * @throws IOException ?? ??   
 */
public boolean copyToCompleteDirectory(FileStatus fileToMove) throws IOException {
    String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim()));
    String completeDirectory = correctPath(jobContext.getValue(local.getCompleteDirectory().trim()));
    FileSystem workingDirectoryFS = getFileSystem(workingDirectory);

    boolean success = false;
    if (local.isRemoveAfterCopy()) {
        logger.info("?  . ? ? '{}'? ."
                + fileToMove.getPath());
        success = workingDirectoryFS.delete(fileToMove.getPath(), false);
        if (!success) {
            logger.info("? ? '{}'? .", fileToMove.getPath());
        }
    } else {
        Path completedPath = new Path(completeDirectory,
                fileToMove.getPath().getName().replaceAll(PROCESSING_FILE_QUALIFIER, ""));
        logger.info(
                "?  . ? ? '{}'? '{}' ??.",
                fileToMove.getPath(), completedPath);
        success = workingDirectoryFS.rename(fileToMove.getPath(), completedPath);
        if (!success) {
            logger.warn("? ??? ? .");
        }
    }
    return success;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »