Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:org.apache.nutch.crawl.LinkDbMerger.java

License:Apache License

public void merge(Path output, Path[] dbs, boolean normalize, boolean filter) throws Exception {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    long start = System.currentTimeMillis();
    LOG.info("LinkDb merge: starting at " + sdf.format(start));

    JobConf job = createMergeJob(getConf(), output, normalize, filter);
    for (int i = 0; i < dbs.length; i++) {
        FileInputFormat.addInputPath(job, new Path(dbs[i], LinkDb.CURRENT_NAME));
    }//from   w  w w.  j a  v  a 2 s. co  m
    JobClient.runJob(job);
    FileSystem fs = FileSystem.get(getConf());
    fs.mkdirs(output);
    fs.rename(FileOutputFormat.getOutputPath(job), new Path(output, LinkDb.CURRENT_NAME));

    long end = System.currentTimeMillis();
    LOG.info("LinkDb merge: finished at " + sdf.format(end) + ", elapsed: "
            + TimingUtil.elapsedTime(start, end));
}

---

From source file:org.apache.oozie.action.hadoop.FsActionExecutor.java

License:Apache License

/**
 * Move source to target/*ww  w . jav a  2  s.  com*/
 *
 * @param context
 * @param fsConf
 * @param nameNodePath
 * @param source
 * @param target
 * @param recovery
 * @throws ActionExecutorException
 */
public void move(Context context, XConfiguration fsConf, Path nameNodePath, Path source, Path target,
        boolean recovery) throws ActionExecutorException {
    try {
        source = resolveToFullPath(nameNodePath, source, true);
        validateSameNN(source, target);
        FileSystem fs = getFileSystemFor(source, context, fsConf);
        Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(source));
        if ((pathArr == null || pathArr.length == 0)) {
            if (!recovery) {
                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS006",
                        "move, source path [{0}] does not exist", source);
            } else {
                return;
            }
        }
        if (pathArr.length > 1 && (!fs.exists(target) || fs.isFile(target))) {
            if (!recovery) {
                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS012",
                        "move, could not rename multiple sources to the same target name");
            } else {
                return;
            }
        }
        checkGlobMax(pathArr);
        for (Path p : pathArr) {
            if (!fs.rename(p, target) && !recovery) {
                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS008",
                        "move, could not move [{0}] to [{1}]", p, target);
            }
        }
    } catch (Exception ex) {
        throw convertException(ex);
    }
}

---

From source file:org.apache.oozie.action.hadoop.JavaActionExecutor.java

License:Apache License

public void prepareActionDir(FileSystem actionFs, Context context) throws ActionExecutorException {
    try {// www. j a  va  2  s .  c  o  m
        Path actionDir = context.getActionDir();
        Path tempActionDir = new Path(actionDir.getParent(), actionDir.getName() + ".tmp");
        if (!actionFs.exists(actionDir)) {
            try {
                actionFs.mkdirs(tempActionDir);
                actionFs.rename(tempActionDir, actionDir);
            } catch (IOException ex) {
                actionFs.delete(tempActionDir, true);
                actionFs.delete(actionDir, true);
                throw ex;
            }
        }
    } catch (Exception ex) {
        throw convertException(ex);
    }
}

---

From source file:org.apache.orc.tools.FileDump.java

License:Apache License

private static void moveFiles(final FileSystem fs, final Path src, final Path dest) throws IOException {
    try {/*from  w ww.jav  a 2 s . c  o  m*/
        // create the dest directory if not exist
        if (!fs.exists(dest.getParent())) {
            fs.mkdirs(dest.getParent());
        }

        // if the destination file exists for some reason delete it
        fs.delete(dest, false);

        if (fs.rename(src, dest)) {
            System.err.println("Moved " + src + " to " + dest);
        } else {
            throw new IOException("Unable to move " + src + " to " + dest);
        }

    } catch (Exception e) {
        throw new IOException("Unable to move " + src + " to " + dest, e);
    }
}

---

From source file:org.apache.ranger.plugin.store.file.BaseFileStore.java

License:Apache License

protected boolean renamePath(Path oldPath, Path newPath) throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> BaseFileStore.renamePath(" + oldPath + "," + newPath + ")");
    }/*  w w w . j  a va2s. c  om*/

    FileSystem fileSystem = getFileSystem(oldPath);

    boolean ret = false;

    if (fileSystem.exists(oldPath)) {
        if (!fileSystem.exists(newPath)) {
            ret = fileSystem.rename(oldPath, newPath);
        } else {
            LOG.warn("target of rename '" + newPath + "' already exists");
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== BaseFileStore.renamePath(" + oldPath + "," + newPath + "): " + ret);
    }

    return ret;
}

---

From source file:org.apache.ranger.plugin.store.file.FileStoreUtil.java

License:Apache License

public boolean renamePath(Path oldPath, Path newPath) throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> FileStoreUtil.renamePath(" + oldPath + "," + newPath + ")");
    }//from   ww  w .j  av a2 s  .  co m

    FileSystem fileSystem = getFileSystem(oldPath);

    boolean ret = false;

    if (fileSystem.exists(oldPath)) {
        if (!fileSystem.exists(newPath)) {
            ret = fileSystem.rename(oldPath, newPath);
        } else {
            LOG.warn("target of rename '" + newPath + "' already exists");
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== FileStoreUtil.renamePath(" + oldPath + "," + newPath + "): " + ret);
    }

    return ret;
}

---

From source file:org.apache.solr.hadoop.ForkedMapReduceIndexerTool.java

License:Apache License

private static boolean rename(Path src, Path dst, FileSystem fs) throws IOException {
    boolean success = fs.rename(src, dst);
    if (!success) {
        LOG.error("Cannot rename " + src + " to " + dst);
    }//from  w w  w  . j  av a 2 s .  co  m
    return success;
}

---

From source file:org.apache.spark.network.yarn.YarnShuffleService.java

License:Apache License

/**
 * Figure out the recovery path and handle moving the DB if YARN NM recovery gets enabled
 * and DB exists in the local dir of NM by old version of shuffle service.
 *///from  w w  w .  j  a v  a2s  .  c o m
protected File initRecoveryDb(String dbName) {
    Preconditions.checkNotNull(_recoveryPath, "recovery path should not be null if NM recovery is enabled");

    File recoveryFile = new File(_recoveryPath.toUri().getPath(), dbName);
    if (recoveryFile.exists()) {
        return recoveryFile;
    }

    // db doesn't exist in recovery path go check local dirs for it
    String[] localDirs = _conf.getTrimmedStrings("yarn.nodemanager.local-dirs");
    for (String dir : localDirs) {
        File f = new File(new Path(dir).toUri().getPath(), dbName);
        if (f.exists()) {
            // If the recovery path is set then either NM recovery is enabled or another recovery
            // DB has been initialized. If NM recovery is enabled and had set the recovery path
            // make sure to move all DBs to the recovery path from the old NM local dirs.
            // If another DB was initialized first just make sure all the DBs are in the same
            // location.
            Path newLoc = new Path(_recoveryPath, dbName);
            Path copyFrom = new Path(f.toURI());
            if (!newLoc.equals(copyFrom)) {
                logger.info("Moving " + copyFrom + " to: " + newLoc);
                try {
                    // The move here needs to handle moving non-empty directories across NFS mounts
                    FileSystem fs = FileSystem.getLocal(_conf);
                    fs.rename(copyFrom, newLoc);
                } catch (Exception e) {
                    // Fail to move recovery file to new path, just continue on with new DB location
                    logger.error("Failed to move recovery file {} to the path {}", dbName,
                            _recoveryPath.toString(), e);
                }
            }
            return new File(newLoc.toUri().getPath());
        }
    }

    return new File(_recoveryPath.toUri().getPath(), dbName);
}

---

From source file:org.apache.sqoop.connector.hdfs.HdfsToDestroyer.java

License:Apache License

/**
 * {@inheritDoc}/*from  w ww  .  j a  v a2 s.co m*/
 */
@Override
public void destroy(DestroyerContext context, LinkConfiguration linkConfig, ToJobConfiguration jobConfig) {
    Configuration configuration = new Configuration();
    HdfsUtils.contextToConfiguration(context.getContext(), configuration);

    String workingDirectory = context.getString(HdfsConstants.WORK_DIRECTORY);
    Path targetDirectory = new Path(jobConfig.toJobConfig.outputDirectory);

    try {
        FileSystem fs = FileSystem.get(configuration);

        // If we succeeded, we need to move all files from working directory
        if (context.isSuccess()) {
            FileStatus[] fileStatuses = fs.listStatus(new Path(workingDirectory));
            for (FileStatus status : fileStatuses) {
                LOG.info("Committing file: " + status.getPath().toString() + " of size " + status.getLen());
                fs.rename(status.getPath(), new Path(targetDirectory, status.getPath().getName()));
            }
        }

        // Clean up working directory
        fs.delete(new Path(workingDirectory), true);
    } catch (IOException e) {
        throw new SqoopException(HdfsConnectorError.GENERIC_HDFS_CONNECTOR_0008, e);
    }
}

---

From source file:org.apache.sqoop.util.AppendUtils.java

License:Apache License

/**
 * Move selected files from source to target using a specified starting partition.
 *
 * Directories are moved without restriction.  Note that the serial
 * number of directories bears no relation to the file partition
 * numbering./*from w  ww.  j a  va  2 s. c om*/
 */
private void moveFiles(FileSystem fs, Path sourceDir, Path targetDir, int partitionStart) throws IOException {

    /* list files in the source dir and check for errors */

    FileStatus[] sourceFiles = fs.listStatus(sourceDir);

    if (null == sourceFiles) {
        // If we've already checked that the dir exists, and now it can't be
        // listed, this is a genuine error (permissions, fs integrity, or other).
        throw new IOException("Could not list files from " + sourceDir);
    }

    /* state used throughout the entire move operation */

    // pad the data partition number thusly
    NumberFormat partFormat = NumberFormat.getInstance();
    partFormat.setMinimumIntegerDigits(PARTITION_DIGITS);
    partFormat.setGroupingUsed(false);

    // where the data partitioning is currently at
    int dataPart = partitionStart;

    /* loop through all top-level files and copy matching ones */

    for (FileStatus fileStatus : sourceFiles) {
        String sourceFilename = fileStatus.getPath().getName();
        StringBuilder destFilename = new StringBuilder();

        if (fileStatus.isDir()) { // move all subdirectories
            // pass target dir as initial dest to prevent nesting inside preexisting dir
            if (fs.rename(fileStatus.getPath(), targetDir)) {
                LOG.debug("Directory: " + sourceFilename + " renamed to: " + sourceFilename);
            } else {
                int dirNumber = 0;
                Path destPath;
                do {
                    // clear the builder in case this isn't the first iteration
                    destFilename.setLength(0);

                    // name-nnnnn?
                    destFilename.append(sourceFilename).append("-").append(partFormat.format(dirNumber++));

                    destPath = new Path(targetDir, destFilename.toString());
                    if (fs.exists(destPath))
                        continue;

                    /*
                     * There's still a race condition right here if an
                     * identically-named directory is created concurrently.
                     * It can be avoided by creating a parent dir for all
                     * migrated dirs, or by an intermediate rename.
                     */

                } while (!fs.rename(fileStatus.getPath(), destPath));

                LOG.debug("Directory: " + sourceFilename + " renamed to: " + destPath.getName());
            }
        } else if (DATA_PART_PATTERN.matcher(sourceFilename).matches()) { // move only matching top-level files
            do {
                // clear the builder in case this isn't the first iteration
                destFilename.setLength(0);

                // name-nnnnn
                destFilename.append(getFilename(sourceFilename)).append(partFormat.format(dataPart++));

                // .ext?
                String extension = getFileExtension(sourceFilename);
                if (extension != null)
                    destFilename.append(getFileExtension(sourceFilename));
            } while (!fs.rename(fileStatus.getPath(), new Path(targetDir, destFilename.toString())));

            LOG.debug("Filename: " + sourceFilename + " repartitioned to: " + destFilename.toString());
        } else { // ignore everything else
            LOG.debug("Filename: " + sourceFilename + " ignored");
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »