Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:com.talis.labs.pagerank.mapreduce.PageRank.java

License:Apache License

private static void swap(FileSystem fs, Path src, Path dst) throws IOException {
    fs.delete(dst, true);/*  ww  w. j a va  2s  .c  o m*/
    fs.rename(src, dst);
}

---

From source file:com.TCG.Nutch_DNS.HostDb.java

License:Apache License

public static void install(JobConf job, Path crawlDb) throws IOException {
    boolean preserveBackup = job.getBoolean("db.preserve.backup", true);

    Path newCrawlDb = FileOutputFormat.getOutputPath(job);
    FileSystem fs = new JobClient(job).getFs();
    Path old = new Path(crawlDb, "old");
    Path current = new Path(crawlDb, CURRENT_NAME);
    if (fs.exists(current)) {
        if (fs.exists(old))
            fs.delete(old, true);//from www.j a  v  a 2  s .  c  o  m
        fs.rename(current, old);
    }
    fs.mkdirs(crawlDb);
    fs.rename(newCrawlDb, current);
    if (!preserveBackup && fs.exists(old))
        fs.delete(old, true);
    Path lock = new Path(crawlDb, LOCK_NAME);
    LockUtil.removeLockFile(fs, lock);
}

---

From source file:com.trace.hadoop.TestDFSRename.java

License:Apache License

public void testRename() throws Exception {
    FileSystem fs = cluster.getFileSystem();
    assertTrue(fs.mkdirs(dir));//from   www  .ja va  2  s  . c  om

    { //test lease
        Path a = new Path(dir, "a");
        Path aa = new Path(dir, "aa");
        Path b = new Path(dir, "b");

        createFile(fs, a);

        //should not have any lease
        assertEquals(0, countLease(cluster));

        createFile(fs, aa);
        DataOutputStream aa_out = fs.create(aa);
        aa_out.writeBytes("something");

        //should have 1 lease
        assertEquals(1, countLease(cluster));
        list(fs, "rename0");
        fs.rename(a, b);
        list(fs, "rename1");
        aa_out.writeBytes(" more");
        aa_out.close();
        list(fs, "rename2");

        //should not have any lease
        assertEquals(0, countLease(cluster));
    }

    { // test non-existent destination
        Path dstPath = new Path("/c/d");
        assertFalse(fs.exists(dstPath));
        assertFalse(fs.rename(dir, dstPath));
    }

    { // dst cannot be a file or directory under src
      // test rename /a/b/foo to /a/b/c
        Path src = new Path("/a/b");
        Path dst = new Path("/a/b/c");

        createFile(fs, new Path(src, "foo"));

        // dst cannot be a file under src
        assertFalse(fs.rename(src, dst));

        // dst cannot be a directory under src
        assertFalse(fs.rename(src.getParent(), dst.getParent()));
    }

    { // dst can start with src, if it is not a directory or file under src
      // test rename /test /testfile
        Path src = new Path("/testPrefix");
        Path dst = new Path("/testPrefixfile");

        createFile(fs, src);
        assertTrue(fs.rename(src, dst));
    }

    { // dst should not be same as src test rename /a/b/c to /a/b/c
        Path src = new Path("/a/b/c");
        createFile(fs, src);
        assertTrue(fs.rename(src, src));
        assertFalse(fs.rename(new Path("/a/b"), new Path("/a/b/")));
        assertTrue(fs.rename(src, new Path("/a/b/c/")));
    }

    fs.delete(dir, true);
}

---

From source file:com.twitter.hraven.etl.JobFilePartitioner.java

License:Apache License

/**
 * @param hdfs/*  w  ww  . j  a v a  2s.  com*/
 *          FileSystem handle
 * @param f
 *          file to process
 * @param outputPath
 * @param conf
 *          configuration to use for copying.
 * @param skipExisting
 *          skip if the file already exist in the target. File will be
 *          overwritten if already there and this argument is false.
 * @retain whether this file should be retained
 * 
 * @throws IOException
 */
private void processHDFSSource(FileSystem hdfs, FileStatus f, Path outputPath, Configuration conf,
        boolean skipExisting, boolean retain) throws IOException {

    long fileModTime = f.getModificationTime();
    Path targetDir = getTargetDirectory(hdfs, outputPath, fileModTime);

    boolean targetExists = false;
    Path target = new Path(targetDir, f.getPath().getName());
    targetExists = hdfs.exists(target);

    if (moveFiles || !retain) {
        if (targetExists) {
            hdfs.delete(f.getPath(), false);
        } else {
            hdfs.rename(f.getPath(), targetDir);
        }
    } else {
        if (targetExists && skipExisting) {
            // Do nothing, target is already there and we're instructed to skip
            // existing records.
        } else {
            copy(hdfs, f, conf, targetDir);
        }
    }
}

---

From source file:com.twitter.pig.backend.hadoop.executionengine.tez.TezJobControlCompiler.java

License:Apache License

/**
 * Walks the temporary directory structure to move (rename) files
 * to their final location.//  w  w  w .ja  v  a  2  s  .  co  m
 */
private void moveResults(Path p, String rem, FileSystem fs) throws IOException {
    for (FileStatus fstat : fs.listStatus(p)) {
        Path src = fstat.getPath();
        if (fstat.isDir()) {
            log.info("mkdir: " + src);
            fs.mkdirs(removePart(src, rem));
            moveResults(fstat.getPath(), rem, fs);
        } else {
            Path dst = removePart(src, rem);
            log.info("mv: " + src + " " + dst);
            fs.rename(src, dst);
        }
    }
}

---

From source file:com.vf.flume.sink.hdfs.BucketWriter.java

License:Apache License

/**
 * Rename bucketPath file from .tmp to permanent location.
 */// w  w  w . ja  va2s . c  o  m
// When this bucket writer is rolled based on rollCount or
// rollSize, the same instance is reused for the new file. But if
// the previous file was not closed/renamed,
// the bucket writer fields no longer point to it and hence need
// to be passed in from the thread attempting to close it. Even
// when the bucket writer is closed due to close timeout,
// this method can get called from the scheduled thread so the
// file gets closed later - so an implicit reference to this
// bucket writer would still be alive in the Callable instance.
private void renameBucket(String bucketPath, String targetPath, final FileSystem fs)
        throws IOException, InterruptedException {
    if (bucketPath.equals(targetPath)) {
        return;
    }

    final Path srcPath = new Path(bucketPath);
    final Path dstPath = new Path(targetPath);

    callWithTimeout(new CallRunner<Void>() {
        @Override
        public Void call() throws Exception {
            if (fs.exists(srcPath)) { // could block
                LOG.info("Renaming " + srcPath + " to " + dstPath);
                renameTries.incrementAndGet();
                fs.rename(srcPath, dstPath); // could block
            }
            return null;
        }
    });
}

---

From source file:com.wipro.ats.bdre.clustermigration.DestTableLoad.java

License:Apache License

public void execute(String[] params) throws IOException {

    CommandLine commandLine = getCommandLine(params, PARAMS_STRUCTURE);
    String src = commandLine.getOptionValue("source-path");
    String dest = commandLine.getOptionValue("dest-path");
    String destFs = commandLine.getOptionValue("dest-fs");

    Configuration config = new Configuration();
    config.set("fs.defaultFS", destFs);
    FileSystem hdfs = FileSystem.get(config);
    Path srcPath = new Path(src);
    RemoteIterator<LocatedFileStatus> srcFiles = hdfs.listFiles(srcPath, true);
    while (srcFiles.hasNext()) {
        String absolutePath = srcFiles.next().getPath().toUri().toString();
        if (absolutePath.endsWith("/"))
            absolutePath = absolutePath.substring(0, absolutePath.length() - 1);
        LOGGER.info("absolutePath of source business partition= " + absolutePath);
        String relativePath = absolutePath.replace(src, "");
        if (relativePath.endsWith("/"))
            relativePath = relativePath.substring(0, relativePath.length() - 1);
        LOGGER.info("relativePath of source business partition= = " + relativePath);
        if (!dest.endsWith("/"))
            dest = dest + "/";
        String destCheckPathString = dest + relativePath;
        Path destCheckPath = new Path(destCheckPathString);
        LOGGER.info("destCheckPath = " + destCheckPath);
        //find first index that contains a "/" from the end of the string, after first find the second such occurrence, finally trim the '/instanceexecid=number/part_0000' from the whole path, do this for both source and dest paths
        int destIndex = destCheckPathString.lastIndexOf("/");
        int secondLastDestIndex = destCheckPath.toString().lastIndexOf("/", destIndex - 1);
        int srcIndex = absolutePath.lastIndexOf("/");
        int secondLastSrcIndex = absolutePath.substring(0, srcIndex).lastIndexOf("/", srcIndex - 1);
        String truncatedSrcPath = absolutePath.substring(0, secondLastSrcIndex);
        LOGGER.info("truncated Src Path = " + truncatedSrcPath);
        String truncatedDestPath = destCheckPath.toString().substring(0, secondLastDestIndex);
        LOGGER.info("truncated Dest Path = " + truncatedDestPath);
        Path existsPathCheck = new Path(truncatedDestPath);
        Path srcPathToMove = new Path(truncatedSrcPath);
        //check if the business partition to be copied already exists inside the destination table, if it does, it has to be overwritten (in this case delete at dest and move from source to dest
        LOGGER.info("Does the business partition exist already inside the table? True/False? = "
                + hdfs.exists(existsPathCheck));
        if (hdfs.exists(existsPathCheck)) {
            LOGGER.info(/*  w  w w  .  j a  v  a  2s  .  c  o m*/
                    "bus partitions to be copied already exist at the destination, hence deleting them at destination");
            hdfs.delete(existsPathCheck, true);
        }
        String destPartitionPath = truncatedDestPath.substring(0, truncatedDestPath.lastIndexOf("/"));
        Path partitionWisePath = new Path(destPartitionPath);
        hdfs.mkdirs(partitionWisePath);
        LOGGER.info("moving the business partitions to the destination table");
        LOGGER.info("moving " + srcPathToMove + " to " + partitionWisePath);
        hdfs.rename(srcPathToMove, partitionWisePath);
    }
    hdfs.delete(srcPath, true);
}

---

From source file:contrail.util.FileHelper.java

License:Open Source License

/**
 * Function moves the contents of old_path into new_path. This is used
 * to save the final graph.//  www  .j  av  a 2  s  .  c  om
 * @param oldPath
 * @param newPath
 */
static public void moveDirectoryContents(Configuration conf, String oldPath, String newPath) {
    // We can't invoke rename directly on old path because it ends up
    // making old_path a subdirectory of new_path.
    FileSystem fs = null;
    try {
        fs = FileSystem.get(conf);
    } catch (IOException e) {
        throw new RuntimeException("Can't get filesystem: " + e.getMessage());
    }
    try {
        Path oldPathObject = new Path(oldPath);
        for (FileStatus status : fs.listStatus(oldPathObject)) {
            Path oldFile = status.getPath();
            Path newFile = new Path(newPath, oldFile.getName());
            fs.rename(oldFile, newFile);
        }
    } catch (IOException e) {
        throw new RuntimeException("Problem moving the files: " + e.getMessage());
    }
}

---

From source file:datafu.hourglass.jobs.StagedOutputJob.java

License:Apache License

/**
 * Run the job and wait for it to complete.  Output will be temporarily stored under the staging path.
 * If the job is successful it will be moved to the final location.
 *///from  ww  w.  j  a  v  a 2  s. co  m
@Override
public boolean waitForCompletion(boolean verbose)
        throws IOException, InterruptedException, ClassNotFoundException {
    final Path actualOutputPath = FileOutputFormat.getOutputPath(this);
    final Path stagedPath = new Path(String.format("%s/%s/staged", _stagingPrefix, System.currentTimeMillis()));

    FileOutputFormat.setOutputPath(this, stagedPath);

    final Thread hook = new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                killJob();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    });

    Runtime.getRuntime().addShutdownHook(hook);

    final boolean retVal = super.waitForCompletion(verbose);
    Runtime.getRuntime().removeShutdownHook(hook);

    if (retVal) {
        FileSystem fs = actualOutputPath.getFileSystem(getConfiguration());

        fs.mkdirs(actualOutputPath);

        _log.info(String.format("Deleting data at old path[%s]", actualOutputPath));
        fs.delete(actualOutputPath, true);

        _log.info(String.format("Moving from staged path[%s] to final resting place[%s]", stagedPath,
                actualOutputPath));
        boolean renamed = fs.rename(stagedPath, actualOutputPath);

        if (renamed && _writeCounters) {
            writeCounters(fs);
        }

        return renamed;
    } else {
        FileSystem fs = actualOutputPath.getFileSystem(getConfiguration());
        _log.info(String.format("Job failed, deleting staged path[%s]", stagedPath));
        try {
            fs.delete(stagedPath, true);
        } catch (IOException e) {
        }
    }

    _log.warn("retVal was false for some reason...");
    return retVal;
}

---

From source file:distributed.hadoop.HDFSUtils.java

License:Open Source License

/**
 * Move a file from one location to another in HDFS
 * //from   www.j  a  va 2 s .  c  o  m
 * @param source the source path in HDFS
 * @param target the target path in HDFS
 * @param config the HDFSConfig with connection details
 * @param env environment variables
 * @throws IOException if a problem occurs
 */
public static void moveInHDFS(String source, String target, HDFSConfig config, Environment env)
        throws IOException {

    createTmpDistributedCacheDirIfNecessary(config);

    Path sourceP = new Path(resolvePath(source, env));
    Path targetP = new Path(resolvePath(target, env));

    Configuration conf = new Configuration();
    config.configureForHadoop(conf, env);

    FileSystem fs = FileSystem.get(conf);

    if (fs.exists(targetP)) {
        fs.delete(targetP, true);
    }

    fs.rename(sourceP, targetP);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »