Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:io.hops.experiments.utils.DFSOperationsUtils.java

License:Apache License

public static boolean renameFile(FileSystem dfs, Path from, Path to) throws IOException {
    if (SERVER_LESS_MODE) {
        serverLessModeRandomWait();/*from   w  w w.  j  a va  2s.  c o m*/
        return true;
    }
    return dfs.rename(from, to);
}

---

From source file:io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore.java

License:Apache License

private static void asyncRename(HdfsEnvironment hdfsEnvironment, Executor executor, AtomicBoolean cancelled,
        List<CompletableFuture<?>> fileRenameFutures, HdfsContext context, Path currentPath, Path targetPath,
        List<String> fileNames) {
    FileSystem fileSystem;
    try {//from   w ww.  j a  va2 s .  co  m
        fileSystem = hdfsEnvironment.getFileSystem(context, currentPath);
    } catch (IOException e) {
        throw new PrestoException(HIVE_FILESYSTEM_ERROR,
                format("Error moving data files to final location. Error listing directory %s", currentPath),
                e);
    }

    for (String fileName : fileNames) {
        Path source = new Path(currentPath, fileName);
        Path target = new Path(targetPath, fileName);
        fileRenameFutures.add(CompletableFuture.runAsync(() -> {
            if (cancelled.get()) {
                return;
            }
            try {
                if (fileSystem.exists(target) || !fileSystem.rename(source, target)) {
                    throw new PrestoException(HIVE_FILESYSTEM_ERROR,
                            format("Error moving data files from %s to final location %s", source, target));
                }
            } catch (IOException e) {
                throw new PrestoException(HIVE_FILESYSTEM_ERROR,
                        format("Error moving data files from %s to final location %s", source, target), e);
            }
        }, executor));
    }
}

---

From source file:kogiri.mapreduce.libra.kmersimilarity_m.KmerSimilarityMap.java

License:Open Source License

private void commit(Path outputPath, Configuration conf) throws IOException {
    FileSystem fs = outputPath.getFileSystem(conf);

    FileStatus status = fs.getFileStatus(outputPath);
    if (status.isDir()) {
        FileStatus[] entries = fs.listStatus(outputPath);
        for (FileStatus entry : entries) {
            Path entryPath = entry.getPath();

            // remove unnecessary outputs
            if (MapReduceHelper.isLogFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else if (MapReduceHelper.isPartialOutputFiles(entryPath)) {
                // rename outputs
                int mapreduceID = MapReduceHelper.getMapReduceID(entryPath);
                String newName = KmerSimilarityHelper.makeKmerSimilarityResultFileName(mapreduceID);
                Path toPath = new Path(entryPath.getParent(), newName);

                LOG.info("output : " + entryPath.toString());
                LOG.info("renamed to : " + toPath.toString());
                fs.rename(entryPath, toPath);
            } else {
                // let it be
            }//from  w  ww. j a  va2 s .  co m
        }
    } else {
        throw new IOException("path not found : " + outputPath.toString());
    }
}

---

From source file:kogiri.mapreduce.preprocess.indexing.stage1.ReadIndexBuilder.java

License:Open Source License

private void commit(Path outputPath, Configuration conf, NamedOutputs namedOutputs) throws IOException {
    FileSystem fs = outputPath.getFileSystem(conf);

    FileStatus status = fs.getFileStatus(outputPath);
    if (status.isDir()) {
        FileStatus[] entries = fs.listStatus(outputPath);
        for (FileStatus entry : entries) {
            Path entryPath = entry.getPath();

            // remove unnecessary outputs
            if (MapReduceHelper.isLogFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else if (MapReduceHelper.isPartialOutputFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else if (KmerHistogramHelper.isKmerHistogramFile(entryPath)) {
                // not necessary
            } else {
                // rename outputs
                NamedOutputRecord namedOutput = namedOutputs.getRecordFromMROutput(entryPath.getName());
                if (namedOutput != null) {
                    Path toPath = new Path(entryPath.getParent(),
                            ReadIndexHelper.makeReadIndexFileName(namedOutput.getFilename()));

                    LOG.info("output : " + entryPath.toString());
                    LOG.info("renamed to : " + toPath.toString());
                    fs.rename(entryPath, toPath);
                }/*w  w  w .  jav  a2  s.  c o m*/
            }
        }
    } else {
        throw new IOException("path not found : " + outputPath.toString());
    }
}

---

From source file:kogiri.mapreduce.preprocess.indexing.stage2.KmerIndexBuilder.java

License:Open Source License

private void commitRoundIndexOutputFiles(Path roundInputPath, Path MROutputPath, Path finalOutputPath,
        Configuration conf, int kmerSize) throws IOException {
    FileSystem fs = MROutputPath.getFileSystem(conf);
    if (!fs.exists(finalOutputPath)) {
        fs.mkdirs(finalOutputPath);/*from   w  w  w . j a v  a 2s .  c o m*/
    }

    FileStatus status = fs.getFileStatus(MROutputPath);
    if (status.isDir()) {
        FileStatus[] entries = fs.listStatus(MROutputPath);
        for (FileStatus entry : entries) {
            Path entryPath = entry.getPath();

            // remove unnecessary outputs
            if (MapReduceHelper.isLogFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else if (MapReduceHelper.isPartialOutputFiles(entryPath)) {
                // rename outputs
                int mapreduceID = MapReduceHelper.getMapReduceID(entryPath);
                Path toPath = new Path(finalOutputPath, KmerIndexHelper
                        .makeKmerIndexPartFileName(roundInputPath.getName(), kmerSize, mapreduceID));

                LOG.info("output : " + entryPath.toString());
                LOG.info("renamed to : " + toPath.toString());
                fs.rename(entryPath, toPath);
            }
        }
    } else {
        throw new IOException("path not found : " + MROutputPath.toString());
    }

    fs.delete(MROutputPath, true);
}

---

From source file:kogiri.mapreduce.readfrequency.kmermatch.KmerMatcher.java

License:Open Source License

private void commit(Path outputPath, Configuration conf) throws IOException {
    FileSystem fs = outputPath.getFileSystem(conf);

    FileStatus status = fs.getFileStatus(outputPath);
    if (status.isDir()) {
        FileStatus[] entries = fs.listStatus(outputPath);
        for (FileStatus entry : entries) {
            Path entryPath = entry.getPath();

            // remove unnecessary outputs
            if (MapReduceHelper.isLogFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else if (MapReduceHelper.isPartialOutputFiles(entryPath)) {
                // rename outputs
                int mapreduceID = MapReduceHelper.getMapReduceID(entryPath);
                String newName = KmerMatchHelper.makeKmerMatchResultFileName(mapreduceID);
                Path toPath = new Path(entryPath.getParent(), newName);

                LOG.info("output : " + entryPath.toString());
                LOG.info("renamed to : " + toPath.toString());
                fs.rename(entryPath, toPath);
            } else {
                // let it be
            }// w  ww.j  av  a  2 s  .  c  om
        }
    } else {
        throw new IOException("path not found : " + outputPath.toString());
    }
}

---

From source file:kogiri.mapreduce.readfrequency.modecount.ModeCounter.java

License:Open Source License

private void commitRoundOutputFiles(Path MROutputPath, Path finalOutputPath, Configuration conf,
        NamedOutputs namedOutputs, int round) throws IOException {
    FileSystem fs = MROutputPath.getFileSystem(conf);
    if (!fs.exists(finalOutputPath)) {
        fs.mkdirs(finalOutputPath);/*from ww  w.  j  a v  a2  s  .  c  om*/
    }

    NamedOutputRecord roundMasterRecord = namedOutputs.getRecordFromID(round);
    Path roundDestPath = new Path(finalOutputPath, roundMasterRecord.getFilename());
    if (!fs.exists(roundDestPath)) {
        fs.mkdirs(roundDestPath);
    }

    FileStatus status = fs.getFileStatus(MROutputPath);
    if (status.isDir()) {
        FileStatus[] entries = fs.listStatus(MROutputPath);
        for (FileStatus entry : entries) {
            Path entryPath = entry.getPath();

            // remove unnecessary outputs
            if (MapReduceHelper.isLogFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else if (MapReduceHelper.isPartialOutputFiles(entryPath)) {
                fs.delete(entryPath, true);
            } else {
                // rename outputs
                NamedOutputRecord namedOutput = namedOutputs.getRecordFromMROutput(entryPath);
                if (namedOutput != null) {
                    Path toPath = new Path(roundDestPath, namedOutput.getFilename() + "."
                            + ReadFrequencyCounterConstants.READ_FREQUENCY_FILENAME_FILENAME_EXTENSION);

                    LOG.info("output : " + entryPath.toString());
                    LOG.info("renamed to : " + toPath.toString());
                    fs.rename(entryPath, toPath);
                }
            }
        }
    } else {
        throw new IOException("path not found : " + MROutputPath.toString());
    }

    fs.delete(MROutputPath, true);
}

---

From source file:net.peacesoft.nutch.crawl.ReCrawlDb.java

License:Apache License

public static void install(JobConf job, Path crawlDb) throws IOException {
    boolean preserveBackup = job.getBoolean("db.preserve.backup", true);

    Path newCrawlDb = FileOutputFormat.getOutputPath(job);
    FileSystem fs = new JobClient(job).getFs();
    Path old = new Path(crawlDb, "old");
    Path current = new Path(crawlDb, CURRENT_NAME);
    if (fs.exists(current)) {
        if (fs.exists(old)) {
            fs.delete(old, true);/*w  w w .j a v a  2 s  . c  o  m*/
        }
        fs.rename(current, old);
    }
    fs.mkdirs(crawlDb);
    fs.rename(newCrawlDb, current);
    if (!preserveBackup && fs.exists(old)) {
        fs.delete(old, true);
    }
    Path lock = new Path(crawlDb, LOCK_NAME);
    LockUtil.removeLockFile(fs, lock);
}

---

From source file:net.peacesoft.nutch.crawl.ReLinkDb.java

License:Apache License

public static void install(JobConf job, Path linkDb) throws IOException {
    Path newLinkDb = FileOutputFormat.getOutputPath(job);
    FileSystem fs = new JobClient(job).getFs();
    Path old = new Path(linkDb, "old");
    Path current = new Path(linkDb, CURRENT_NAME);
    if (fs.exists(current)) {
        if (fs.exists(old)) {
            fs.delete(old, true);/* w  ww  .ja va 2 s. co  m*/
        }
        fs.rename(current, old);
    }
    fs.mkdirs(linkDb);
    fs.rename(newLinkDb, current);
    if (fs.exists(old)) {
        fs.delete(old, true);
    }
    LockUtil.removeLockFile(fs, new Path(linkDb, LOCK_NAME));
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.MapReduceJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    FileSystem dfs = FileSystem.get(getConf());
    String inPath = inputPath;//from  w ww  . j  a v a 2 s.  c  o  m

    while (!isFinished()) {
        iteration++;

        // Prepare job configuration
        JobConf jobConfiguration = new JobConf(this.getConf());
        jobConfiguration.setJarByClass(this.getClass());

        jobConfiguration.setMapOutputKeyClass(getMapOutputKeyClass());
        jobConfiguration.setMapOutputValueClass(getMapOutputValueClass());

        jobConfiguration.setMapperClass(getMapperClass());
        if (getCombinerClass() != null)
            jobConfiguration.setCombinerClass(getCombinerClass());
        jobConfiguration.setReducerClass(getReducerClass());

        jobConfiguration.setOutputKeyClass(getOutputKeyClass());
        jobConfiguration.setOutputValueClass(getOutputValueClass());

        jobConfiguration.setInputFormat(getInputFormatClass());
        jobConfiguration.setOutputFormat(getOutputFormatClass());

        if (getNumMappers() != -1)
            jobConfiguration.setNumMapTasks(getNumMappers());
        if (getNumReducers() != -1)
            jobConfiguration.setNumReduceTasks(getNumReducers());

        setConfigurationParameters(jobConfiguration);

        // Set the input and output paths
        String outPath = intermediatePath + "/iteration-" + iteration;
        FileInputFormat.addInputPath(jobConfiguration, new Path(inPath));
        FileOutputFormat.setOutputPath(jobConfiguration, new Path(outPath));

        // Execute the current iteration
        RunningJob jobExecution = JobClient.runJob(jobConfiguration);
        jobExecution.waitForCompletion();

        // Remove the output of the previous job (unless it is the input graph)
        if (iteration != 1) {
            dfs.delete(new Path(inPath), true);
        }
        inPath = outPath;

        processJobOutput(jobExecution);
    }

    // Rename the last job output to the specified output path
    try {
        dfs.mkdirs(new Path(outputPath).getParent());
        dfs.rename(new Path(inPath), new Path(outputPath));
    } catch (Exception e) {
        LOG.warn("Failed to rename MapReduce job output.", e);
    }

    return 0;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »