Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:org.apache.blur.thrift.TableAdmin.java

License:Apache License

private void loadShard(Path newLoadShardPath, FileSystem fileSystem, Path tablePath) throws IOException {
    Path shardPath = new Path(tablePath, newLoadShardPath.getName());
    FileStatus[] listStatus = fileSystem.listStatus(newLoadShardPath, new PathFilter() {
        @Override//from  w w w .ja  va  2 s .  c  om
        public boolean accept(Path path) {
            return path.getName().endsWith(".commit");
        }
    });

    for (FileStatus fileStatus : listStatus) {
        Path src = fileStatus.getPath();
        Path dst = new Path(shardPath, src.getName());
        if (fileSystem.rename(src, dst)) {
            LOG.info("Successfully moved [{0}] to [{1}].", src, dst);
        } else {
            LOG.info("Could not move [{0}] to [{1}].", src, dst);
            throw new IOException("Could not move [" + src + "] to [" + dst + "].");
        }
    }
}

---

From source file:org.apache.carbondata.core.datastorage.store.filesystem.AbstractDFSCarbonFile.java

License:Apache License

public boolean renameTo(String changetoName) {
    FileSystem fs;
    try {//from w w w . j a  v  a  2 s  . com
        fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
        return fs.rename(fileStatus.getPath(), new Path(changetoName));
    } catch (IOException e) {
        LOGGER.error("Exception occured:" + e.getMessage());
        return false;
    }
}

---

From source file:org.apache.carbondata.core.datastorage.store.filesystem.ViewFSCarbonFile.java

License:Apache License

@Override
public boolean renameForce(String changetoName) {
    FileSystem fs;
    try {/*from w w  w  . ja v a2 s.  c o m*/
        fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
        if (fs instanceof ViewFileSystem) {
            fs.delete(new Path(changetoName), true);
            fs.rename(fileStatus.getPath(), new Path(changetoName));
            return true;
        } else {
            return false;
        }
    } catch (IOException e) {
        LOGGER.error("Exception occured" + e.getMessage());
        return false;
    }
}

---

From source file:org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.java

License:Apache License

/**
  TODO: The current implementation of renameForce is not correct as it deletes the destination
  object and then performs rename(copy).
  If the copy fails then there is not way to recover the old file.
   This can happen when tablestatus.write is renamed to tablestatus.
  One solution can be to read the content and rewrite the file as write with the same name
  will overwrite the file by default. Need to discuss.
  Refer CARBONDATA-2670 for tracking this.
 *///from   w  w  w  . ja  v a 2  s.  c  om
@Override
public boolean renameForce(String changeToName) {
    FileSystem fs;
    try {
        deleteFile(changeToName, FileFactory.getFileType(changeToName));
        fs = fileStatus.getPath().getFileSystem(hadoopConf);
        return fs.rename(fileStatus.getPath(), new Path(changeToName));
    } catch (IOException e) {
        LOGGER.error("Exception occured: " + e.getMessage());
        return false;
    }
}

---

From source file:org.apache.crunch.impl.mr.exec.CrunchJob.java

License:Apache License

private synchronized void handleMultiPaths() throws IOException {
    if (!multiPaths.isEmpty()) {
        // Need to handle moving the data from the output directory of the
        // job to the output locations specified in the paths.
        FileSystem fs = FileSystem.get(job.getConfiguration());
        for (int i = 0; i < multiPaths.size(); i++) {
            Path src = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + i + "-*");
            Path[] srcs = FileUtil.stat2Paths(fs.globStatus(src), src);
            Path dst = multiPaths.get(i);
            if (!fs.exists(dst)) {
                fs.mkdirs(dst);/*from  w ww  .j  av a2  s  .com*/
            }
            int minPartIndex = getMinPartIndex(dst, fs);
            for (Path s : srcs) {
                fs.rename(s, getDestFile(s, dst, minPartIndex++));
            }
        }
    }
}

---

From source file:org.apache.crunch.io.avro.AvroPathPerKeyTarget.java

License:Apache License

@Override
public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException {
    FileSystem srcFs = workingPath.getFileSystem(conf);
    Path base = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + index);
    Path[] keys = FileUtil.stat2Paths(srcFs.listStatus(base), base);
    FileSystem dstFs = path.getFileSystem(conf);
    if (!dstFs.exists(path)) {
        dstFs.mkdirs(path);//from ww  w.  j av a 2 s .c  o m
    }
    boolean sameFs = isCompatible(srcFs, path);
    for (Path key : keys) {
        Path[] srcs = FileUtil.stat2Paths(srcFs.listStatus(key), key);
        Path targetPath = new Path(path, key.getName());
        dstFs.mkdirs(targetPath);
        for (Path s : srcs) {
            Path d = getDestFile(conf, s, targetPath, s.getName().contains("-m-"));
            if (sameFs) {
                srcFs.rename(s, d);
            } else {
                FileUtil.copy(srcFs, s, dstFs, d, true, true, conf);
            }
        }
    }
    dstFs.create(getSuccessIndicator(), true).close();
}

---

From source file:org.apache.crunch.io.impl.FileTargetImpl.java

License:Apache License

@Override
public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException {
    FileSystem srcFs = workingPath.getFileSystem(conf);
    Path src = getSourcePattern(workingPath, index);
    Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(src), src);
    FileSystem dstFs = path.getFileSystem(conf);
    if (!dstFs.exists(path)) {
        dstFs.mkdirs(path);//  ww w.  java 2 s .c om
    }
    boolean sameFs = isCompatible(srcFs, path);
    for (Path s : srcs) {
        Path d = getDestFile(conf, s, path, s.getName().contains("-m-"));
        if (sameFs) {
            srcFs.rename(s, d);
        } else {
            FileUtil.copy(srcFs, s, dstFs, d, true, true, conf);
        }
    }
    dstFs.create(getSuccessIndicator(), true).close();
}

---

From source file:org.apache.druid.indexer.JobHelper.java

License:Apache License

static void addJarToClassPath(File jarFile, Path distributedClassPath, Path intermediateClassPath,
        FileSystem fs, Job job) throws IOException {
    // Create distributed directory if it does not exist.
    // rename will always fail if destination does not exist.
    fs.mkdirs(distributedClassPath);/*from w  w  w  .ja v a2  s.c om*/

    // Non-snapshot jar files are uploaded to the shared classpath.
    final Path hdfsPath = new Path(distributedClassPath, jarFile.getName());
    if (shouldUploadOrReplace(jarFile, hdfsPath, fs)) {
        // Muliple jobs can try to upload the jar here,
        // to avoid them from overwriting files, first upload to intermediateClassPath and then rename to the distributedClasspath.
        final Path intermediateHdfsPath = new Path(intermediateClassPath, jarFile.getName());
        uploadJar(jarFile, intermediateHdfsPath, fs);
        IOException exception = null;
        try {
            log.info("Renaming jar to path[%s]", hdfsPath);
            fs.rename(intermediateHdfsPath, hdfsPath);
            if (!fs.exists(hdfsPath)) {
                throw new IOE("File does not exist even after moving from[%s] to [%s]", intermediateHdfsPath,
                        hdfsPath);
            }
        } catch (IOException e) {
            // rename failed, possibly due to race condition. check if some other job has uploaded the jar file.
            try {
                if (!fs.exists(hdfsPath)) {
                    log.error(e, "IOException while Renaming jar file");
                    exception = e;
                }
            } catch (IOException e1) {
                e.addSuppressed(e1);
                exception = e;
            }
        } finally {
            try {
                if (fs.exists(intermediateHdfsPath)) {
                    fs.delete(intermediateHdfsPath, false);
                }
            } catch (IOException e) {
                if (exception == null) {
                    exception = e;
                } else {
                    exception.addSuppressed(e);
                }
            }
            if (exception != null) {
                throw exception;
            }
        }
    }
    job.addFileToClassPath(hdfsPath);
}

---

From source file:org.apache.druid.indexer.JobHelper.java

License:Apache License

/**
 * Rename the files. This works around some limitations of both FileContext (no s3n support) and NativeS3FileSystem.rename
 * which will not overwrite//from   ww w  .j  av  a 2s . c o  m
 *
 * @param outputFS              The output fs
 * @param indexZipFilePath      The original file path
 * @param finalIndexZipFilePath The to rename the original file to
 *
 * @return False if a rename failed, true otherwise (rename success or no rename needed)
 */
private static boolean renameIndexFiles(final FileSystem outputFS, final Path indexZipFilePath,
        final Path finalIndexZipFilePath) {
    try {
        return RetryUtils.retry(() -> {
            final boolean needRename;

            if (outputFS.exists(finalIndexZipFilePath)) {
                // NativeS3FileSystem.rename won't overwrite, so we might need to delete the old index first
                final FileStatus zipFile = outputFS.getFileStatus(indexZipFilePath);
                final FileStatus finalIndexZipFile = outputFS.getFileStatus(finalIndexZipFilePath);

                if (zipFile.getModificationTime() >= finalIndexZipFile.getModificationTime()
                        || zipFile.getLen() != finalIndexZipFile.getLen()) {
                    log.info("File[%s / %s / %sB] existed, but wasn't the same as [%s / %s / %sB]",
                            finalIndexZipFile.getPath(), DateTimes.utc(finalIndexZipFile.getModificationTime()),
                            finalIndexZipFile.getLen(), zipFile.getPath(),
                            DateTimes.utc(zipFile.getModificationTime()), zipFile.getLen());
                    outputFS.delete(finalIndexZipFilePath, false);
                    needRename = true;
                } else {
                    log.info("File[%s / %s / %sB] existed and will be kept", finalIndexZipFile.getPath(),
                            DateTimes.utc(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen());
                    needRename = false;
                }
            } else {
                needRename = true;
            }

            if (needRename) {
                log.info("Attempting rename from [%s] to [%s]", indexZipFilePath, finalIndexZipFilePath);
                return outputFS.rename(indexZipFilePath, finalIndexZipFilePath);
            } else {
                return true;
            }
        }, FileUtils.IS_EXCEPTION, NUM_RETRIES);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---

From source file:org.apache.flink.api.java.hadoop.mapreduce.HadoopOutputFormatBase.java

License:Apache License

/**
 * commit the task by moving the output file out from the temporary directory.
 * @throws java.io.IOException/*from   w  w  w .  ja  v a 2s  . c om*/
 */
@Override
public void close() throws IOException {

    // enforce sequential close() calls
    synchronized (CLOSE_MUTEX) {
        try {
            this.recordWriter.close(this.context);
        } catch (InterruptedException e) {
            throw new IOException("Could not close RecordReader.", e);
        }

        if (this.outputCommitter.needsTaskCommit(this.context)) {
            this.outputCommitter.commitTask(this.context);
        }

        Path outputPath = new Path(this.configuration.get("mapred.output.dir"));

        // rename tmp-file to final name
        FileSystem fs = FileSystem.get(outputPath.toUri(), this.configuration);

        String taskNumberStr = Integer.toString(this.taskNumber);
        String tmpFileTemplate = "tmp-r-00000";
        String tmpFile = tmpFileTemplate.substring(0, 11 - taskNumberStr.length()) + taskNumberStr;

        if (fs.exists(new Path(outputPath.toString() + "/" + tmpFile))) {
            fs.rename(new Path(outputPath.toString() + "/" + tmpFile),
                    new Path(outputPath.toString() + "/" + taskNumberStr));
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »