List of usage examples for org.apache.hadoop.fs FileSystem rename
public abstract boolean rename(Path src, Path dst) throws IOException;
From source file:gobblin.util.HadoopUtils.java
License:Apache License
/** * Renames a src {@link Path} on fs {@link FileSystem} to a dst {@link Path}. If fs is a {@link LocalFileSystem} and * src is a directory then {@link File#renameTo} is called directly to avoid a directory rename race condition where * {@link org.apache.hadoop.fs.RawLocalFileSystem#rename} copies the conflicting src directory into dst resulting in * an extra nested level, such as /root/a/b/c/e/e where e is repeated. * * @param fs the {@link FileSystem} where the src {@link Path} exists * @param src the source {@link Path} which will be renamed * @param dst the {@link Path} to rename to * @return true if rename succeeded, false if rename failed. * @throws IOException if rename failed for reasons other than target exists. *//* w w w . j ava2 s . c o m*/ public static boolean renamePathHandleLocalFSRace(FileSystem fs, Path src, Path dst) throws IOException { if (DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem && fs.isDirectory(src)) { LocalFileSystem localFs = (LocalFileSystem) DecoratorUtils.resolveUnderlyingObject(fs); File srcFile = localFs.pathToFile(src); File dstFile = localFs.pathToFile(dst); return srcFile.renameTo(dstFile); } else { return fs.rename(src, dst); } }
From source file:gobblin.util.HadoopUtils.java
License:Apache License
/** * A wrapper around {@link FileSystem#rename(Path, Path)} which throws {@link IOException} if * {@link FileSystem#rename(Path, Path)} returns False. *///from w ww .ja va 2s . c o m public static void renamePath(FileSystem fs, Path oldName, Path newName, boolean overwrite) throws IOException { if (!fs.exists(oldName)) { throw new FileNotFoundException( String.format("Failed to rename %s to %s: src not found", oldName, newName)); } if (fs.exists(newName)) { if (overwrite) { if (!fs.delete(newName, true)) { throw new IOException(String.format("Failed to delete %s while renaming %s to %s", newName, oldName, newName)); } } else { throw new FileAlreadyExistsException( String.format("Failed to rename %s to %s: dst already exists", oldName, newName)); } } if (!fs.rename(oldName, newName)) { throw new IOException(String.format("Failed to rename %s to %s", oldName, newName)); } }
From source file:gobblin.util.HadoopUtils.java
License:Apache License
/** * A thread safe variation of {@link #renamePath(FileSystem, Path, Path)} which can be used in * multi-threaded/multi-mapper environment. The rename operation always happens at file level hence directories are * not overwritten under the 'to' path./*w w w . j a v a 2s . c o m*/ * * <p> * If the contents of destination 'to' path is not expected to be modified concurrently, use * {@link #renamePath(FileSystem, Path, Path)} which is faster and more optimized * </p> * * <b>NOTE: This does not seem to be working for all {@link FileSystem} implementations. Use * {@link #renameRecursively(FileSystem, Path, Path)}</b> * * @param fileSystem on which the data needs to be moved * @param from path of the data to be moved * @param to path of the data to be moved * */ public static void safeRenameRecursively(FileSystem fileSystem, Path from, Path to) throws IOException { for (FileStatus fromFile : FileListUtils.listFilesRecursively(fileSystem, from)) { Path relativeFilePath = new Path( StringUtils.substringAfter(fromFile.getPath().toString(), from.toString() + Path.SEPARATOR)); Path toFilePath = new Path(to, relativeFilePath); if (!fileSystem.exists(toFilePath)) { boolean renamed = false; // underlying file open can fail with file not found error due to some race condition // when the parent directory is created in another thread, so retry a few times for (int i = 0; !renamed && i < MAX_RENAME_TRIES; i++) { try { renamed = fileSystem.rename(fromFile.getPath(), toFilePath); break; } catch (FileNotFoundException e) { if (i + 1 >= MAX_RENAME_TRIES) { throw e; } } } if (!renamed) { throw new IOException( String.format("Failed to rename %s to %s.", fromFile.getPath(), toFilePath)); } log.info(String.format("Renamed %s to %s", fromFile.getPath(), toFilePath)); } else { log.info(String.format("File already exists %s. Will not rewrite", toFilePath)); } } }
From source file:gobblin.util.HadoopUtilsTest.java
License:Apache License
@Test(groups = { "performance" })
public void testRenamePerformance() throws Exception {
FileSystem fs = Mockito.mock(FileSystem.class);
Path sourcePath = new Path("/source");
Path s1 = new Path(sourcePath, "d1");
FileStatus[] sourceStatuses = new FileStatus[10000];
FileStatus[] targetStatuses = new FileStatus[1000];
for (int i = 0; i < sourceStatuses.length; i++) {
sourceStatuses[i] = getFileStatus(new Path(s1, "path" + i), false);
}//from ww w. ja va 2 s. c om
for (int i = 0; i < targetStatuses.length; i++) {
targetStatuses[i] = getFileStatus(new Path(s1, "path" + i), false);
}
Mockito.when(fs.getUri()).thenReturn(new URI("file:///"));
Mockito.when(fs.getFileStatus(sourcePath)).thenAnswer(getDelayedAnswer(getFileStatus(sourcePath, true)));
Mockito.when(fs.exists(sourcePath)).thenAnswer(getDelayedAnswer(true));
Mockito.when(fs.listStatus(sourcePath))
.thenAnswer(getDelayedAnswer(new FileStatus[] { getFileStatus(s1, true) }));
Mockito.when(fs.exists(s1)).thenAnswer(getDelayedAnswer(true));
Mockito.when(fs.listStatus(s1)).thenAnswer(getDelayedAnswer(sourceStatuses));
Path target = new Path("/target");
Path s1Target = new Path(target, "d1");
Mockito.when(fs.exists(target)).thenAnswer(getDelayedAnswer(true));
Mockito.when(fs.exists(s1Target)).thenAnswer(getDelayedAnswer(true));
Mockito.when(fs.mkdirs(Mockito.any(Path.class))).thenAnswer(getDelayedAnswer(true));
Mockito.when(fs.rename(Mockito.any(Path.class), Mockito.any(Path.class)))
.thenAnswer(getDelayedAnswer(true));
HadoopUtils.renameRecursively(fs, sourcePath, target);
}
From source file:hadoopdemo.Hadoop.java
private void renameButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_renameButtonActionPerformed try {// ww w . java 2 s.c o m String oldName = oldNameTextField.getText(); String newName = newNameTextFIeld.getText(); Configuration conf = new Configuration(); conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/core-site.xml")); conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/hdfs-site.xml")); conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/mapred-site.xml")); FileSystem fileSystem = FileSystem.get(conf); fileSystem.rename(new Path(oldName), new Path(newName)); } catch (IOException ex) { Logger.getLogger(Hadoop.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:hibench.DataPaths.java
License:Apache License
public static void moveFilesToParent(Path src) throws IOException { FileSystem fs = src.getFileSystem(new Configuration()); Path parent = src.getParent(); FileStatus[] flist = fs.listStatus(src); for (FileStatus file : flist) { if (null != file) { fs.rename(file.getPath(), new Path(parent, file.getPath().getName())); }//ww w . j a v a2 s .c o m } fs.delete(src, true); fs.close(); }
From source file:hitune.analysis.mapreduce.processor.AnalysisProcessor.java
License:Apache License
/** * Move the TEMP output folder to final one(user defined one); * If there are multiple files under one job's output folder, it should merge the output into one file. * Then rename the folder to the final one. * @param job/* w ww.j ava 2 s. com*/ * @param output * @param result */ protected void moveResults(JobConf job, String output, String result) { try { FileSystem fs = FileSystem.get(job); log.debug("move results: " + result); Path src = new Path(result + "/" + "*.csv*"); Path dst = new Path(output); if (!fs.exists(dst)) { fs.mkdirs(dst); } FileStatus[] matches = fs.globStatus(src, new PathFilter() { @Override public boolean accept(Path path) { // TODO Auto-generated method stub return true; } }); if (matches != null && matches.length != 0) { if (matches.length > 1) { //multiple output files String[] args = new String[2]; args[0] = result; args[1] = "_" + result; fs.delete(new Path("_" + result)); //merge multiple output files into one file ToolRunner.run(new MergeOutput(this.conf), args); fs.delete(new Path(result)); fs.rename(new Path("_" + result), new Path(result)); } matches = fs.globStatus(src, new PathFilter() { @Override public boolean accept(Path path) { // TODO Auto-generated method stub return true; } }); for (FileStatus file : matches) { String filename = file.getPath().getName(); filename = filename.substring(0, filename.indexOf("-")); log.debug("move file:" + filename); Path toFile = new Path(output + "/" + filename); if (fs.exists(toFile)) { fs.delete(toFile); } fs.rename(file.getPath(), toFile); fs.delete(file.getPath().getParent(), true); FileStatus[] tmpDirs = fs.listStatus(file.getPath().getParent().getParent()); if (tmpDirs == null || tmpDirs.length == 0) { fs.delete(file.getPath().getParent().getParent(), true); } break; } } else { MOVE_DONE = false; } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); MOVE_DONE = false; } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } MOVE_DONE = true; }
From source file:io.druid.indexer.JobHelper.java
License:Apache License
static void addJarToClassPath(File jarFile, Path distributedClassPath, Path intermediateClassPath, FileSystem fs, Job job) throws IOException { // Create distributed directory if it does not exist. // rename will always fail if destination does not exist. fs.mkdirs(distributedClassPath);/* w ww . j ava2 s.co m*/ // Non-snapshot jar files are uploaded to the shared classpath. final Path hdfsPath = new Path(distributedClassPath, jarFile.getName()); if (!fs.exists(hdfsPath)) { // Muliple jobs can try to upload the jar here, // to avoid them from overwriting files, first upload to intermediateClassPath and then rename to the distributedClasspath. final Path intermediateHdfsPath = new Path(intermediateClassPath, jarFile.getName()); uploadJar(jarFile, intermediateHdfsPath, fs); IOException exception = null; try { log.info("Renaming jar to path[%s]", hdfsPath); fs.rename(intermediateHdfsPath, hdfsPath); if (!fs.exists(hdfsPath)) { throw new IOException(String.format("File does not exist even after moving from[%s] to [%s]", intermediateHdfsPath, hdfsPath)); } } catch (IOException e) { // rename failed, possibly due to race condition. check if some other job has uploaded the jar file. try { if (!fs.exists(hdfsPath)) { log.error(e, "IOException while Renaming jar file"); exception = e; } } catch (IOException e1) { e.addSuppressed(e1); exception = e; } } finally { try { if (fs.exists(intermediateHdfsPath)) { fs.delete(intermediateHdfsPath, false); } } catch (IOException e) { if (exception == null) { exception = e; } else { exception.addSuppressed(e); } } if (exception != null) { throw exception; } } } job.addFileToClassPath(hdfsPath); }
From source file:io.druid.indexer.JobHelper.java
License:Apache License
/** * Rename the files. This works around some limitations of both FileContext (no s3n support) and NativeS3FileSystem.rename * which will not overwrite/* w w w. ja v a 2s . c om*/ * * @param outputFS The output fs * @param indexZipFilePath The original file path * @param finalIndexZipFilePath The to rename the original file to * * @return False if a rename failed, true otherwise (rename success or no rename needed) */ private static boolean renameIndexFiles(final FileSystem outputFS, final Path indexZipFilePath, final Path finalIndexZipFilePath) { try { return RetryUtils.retry(new Callable<Boolean>() { @Override public Boolean call() throws Exception { final boolean needRename; if (outputFS.exists(finalIndexZipFilePath)) { // NativeS3FileSystem.rename won't overwrite, so we might need to delete the old index first final FileStatus zipFile = outputFS.getFileStatus(indexZipFilePath); final FileStatus finalIndexZipFile = outputFS.getFileStatus(finalIndexZipFilePath); if (zipFile.getModificationTime() >= finalIndexZipFile.getModificationTime() || zipFile.getLen() != finalIndexZipFile.getLen()) { log.info("File[%s / %s / %sB] existed, but wasn't the same as [%s / %s / %sB]", finalIndexZipFile.getPath(), new DateTime(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen(), zipFile.getPath(), new DateTime(zipFile.getModificationTime()), zipFile.getLen()); outputFS.delete(finalIndexZipFilePath, false); needRename = true; } else { log.info("File[%s / %s / %sB] existed and will be kept", finalIndexZipFile.getPath(), new DateTime(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen()); needRename = false; } } else { needRename = true; } if (needRename) { log.info("Attempting rename from [%s] to [%s]", indexZipFilePath, finalIndexZipFilePath); return outputFS.rename(indexZipFilePath, finalIndexZipFilePath); } else { return true; } } }, FileUtils.IS_EXCEPTION, NUM_RETRIES); } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:io.hops.experiments.controller.MoveThenDelete.java
License:Apache License
private void start(String configFilePath, String hdfsFolder) { try {//from ww w. j a v a 2s.c o m args = new Configuration(configFilePath); org.apache.hadoop.conf.Configuration conf = createHdfsConf(); // do shit here FileSystem dfs = DFSOperationsUtils.getDFSClient(conf); Path from = new Path(hdfsFolder); Path to = new Path(hdfsFolder + "1"); Long startTime = System.currentTimeMillis(); dfs.rename(from, to); Long endTime = System.currentTimeMillis(); System.out.println("Move time taken in ms " + (endTime - startTime)); System.out.println("Move time taken in minutes " + (endTime - startTime) / (double) (1000 * 60)); startTime = System.currentTimeMillis(); dfs.delete(to, true); endTime = System.currentTimeMillis(); System.out.println("Delete time taken in ms " + (endTime - startTime)); System.out.println("Delete time taken in minutes " + (endTime - startTime) / (double) (1000 * 60)); } catch (Throwable e) { System.out.println(e); } finally { System.out.println("Exiting ... "); System.exit(0); } }