List of usage examples for org.apache.hadoop.fs FileSystem rename
public abstract boolean rename(Path src, Path dst) throws IOException;
From source file:org.apache.hcatalog.mapreduce.FileOutputCommitterContainer.java
License:Apache License
/** * Move all of the files from the temp directory to the final location * @param fs the output file system// ww w . ja v a 2 s .co m * @param file the file to move * @param srcDir the source directory * @param destDir the target directory * @param dryRun - a flag that simply tests if this move would succeed or not based * on whether other files exist where we're trying to copy * @throws java.io.IOException */ private void moveTaskOutputs(FileSystem fs, Path file, Path srcDir, Path destDir, final boolean dryRun) throws IOException { if (file.getName().equals(TEMP_DIR_NAME) || file.getName().equals(LOGS_DIR_NAME) || file.getName().equals(SUCCEEDED_FILE_NAME)) { return; } final Path finalOutputPath = getFinalPath(file, srcDir, destDir); if (fs.isFile(file)) { if (dryRun) { if (LOG.isDebugEnabled()) { LOG.debug("Testing if moving file: [" + file + "] to [" + finalOutputPath + "] would cause a problem"); } if (fs.exists(finalOutputPath)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Data already exists in " + finalOutputPath + ", duplicate publish not possible."); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Moving file: [ " + file + "] to [" + finalOutputPath + "]"); } // Make sure the parent directory exists. It is not an error // to recreate an existing directory fs.mkdirs(finalOutputPath.getParent()); if (!fs.rename(file, finalOutputPath)) { if (!fs.delete(finalOutputPath, true)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Failed to delete existing path " + finalOutputPath); } if (!fs.rename(file, finalOutputPath)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Failed to move output to " + finalOutputPath); } } } } else if (fs.getFileStatus(file).isDir()) { FileStatus[] children = fs.listStatus(file); FileStatus firstChild = null; if (children != null) { int index = 0; while (index < children.length) { if (!children[index].getPath().getName().equals(TEMP_DIR_NAME) && !children[index].getPath().getName().equals(LOGS_DIR_NAME) && !children[index].getPath().getName().equals(SUCCEEDED_FILE_NAME)) { firstChild = children[index]; break; } index++; } } if (firstChild != null && firstChild.isDir()) { // If the first child is directory, then rest would be directory too according to HCatalog dir structure // recurse in that case for (FileStatus child : children) { moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun); } } else { if (!dryRun) { if (dynamicPartitioningUsed) { // Optimization: if the first child is file, we have reached the leaf directory, move the parent directory itself // instead of moving each file under the directory. See HCATALOG-538 final Path parentDir = finalOutputPath.getParent(); // Create the directory Path placeholder = new Path(parentDir, "_placeholder"); if (fs.mkdirs(parentDir)) { // It is weired but we need a placeholder, // otherwise rename cannot move file to the right place fs.create(placeholder).close(); } if (LOG.isDebugEnabled()) { LOG.debug("Moving directory: " + file + " to " + parentDir); } if (!fs.rename(file, parentDir)) { final String msg = "Failed to move file: " + file + " to " + parentDir; LOG.error(msg); throw new HCatException(ErrorType.ERROR_MOVE_FAILED, msg); } fs.delete(placeholder, false); } else { // In case of no partition we have to move each file for (FileStatus child : children) { moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun); } } } else { if (fs.exists(finalOutputPath)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Data already exists in " + finalOutputPath + ", duplicate publish not possible."); } } } } else { // Should never happen final String msg = "Unknown file type being asked to be moved, erroring out"; throw new HCatException(ErrorType.ERROR_MOVE_FAILED, msg); } }
From source file:org.apache.hive.hcatalog.mapreduce.FileOutputCommitterContainer.java
License:Apache License
/** * Move all of the files from the temp directory to the final location * @param fs the output file system//w ww . j a v a 2s .co m * @param file the file to move * @param srcDir the source directory * @param destDir the target directory * @param dryRun - a flag that simply tests if this move would succeed or not based * on whether other files exist where we're trying to copy * @throws java.io.IOException */ private void moveTaskOutputs(FileSystem fs, Path file, Path srcDir, Path destDir, final boolean dryRun, boolean immutable) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("moveTaskOutputs " + file + " from: " + srcDir + " to: " + destDir + " dry: " + dryRun + " immutable: " + immutable); } if (dynamicPartitioningUsed) { immutable = true; // Making sure we treat dynamic partitioning jobs as if they were immutable. } if (file.getName().equals(TEMP_DIR_NAME) || file.getName().equals(LOGS_DIR_NAME) || file.getName().equals(SUCCEEDED_FILE_NAME)) { return; } final Path finalOutputPath = getFinalPath(fs, file, srcDir, destDir, immutable); FileStatus fileStatus = fs.getFileStatus(file); if (!fileStatus.isDir()) { if (dryRun) { if (immutable) { // Dryrun checks are meaningless for mutable table - we should always succeed // unless there is a runtime IOException. if (LOG.isDebugEnabled()) { LOG.debug("Testing if moving file: [" + file + "] to [" + finalOutputPath + "] would cause a problem"); } if (fs.exists(finalOutputPath)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Data already exists in " + finalOutputPath + ", duplicate publish not possible."); } } } else { if (LOG.isDebugEnabled()) { LOG.debug("Moving file: [ " + file + "] to [" + finalOutputPath + "]"); } // Make sure the parent directory exists. It is not an error // to recreate an existing directory fs.mkdirs(finalOutputPath.getParent()); if (!fs.rename(file, finalOutputPath)) { if (!fs.delete(finalOutputPath, true)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Failed to delete existing path " + finalOutputPath); } if (!fs.rename(file, finalOutputPath)) { throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Failed to move output to " + finalOutputPath); } } } } else { FileStatus[] children = fs.listStatus(file); FileStatus firstChild = null; if (children != null) { int index = 0; while (index < children.length) { if (!children[index].getPath().getName().equals(TEMP_DIR_NAME) && !children[index].getPath().getName().equals(LOGS_DIR_NAME) && !children[index].getPath().getName().equals(SUCCEEDED_FILE_NAME)) { firstChild = children[index]; break; } index++; } } if (firstChild != null && firstChild.isDir()) { // If the first child is directory, then rest would be directory too according to HCatalog dir structure // recurse in that case for (FileStatus child : children) { moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); } } else { if (!dryRun) { if (dynamicPartitioningUsed) { // Optimization: if the first child is file, we have reached the leaf directory, move the parent directory itself // instead of moving each file under the directory. See HCATALOG-538 // Note for future Append implementation : This optimization is another reason dynamic // partitioning is currently incompatible with append on mutable tables. final Path parentDir = finalOutputPath.getParent(); // Create the directory Path placeholder = new Path(parentDir, "_placeholder"); if (fs.mkdirs(parentDir)) { // It is weired but we need a placeholder, // otherwise rename cannot move file to the right place fs.create(placeholder).close(); } if (LOG.isDebugEnabled()) { LOG.debug("Moving directory: " + file + " to " + parentDir); } // If custom dynamic location provided, need to rename to final output path Path dstPath = !customDynamicLocationUsed ? parentDir : finalOutputPath; if (!fs.rename(file, dstPath)) { final String msg = "Failed to move file: " + file + " to " + dstPath; LOG.error(msg); throw new HCatException(ErrorType.ERROR_MOVE_FAILED, msg); } fs.delete(placeholder, false); } else { // In case of no partition we have to move each file for (FileStatus child : children) { moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); } } } else { if (immutable && fs.exists(finalOutputPath) && !MetaStoreUtils.isDirEmpty(fs, finalOutputPath)) { throw new HCatException(ErrorType.ERROR_DUPLICATE_PARTITION, "Data already exists in " + finalOutputPath + ", duplicate publish not possible."); } } } } }
From source file:org.apache.hive.hcatalog.streaming.TestStreaming.java
License:Apache License
private void corruptDataFile(final String file, final Configuration conf, final int addRemoveBytes) throws Exception { Path bPath = new Path(file); Path cPath = new Path(bPath.getParent(), bPath.getName() + ".corrupt"); FileSystem fs = bPath.getFileSystem(conf); FileStatus fileStatus = fs.getFileStatus(bPath); int len = addRemoveBytes == Integer.MIN_VALUE ? 0 : (int) fileStatus.getLen() + addRemoveBytes; byte[] buffer = new byte[len]; FSDataInputStream fdis = fs.open(bPath); fdis.readFully(0, buffer, 0, (int) Math.min(fileStatus.getLen(), buffer.length)); fdis.close();/*from w w w . j ava2s . c o m*/ FSDataOutputStream fdos = fs.create(cPath, true); fdos.write(buffer, 0, buffer.length); fdos.close(); fs.delete(bPath, false); fs.rename(cPath, bPath); }
From source file:org.apache.hive.hcatalog.streaming.TestStreaming.java
License:Apache License
private void corruptSideFile(final String file, final HiveConf conf, final Map<String, List<Long>> offsetMap, final String key, final int numEntries) throws IOException { Path dataPath = new Path(file); Path sideFilePath = OrcAcidUtils.getSideFile(dataPath); Path cPath = new Path(sideFilePath.getParent(), sideFilePath.getName() + ".corrupt"); FileSystem fs = sideFilePath.getFileSystem(conf); List<Long> offsets = offsetMap.get(key); long lastOffset = offsets.get(offsets.size() - 1); FSDataOutputStream fdos = fs.create(cPath, true); // corrupt last entry if (numEntries < 0) { byte[] lastOffsetBytes = longToBytes(lastOffset); for (int i = 0; i < offsets.size() - 1; i++) { fdos.writeLong(offsets.get(i)); }/*w ww.ja va 2s .com*/ fdos.write(lastOffsetBytes, 0, 3); } else if (numEntries > 0) { int firstRun = Math.min(offsets.size(), numEntries); // add original entries for (int i = 0; i < firstRun; i++) { fdos.writeLong(offsets.get(i)); } // add fake entries int remaining = numEntries - firstRun; for (int i = 0; i < remaining; i++) { fdos.writeLong(lastOffset + ((i + 1) * 100)); } } fdos.close(); fs.delete(sideFilePath, false); fs.rename(cPath, sideFilePath); }
From source file:org.apache.impala.common.FileSystemUtil.java
License:Apache License
/** * Relocates the given file to a new location (either another directory or a * file in the same or different filesystem). The file is generally moved (renamed) to * the new location. However, the file is copied if the source and destination are in * different encryption zones so that the file can be decrypted and/or encrypted, or if * the source and destination are in different filesystems. If renameIfAlreadyExists is * true, no error will be thrown if a file with the same name already exists in the * destination location. Instead, a UUID will be appended to the base file name, * preserving the existing file extension. If renameIfAlreadyExists is false, an * IOException will be thrown if there is a file name conflict. */// www . j av a2 s.c o m public static void relocateFile(Path sourceFile, Path dest, boolean renameIfAlreadyExists) throws IOException { FileSystem destFs = dest.getFileSystem(CONF); FileSystem sourceFs = sourceFile.getFileSystem(CONF); Path destFile = destFs.isDirectory(dest) ? new Path(dest, sourceFile.getName()) : dest; // If a file with the same name does not already exist in the destination location // then use the same file name. Otherwise, generate a unique file name. if (renameIfAlreadyExists && destFs.exists(destFile)) { Path destDir = destFs.isDirectory(dest) ? dest : dest.getParent(); destFile = new Path(destDir, appendToBaseFileName(destFile.getName(), UUID.randomUUID().toString())); } boolean sameFileSystem = isPathOnFileSystem(sourceFile, destFs); boolean destIsDfs = isDistributedFileSystem(destFs); // If the source and the destination are on different file systems, or in different // encryption zones, files can't be moved from one location to the other and must be // copied instead. boolean sameEncryptionZone = arePathsInSameHdfsEncryptionZone(destFs, sourceFile, destFile); // We can do a rename if the src and dst are in the same encryption zone in the same // distributed filesystem. boolean doRename = destIsDfs && sameFileSystem && sameEncryptionZone; // Alternatively, we can do a rename if the src and dst are on the same // non-distributed filesystem. if (!doRename) doRename = !destIsDfs && sameFileSystem; if (doRename) { if (LOG.isTraceEnabled()) { LOG.trace(String.format("Moving '%s' to '%s'", sourceFile.toString(), destFile.toString())); } // Move (rename) the file. destFs.rename(sourceFile, destFile); return; } if (destIsDfs && sameFileSystem) { Preconditions.checkState(!doRename); // We must copy rather than move if the source and dest are in different // encryption zones. A move would return an error from the NN because a move is a // metadata-only operation and the files would not be encrypted/decrypted properly // on the DNs. if (LOG.isTraceEnabled()) { LOG.trace(String.format("Copying source '%s' to '%s' because HDFS encryption zones are different.", sourceFile, destFile)); } } else { Preconditions.checkState(!sameFileSystem); if (LOG.isTraceEnabled()) { LOG.trace(String.format("Copying '%s' to '%s' between filesystems.", sourceFile, destFile)); } } FileUtil.copy(sourceFs, sourceFile, destFs, destFile, true, true, CONF); }
From source file:org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter.java
License:Apache License
@Override public void start() { final Path tmpPath = new Path(pathString + ".tmp"); URI tmpPathURI = tmpPath.toUri(); final FileSystem fs; try {//from w w w . j av a 2 s.co m if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) { //default local fs = FileSystem.getLocal(new Configuration()); } else { fs = FileSystem.get(tmpPathURI, new Configuration()); } } catch (IOException e) { LOGGER.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e); return; } Runnable task = new Runnable() { public void run() { try { String json = null; try { json = jsonWriter.writeValueAsString(metricRegistry); } catch (JsonProcessingException e) { LOGGER.error("Unable to convert json to string ", e); return; } BufferedWriter bw = null; try { fs.delete(tmpPath, true); bw = new BufferedWriter( new OutputStreamWriter(fs.create(tmpPath, true), StandardCharsets.UTF_8)); bw.write(json); fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644)); } catch (IOException e) { LOGGER.error("Unable to write to temp file " + tmpPath, e); return; } finally { if (bw != null) { bw.close(); } } try { fs.rename(tmpPath, path); fs.setPermission(path, FsPermission.createImmutable((short) 0644)); } catch (IOException e) { LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e); return; } } catch (Throwable t) { // catch all errors (throwable and execptions to prevent subsequent tasks from being suppressed) LOGGER.error("Error executing scheduled task ", t); } } }; executorService.scheduleWithFixedDelay(task, 0, frequency, TimeUnit.MILLISECONDS); }
From source file:org.apache.kylin.dict.AppendTrieDictionaryTest.java
License:Apache License
private void convertDirToOldFormat(String baseDir) throws IOException { Path basePath = new Path(baseDir); FileSystem fs = HadoopUtil.getFileSystem(basePath); // move version dir to base dir, to simulate the older format GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir); Long[] versions = store.listAllVersions(); Path versionPath = store.getVersionDir(versions[versions.length - 1]); Path tmpVersionPath = new Path(versionPath.getParent().getParent(), versionPath.getName()); fs.rename(versionPath, tmpVersionPath); fs.delete(new Path(baseDir), true); fs.rename(tmpVersionPath, new Path(baseDir)); }
From source file:org.apache.kylin.dict.AppendTrieDictionaryTest.java
License:Apache License
private void convertIndexToOldFormat(String baseDir) throws IOException { Path basePath = new Path(baseDir); FileSystem fs = HadoopUtil.getFileSystem(basePath); GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir); Long[] versions = store.listAllVersions(); GlobalDictMetadata metadata = store.getMetadata(versions[versions.length - 1]); //convert v2 index to v1 index Path versionPath = store.getVersionDir(versions[versions.length - 1]); Path v2IndexFile = new Path(versionPath, V2_INDEX_NAME); fs.delete(v2IndexFile, true);/* ww w . j av a 2 s .c o m*/ GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs, HadoopUtil.getCurrentConfiguration()); indexFormatV1.writeIndexFile(versionPath, metadata); //convert v2 fileName format to v1 fileName format for (Map.Entry<AppendDictSliceKey, String> entry : metadata.sliceFileMap.entrySet()) { fs.rename(new Path(versionPath, entry.getValue()), new Path(versionPath, "cached_" + entry.getKey())); } }
From source file:org.apache.lens.lib.query.HadoopFileFormatter.java
License:Apache License
@Override public void commit() throws IOException { rowWriter.close(Reporter.NULL);/*from ww w. j a v a 2 s . co m*/ if (outputPath != null && rowWriter.getTmpPath() != null) { FileSystem fs = outputPath.getFileSystem(ctx.getConf()); finalPath = outputPath; if (rowWriter.getExtn() != null) { finalPath = new Path(outputPath + rowWriter.getExtn()); } finalPath = finalPath.makeQualified(fs); fs.rename(rowWriter.getTmpPath(), finalPath); ctx.setResultSetPath(finalPath.toString()); fileSize = fs.getFileStatus(finalPath).getLen(); } }
From source file:org.apache.mahout.clustering.topdown.postprocessor.ClusterOutputPostProcessorDriver.java
License:Apache License
/** * Using @FileSystem rename method to move the file. */// w w w . j av a 2 s . c om private static void renameFile(Writable key, FileStatus fileStatus, Configuration conf) throws IOException { Path path = fileStatus.getPath(); FileSystem fileSystem = path.getFileSystem(conf); Path subDir = new Path(key.toString()); Path renameTo = new Path(path.getParent(), subDir); fileSystem.mkdirs(renameTo); fileSystem.rename(path, renameTo); }