List of usage examples for org.apache.hadoop.fs FileUtil copy
public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst, boolean deleteSource, boolean overwrite, Configuration conf) throws IOException
From source file:org.apache.crunch.io.impl.FileTargetImpl.java
License:Apache License
@Override public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException { FileSystem srcFs = workingPath.getFileSystem(conf); Path src = getSourcePattern(workingPath, index); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(src), src); FileSystem dstFs = path.getFileSystem(conf); if (!dstFs.exists(path)) { dstFs.mkdirs(path);//ww w . j ava2 s . c o m } boolean sameFs = isCompatible(srcFs, path); for (Path s : srcs) { Path d = getDestFile(conf, s, path, s.getName().contains("-m-")); if (sameFs) { srcFs.rename(s, d); } else { FileUtil.copy(srcFs, s, dstFs, d, true, true, conf); } } dstFs.create(getSuccessIndicator(), true).close(); }
From source file:org.apache.drill.test.framework.TestDriver.java
License:Apache License
public void generateReports(List<DrillTest> tests, int iteration) { try {/* ww w. j a va 2 s . c o m*/ if (drillReportsDir == null) { drillReportsDir = CWD; } File drillReportDir = new File(drillReportsDir); FileSystem localFS = FileSystem.getLocal(conf); FileSystem DFS = FileSystem.get(conf); if (!drillReportDir.exists()) { if (!drillReportDir.mkdir()) { LOG.debug("Cannot create directory " + drillReportsDir + ". Using current working directory for drill output"); drillReportsDir = CWD; } } File reportFile = new File(drillReportsDir + "/apache-drill-" + version + "_" + commitId + "_" + "report_" + new Date().toString().replace(' ', '_').replace(':', '_') + ".json"); BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(reportFile)); Document document; for (DrillTest test : tests) { document = Json.newDocument(); document.set("_id", test.getTestId() + "_" + new File(test.getInputFile()).getName() + "_" + test.getCloneId() + "_" + iteration); document.set("queryFilepath", test.getInputFile().substring(test.getInputFile().indexOf("resources/") + 10)); String query = test.getQuery(); if (query != null) { query.replaceAll("\n", ""); } document.set("query", query); document.set("status", test.getTestStatus().toString()); if (test.getTestStatus().equals(TestStatus.EXECUTION_FAILURE) || test.getTestStatus().equals(TestStatus.VERIFICATION_FAILURE)) { document.set("errorMessage", test.getException().toString().replaceAll("\n", "")); } else { document.set("errorMessage", "N/A"); } document.set("queryExecutionTime", test.getDuration().toString()); document.set("drillVersion", version); document.set("commitId", commitId); bufferedWriter.write(document.toString()); bufferedWriter.newLine(); } bufferedWriter.flush(); bufferedWriter.close(); // Upload report to DFS if the drillReportsDFSDir variable is set if (drillReportsDFSDir != null) { FileUtil.copy(localFS, new Path(reportFile.getAbsolutePath()), DFS, new Path(drillReportsDFSDir + "/" + reportFile.getName()), true, false, DFS.getConf()); } } catch (Exception e) { e.printStackTrace(); } }
From source file:org.apache.falcon.regression.hive.dr.HiveObjectCreator.java
License:Apache License
static void bootstrapCopy(Connection srcConnection, FileSystem srcFs, String srcTable, Connection dstConnection, FileSystem dstFs, String dstTable) throws Exception { LOGGER.info("Starting bootstrap..."); final String dumpPath = HDFS_TMP_DIR + srcTable + "/"; HadoopUtil.recreateDir(srcFs, dumpPath); runSqlQuietly(srcConnection, "dfs -chmod -R 777 " + dumpPath); HadoopUtil.deleteDirIfExists(dumpPath, dstFs); runSql(srcConnection, "export table " + srcTable + " to '" + dumpPath + "' FOR REPLICATION('ignore')"); FileUtil.copy(srcFs, new Path(dumpPath), dstFs, new Path(dumpPath), false, true, new Configuration()); runSqlQuietly(dstConnection, "dfs -chmod -R 777 " + dumpPath); runSql(dstConnection, "import table " + dstTable + " from '" + dumpPath + "'"); HadoopUtil.deleteDirIfExists(dumpPath, srcFs); HadoopUtil.deleteDirIfExists(dumpPath, dstFs); LOGGER.info("Finished bootstrap"); }
From source file:org.apache.falcon.service.SharedLibraryHostingService.java
License:Apache License
private void pushExtensionArtifactsToCluster(final Cluster cluster, final FileSystem clusterFs) throws FalconException { if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) { LOG.info("ExtensionService not registered, return"); return;// www .j a va2s. co m } ExtensionStore store = ExtensionStore.get(); if (!store.isExtensionStoreInitialized()) { LOG.info( "Extension store not initialized by Extension service. Make sure Extension service is added in " + "start up properties"); return; } final String filterPath = "/apps/falcon/extensions/mirroring/"; Path extensionStorePath = store.getExtensionStorePath(); LOG.info("extensionStorePath :{}", extensionStorePath); FileSystem falconFileSystem = HadoopClientFactory.get().createFalconFileSystem(extensionStorePath.toUri()); String nameNode = StringUtils .removeEnd(falconFileSystem.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY), File.separator); String clusterStorageUrl = StringUtils.removeEnd(ClusterHelper.getStorageUrl(cluster), File.separator); // If default fs for Falcon server is same as cluster fs abort copy if (nameNode.equalsIgnoreCase(clusterStorageUrl)) { LOG.info("clusterStorageUrl :{} same return", clusterStorageUrl); return; } try { RemoteIterator<LocatedFileStatus> fileStatusListIterator = falconFileSystem .listFiles(extensionStorePath, true); while (fileStatusListIterator.hasNext()) { LocatedFileStatus srcfileStatus = fileStatusListIterator.next(); Path filePath = Path.getPathWithoutSchemeAndAuthority(srcfileStatus.getPath()); if (filePath != null && filePath.toString().startsWith(filterPath)) { /* HiveDR uses filter path as store path in DRStatusStore, so skip it. Copy only the extension artifacts */ continue; } if (srcfileStatus.isDirectory()) { if (!clusterFs.exists(filePath)) { HadoopClientFactory.mkdirs(clusterFs, filePath, srcfileStatus.getPermission()); } } else { if (clusterFs.exists(filePath)) { FileStatus targetfstat = clusterFs.getFileStatus(filePath); if (targetfstat.getLen() == srcfileStatus.getLen()) { continue; } } Path parentPath = filePath.getParent(); if (!clusterFs.exists(parentPath)) { FsPermission dirPerm = falconFileSystem.getFileStatus(parentPath).getPermission(); HadoopClientFactory.mkdirs(clusterFs, parentPath, dirPerm); } FileUtil.copy(falconFileSystem, srcfileStatus, clusterFs, filePath, false, true, falconFileSystem.getConf()); FileUtil.chmod(clusterFs.makeQualified(filePath).toString(), srcfileStatus.getPermission().toString()); } } } catch (IOException | InterruptedException e) { throw new FalconException("Failed to copy extension artifacts to cluster" + cluster.getName(), e); } }
From source file:org.apache.hoya.tools.HoyaUtils.java
License:Apache License
/** * Copy a directory to a new FS -both paths must be qualified. If * a directory needs to be created, supplied permissions can override * the default values. Existing directories are not touched * @param conf conf file/*from ww w.j av a2s . com*/ * @param srcDirPath src dir * @param destDirPath dest dir * @param permission permission for the dest directory; null means "default" * @return # of files copies */ public static int copyDirectory(Configuration conf, Path srcDirPath, Path destDirPath, FsPermission permission) throws IOException, BadClusterStateException { FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf); FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf); //list all paths in the src. if (!srcFS.exists(srcDirPath)) { throw new FileNotFoundException("Source dir not found " + srcDirPath); } if (!srcFS.isDirectory(srcDirPath)) { throw new FileNotFoundException("Source dir not a directory " + srcDirPath); } FileStatus[] entries = srcFS.listStatus(srcDirPath); int srcFileCount = entries.length; if (srcFileCount == 0) { return 0; } if (permission == null) { permission = FsPermission.getDirDefault(); } if (!destFS.exists(destDirPath)) { new HoyaFileSystem(destFS, conf).createWithPermissions(destDirPath, permission); } Path[] sourcePaths = new Path[srcFileCount]; for (int i = 0; i < srcFileCount; i++) { FileStatus e = entries[i]; Path srcFile = e.getPath(); if (srcFS.isDirectory(srcFile)) { throw new IOException("Configuration dir " + srcDirPath + " contains a directory " + srcFile); } log.debug("copying src conf file {}", srcFile); sourcePaths[i] = srcFile; } log.debug("Copying {} files from {} to dest {}", srcFileCount, srcDirPath, destDirPath); FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf); return srcFileCount; }
From source file:org.apache.ignite.igfs.IgfsHadoopFileSystemAbstractSelfTest.java
License:Apache License
/** * Copy files from one FS to another./*from w w w . ja v a 2s .co m*/ * * @param msg Info message to display after copying finishes. * @param srcFs Source file system. * @param src Source path to copy from. * @param destFs Destination file system. * @param dest Destination path to copy to. * @throws IOException If failed. */ private void copy(String msg, FileSystem srcFs, Path src, FileSystem destFs, Path dest) throws IOException { assert destFs.delete(dest, true) || !destFs.exists(dest) : "Failed to remove: " + dest; destFs.mkdirs(dest); Configuration conf = new Configuration(true); long time = System.currentTimeMillis(); FileUtil.copy(srcFs, src, destFs, dest, false, true, conf); time = System.currentTimeMillis() - time; info("Copying finished, " + msg + " [time=" + time + "ms, src=" + src + ", dest=" + dest + ']'); }
From source file:org.apache.impala.common.FileSystemUtil.java
License:Apache License
/** * Relocates the given file to a new location (either another directory or a * file in the same or different filesystem). The file is generally moved (renamed) to * the new location. However, the file is copied if the source and destination are in * different encryption zones so that the file can be decrypted and/or encrypted, or if * the source and destination are in different filesystems. If renameIfAlreadyExists is * true, no error will be thrown if a file with the same name already exists in the * destination location. Instead, a UUID will be appended to the base file name, * preserving the existing file extension. If renameIfAlreadyExists is false, an * IOException will be thrown if there is a file name conflict. *///w w w. java 2 s. co m public static void relocateFile(Path sourceFile, Path dest, boolean renameIfAlreadyExists) throws IOException { FileSystem destFs = dest.getFileSystem(CONF); FileSystem sourceFs = sourceFile.getFileSystem(CONF); Path destFile = destFs.isDirectory(dest) ? new Path(dest, sourceFile.getName()) : dest; // If a file with the same name does not already exist in the destination location // then use the same file name. Otherwise, generate a unique file name. if (renameIfAlreadyExists && destFs.exists(destFile)) { Path destDir = destFs.isDirectory(dest) ? dest : dest.getParent(); destFile = new Path(destDir, appendToBaseFileName(destFile.getName(), UUID.randomUUID().toString())); } boolean sameFileSystem = isPathOnFileSystem(sourceFile, destFs); boolean destIsDfs = isDistributedFileSystem(destFs); // If the source and the destination are on different file systems, or in different // encryption zones, files can't be moved from one location to the other and must be // copied instead. boolean sameEncryptionZone = arePathsInSameHdfsEncryptionZone(destFs, sourceFile, destFile); // We can do a rename if the src and dst are in the same encryption zone in the same // distributed filesystem. boolean doRename = destIsDfs && sameFileSystem && sameEncryptionZone; // Alternatively, we can do a rename if the src and dst are on the same // non-distributed filesystem. if (!doRename) doRename = !destIsDfs && sameFileSystem; if (doRename) { if (LOG.isTraceEnabled()) { LOG.trace(String.format("Moving '%s' to '%s'", sourceFile.toString(), destFile.toString())); } // Move (rename) the file. destFs.rename(sourceFile, destFile); return; } if (destIsDfs && sameFileSystem) { Preconditions.checkState(!doRename); // We must copy rather than move if the source and dest are in different // encryption zones. A move would return an error from the NN because a move is a // metadata-only operation and the files would not be encrypted/decrypted properly // on the DNs. if (LOG.isTraceEnabled()) { LOG.trace(String.format("Copying source '%s' to '%s' because HDFS encryption zones are different.", sourceFile, destFile)); } } else { Preconditions.checkState(!sameFileSystem); if (LOG.isTraceEnabled()) { LOG.trace(String.format("Copying '%s' to '%s' between filesystems.", sourceFile, destFile)); } } FileUtil.copy(sourceFs, sourceFile, destFs, destFile, true, true, CONF); }
From source file:org.apache.kylin.common.persistence.JDBCResourceDAO.java
License:Apache License
public void rollbackLargeCellFromHdfs(String resPath) throws SQLException { Path redirectPath = bigCellHDFSPath(resPath); Path oldPath = new Path(redirectPath.toString() + "_old"); try {/*from w w w. j a va 2s . c o m*/ if (redirectFileSystem.exists(oldPath)) { FileUtil.copy(redirectFileSystem, oldPath, redirectFileSystem, redirectPath, true, true, HadoopUtil.getCurrentConfiguration()); logger.info("roll back hdfs file {}", resPath); } else { redirectFileSystem.delete(redirectPath, true); logger.warn("no backup for hdfs file {} is found, clean it", resPath); } } catch (Exception e) { try { //last try to delete redirectPath, because we prefer a deleted rather than incomplete redirectFileSystem.delete(redirectPath, true); } catch (Exception ex) { logger.error("fail to delete resource " + redirectPath + " in hdfs", ex); } throw new SQLException(e); } }
From source file:org.apache.kylin.dict.CachedTreeMap.java
License:Apache License
private CachedTreeMap(int maxCount, Class<K> keyClazz, Class<V> valueClazz, String baseDir, boolean persistent, boolean immutable) throws IOException { super();//from ww w . j ava 2 s . c o m this.keyClazz = keyClazz; this.valueClazz = valueClazz; this.fileList = new TreeSet<>(); this.conf = new Configuration(); if (baseDir.endsWith("/")) { this.baseDir = baseDir.substring(0, baseDir.length() - 1); } else { this.baseDir = baseDir; } this.tmpDir = this.baseDir + ".tmp"; this.fs = FileSystem.get(new Path(baseDir).toUri(), conf); this.persistent = persistent; this.immutable = immutable; CacheBuilder builder = CacheBuilder.newBuilder().removalListener(new RemovalListener<K, V>() { @Override public void onRemoval(RemovalNotification<K, V> notification) { logger.info(String.format("Evict cache key %s(%d) with value %s caused by %s, size %d/%d ", notification.getKey(), notification.getKey().hashCode(), notification.getValue(), notification.getCause(), size(), valueCache.size())); switch (notification.getCause()) { case SIZE: writeValue(notification.getKey(), notification.getValue()); break; case EXPLICIT: deleteValue(notification.getKey()); break; default: throw new RuntimeException("unexpected evict reason " + notification.getCause()); } } }); // For immutable values, load all values as much as possible, and evict by soft reference to free memory when gc if (this.immutable) { builder.softValues(); } else { builder.maximumSize(maxCount); // For mutable map, copy all data into tmp and modify on tmp data, avoiding suddenly server crash made data corrupt if (fs.exists(new Path(tmpDir))) { fs.delete(new Path(tmpDir), true); } if (fs.exists(new Path(this.baseDir))) { FileUtil.copy(fs, new Path(this.baseDir), fs, new Path(tmpDir), false, true, conf); } else { fs.mkdirs(new Path(this.baseDir)); } } this.valueCache = builder.build(new CacheLoader<K, V>() { @Override public V load(K key) throws Exception { V value = readValue(key); logger.info(String.format("Load cache by key %s(%d) with value %s", key, key.hashCode(), value)); return value; } }); }
From source file:org.apache.kylin.dict.CachedTreeMap.java
License:Apache License
public void commit(boolean stillMutable) throws IOException { assert !immutable : "Only support commit method with immutable false"; Path basePath = new Path(baseDir); Path backupPath = new Path(baseDir + ".bak"); Path tmpPath = new Path(tmpDir); try {/* w ww . jav a 2 s . c o m*/ fs.rename(basePath, backupPath); } catch (IOException e) { logger.info("CachedTreeMap commit backup basedir failed, " + e, e); throw e; } try { if (stillMutable) { FileUtil.copy(fs, tmpPath, fs, basePath, false, true, conf); } else { fs.rename(tmpPath, basePath); } fs.delete(backupPath, true); } catch (IOException e) { fs.rename(backupPath, basePath); logger.info("CachedTreeMap commit move/copy tmpdir failed, " + e, e); throw e; } }