List of usage examples for org.apache.hadoop.fs FileUtil copy
private static boolean copy(FileSystem srcFS, FileStatus srcStatus, File dst, boolean deleteSource, Configuration conf) throws IOException
From source file:cascading.ClusterTestCase.java
License:Open Source License
protected void copyFromLocal(String inputFile) throws IOException { if (!enableCluster) return;// ww w . ja v a2 s . co m Path path = new Path(inputFile); if (!fileSys.exists(path)) FileUtil.copy(new File(inputFile), fileSys, path, false, jobConf); }
From source file:cascading.platform.hadoop.BaseHadoopPlatform.java
License:Open Source License
@Override public void copyFromLocal(String inputFile) throws IOException { if (!new File(inputFile).exists()) throw new FileNotFoundException("data file not found: " + inputFile); if (!isUseCluster()) return;/*from w ww. j a v a 2s . c om*/ Path path = new Path(safeFileName(inputFile)); if (!fileSys.exists(path)) FileUtil.copy(new File(inputFile), fileSys, path, false, configuration); }
From source file:cascading.platform.hadoop.BaseHadoopPlatform.java
License:Open Source License
@Override public void copyToLocal(String outputFile) throws IOException { if (!isUseCluster()) return;/*from w ww .j a va 2s. c o m*/ Path path = new Path(safeFileName(outputFile)); if (!fileSys.exists(path)) throw new FileNotFoundException("data file not found: " + outputFile); File file = new File(outputFile); if (file.exists()) file.delete(); if (fileSys.isFile(path)) { // its a file, so just copy it over FileUtil.copy(fileSys, path, file, false, configuration); return; } // it's a directory file.mkdirs(); FileStatus contents[] = fileSys.listStatus(path); for (FileStatus fileStatus : contents) { Path currentPath = fileStatus.getPath(); if (currentPath.getName().startsWith("_")) // filter out temp and log dirs continue; FileUtil.copy(fileSys, currentPath, new File(file, currentPath.getName()), false, configuration); } }
From source file:com.baifendian.swordfish.common.hadoop.HdfsClient.java
License:Apache License
/** * ? hdfs // www .j ava2 s . c o m * * @param srcPath * @param dstPath * @param deleteSource * @param overwrite * @return * @throws HdfsException */ public boolean copyHdfsToLocal(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws HdfsException { Path srcPathObj = new Path(srcPath); File dstFile = new File(dstPath); try { if (dstFile.exists()) { if (dstFile.isFile()) { if (overwrite) { dstFile.delete(); } } else { throw new HdfsException("Destination must not be a dir"); } } // ? dstFile.getParentFile().mkdirs(); // hdfs ? FileUtil.copy(fileSystem, srcPathObj, dstFile, deleteSource, fileSystem.getConf()); } catch (IOException e) { LOGGER.error("Copy exception", e); throw new HdfsException("Copy exception", e); } return true; }
From source file:com.cloudera.cdk.maven.plugins.DeployAppMojo.java
License:Apache License
public void execute() throws MojoExecutionException, MojoFailureException { try {/* w ww . j a v a 2 s.c o m*/ Configuration conf = new Configuration(); Path appPath = getAppPath(); getLog().info("Deploying " + localApplicationFile + " to " + appPath); FileSystem destFileSystem = FileSystem.get(new URI(deployFileSystem), conf); if (destFileSystem.exists(appPath)) { if (!updateApplication) { throw new MojoExecutionException("Application already exists at " + appPath + ". Use 'updateApplication' option to force deployment."); } boolean success = destFileSystem.delete(appPath, true); if (!success) { throw new MojoExecutionException("Error deleting existing application at " + appPath); } } boolean success = FileUtil.copy(localApplicationFile, destFileSystem, appPath, false, conf); if (!success) { throw new MojoExecutionException( "Error creating parent directories " + "for deploying Oozie application"); } } catch (URISyntaxException e) { throw new MojoExecutionException("Syntax error in 'deployFileSystem': " + deployFileSystem, e); } catch (IOException e) { throw new MojoExecutionException("Error deploying application", e); } }
From source file:com.datatorrent.stram.util.FSUtil.java
License:Apache License
/** * Download the file from dfs to local file. * * @param fs//from ww w .ja va2 s .com * @param destinationFile * @param dfsFile * @param conf * @return * @throws IOException */ public static File copyToLocalFileSystem(FileSystem fs, String destinationPath, String destinationFile, String dfsFile, Configuration conf) throws IOException { File destinationDir = new File(destinationPath); if (!destinationDir.exists() && !destinationDir.mkdirs()) { throw new RuntimeException("Unable to create local directory"); } RawLocalFileSystem localFileSystem = new RawLocalFileSystem(); try { // allow app user to access local dir FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE); localFileSystem.setPermission(new Path(destinationDir.getAbsolutePath()), permissions); Path dfsFilePath = new Path(dfsFile); File localFile = new File(destinationDir, destinationFile); FileUtil.copy(fs, dfsFilePath, localFile, false, conf); // set permissions on actual file to be read-only for user permissions = new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE); localFileSystem.setPermission(new Path(localFile.getAbsolutePath()), permissions); return localFile; } finally { localFileSystem.close(); } }
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
public static void copyNameDirs(Collection<URI> srcDirs, Collection<URI> dstDirs, Configuration dstConf) throws IOException { URI srcDir = Lists.newArrayList(srcDirs).get(0); FileSystem dstFS = FileSystem.getLocal(dstConf).getRaw(); for (URI dstDir : dstDirs) { Preconditions.checkArgument(!dstDir.equals(srcDir), "src and dst are the same: " + dstDir); File dstDirF = new File(dstDir); if (dstDirF.exists()) { if (!FileUtil.fullyDelete(dstDirF)) { throw new IOException("Unable to delete: " + dstDirF); }// w w w .j a v a 2 s . com } LOG.info("Copying namedir from primary node dir " + srcDir + " to " + dstDir); FileUtil.copy(new File(srcDir), dstFS, new Path(dstDir), false, dstConf); } }
From source file:com.moz.fiji.schema.testutil.IntegrationHelper.java
License:Apache License
/** * Copies a local file into the default filesystem (which is HDFS if you've started an * HBase cluster)./*from ww w . ja va 2 s .c o m*/ * * @param localFile The file to copy. * @param destPath A relative destination path to be used within the shared tmp dir. * @return The path of the file in HDFS. */ public Path copyToDfs(File localFile, String destPath) throws IOException { Path target = getDfsPath(destPath); FileSystem fs = FileSystem.get(getConf()); if (!fs.exists(target)) { // Only copy if it doesn't already exist. FileUtil.copy(localFile, fs, target, false, getConf()); } return target; }
From source file:com.splout.db.common.SploutHadoopConfiguration.java
License:Apache License
/** * Adds the SQLite native libraries to the DistributedCache so that they will be present in the java.library.path * of the child's Hadoop task./* w w w .ja v a2 s . c o m*/ * <p/> * Usually you don't need to do this as the task will already try to load them from the job's uncompressed JAR, however * it is not assured that all Hadoop versions do the uncompressing of the JAR so in this case it's safer to use this. */ public static void addSQLite4JavaNativeLibsToDC(Configuration conf, File nativeLibsLocalPath) throws IOException, URISyntaxException { Path nativeLibHdfs = new Path("splout-native"); FileSystem fS = FileSystem.get(conf); if (fS.exists(nativeLibHdfs)) { fS.delete(nativeLibHdfs, true); } fS.mkdirs(nativeLibHdfs); // Copy native libs to HDFS File[] natives = nativeLibsLocalPath.listFiles(); if (natives == null) { throw new RuntimeException( "natives lib folder not present in local working directory! Are you in SPLOUT_HOME?"); } for (File nativeLib : natives) { FileUtil.copy(nativeLib, fS, nativeLibHdfs, false, conf); } for (FileStatus nativeLibInHdfs : fS.listStatus(nativeLibHdfs)) { // http://hadoop.apache.org/docs/r0.20.2/native_libraries.html#Loading+native+libraries+through+DistributedCache DistributedCache.createSymlink(conf); URI uriToAdd = new URI( nativeLibInHdfs.getPath().makeQualified(fS) + "#" + nativeLibInHdfs.getPath().getName()); DistributedCache.addCacheFile(uriToAdd, conf); log.info("Adding to distributed cache: " + uriToAdd); } }
From source file:fr.ens.biologie.genomique.eoulsan.util.hadoop.PathUtils.java
License:LGPL
/** * Copy a file from a path to a local file * @param srcPath Path of the file to copy * @param destFile Destination file/* ww w. j a va 2 s . co m*/ * @param removeOriginalFile true if the original file must be deleted * @param conf Configuration object * @return true if the copy is successful * @throws IOException if an error occurs while copying file */ public static boolean copyFromPathToLocalFile(final Path srcPath, final File destFile, final boolean removeOriginalFile, final Configuration conf) throws IOException { if (srcPath == null) { throw new NullPointerException("The source path is null"); } if (destFile == null) { throw new NullPointerException("The destination file is null"); } if (conf == null) { throw new NullPointerException("The configuration object is null"); } final FileSystem fs = FileSystem.get(srcPath.toUri(), conf); return FileUtil.copy(fs, srcPath, destFile, removeOriginalFile, conf); }