List of usage examples for org.apache.hadoop.fs FileUtil copy
private static boolean copy(FileSystem srcFS, FileStatus srcStatus, File dst, boolean deleteSource, Configuration conf) throws IOException
From source file:fr.ens.biologie.genomique.eoulsan.util.hadoop.PathUtils.java
License:LGPL
/** * Copy a local file to a path/* w w w . jav a2 s .c o m*/ * @param srcFile source file * @param destPath destination path * @param removeSrcFile true if the source file must be removed * @param conf Configuration object * @return true if the copy is successful * @throws IOException if an error occurs while copying file */ public static boolean copyLocalFileToPath(final File srcFile, final Path destPath, final boolean removeSrcFile, final Configuration conf) throws IOException { if (srcFile == null) { throw new NullPointerException("The source file is null"); } if (destPath == null) { throw new NullPointerException("The destination path is null"); } if (conf == null) { throw new NullPointerException("The configuration object is null"); } return FileUtil.copy(srcFile, FileSystem.get(destPath.toUri(), conf), destPath, removeSrcFile, conf); }
From source file:io.druid.storage.hdfs.tasklog.HdfsTaskLogs.java
License:Apache License
@Override public void pushTaskLog(String taskId, File logFile) throws IOException { final Path path = getTaskLogFileFromId(taskId); log.info("Writing task log to: %s", path); final FileSystem fs = path.getFileSystem(hadoopConfig); FileUtil.copy(logFile, fs, path, false, hadoopConfig); log.info("Wrote task log to: %s", path); }
From source file:org.apache.hdt.core.cluster.HadoopJob.java
License:Apache License
/** * Try to locate and load the JobConf file for this job so to get more * details on the job (number of maps and of reduces) *///from w ww.jav a2 s . com private void loadJobFile() { try { String jobFile = getJobFile(); FileSystem fs = location.getDFS(); File tmp = File.createTempFile(getJobID().toString(), ".xml"); if (FileUtil.copy(fs, new Path(jobFile), tmp, false, location.getConfiguration())) { this.jobConf = new JobConf(tmp.toString()); this.totalMaps = jobConf.getNumMapTasks(); this.totalReduces = jobConf.getNumReduceTasks(); } } catch (IOException ioe) { ioe.printStackTrace(); } }
From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2JobResourceManager.java
License:Apache License
/** * Prepare job resources. Resolve the classpath list and download it if needed. * * @param download {@code true} If need to download resources. * @param jobLocDir Work directory for the job. * @throws IgniteCheckedException If failed. *//*from w ww .ja v a 2 s.co m*/ public void prepareJobEnvironment(boolean download, File jobLocDir) throws IgniteCheckedException { try { if (jobLocDir.exists()) throw new IgniteCheckedException( "Local job directory already exists: " + jobLocDir.getAbsolutePath()); JobConf cfg = ctx.getJobConf(); Collection<URL> clsPathUrls = new ArrayList<>(); String mrDir = cfg.get(MRJobConfig.MAPREDUCE_JOB_DIR); if (mrDir != null) { stagingDir = new Path(new URI(mrDir)); if (download) { FileSystem fs = job.fileSystem(stagingDir.toUri(), cfg); if (!fs.exists(stagingDir)) throw new IgniteCheckedException("Failed to find map-reduce submission " + "directory (does not exist): " + stagingDir); if (!FileUtil.copy(fs, stagingDir, jobLocDir, false, cfg)) throw new IgniteCheckedException("Failed to copy job submission directory " + "contents to local file system " + "[path=" + stagingDir + ", locDir=" + jobLocDir.getAbsolutePath() + ", jobId=" + jobId + ']'); } File jarJobFile = new File(jobLocDir, "job.jar"); clsPathUrls.add(jarJobFile.toURI().toURL()); rsrcSet.add(jarJobFile); rsrcSet.add(new File(jobLocDir, "job.xml")); } else if (!jobLocDir.mkdirs()) throw new IgniteCheckedException( "Failed to create local job directory: " + jobLocDir.getAbsolutePath()); processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES); processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES); processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null); processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null); if (!clsPathUrls.isEmpty()) clsPath = clsPathUrls.toArray(new URL[clsPathUrls.size()]); setLocalFSWorkingDirectory(jobLocDir); } catch (URISyntaxException | IOException e) { throw new IgniteCheckedException(e); } }
From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2JobResourceManager.java
License:Apache License
/** * Prepare job resources. Resolve the classpath list and download it if needed. * * @param download {@code true} If need to download resources. * @param jobLocDir Work directory for the job. * @throws IgniteCheckedException If failed. *//*from www . j a va 2 s . c o m*/ public void prepareJobEnvironment(boolean download, File jobLocDir) throws IgniteCheckedException { try { if (jobLocDir.exists()) throw new IgniteCheckedException( "Local job directory already exists: " + jobLocDir.getAbsolutePath()); JobConf cfg = ctx.getJobConf(); String mrDir = cfg.get("mapreduce.job.dir"); if (mrDir != null) { stagingDir = new Path(new URI(mrDir)); if (download) { FileSystem fs = FileSystem.get(stagingDir.toUri(), cfg); if (!fs.exists(stagingDir)) throw new IgniteCheckedException( "Failed to find map-reduce submission directory (does not exist): " + stagingDir); if (!FileUtil.copy(fs, stagingDir, jobLocDir, false, cfg)) throw new IgniteCheckedException( "Failed to copy job submission directory contents to local file system " + "[path=" + stagingDir + ", locDir=" + jobLocDir.getAbsolutePath() + ", jobId=" + jobId + ']'); } File jarJobFile = new File(jobLocDir, "job.jar"); Collection<URL> clsPathUrls = new ArrayList<>(); clsPathUrls.add(jarJobFile.toURI().toURL()); rsrcSet.add(jarJobFile); rsrcSet.add(new File(jobLocDir, "job.xml")); processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES); processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES); processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null); processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null); if (!clsPathUrls.isEmpty()) { clsPath = new URL[clsPathUrls.size()]; clsPathUrls.toArray(clsPath); } } else if (!jobLocDir.mkdirs()) throw new IgniteCheckedException( "Failed to create local job directory: " + jobLocDir.getAbsolutePath()); setLocalFSWorkingDirectory(jobLocDir); } catch (URISyntaxException | IOException e) { throw new IgniteCheckedException(e); } }
From source file:org.apache.reef.runtime.mesos.driver.MesosResourceLaunchHandler.java
License:Apache License
@Override public void onNext(final DriverRuntimeProtocol.ResourceLaunchProto resourceLaunchProto) { try {/*from ww w . ja va2s . c om*/ LOG.log(Level.INFO, "resourceLaunchProto. {0}", resourceLaunchProto.toString()); final File localStagingFolder = Files.createTempDirectory(this.fileNames.getEvaluatorFolderPrefix()) .toFile(); final Configuration evaluatorConfiguration = Tang.Factory.getTang() .newConfigurationBuilder( this.configurationSerializer.fromString(resourceLaunchProto.getEvaluatorConf())) .bindImplementation(TempFileCreator.class, WorkingDirectoryTempFileCreator.class).build(); final File configurationFile = new File(localStagingFolder, this.fileNames.getEvaluatorConfigurationName()); this.configurationSerializer.toFile(evaluatorConfiguration, configurationFile); JobJarMaker.copy(resourceLaunchProto.getFileList(), localStagingFolder); final FileSystem fileSystem = FileSystem.get(new org.apache.hadoop.conf.Configuration()); final Path hdfsFolder = new Path(fileSystem.getUri() + "/" + resourceLaunchProto.getIdentifier() + "/"); FileUtil.copy(localStagingFolder, fileSystem, hdfsFolder, false, new org.apache.hadoop.conf.Configuration()); // TODO: Replace REEFExecutor with a simple launch command (we only need to launch REEFExecutor) final LaunchCommandBuilder commandBuilder; switch (resourceLaunchProto.getType()) { case JVM: commandBuilder = new JavaLaunchCommandBuilder() .setClassPath(this.classpath.getEvaluatorClasspath()); break; case CLR: commandBuilder = new CLRLaunchCommandBuilder(); break; default: throw new IllegalArgumentException("Unsupported container type"); } final List<String> command = commandBuilder.setErrorHandlerRID(this.remoteManager.getMyIdentifier()) .setLaunchID(resourceLaunchProto.getIdentifier()) .setConfigurationFileName(this.fileNames.getEvaluatorConfigurationPath()) .setMemory((int) (this.jvmHeapFactor * this.executors.getMemory(resourceLaunchProto.getIdentifier()))) .build(); this.executors.launchEvaluator( new EvaluatorLaunch(resourceLaunchProto.getIdentifier(), StringUtils.join(command, ' '))); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.apache.reef.runtime.mesos.evaluator.REEFExecutor.java
License:Apache License
public final void onEvaluatorLaunch(final EvaluatorLaunch evaluatorLaunch) { LOG.log(Level.INFO, "Launch!!!! {0}", evaluatorLaunch.toString()); assert (evaluatorLaunch.getIdentifier().toString().equals(this.mesosExecutorId)); final ExecutorService evaluatorLaunchExecutorService = Executors.newSingleThreadExecutor(); evaluatorLaunchExecutorService.submit(new Thread() { public void run() { try { final List<String> command = Arrays.asList(evaluatorLaunch.getCommand().toString().split(" ")); LOG.log(Level.INFO, "Command!!!! {0}", command); final FileSystem fileSystem = FileSystem.get(new Configuration()); final Path hdfsFolder = new Path(fileSystem.getUri() + "/" + mesosExecutorId); final File localFolder = new File(fileNames.getREEFFolderName(), fileNames.getLocalFolderName()); FileUtil.copy(fileSystem, hdfsFolder, localFolder, true, new Configuration()); evaluatorProcess = new ProcessBuilder().command(command) .redirectError(new File(fileNames.getEvaluatorStderrFileName())) .redirectOutput(new File(fileNames.getEvaluatorStdoutFileName())).start(); evaluatorProcessExitValue = evaluatorProcess.waitFor(); fileSystem.close();// w ww.j a v a2 s . co m } catch (IOException | InterruptedException e) { throw new RuntimeException(e); } } }); evaluatorLaunchExecutorService.shutdown(); }
From source file:org.elasticsearch.hadoop.yarn.cli.YarnBootstrap.java
License:Apache License
private void install(File src, String dst, Configuration cfg) { Path target = new Path(dst); try {//from w w w . j a v a 2 s .c o m FileSystem fs = FileSystem.get(URI.create("hdfs:///"), cfg); if (fs.exists(target)) { fs.delete(target, true); } FileUtil.copy(src, fs, target, false, cfg); FileStatus stats = fs.getFileStatus(target); System.out.println(String.format("Uploaded %s to HDFS at %s", src.getAbsolutePath(), stats.getPath())); } catch (IOException ex) { throw new IllegalStateException( String.format("Cannot upload %s in HDFS at %s", src.getAbsolutePath(), dst), ex); } }
From source file:org.gbif.occurrence.download.oozie.ArchiveBuilder.java
License:Creative Commons License
/** * Copies and merges the hive query results files into a single, local occurrence data file. *///www.j a va2 s .co m private void addOccurrenceDataFile(String dataTable, String headerFileName, String destFileName) throws IOException { LOG.info("Copy-merge occurrence data hdfs file {} to local filesystem", dataTable); final Path dataSrc = new Path(hdfsPath + Path.SEPARATOR + dataTable); boolean hasRecords = hdfs.exists(dataSrc); if (!hasRecords) { hdfs.create(dataSrc); } if (!isSmallDownload && hasRecords) { // small downloads already include the headers FileUtil.copy(new File(headerFileName), hdfs, new Path(dataSrc + Path.SEPARATOR + HEADERS_FILENAME), false, conf); } File rawDataResult = new File(archiveDir, destFileName); Path dataDest = new Path(rawDataResult.toURI()); FileUtil.copyMerge(hdfs, dataSrc, localfs, dataDest, false, conf, null); // remove the CRC file created by copyMerge method removeDataCRCFile(destFileName); }
From source file:org.godhuli.rhipe.FileUtils.java
License:Apache License
public void copyToLocal(FileSystem srcFS, Path src, File dst) throws IOException { if (!srcFS.getFileStatus(src).isDir()) { File tmp = FileUtil.createLocalTempFile(dst.getAbsoluteFile(), COPYTOLOCAL_PREFIX, true); if (!FileUtil.copy(srcFS, src, tmp, false, srcFS.getConf())) { throw new IOException("Failed to copy " + src + " to " + dst); }/* w w w.j ava 2s . co m*/ if (!tmp.renameTo(dst)) { throw new IOException( "Failed to rename tmp file " + tmp + " to local destination \"" + dst + "\"."); } } else { dst.mkdirs(); for (FileStatus path : srcFS.listStatus(src)) { copyToLocal(srcFS, path.getPath(), new File(dst, path.getPath().getName())); } } }