List of usage examples for org.apache.hadoop.fs FileSystem copyToLocalFile
public void copyToLocalFile(Path src, Path dst) throws IOException
From source file:org.apache.falcon.ExtensionHandler.java
License:Apache License
private static List<URL> copyExtensionPackage(String extensionBuildUrl, String stagePath) throws IOException, FalconException, URISyntaxException { FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(new URI(extensionBuildUrl)); Path libsPath = new Path(extensionBuildUrl, FalconExtensionConstants.LIBS); Path buildLibsPath = new Path(libsPath, FalconExtensionConstants.BUILD); Path localStagePath = new Path(stagePath); Path localBuildLibsPath = new Path(localStagePath, FalconExtensionConstants.LIBS); LOG.info("Copying build time libs from {} to {}", buildLibsPath, localBuildLibsPath); fs.copyToLocalFile(buildLibsPath, localBuildLibsPath); Path resourcesPath = new Path(extensionBuildUrl, FalconExtensionConstants.RESOURCES); Path buildResourcesPath = new Path(resourcesPath, FalconExtensionConstants.BUILD); Path localBuildResourcesPath = new Path(localStagePath, FalconExtensionConstants.RESOURCES); LOG.info("Copying build time resources from {} to {}", buildLibsPath, localBuildResourcesPath); fs.copyToLocalFile(buildResourcesPath, localBuildResourcesPath); List<URL> urls = new ArrayList<>(); urls.addAll(getFilesInPath(localBuildLibsPath.toUri().toURL())); urls.add(localBuildResourcesPath.toUri().toURL()); return urls;/*w ww . j a va2 s . c om*/ }
From source file:org.apache.falcon.util.HdfsClassLoader.java
License:Apache License
private static URL[] copyHdfsJarFilesToTempDir(String databaseName, List<String> jars) throws IOException { List<URL> urls = new ArrayList<URL>(); final Configuration conf = new Configuration(); Path localPath = createTempDir(databaseName, conf); for (String jar : jars) { Path jarPath = new Path(jar); final FileSystem fs = jarPath.getFileSystem(conf); if (fs.isFile(jarPath) && jarPath.getName().endsWith(".jar")) { LOG.info("Copying jarFile = " + jarPath); fs.copyToLocalFile(jarPath, localPath); }/* w w w .j av a 2 s. c o m*/ } urls.addAll(getJarsInPath(localPath.toUri().toURL())); return urls.toArray(new URL[urls.size()]); }
From source file:org.apache.flink.streaming.util.HDFSCopyToLocal.java
License:Apache License
public static void copyToLocal(final URI remotePath, final File localPath) throws Exception { // Do it in another Thread because HDFS can deadlock if being interrupted while copying String threadName = "HDFS Copy from " + remotePath + " to " + localPath; final Tuple1<Exception> asyncException = Tuple1.of(null); Thread copyThread = new Thread(threadName) { @Override/*from ww w. ja va2s. c om*/ public void run() { try { Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration(); FileSystem fs = FileSystem.get(remotePath, hadoopConf); fs.copyToLocalFile(new Path(remotePath), new Path(localPath.getAbsolutePath())); } catch (Exception t) { asyncException.f0 = t; } } }; copyThread.setDaemon(true); copyThread.start(); copyThread.join(); if (asyncException.f0 != null) { throw asyncException.f0; } }
From source file:org.apache.flink.util.HDFSCopyToLocal.java
License:Apache License
public static void main(String[] args) throws Exception { String backupUri = args[0];/* w w w.j ava 2 s .c om*/ String dbPath = args[1]; FileSystem fs = FileSystem.get(new URI(backupUri), new Configuration()); fs.copyToLocalFile(new Path(backupUri), new Path(dbPath)); }
From source file:org.apache.giraph.debugger.gui.ServerUtils.java
License:Apache License
/** * @param jobId id of the job, whose jar path will be returned. * @return a url wrapped inside an array for convenience. *///from ww w .j ava2s . co m public static URL[] getCachedJobJarPath(String jobId) { // read the jar signature file under the TRACE_ROOT/jobId/ Path jarSignaturePath = new Path(DebuggerUtils.getTraceFileRoot(jobId) + "/" + "jar.signature"); try { FileSystem fs = getFileSystem(); try (FSDataInputStream jarSignatureInput = fs.open(jarSignaturePath)) { List<String> lines = IOUtils.readLines(jarSignatureInput); if (lines.size() > 0) { String jarSignature = lines.get(0); // check if jar is already in JARCACHE_LOCAL File localFile = new File(DebuggerUtils.JARCACHE_LOCAL + "/" + jarSignature + ".jar"); if (!localFile.exists()) { // otherwise, download from HDFS Path hdfsPath = new Path( fs.getUri().resolve(DebuggerUtils.JARCACHE_HDFS + "/" + jarSignature + ".jar")); Logger.getLogger(ServerUtils.class) .info("Copying from HDFS: " + hdfsPath + " to " + localFile); localFile.getParentFile().mkdirs(); fs.copyToLocalFile(hdfsPath, new Path(localFile.toURI())); } return new URL[] { localFile.toURI().toURL() }; } } } catch (IOException e) { // gracefully ignore if we failed to read the jar.signature LOG.warn("An IOException is thrown but will be ignored: " + e.toString()); } return new URL[0]; }
From source file:org.apache.hama.bsp.GroomServer.java
License:Apache License
private void localizeJob(TaskInProgress tip) throws IOException { Task task = tip.getTask();//w w w . java2 s . c o m conf.addResource(task.getJobFile()); BSPJob defaultJobConf = new BSPJob((HamaConfiguration) conf); Path localJobFile = defaultJobConf.getLocalPath(SUBDIR + "/" + task.getTaskID() + "/" + "job.xml"); RunningJob rjob = addTaskToJob(task.getJobID(), localJobFile, tip); BSPJob jobConf = null; synchronized (rjob) { if (!rjob.localized) { FileSystem dfs = FileSystem.get(conf); FileSystem localFs = FileSystem.getLocal(conf); Path jobDir = localJobFile.getParent(); if (localFs.exists(jobDir)) { localFs.delete(jobDir, true); boolean b = localFs.mkdirs(jobDir); if (!b) throw new IOException("Not able to create job directory " + jobDir.toString()); } Path localJarFile = defaultJobConf.getLocalPath(SUBDIR + "/" + task.getTaskID() + "/" + "job.jar"); Path jobFilePath = new Path(task.getJobFile()); //wait a while for file to finish being written for (int i = 0; i < 300 & !dfs.exists(jobFilePath); i++) { try { Thread.sleep(100); } catch (InterruptedException e) { LOG.warn("Sleep failed", e); } } dfs.copyToLocalFile(jobFilePath, localJobFile); HamaConfiguration conf = new HamaConfiguration(); conf.addResource(localJobFile); jobConf = new BSPJob(conf, task.getJobID().toString()); Path jarFile = null; if (jobConf.getJar() != null) { jarFile = new Path(jobConf.getJar()); } else { LOG.warn("No jar file for job " + task.getJobID() + " has been defined!"); } jobConf.setJar(localJarFile.toString()); if (jarFile != null) { dfs.copyToLocalFile(jarFile, localJarFile); // also unjar the job.jar files in workdir File workDir = new File(new File(localJobFile.toString()).getParent(), "work"); if (!workDir.mkdirs()) { if (!workDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + workDir.toString()); } } RunJar.unJar(new File(localJarFile.toString()), workDir); } rjob.localized = true; } else { HamaConfiguration conf = new HamaConfiguration(); conf.addResource(rjob.getJobFile()); jobConf = new BSPJob(conf, rjob.getJobId().toString()); } } launchTaskForJob(tip, jobConf); }
From source file:org.apache.hama.bsp.JobInProgress.java
License:Apache License
public JobInProgress(BSPJobID jobId, Path jobFile, BSPMaster master, Configuration conf) throws IOException { this.conf = conf; this.jobId = jobId; this.localFs = FileSystem.getLocal(conf); this.jobFile = jobFile; this.master = master; this.status = new JobStatus(jobId, null, 0L, 0L, JobStatus.State.PREP.value(), counters); this.startTime = System.currentTimeMillis(); this.superstepCounter = 0; this.restartCount = 0; this.localJobFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId + ".xml"); this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId + ".jar"); Path jobDir = master.getSystemDirectoryForJob(jobId); FileSystem fs = jobDir.getFileSystem(conf); fs.copyToLocalFile(jobFile, localJobFile); BSPJob job = new BSPJob(jobId, localJobFile.toString()); this.jobSplit = job.getConfiguration().get("bsp.job.split.file"); this.numBSPTasks = job.getNumBspTask(); this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>(numBSPTasks + 10); this.maxTaskAttempts = job.getConfiguration().getInt(Constants.MAX_TASK_ATTEMPTS, Constants.DEFAULT_MAX_TASK_ATTEMPTS); this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(), job.getJobName()); this.setJobName(job.getJobName()); status.setUsername(job.getUser());/*from www.j a v a 2 s. com*/ status.setStartTime(startTime); String jarFile = job.getJar(); if (jarFile != null) { fs.copyToLocalFile(new Path(jarFile), localJarFile); } failedTasksTillNow = new HashSet<Task>(2 * tasks.length); }
From source file:org.apache.hama.pipes.util.DistributedCacheUtil.java
License:Apache License
/** * Transfers DistributedCache files into the local cache files. Also creates * symbolic links for URIs specified with a fragment if * DistributedCache.getSymlinks() is true. * //www . jav a2 s . co m * @param conf The job's configuration * @throws IOException If a DistributedCache file cannot be found. */ public static final void moveLocalFiles(Configuration conf) throws IOException { StringBuilder files = new StringBuilder(); boolean first = true; if (DistributedCache.getCacheFiles(conf) != null) { for (URI uri : DistributedCache.getCacheFiles(conf)) { if (uri != null) { if (!first) { files.append(","); } if (null != uri.getFragment() && DistributedCache.getSymlink(conf)) { FileUtil.symLink(uri.getPath(), uri.getFragment()); files.append(uri.getFragment()).append(","); } FileSystem hdfs = FileSystem.get(conf); Path pathSrc = new Path(uri.getPath()); // LOG.info("pathSrc: " + pathSrc); if (hdfs.exists(pathSrc)) { LocalFileSystem local = LocalFileSystem.getLocal(conf); Path pathDst = new Path(local.getWorkingDirectory(), pathSrc.getName()); // LOG.info("user.dir: "+System.getProperty("user.dir")); // LOG.info("WorkingDirectory: "+local.getWorkingDirectory()); // LOG.info("pathDst: " + pathDst); LOG.debug("copyToLocalFile: " + pathDst); hdfs.copyToLocalFile(pathSrc, pathDst); local.deleteOnExit(pathDst); files.append(pathDst.toUri().getPath()); } first = false; } } } if (files.length() > 0) { // I've replaced the use of the missing setLocalFiles and // addLocalFiles methods (hadoop 0.23.x) with our own DistCacheUtils // methods which set the cache configurations directly. DistCacheUtils.addLocalFiles(conf, files.toString()); } }
From source file:org.apache.hama.pipes.util.DistributedCacheUtil.java
License:Apache License
/** * Add the JARs from the given HDFS paths to the Classpath * //from www . ja v a 2s. c o m * @param conf The job's configuration */ public static URL[] addJarsToJobClasspath(Configuration conf) { URL[] classLoaderURLs = ((URLClassLoader) conf.getClassLoader()).getURLs(); String files = conf.get("tmpjars", ""); if (!files.isEmpty()) { String[] fileArr = files.split(","); URL[] libjars = new URL[fileArr.length + classLoaderURLs.length]; for (int i = 0; i < fileArr.length; i++) { String tmp = fileArr[i]; URI pathURI; try { pathURI = new URI(tmp); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } try { FileSystem hdfs = FileSystem.get(conf); Path pathSrc = new Path(pathURI.getPath()); // LOG.info("pathSrc: " + pathSrc); if (hdfs.exists(pathSrc)) { LocalFileSystem local = LocalFileSystem.getLocal(conf); // File dst = File.createTempFile(pathSrc.getName() + "-", ".jar"); Path pathDst = new Path(local.getWorkingDirectory(), pathSrc.getName()); LOG.debug("copyToLocalFile: " + pathDst); hdfs.copyToLocalFile(pathSrc, pathDst); local.deleteOnExit(pathDst); libjars[i] = new URL(local.makeQualified(pathDst).toString()); } } catch (IOException ex) { throw new RuntimeException("Error setting up classpath", ex); } } // Add old classLoader entries int index = fileArr.length; for (int i = 0; i < classLoaderURLs.length; i++) { libjars[index] = classLoaderURLs[i]; index++; } // Set classloader in current conf/thread conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); Thread.currentThread().setContextClassLoader( new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader())); // URL[] urls = ((URLClassLoader) conf.getClassLoader()).getURLs(); // for (URL u : urls) // LOG.info("newClassLoader: " + u.getPath()); // Set tmpjars // hdfs to local path String jars = ""; for (int i = 0; i < fileArr.length; i++) { URL url = libjars[i]; if (jars.length() > 0) { jars += ","; } jars += url.toString(); } conf.set("tmpjars", jars); return libjars; } return null; }
From source file:org.apache.hive.hcatalog.templeton.tool.LaunchMapper.java
License:Apache License
private void copyLocal(String var, Configuration conf) throws IOException { String[] filenames = TempletonUtils.decodeArray(conf.get(var)); if (filenames != null) { for (String filename : filenames) { Path src = new Path(filename); Path dst = new Path(src.getName()); FileSystem fs = src.getFileSystem(conf); LOG.info("templeton: copy " + src + " => " + dst); fs.copyToLocalFile(src, dst); }/*from ww w.j a va 2s .c om*/ } }