List of usage examples for org.apache.hadoop.fs FileSystem copyFromLocalFile
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
private Path copyToHDFS(FileSystem fs, String name, URI uri) throws Exception { // will throw exception if the file name is without extension String extension = uri.getPath().substring(uri.getPath().lastIndexOf(".") + 1); String pathSuffix = _applicationSpec.getAppName() + "/" + _appId.getId() + "/" + name + "." + extension; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); Path src = new Path(uri); fs.copyFromLocalFile(false, true, src, dst); return dst;//from ww w .j a v a2 s.com }
From source file:org.apache.hive.hplsql.Copy.java
License:Apache License
/** * Run COPY FROM LOCAL statement/*w w w . j a va 2 s .co m*/ */ public Integer runFromLocal(HplsqlParser.Copy_from_local_stmtContext ctx) { trace(ctx, "COPY FROM LOCAL"); initFileOptions(ctx.copy_file_option()); HashMap<String, Pair<String, Long>> srcFiles = new HashMap<String, Pair<String, Long>>(); String src = evalPop(ctx.copy_source(0)).toString(); String dest = evalPop(ctx.copy_target()).toString(); int srcItems = ctx.copy_source().size(); for (int i = 0; i < srcItems; i++) { createLocalFileList(srcFiles, evalPop(ctx.copy_source(i)).toString(), null); } if (info) { info(ctx, "Files to copy: " + srcFiles.size() + " (" + Utils.formatSizeInBytes(srcSizeInBytes) + ")"); } if (srcFiles.size() == 0) { exec.setHostCode(2); return 2; } timer.start(); File file = new File(); FileSystem fs = null; int succeed = 0; int failed = 0; long copiedSize = 0; try { fs = file.createFs(); boolean multi = false; if (srcFiles.size() > 1) { multi = true; } for (Map.Entry<String, Pair<String, Long>> i : srcFiles.entrySet()) { try { Path s = new Path(i.getKey()); Path d = null; if (multi) { String relativePath = i.getValue().getLeft(); if (relativePath == null) { d = new Path(dest, s.getName()); } else { d = new Path(dest, relativePath + Path.SEPARATOR + s.getName()); } } else { // Path to file is specified (can be relative), so treat target as a file name (hadoop fs -put behavior) if (srcItems == 1 && i.getKey().endsWith(src)) { d = new Path(dest); } // Source directory is specified, so treat the target as a directory else { d = new Path(dest + Path.SEPARATOR + s.getName()); } } fs.copyFromLocalFile(delete, overwrite, s, d); succeed++; long size = i.getValue().getRight(); copiedSize += size; if (info) { info(ctx, "Copied: " + file.resolvePath(d) + " (" + Utils.formatSizeInBytes(size) + ")"); } } catch (IOException e) { failed++; if (!ignore) { throw e; } } } } catch (IOException e) { exec.signal(e); exec.setHostCode(1); return 1; } finally { long elapsed = timer.stop(); if (info) { info(ctx, "COPY completed: " + succeed + " succeed, " + failed + " failed, " + timer.format() + ", " + Utils.formatSizeInBytes(copiedSize) + ", " + Utils.formatBytesPerSec(copiedSize, elapsed)); } if (failed == 0) { exec.setHostCode(0); } else { exec.setHostCode(1); } file.close(); } return 0; }
From source file:org.apache.ignite.yarn.utils.IgniteYarnUtils.java
License:Apache License
/** * @param fs File system./*from w w w.j a va2 s . c o m*/ * @param src Source path. * @param dst Destination path. * @return Path to file to hdfs file system. */ public static Path copyLocalToHdfs(FileSystem fs, String src, String dst) throws Exception { Path dstPath = new Path(dst); // Local file isn't removed, dst file override. fs.copyFromLocalFile(false, true, new Path(src), dstPath); return dstPath; }
From source file:org.apache.ivory.service.SharedLibraryHostingService.java
License:Apache License
public static void pushLibsToHDFS(String path, Cluster cluster, PathFilter pathFilter) throws IOException { Configuration conf = ClusterHelper.getConfiguration(cluster); FileSystem fs = FileSystem.get(conf); String localPaths = StartupProperties.get().getProperty("system.lib.location"); assert localPaths != null && !localPaths.isEmpty() : "Invalid value for system.lib.location"; if (!new File(localPaths).isDirectory()) { LOG.warn(localPaths + " configured for system.lib.location doesn't contain any valid libs"); return;//from w w w. j a v a 2 s . c om } for (File localFile : new File(localPaths).listFiles()) { Path clusterFile = new Path(path, localFile.getName()); if (!pathFilter.accept(clusterFile)) continue; if (fs.exists(clusterFile)) { FileStatus fstat = fs.getFileStatus(clusterFile); if (fstat.getLen() == localFile.length() && fstat.getModificationTime() == localFile.lastModified()) continue; } fs.copyFromLocalFile(false, true, new Path(localFile.getAbsolutePath()), clusterFile); fs.setTimes(clusterFile, localFile.lastModified(), System.currentTimeMillis()); LOG.info("Copied " + localFile.getAbsolutePath() + " to " + path + " in " + fs.getUri()); } }
From source file:org.apache.metron.dataloads.nonbulk.geo.GeoEnrichmentLoader.java
License:Apache License
protected void putDbFile(Path src, Path dst) throws IOException { Configuration conf = new Configuration(); FileSystem fileSystem = FileSystem.get(conf); System.out.println("Putting: " + src + " onto HDFS at: " + dst); fileSystem.mkdirs(dst);/* w w w . j ava 2 s . c o m*/ fileSystem.copyFromLocalFile(true, true, src, dst); System.out.println("Done putting GeoLite file into HDFS"); }
From source file:org.apache.mrql.Plan.java
License:Apache License
/** put the jar file that contains the compiled MR functional parameters into the TaskTracker classpath */ final static void distribute_compiled_arguments(Configuration conf) { try {/* www . j a v a 2s. com*/ if (!Config.compile_functional_arguments) return; Path local_path = new Path("file://" + Compiler.jar_path); if (Config.flink_mode) conf.set("mrql.jar.path", Compiler.jar_path); else if (Config.spark_mode) conf.set("mrql.jar.path", local_path.toString()); else { // distribute the jar file with the compiled arguments to all clients Path hdfs_path = new Path("mrql-tmp/class" + random_generator.nextInt(1000000) + ".jar"); FileSystem fs = hdfs_path.getFileSystem(conf); fs.copyFromLocalFile(false, true, local_path, hdfs_path); temporary_paths.add(hdfs_path.toString()); conf.set("mrql.jar.path", hdfs_path.toString()); } } catch (Exception ex) { throw new Error(ex); } }
From source file:org.apache.oozie.action.hadoop.GitOperations.java
License:Apache License
/** * Clone a Git repo up to a FileSystem/*from w w w. j a v a 2s . com*/ * * @param destination - Hadoop FileSystem path to which repository should be cloned * @throws GitOperationsException if the Git operations fail * @throws IOException if the HDFS or local file system operations fail */ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "Path is created runtime") String cloneRepoToFS(final Path destination) throws IOException, GitOperationsException { final File tempDir = GitMain.createTempDir("git"); final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.get(destination.toUri(), conf); cloneRepo(tempDir); // create a list of files and directories to upload final File src = new File(tempDir.getAbsolutePath()); final ArrayList<Path> srcs = new ArrayList<Path>(1000); final File[] sourceFiles = src.listFiles(); if (sourceFiles != null) { for (final File sourceFile : sourceFiles) { srcs.add(new Path(sourceFile.toString())); } } System.out.println("Finished cloning to local"); fs.mkdirs(destination); fs.copyFromLocalFile(false, true, srcs.toArray(new Path[0]), destination); System.out.println("Finished copying to " + destination.toString()); return destination.toString(); }
From source file:org.apache.pig.backend.hadoop.streaming.HadoopExecutableManager.java
License:Apache License
public void close() throws IOException { try {/*from w w w. j av a2 s . c o m*/ super.close(); // Copy the secondary outputs of the task to HDFS if (this.scriptOutputDir == null) { return; } Path scriptOutputDir = new Path(this.scriptOutputDir); FileSystem fs = scriptOutputDir.getFileSystem(job); List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT); if (outputSpecs != null) { for (int i = 1; i < outputSpecs.size(); ++i) { String fileName = outputSpecs.get(i).getName(); try { int partition = job.getInt(MRConfiguration.TASK_PARTITION, -1); Path dst = new Path(new Path(scriptOutputDir, fileName), getOutputName(partition)); fs.copyFromLocalFile(false, true, new Path(fileName), dst); fs.setReplication(dst, (short) job.getInt(MRConfiguration.SUMIT_REPLICATION, 3)); } catch (IOException ioe) { int errCode = 6014; String msg = "Failed to save secondary output '" + fileName + "' of task: " + taskId; throw new ExecException(msg, errCode, PigException.REMOTE_ENVIRONMENT, ioe); } } } } finally { // Footer for stderr file of the task writeDebugFooter(); // Close the stderr file on HDFS if (errorStream != null) { errorStream.close(); } } }
From source file:org.apache.pig.pigunit.Cluster.java
License:Apache License
public void copyFromLocalFile(Path local, Path destination, boolean overwrite) throws IOException { FileSystem fs = local.getFileSystem(configuration); fs.copyFromLocalFile(false, overwrite, local, destination); }
From source file:org.apache.rya.reasoning.mr.MRReasoningUtils.java
License:Apache License
/** * If a local input path was given, upload it to HDFS and configure file * input. Useful for automating tests against small inputs. */// w w w .j a va 2 s . c o m static boolean uploadIfNecessary(Configuration conf) throws IOException { String local = conf.get(LOCAL_INPUT); if (local == null) { return false; } FileSystem fs = FileSystem.get(conf); String current = new File("").getAbsolutePath(); Path sourcePath = new Path(current, local); Path destPath = getOutputPath(conf, "input"); fs.copyFromLocalFile(false, true, sourcePath, destPath); conf.set(MRUtils.INPUT_PATH, destPath.toString()); return true; }