List of usage examples for org.apache.hadoop.fs FileSystem copyFromLocalFile
public void copyFromLocalFile(Path src, Path dst) throws IOException
From source file:pl.edu.icm.coansys.heeut.TestMapReduce.java
License:Apache License
@Test(timeout = 1800000) public void testWordCountDiff() throws Exception { String prefix = getCurrentDateAppended("wordcount"); String inputDirName = prefix + "-input"; String outputDirName = prefix + "-output"; FileSystem dfs = UTIL.getDFSCluster().getFileSystem(); Path inputDir = new Path(inputDirName); Path qualifiedInputDir = dfs.makeQualified(inputDir); String inputFileName = "src/test/resource/input/wordcount/apache_projects.dat"; dfs.copyFromLocalFile(new Path(inputFileName), qualifiedInputDir); ToolRunner.run(UTIL.getConfiguration(), new WordCount(), new String[] { inputDirName, outputDirName }); Path outputDir = new Path(outputDirName); Path qualifiedOutputDir = dfs.makeQualified(outputDir); String localOutputDir = "src/test/resource/output/wordcount/" + prefix; dfs.copyToLocalFile(qualifiedOutputDir, new Path(localOutputDir)); File outputFile = new File(localOutputDir + "/part-00000"); File expectedFile = new File("src/test/resource/exp/wordcount/apache_projects.exp"); boolean isEqual = FileUtils.contentEquals(outputFile, expectedFile); Assert.assertTrue(isEqual);// w w w . ja va 2 s . co m }
From source file:sharedsidefunctions.CopyFromLocal.java
License:Apache License
public static void copyFromLocal(Configuration conf, String diamond, String query, String dataBase, String userName) throws IOException { FileSystem fs = FileSystem.get(conf); fs.moveFromLocalFile(new Path(dataBase + ".dmnd"), new Path("/user/" + userName + "/Hamond")); fs.copyFromLocalFile(new Path(query), new Path("/user/" + userName + "/Hamond")); // fs.copyFromLocalFile(new Path(System.getProperty("user.dir")+"/diamond"), new Path(userName)); fs.copyFromLocalFile(new Path(diamond), new Path("/user/" + userName + "/Hamond")); //close file system fs.close();/*w w w . ja va2 s. co m*/ }
From source file:smile.wide.algorithms.SMILEBSjob.java
License:Apache License
/** Sets up the hadoop job and sends it to the cluster * waits for the job to be completed.*/ @Override//from w w w . j a va 2s . com public int run(String[] params) throws Exception { //params: <trainfile> <output_path> <number of seeds> Configuration conf = super.getConf(); conf.set("trainfile", params[0]); //distributed cache initialization DistributedCache.createSymlink(conf); DistributedCache.addFileToClassPath(new Path(libHDFSPath_ + "/smile.jar"), conf); DistributedCache.addCacheFile(new URI(libHDFSPath_ + "/libjsmile.so#libjsmile.so"), conf); //upload data file to HDFS and add it to the distributed cache FileSystem dfs = FileSystem.get(conf); dfs.copyFromLocalFile(new Path(params[0]), new Path(dataHDFSPath_)); DistributedCache.addCacheFile(new URI(dataHDFSPath_ + basename(params[0]) + "#" + basename(params[0])), conf); //for now, keep the Bayesian search parameters constant conf.setInt("iterationCount", iterationCount); conf.setFloat("linkProbability", linkProbability); conf.setInt("maxParents", maxParents); conf.setInt("maxSearchTime", maxSearchTime); conf.setFloat("priorLinkProbability", priorLinkProbability); conf.setInt("priorSampleSize", priorSampleSize); // conf.setInt(RandSeedInputFormat.CONFKEY_SEED_COUNT, Integer.parseInt(params[2])); conf.setInt(RandSeedInputFormat.CONFKEY_WARMUP_ITER, 100000); conf.setLong("mapred.task.timeout", 3600000); Job job = new Job(conf); job.setJobName("Distributed Bayesian Search"); job.setJarByClass(SMILEBSjob.class); job.setMapperClass(SMILEBSMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(StrucLogLikeWritable.class); job.setReducerClass(SMILEBSReducer.class); job.setNumReduceTasks(1); job.setInputFormatClass(RandSeedInputFormat.class); Path outputPath = new Path(params[1]); FileOutputFormat.setOutputPath(job, outputPath); outputPath.getFileSystem(conf).delete(outputPath, true); //Run the job job.waitForCompletion(true); //now download result outputPath.suffix("/part-r-00000"); dfs.copyToLocalFile(outputPath.suffix("/part-r-00000"), new Path("./smile-output.txt")); return 0; }
From source file:tajo.TajoTestingUtility.java
License:Apache License
public static ResultSet run(String[] names, String[] tablepaths, Schema[] schemas, Options option, String query) throws Exception { TajoTestingUtility util = new TajoTestingUtility(); util.startMiniCluster(1);//w ww . ja v a2s . com TajoConf conf = util.getConfiguration(); TajoClient client = new TajoClient(conf); FileSystem fs = util.getDefaultFileSystem(); Path rootDir = util.getMiniTajoCluster().getMaster().getStorageManager().getDataRoot(); fs.mkdirs(rootDir); for (int i = 0; i < tablepaths.length; i++) { Path localPath = new Path(tablepaths[i]); Path tablePath = new Path(rootDir, names[i]); fs.mkdirs(tablePath); Path dataPath = new Path(tablePath, "data"); fs.mkdirs(dataPath); Path dfsPath = new Path(dataPath, localPath.getName()); fs.copyFromLocalFile(localPath, dfsPath); TableMeta meta = TCatUtil.newTableMeta(schemas[i], CatalogProtos.StoreType.CSV, option); client.createTable(names[i], tablePath, meta); } Thread.sleep(1000); ResultSet res = client.executeQuery(query); util.shutdownMiniCluster(); return res; }
From source file:terasort.utils.Utils.java
License:Apache License
public static Map<String, LocalResource> getLocalResources(List<Class> classList, TezConfiguration tezConf) throws IOException, URISyntaxException { Map<String, LocalResource> localResources = Maps.newHashMap(); Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf); // staging dir FileSystem fs = FileSystem.get(tezConf); String uuid = UUID.randomUUID().toString(); Path jobJar = new Path(stagingDir, uuid + "_job.jar"); if (fs.exists(jobJar)) { fs.delete(jobJar, false);/*w w w .jav a 2 s . c om*/ } for (Class clazz : classList) { Path path = getCurrentJarURL(clazz); fs.copyFromLocalFile(path, jobJar); System.out.println("Path : " + path); } localResources.put(uuid + "_job.jar", createLocalResource(fs, jobJar)); return localResources; }
From source file:terasort.utils.Utils.java
License:Apache License
public static Map<String, LocalResource> getLocalResources(TezConfiguration tezConf) throws IOException, URISyntaxException { Map<String, LocalResource> localResources = Maps.newHashMap(); Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf); // staging dir FileSystem fs = FileSystem.get(tezConf); String uuid = UUID.randomUUID().toString(); Path jobJar = new Path(stagingDir, uuid + "_job.jar"); if (fs.exists(jobJar)) { fs.delete(jobJar, false);/*ww w . j a va 2s .c o m*/ } fs.copyFromLocalFile(getCurrentJarURL(TeraSort.class), jobJar); localResources.put(uuid + "_job.jar", createLocalResource(fs, jobJar)); return localResources; }
From source file:timo.yarn_app_call_java_app.Client.java
License:Apache License
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = Constants.APP_NAME + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try {//w ww. java 2 s . c om ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
From source file:timo.yarn_app_call_java_daemon.Client.java
License:Apache License
/** * addToLocalResources// ww w.ja v a 2 s. com * * @param fs * @param fileSrcPath * @param fileDstPath * @param appId * @param localResources * @param resources * @throws IOException */ private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = Constants.APP_NAME + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
From source file:x10.x10rt.yarn.Client.java
License:Open Source License
private String addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = appName + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try {/*from w w w. j a v a2 s .c om*/ ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); return ("/user/" + System.getProperty("user.name") + '/' + suffix); }
From source file:yarnkit.utils.YarnUtils.java
License:Apache License
public static LocalResource copyToHDFS(@Nonnull FileSystem fs, @Nonnull Path srcLocalPath, @Nonnull Path dstHdfsPath) throws IOException { if (!"hdfs".equals(srcLocalPath.toUri().getScheme())) { fs.copyFromLocalFile(srcLocalPath, dstHdfsPath); }// w w w .j a v a 2 s. c o m return createLocalResource(fs, dstHdfsPath); }