List of usage examples for org.apache.hadoop.fs FileSystem copyToLocalFile
public void copyToLocalFile(Path src, Path dst) throws IOException
From source file:org.opencb.hpg.bigdata.app.cli.hadoop.AlignmentCommandExecutor.java
License:Apache License
private void stats() throws Exception { String input = alignmentCommandOptions.statsAlignmentCommandOptions.input; String output = alignmentCommandOptions.statsAlignmentCommandOptions.output; // prepare the HDFS output folder Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); String outHdfsDirname = Long.toString(new Date().getTime()); // run MapReduce job to compute stats ReadAlignmentStatsMR.run(input, outHdfsDirname); // post-processing Path outFile = new Path(outHdfsDirname + "/part-r-00000"); try {/*from ww w . j a v a 2 s . c o m*/ if (!fs.exists(outFile)) { logger.error("Stats results file not found: {}", outFile.getName()); } else { String outRawFileName = output + "/stats.json"; fs.copyToLocalFile(outFile, new Path(outRawFileName)); //Utils.parseStatsFile(outRawFileName, out); } fs.delete(new Path(outHdfsDirname), true); } catch (IOException e) { throw e; } }
From source file:org.opencb.hpg.bigdata.app.cli.hadoop.SequenceCommandExecutor.java
License:Apache License
private void stats() throws Exception { CliOptionsParser.StatsSequenceCommandOptions statsSequenceCommandOptions = sequenceCommandOptions.statsSequenceCommandOptions; // get input parameters String input = statsSequenceCommandOptions.input; String output = statsSequenceCommandOptions.output; int kvalue = statsSequenceCommandOptions.kmers; // prepare the HDFS output folder Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); String outHdfsDirname = Long.toString(new Date().getTime()); // run MapReduce job to compute stats ReadStatsMR.run(input, outHdfsDirname, kvalue); // post-processing Path outFile = new Path(outHdfsDirname + "/part-r-00000"); try {//from w w w. j ava2 s . c o m if (!fs.exists(outFile)) { logger.error("Stats results file not found: {}", outFile.getName()); } else { String outRawFileName = output + "/stats.json"; fs.copyToLocalFile(outFile, new Path(outRawFileName)); //Utils.parseStatsFile(outRawFileName, out); } fs.delete(new Path(outHdfsDirname), true); } catch (IOException e) { throw e; } }
From source file:org.trommel.trommel.mapreduce.TrommelDriver.java
License:Apache License
private static int processScript(Level logLevel, int numOfReducers, String trommelScriptFilePath) throws Exception { int exitCode = 0; FrontEndInterpreter frontEndInterpreter = null; FileSystem fileSystem = null; Path cachedScript = null;// www. j ava2 s . com logger.setLevel(logLevel); try { logger.info(String.format("Loading and parsing TommelScript file %1$s ...", trommelScriptFilePath)); Lexer lexer = new Lexer( new PushbackReader(new BufferedReader(new FileReader(trommelScriptFilePath)), 4096)); Parser parser = new Parser(lexer); Start ast = parser.parse(); ValidationInterpreter validationInterpreter = new ValidationInterpreter(); logger.info(String.format("Validating TommelScript...", trommelScriptFilePath)); ast.apply(validationInterpreter); if (validationInterpreter.getSemanticErrors().size() != 0) { // Validation of script failed logger.info(String.format("TommelScript failed validation with following errors:", trommelScriptFilePath)); for (String errorMessage : validationInterpreter.getSemanticErrors()) { logger.info(errorMessage); } return exitCode; } logger.info(String.format("Interpreting TommelScript...", trommelScriptFilePath)); frontEndInterpreter = new FrontEndInterpreter(logger, DEFAULT_HDFS_PATH); ast.apply(frontEndInterpreter); logger.debug("Creating Job object"); Job job = new Job(); job.setJarByClass(TrommelDriver.class); // Copy TrommelScript file from local file system to HDFS and added to distributed cache fileSystem = FileSystem.get(job.getConfiguration()); Path src = new Path(trommelScriptFilePath); cachedScript = new Path(String.format("/tmp/%1$s_%2$s", src.getName(), UUID.randomUUID().toString())); fileSystem.copyFromLocalFile(src, cachedScript); logger.debug(String.format("Moved TrommelScript file to HDFS as %1$s.", cachedScript.toString())); logger.debug("Adding TrommelScript file to DistibutedCachce."); DistributedCache.addCacheFile(new URI(cachedScript.toString()), job.getConfiguration()); logger.debug(String.format("Setting LOGGING_LEVEL_CONFIG_PROP to %1$s", logLevel.toString())); job.getConfiguration().set(LOGGING_LEVEL_CONFIG_PROP, logLevel.toString()); // Specify HDFS input/output locations logger.debug(String.format("Calling FileInputFormat.addInputPath() with %1$s.", frontEndInterpreter.getHdfsInputFilePath())); FileInputFormat.addInputPath(job, new Path(frontEndInterpreter.getHdfsInputFilePath())); logger.debug(String.format("Calling FileOutputFormat.setOutputPath() with %1$s.", frontEndInterpreter.getHdfsOutputFilePath())); FileOutputFormat.setOutputPath(job, new Path(frontEndInterpreter.getHdfsOutputFilePath())); // Hadoop setup job.setMapperClass(TrommelMapper.class); if (frontEndInterpreter.samplingData()) { logger.debug("Trommel is sampling data, 0 Reducers set."); job.setNumReduceTasks(0); } else { logger.debug(String.format("Setting number of Reducers to %1$s.", numOfReducers)); job.setReducerClass(TrommelReducer.class); job.setNumReduceTasks(numOfReducers); } job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); logger.debug("Running job"); if (!job.waitForCompletion(true)) { exitCode = 1; } else if (frontEndInterpreter.getLocalFilePath() != null) { // User would like data exported to local file system logger.debug(String.format("Exporting Trommel output from %1$s to %2$s.", frontEndInterpreter.getHdfsOutputFilePath(), frontEndInterpreter.getLocalFilePath())); Path mergeFilePath = new Path(String.format("/tmp/%1$s", UUID.randomUUID())); FSDataOutputStream mergeFileStream = fileSystem.create(mergeFilePath); Path localFilePath = new Path(frontEndInterpreter.getLocalFilePath()); FileStatus[] outputFileStatuses = fileSystem .listStatus(new Path(frontEndInterpreter.getHdfsOutputFilePath())); FSDataInputStream outputFileStream = null; String fileNameFilter = (frontEndInterpreter.samplingData() ? "part-m" : "part-r"); try { // Loop through the output, merging any reducer output file for export to local file system for (FileStatus outputFileStatus : outputFileStatuses) { if (!outputFileStatus.isDir() && outputFileStatus.getPath().getName().contains(fileNameFilter)) { logger.debug(String.format("Merging file %1$s into local file system output.", outputFileStatus.getPath().toString())); outputFileStream = fileSystem.open(outputFileStatus.getPath()); byte[] buffer = new byte[(int) outputFileStatus.getLen()]; outputFileStream.read(buffer); mergeFileStream.write(buffer); outputFileStream.close(); } } } finally { if (mergeFileStream != null) { mergeFileStream.close(); fileSystem.copyToLocalFile(mergeFilePath, localFilePath); fileSystem.delete(mergeFilePath, true); } } } } finally { try { if (fileSystem != null) { // Clean up any temp files if needed if (frontEndInterpreter.getHdfsOutputFilePath().equals(DEFAULT_HDFS_PATH)) { logger.debug(String.format("Deleting temp files from /tmp/Trommel")); fileSystem.delete(new Path(DEFAULT_HDFS_PATH), true); } // Clean up the cached file logger.debug(String.format("Deleting cached TrommelScript file %1$s", cachedScript.toString())); fileSystem.delete(cachedScript, true); } } catch (IOException ioe) { // Couldn't delete file for some reason, alert user logger.error(String.format( "Exception encountered deleting cached TommelScript file %1$s. Error message: %2$s", cachedScript.toString(), ioe.getMessage())); } } return exitCode; }
From source file:pegasus.ResultInfo.java
License:Apache License
public int run(final String[] args) throws Exception { if (args.length != 9) { return printUsage(); }/*ww w. ja va 2s. c o m*/ edge_path = new Path(args[0]); curbm_path = new Path(args[1]); tempbm_path = new Path(args[2]); nextbm_path = new Path(args[3]); output_path = new Path(args[4]); summaryout_path = new Path("concmpt_summaryout"); number_nodes = Integer.parseInt(args[5]); nreducers = Integer.parseInt(args[6]); if (args[7].compareTo("new") == 0) start_from_newbm = 1; else { // args[7] == contNN e.g.) cont10 start_from_newbm = 0; cur_iter = Integer.parseInt(args[7].substring(4)); System.out.println("Starting from cur_iter = " + cur_iter); } if (args[8].compareTo("makesym") == 0) make_symmetric = 1; else make_symmetric = 0; System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n"); System.out.println("[PEGASUS] Computing connected component. Edge path = " + args[0] + ", Newbm = " + args[7] + ", Reducers = " + nreducers); local_output_path = args[4] + "_temp"; if (start_from_newbm == 1) { System.out.print("Generating initial component vector for " + number_nodes + " nodes "); // create bitmask generate command file, and copy to curbm_path gen_component_vector_file(number_nodes, curbm_path); System.out.println(" done"); } else { System.out.println("Resuming from current component vector at radius(" + cur_iter + ")"); } // Iteratively calculate neighborhood function. for (int i = cur_iter; i < MAX_ITERATIONS; i++) { cur_iter++; JobClient.runJob(configStage1()); JobClient.runJob(configStage2()); JobClient.runJob(configStage3()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); final FileSystem fs = FileSystem.get(getConf()); // copy neighborhood information from HDFS to local disk, and read it! String new_path = local_output_path + "/" + i; fs.copyToLocalFile(output_path, new Path(new_path)); ResultInfo ri = readIterationOutput(new_path); changed_nodes[iter_counter] = ri.changed; changed_nodes[iter_counter] = ri.unchanged; iter_counter++; System.out.println("Hop " + i + " : changed = " + ri.changed + ", unchanged = " + ri.unchanged); // Stop when the minimum neighborhood doesn't change if (ri.changed == 0) { System.out.println("All the component ids converged. Finishing..."); fs.delete(curbm_path); fs.delete(tempbm_path); fs.delete(output_path); fs.rename(nextbm_path, curbm_path); break; } // rotate directory fs.delete(curbm_path); fs.delete(tempbm_path); fs.delete(output_path); fs.rename(nextbm_path, curbm_path); } FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // calculate summary information using an additional stage System.out.println("Summarizing connected components information..."); JobClient.runJob(configStage4()); // finishing. System.out.println("\n[PEGASUS] Connected component computed."); System.out.println("[PEGASUS] Total Iteration = " + iter_counter); System.out.println( "[PEGASUS] Connected component information is saved in the HDFS concmpt_curbm as\n\"node_id 'msf'component_id\" format"); System.out.println( "[PEGASUS] Connected component distribution is saved in the HDFS concmpt_summaryout as\n\"component_id number_of_nodes\" format.\n"); return 0; }
From source file:pegasus.ConCmptBlock.java
License:Apache License
public int run(final String[] args) throws Exception { if (args.length != 9) { return printUsage(); }/*from ww w.j a v a 2 s . co m*/ int i; edge_path = new Path(args[0]); curbm_path = new Path(args[1]); tempbm_path = new Path(args[2]); nextbm_path = new Path(args[3]); output_path = new Path(args[4]); curbm_unfold_path = new Path("concmpt_curbm"); summaryout_path = new Path("concmpt_summaryout"); number_nodes = Integer.parseInt(args[5]); nreducers = Integer.parseInt(args[6]); if (args[7].compareTo("fast") == 0) recursive_diagmult = 1; else recursive_diagmult = 0; block_width = Integer.parseInt(args[8]); System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n"); System.out.println("[PEGASUS] Computing connected component using block method. Reducers = " + nreducers + ", block_width = " + block_width); local_output_path = args[4] + "_temp"; // Iteratively calculate neighborhood function. for (i = cur_radius; i < MAX_ITERATIONS; i++) { cur_radius++; iter_counter++; JobClient.runJob(configStage1()); JobClient.runJob(configStage2()); JobClient.runJob(configStage3()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); final FileSystem fs = FileSystem.get(getConf()); // copy neighborhood information from HDFS to local disk, and read it! String new_path = local_output_path + "/" + i; fs.copyToLocalFile(output_path, new Path(new_path)); ResultInfo ri = ConCmpt.readIterationOutput(new_path); changed_nodes[iter_counter] = ri.changed; changed_nodes[iter_counter] = ri.unchanged; System.out.println("Hop " + i + " : changed = " + ri.changed + ", unchanged = " + ri.unchanged); // Stop when the minimum neighborhood doesn't change if (ri.changed == 0) { System.out.println("All the component ids converged. Finishing..."); fs.delete(curbm_path); fs.delete(tempbm_path); fs.delete(output_path); fs.rename(nextbm_path, curbm_path); System.out.println("Unfolding the block structure for easy lookup..."); JobClient.runJob(configStage4()); break; } // rotate directory fs.delete(curbm_path); fs.delete(tempbm_path); fs.delete(output_path); fs.rename(nextbm_path, curbm_path); } FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // calculate summary information using an additional pass System.out.println("Summarizing connected components information..."); JobClient.runJob(configStage5()); // finishing. System.out.println("\n[PEGASUS] Connected component computed."); System.out.println("[PEGASUS] Total Iteration = " + iter_counter); System.out.println( "[PEGASUS] Connected component information is saved in the HDFS concmpt_curbm as\n\"node_id 'msf'component_id\" format"); System.out.println( "[PEGASUS] Connected component distribution is saved in the HDFS concmpt_summaryout as\n\"component_id number_of_nodes\" format.\n"); return 0; }
From source file:pegasus.hadi.Hadi.java
License:Apache License
public int run(final String[] args) throws Exception { int i;/*from w ww .j a va 2s . c o m*/ int max_iteration = MAX_ITERATIONS; if (args.length != 12) { return printUsage(); } edge_path = new Path(args[0]); curbm_path = new Path(args[1]); tempbm_path = new Path(args[2]); nextbm_path = new Path(args[3]); output_path = new Path(args[4]); number_nodes = Integer.parseInt(args[5]); radius_path = new Path("hadi_radius"); radius_summary_path = new Path("hadi_radius_summary"); nreplication = Integer.parseInt(args[6]); nreducer = Integer.parseInt(args[7]); if (args[8].compareTo("enc") == 0) encode_bitmask = 1; if (args[9].compareTo("newbm") == 0) { start_from_newbm = 1; } else if (args[9].startsWith("cont")) { start_from_newbm = 0; cur_radius = Integer.parseInt(args[9].substring(4)); } if (args[10].compareTo("makesym") == 0) make_symmetric = 1; else make_symmetric = 0; if (args[11].compareTo("max") != 0) max_iteration = Integer.parseInt(args[11]); System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n"); System.out.println("[PEGASUS] Computing Radii/Diameter. Current hop: " + cur_radius + ", " + "edge_path: " + args[0] + ", encode: " + encode_bitmask + ", # reducers: " + nreducer + ", makesym: " + make_symmetric + ", max_iteration: " + max_iteration + "\n"); local_output_path = args[4] + number_nodes + "_temp"; if (start_from_newbm == 1) { System.out.print("Generating initial bitstrings for " + number_nodes + " nodes "); // create bitmask generate command file, and copy to curbm_path gen_bitmask_cmd_file(number_nodes, nreplication, curbm_path); System.out.println(" done"); } else { System.out.println("Resuming from current hadi_curbm which contains up to N(" + (cur_radius - 1) + ")"); } N[0] = number_nodes; boolean eff_diameter_computed = false; // Iteratively run Stage1 to Stage3. for (i = cur_radius; i <= max_iteration; i++) { JobClient.runJob(configStage1(edge_type)); JobClient.runJob(configStage2()); JobClient.runJob(configStage3()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); final FileSystem fs = FileSystem.get(getConf()); // copy neighborhood information from HDFS to local disk, and read it! String new_path = local_output_path + "/" + i; fs.copyToLocalFile(output_path, new Path(new_path)); HadiResultInfo ri = HadiUtils.readNhoodOutput(new_path); N[i] = ri.nh; iter_counter++; System.out.println( "Nh(" + i + "):\t" + N[i] + "\tGuessed Radius(" + (i - 1) + "):\t" + ri.converged_nodes); // Stop when all radii converged. if (ri.changed_nodes == 0) {//if( i > 1 && N[i] == N[i-1] ) { System.out.println("All the bitstrings converged. Finishing..."); fs.delete(curbm_path); fs.delete(tempbm_path); fs.rename(nextbm_path, curbm_path); System.out.println("Calculating the effective diameter..."); JobClient.runJob(configStage4()); eff_diameter_computed = true; break; } // rotate directory. fs.delete(curbm_path); fs.delete(tempbm_path); if (i < MAX_ITERATIONS - 1) fs.delete(output_path); fs.rename(nextbm_path, curbm_path); cur_radius++; } if (eff_diameter_computed == false) { System.out.println("Calculating the effective diameter..."); JobClient.runJob(configStage4()); } // Summarize Radius Information System.out.println("Summarizing radius information..."); JobClient.runJob(configStage5()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // print summary information if (i > max_iteration) System.out.println("Reached Max Iteartion " + max_iteration); System.out.println("Total Iteration = " + iter_counter + "."); System.out.println("Neighborhood Summary:"); for (int j = 0; j <= (i); j++) System.out.println("\tNh(" + (j) + "):\t" + N[j]); System.out.println("\n[PEGASUS] Radii and diameter computed."); System.out.println("[PEGASUS] Maximum diameter: " + (cur_radius - 1)); System.out.println("[PEGASUS] Average diameter: " + HadiUtils.average_diameter(N, cur_radius - 1)); System.out.println("[PEGASUS] 90% Effective diameter: " + HadiUtils.effective_diameter(N, cur_radius - 1)); System.out.println("[PEGASUS] Radii are saved in the HDFS " + radius_path.getName()); System.out.println("[PEGASUS] Radii summary is saved in the HDFS " + radius_summary_path.getName() + "\n"); return 0; }
From source file:pegasus.hadi.HadiBlock.java
License:Apache License
public int run(final String[] args) throws Exception { int i;//ww w . jav a2 s . c o m int max_iteration = MAX_ITERATIONS; if (args.length != 12) { return printUsage(); } edge_path = new Path(args[0]); curbm_path = new Path(args[1]); tempbm_path = new Path(args[2]); nextbm_path = new Path(args[3]); output_path = new Path(args[4]); number_nodes = Integer.parseInt(args[5]); radius_path = new Path("hadi_radius_block"); radius_summary_path = new Path("hadi_radius_block_summary"); nreplication = Integer.parseInt(args[6]); nreducer = Integer.parseInt(args[7]); if (args[8].compareTo("enc") == 0) encode_bitmask = 1; if (args[9].compareTo("newbm") == 0) start_from_newbm = 1; else { start_from_newbm = 0; cur_radius = Integer.parseInt(args[9].substring(4)); } block_width = Integer.parseInt(args[10]); if (args[11].compareTo("max") != 0) max_iteration = Integer.parseInt(args[11]); System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n"); System.out.println("[PEGASUS] Computing Radii/Diameter using block method. Current hop: " + cur_radius + ", edge_path: " + args[0] + ", encode: " + encode_bitmask + ", # reducers: " + nreducer + ", block width: " + block_width + ", max_iteration: " + max_iteration + "\n"); local_output_path = args[4] + number_nodes + "_tempblk"; N[0] = number_nodes; // Iteratively run Stage1 to Stage3. for (i = cur_radius; i <= max_iteration; i++) { JobClient.runJob(configStage1()); JobClient.runJob(configStage2()); JobClient.runJob(configStage3()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); final FileSystem fs = FileSystem.get(getConf()); // copy neighborhood information from HDFS to local disk, and read it! String new_path = local_output_path + "/" + i; fs.copyToLocalFile(output_path, new Path(new_path)); HadiResultInfo ri = HadiUtils.readNhoodOutput(new_path); N[i] = ri.nh; iter_counter++; System.out.println( "Nh(" + i + "):\t" + N[i] + "\tGuessed Radius(" + (i - 1) + "):\t" + ri.converged_nodes); // Stop when all radii converged. if (ri.changed_nodes == 0) {//if( i > 1 && N[i] == N[i-1] ) { System.out.println("All the bitstrings converged. Finishing..."); fs.delete(curbm_path); fs.delete(tempbm_path); fs.rename(nextbm_path, curbm_path); break; } // rotate directory fs.delete(curbm_path); fs.delete(tempbm_path); if (i < MAX_ITERATIONS - 1) fs.delete(output_path); fs.rename(nextbm_path, curbm_path); cur_radius++; } // Summarize Radius Information System.out.println("Calculating the effective diameter..."); JobClient.runJob(configStage4()); // Summarize Radius Information System.out.println("Summarizing radius information..."); JobClient.runJob(configStage5()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // print summary information if (i > max_iteration) System.out.println("Reached Max Iteartion " + max_iteration); System.out.println("Total Iteration = " + iter_counter + "."); System.out.println("Neighborhood Summary:"); for (int j = 0; j <= (i); j++) System.out.println("\tNh(" + (j) + "):\t" + N[j]); System.out.println("\n[PEGASUS] Radii and diameter computed."); System.out.println("[PEGASUS] Maximum diameter: " + (cur_radius - 1)); System.out.println("[PEGASUS] Average diameter: " + HadiUtils.average_diameter(N, cur_radius - 1)); System.out.println("[PEGASUS] 90% Effective diameter: " + HadiUtils.effective_diameter(N, cur_radius - 1)); System.out.println("[PEGASUS] Radii are saved in the HDFS " + radius_path.getName()); System.out.println("[PEGASUS] Radii summary is saved in the HDFS " + radius_summary_path.getName() + "\n"); return 0; }
From source file:pegasus.HadiResultInfo.java
License:Apache License
public int run(final String[] args) throws Exception { int i;/*from w w w . j av a 2 s . co m*/ int max_iteration = MAX_ITERATIONS; if (args.length != 12) { return printUsage(); } edge_path = new Path(args[0]); curbm_path = new Path(args[1]); tempbm_path = new Path(args[2]); nextbm_path = new Path(args[3]); output_path = new Path(args[4]); number_nodes = Integer.parseInt(args[5]); radius_path = new Path("hadi_radius"); radius_summary_path = new Path("hadi_radius_summary"); nreplication = Integer.parseInt(args[6]); nreducer = Integer.parseInt(args[7]); if (args[8].compareTo("enc") == 0) encode_bitmask = 1; if (args[9].compareTo("newbm") == 0) { start_from_newbm = 1; } else if (args[9].startsWith("cont")) { start_from_newbm = 0; cur_radius = Integer.parseInt(args[9].substring(4)); } if (args[10].compareTo("makesym") == 0) make_symmetric = 1; else make_symmetric = 0; if (args[11].compareTo("max") != 0) max_iteration = Integer.parseInt(args[11]); System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n"); System.out.println("[PEGASUS] Computing Radii/Diameter. Current hop: " + cur_radius + ", edge_path: " + args[0] + ", encode: " + encode_bitmask + ", # reducers: " + nreducer + ", makesym: " + make_symmetric + ", max_iteration: " + max_iteration + "\n"); local_output_path = args[4] + number_nodes + "_temp"; if (start_from_newbm == 1) { System.out.print("Generating initial bitstrings for " + number_nodes + " nodes "); // create bitmask generate command file, and copy to curbm_path gen_bitmask_cmd_file(number_nodes, nreplication, curbm_path); System.out.println(" done"); } else { System.out.println("Resuming from current hadi_curbm which contains up to N(" + (cur_radius - 1) + ")"); } N[0] = number_nodes; boolean eff_diameter_computed = false; // Iteratively run Stage1 to Stage3. for (i = cur_radius; i <= max_iteration; i++) { JobClient.runJob(configStage1(edge_type)); JobClient.runJob(configStage2()); JobClient.runJob(configStage3()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); final FileSystem fs = FileSystem.get(getConf()); // copy neighborhood information from HDFS to local disk, and read it! String new_path = local_output_path + "/" + i; fs.copyToLocalFile(output_path, new Path(new_path)); HadiResultInfo ri = HadiUtils.readNhoodOutput(new_path); N[i] = ri.nh; iter_counter++; System.out.println( "Nh(" + i + "):\t" + N[i] + "\tGuessed Radius(" + (i - 1) + "):\t" + ri.converged_nodes); // Stop when all radii converged. if (ri.changed_nodes == 0) {//if( i > 1 && N[i] == N[i-1] ) { System.out.println("All the bitstrings converged. Finishing..."); fs.delete(curbm_path); fs.delete(tempbm_path); fs.rename(nextbm_path, curbm_path); System.out.println("Calculating the effective diameter..."); JobClient.runJob(configStage4()); eff_diameter_computed = true; break; } // rotate directory. fs.delete(curbm_path); fs.delete(tempbm_path); if (i < MAX_ITERATIONS - 1) fs.delete(output_path); fs.rename(nextbm_path, curbm_path); cur_radius++; } if (eff_diameter_computed == false) { System.out.println("Calculating the effective diameter..."); JobClient.runJob(configStage4()); } // Summarize Radius Information System.out.println("Summarizing radius information..."); JobClient.runJob(configStage5()); FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); // print summary information if (i > max_iteration) System.out.println("Reached Max Iteartion " + max_iteration); System.out.println("Total Iteration = " + iter_counter + "."); System.out.println("Neighborhood Summary:"); for (int j = 0; j <= (i); j++) System.out.println("\tNh(" + (j) + "):\t" + N[j]); System.out.println("\n[PEGASUS] Radii and diameter computed."); System.out.println("[PEGASUS] Maximum diameter: " + (cur_radius - 1)); System.out.println("[PEGASUS] Average diameter: " + HadiUtils.average_diameter(N, cur_radius - 1)); System.out.println("[PEGASUS] 90% Effective diameter: " + HadiUtils.effective_diameter(N, cur_radius - 1)); System.out.println("[PEGASUS] Radii are saved in the HDFS " + radius_path.getName()); System.out.println("[PEGASUS] Radii summary is saved in the HDFS " + radius_summary_path.getName() + "\n"); return 0; }
From source file:pegasus.pegasus.PegasusUtils.java
License:Apache License
public static void copyToLocalFile(Configuration conf, Path hdfs_path, Path local_path) throws Exception { FileSystem fs = FileSystem.get(conf); // read the result fs.copyToLocalFile(hdfs_path, local_path); }
From source file:pegasus.pegasus.PegasusUtils.java
License:Apache License
public static double read_l1norm_result(Configuration conf) throws Exception { Path l2norm_output = new Path("l1norm_output"); FileSystem lfs = FileSystem.getLocal(conf); // read the result String local_output_path = "l1norm"; lfs.delete(new Path("l1norm/"), true); FileSystem fs = FileSystem.get(conf); fs.copyToLocalFile(l2norm_output, new Path(local_output_path)); double result = PegasusUtils.readLocaldirOneline(local_output_path); lfs.delete(new Path("l1norm/"), true);//FileUtil.fullyDelete( fs.getLocal(conf), new Path(local_output_path)); //FileUtil.fullyDelete( FileSystem.getLocal(conf), new Path("lanczos")); return result; }