List of usage examples for org.apache.hadoop.fs FileSystem copyFromLocalFile
public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException
From source file:org.opencb.opencga.storage.hadoop.variant.AbstractHadoopVariantStoragePipeline.java
License:Apache License
@Override public URI preLoad(URI input, URI output) throws StorageEngineException { boolean loadArch = options.getBoolean(HADOOP_LOAD_ARCHIVE); boolean loadVar = options.getBoolean(HADOOP_LOAD_VARIANT); if (!loadArch && !loadVar) { loadArch = true;//www. j a va2 s. c o m loadVar = true; options.put(HADOOP_LOAD_ARCHIVE, loadArch); options.put(HADOOP_LOAD_VARIANT, loadVar); } if (loadArch) { super.preLoad(input, output); if (needLoadFromHdfs() && !input.getScheme().equals("hdfs")) { if (!StringUtils.isEmpty(options.getString(OPENCGA_STORAGE_HADOOP_INTERMEDIATE_HDFS_DIRECTORY))) { output = URI.create(options.getString(OPENCGA_STORAGE_HADOOP_INTERMEDIATE_HDFS_DIRECTORY)); } if (output.getScheme() != null && !output.getScheme().equals("hdfs")) { throw new StorageEngineException("Output must be in HDFS"); } try { long startTime = System.currentTimeMillis(); // Configuration conf = getHadoopConfiguration(options); FileSystem fs = FileSystem.get(conf); org.apache.hadoop.fs.Path variantsOutputPath = new org.apache.hadoop.fs.Path( output.resolve(Paths.get(input.getPath()).getFileName().toString())); logger.info("Copy from {} to {}", new org.apache.hadoop.fs.Path(input).toUri(), variantsOutputPath.toUri()); fs.copyFromLocalFile(false, new org.apache.hadoop.fs.Path(input), variantsOutputPath); logger.info("Copied to hdfs in {}s", (System.currentTimeMillis() - startTime) / 1000.0); startTime = System.currentTimeMillis(); URI fileInput = URI.create(VariantReaderUtils.getMetaFromTransformedFile(input.toString())); org.apache.hadoop.fs.Path fileOutputPath = new org.apache.hadoop.fs.Path( output.resolve(Paths.get(fileInput.getPath()).getFileName().toString())); logger.info("Copy from {} to {}", new org.apache.hadoop.fs.Path(fileInput).toUri(), fileOutputPath.toUri()); fs.copyFromLocalFile(false, new org.apache.hadoop.fs.Path(fileInput), fileOutputPath); logger.info("Copied to hdfs in {}s", (System.currentTimeMillis() - startTime) / 1000.0); input = variantsOutputPath.toUri(); } catch (IOException e) { e.printStackTrace(); } } } try { ArchiveDriver.createArchiveTableIfNeeded(dbAdaptor.getGenomeHelper(), archiveTableCredentials.getTable(), dbAdaptor.getConnection()); } catch (IOException e) { throw new StorageHadoopException("Issue creating table " + archiveTableCredentials.getTable(), e); } try { VariantTableDriver.createVariantTableIfNeeded(dbAdaptor.getGenomeHelper(), variantsTableCredentials.getTable(), dbAdaptor.getConnection()); } catch (IOException e) { throw new StorageHadoopException("Issue creating table " + variantsTableCredentials.getTable(), e); } if (loadVar) { preMerge(input); } return input; }
From source file:pegasus.ResultInfo.java
License:Apache License
public void gen_one_file(int number_nodes, int start_pos, int len, Path curbm_path) throws IOException { // generate a temporary local bitmask command file int i, j = 0, threshold = 0, count = 0; String file_name = "component_vector.temp." + start_pos; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); out.write("# component vector file - hadoop\n"); out.write("# number of nodes in graph = " + number_nodes + ", start_pos=" + start_pos + "\n"); System.out.println("creating bitmask generation cmd for node " + start_pos + " ~ " + (start_pos + len)); for (i = 0; i < number_nodes; i++) { int cur_nodeid = start_pos + i; out.write(cur_nodeid + "\tmsi" + cur_nodeid + "\n"); if (++j > len / 10) { System.out.print("."); j = 0;/*from ww w . jav a 2s .c o m*/ } if (++count >= len) break; } out.close(); System.out.println(""); // copy it to curbm_path, and delete temporary local file. final FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), new Path(curbm_path.toString() + "/" + file_name)); }
From source file:pegasus.ConCmptIVGen.java
License:Apache License
public void gen_cmd_file(int num_nodes, int num_reducers, Path input_path) throws IOException { // generate a temporary local command file int i;/*from w w w. ja v a 2 s .c o m*/ String file_name = "component_iv.temp"; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); out.write("# component vector file from ConCmptIVGen\n"); out.write("# number of nodes in graph = " + number_nodes + "\n"); System.out.print("creating initial vector generation cmd..."); int step = num_nodes / num_reducers; int start_node, end_node; for (i = 0; i < num_reducers; i++) { start_node = i * step; if (i < num_reducers - 1) end_node = step * (i + 1) - 1; else end_node = num_nodes - 1; out.write(i + "\t" + start_node + "\t" + end_node + "\n"); } out.close(); System.out.println("done."); // copy it to curbm_path, and delete the temporary local file. final FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), new Path(input_path.toString() + "/" + file_name)); }
From source file:pegasus.hadi.Hadi.java
License:Apache License
public void gen_bitmask_cmd_file(int number_nodes, int start_pos, int len, int nreplication, Path curbm_path) throws IOException { // generate a temporary local bitmask command file int i, j = 0, threshold = 0, count = 0; String file_name = "bitmask_cmd.hadi." + number_nodes + "." + start_pos; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); out.write("# bitmask command file for HADI\n"); out.write("# number of nodes in graph = " + number_nodes + ", start_pos=" + start_pos + "\n"); System.out.println("creating bitmask generation cmd for node " + start_pos + " ~ " + (start_pos + len)); for (i = 0; i < number_nodes; i++) { int cur_nodeid = start_pos + i; out.write(cur_nodeid + "\tc\n"); if (++j > len / 10) { System.out.print("."); j = 0;//from w w w . j a v a 2s . com } if (++count >= len) break; } out.close(); System.out.println(""); // copy it to curbm_path, and delete temporary local file. final FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), new Path(curbm_path.toString() + "/" + file_name)); }
From source file:pegasus.hadi.HadiIVGen.java
License:Apache License
public void gen_cmd_file(int num_nodes, int num_reducers, int nreplication, Path input_path) throws IOException { // generate a temporary local command file int i;//from w w w. j av a 2 s .com String file_name = "hadi_iv.temp"; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); out.write("# component vector file from HadiIVGen\n"); out.write("# number of nodes in graph = " + number_nodes + "\n"); System.out.print("creating initial vector generation cmd..."); int step = num_nodes / num_reducers; int start_node, end_node; for (i = 0; i < num_reducers; i++) { start_node = i * step; if (i < num_reducers - 1) end_node = step * (i + 1) - 1; else end_node = num_nodes - 1; out.write(i + "\t" + start_node + "\t" + end_node + "\n"); } out.close(); System.out.println("done."); // copy it to curbm_path, and delete temporary local file. final FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), new Path(input_path.toString() + "/" + file_name)); }
From source file:pegasus.pagerank.PagerankInitVector.java
License:Apache License
public void gen_initial_pagerank_file(int number_nodes, int nmachines, Path initial_input_path) throws IOException { int gap = number_nodes / nmachines; int i;/*from www. j ava 2s .com*/ int start_node, end_node; String file_name = "pagerank.initial_rank.temp"; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); out.write("# number of nodes in graph = " + number_nodes + "\n"); System.out.println("creating initial pagerank (total nodes = " + number_nodes + ")"); for (i = 0; i < nmachines; i++) { start_node = i * gap; if (i < nmachines - 1) end_node = (i + 1) * gap - 1; else end_node = number_nodes - 1; out.write("" + i + "\t" + start_node + "\t" + end_node + "\n"); } out.close(); // copy it to initial_input_path, and delete the temporary local file. final FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), new Path(initial_input_path.toString() + "/" + file_name)); }
From source file:pegasus.pagerank.PagerankNaive.java
License:Apache License
public void gen_initial_vector(int number_nodes, Path vector_path) throws IOException { int i, j = 0; int milestone = number_nodes / 10; String file_name = "pagerank_init_vector.temp"; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); System.out.print("Creating initial pagerank vectors..."); double initial_rank = 1.0 / (double) number_nodes; for (i = 0; i < number_nodes; i++) { out.write(i + "\tv" + initial_rank + "\n"); if (++j > milestone) { System.out.print("."); j = 0;// ww w . j a va 2 s .c o m } } out.close(); System.out.println(""); // copy it to curbm_path, and delete temporary local file. Path destination = new Path(vector_path.toString() + "/" + file_name); final FileSystem fs = FileSystem.get(destination.toUri(), getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), destination); }
From source file:pic.MinMaxInfo.java
License:Apache License
public void gen_initial_vector(int number_nodes, Path vector_path, boolean start1) throws IOException { int i, j = 0; int milestone = number_nodes / 10; String file_name = "pic_init_vector.temp"; FileWriter file = new FileWriter(file_name); BufferedWriter out = new BufferedWriter(file); System.out.print("Creating initial pic vectors..."); double initial_rank; if (!start1)/*from www . j a v a2 s. c o m*/ initial_rank = 1.0 / (double) number_nodes; else initial_rank = 1.0 / (double) (number_nodes - 1); // TODO: matlab label starts from 1 i = start1 ? 1 : 0; for (; i < number_nodes; i++) { out.write(i + "\tv" + initial_rank + "\n"); if (++j > milestone) { System.out.print("."); j = 0; } } out.close(); System.out.println(""); // copy it to curbm_path, and delete temporary local file. final FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path("./" + file_name), new Path(vector_path.toString() + "/" + file_name)); }