Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:pegasus.rwr.RWRBlock.java

License:Apache License

public static Path SaxpyBlock(Configuration conf, int nreducer, Path py, Path px, Path out_path, double a,
        int block_width) throws Exception {
    //System.out.println("Running Saxpy: py=" + py.getName() + ", px=" + px.getName() + ",
    // a=" +a);//from w w w.  j  a  va2 s  . c o m

    String[] args = new String[5];
    args[0] = new String("" + nreducer);
    args[1] = new String(py.getName());
    args[2] = new String(px.getName());
    args[3] = new String("" + a);
    args[4] = new String("" + block_width);
    int saxpy_result = ToolRunner.run(conf, new SaxpyBlock(), args);

    //Path ret_path = null;
    FileSystem fs = FileSystem.get(conf);
    fs.delete(out_path, true);

    if (saxpy_result == 1)
        fs.rename(new Path("saxpy_output1"), out_path);
    else
        fs.rename(new Path("saxpy_output"), out_path);

    return out_path;
}

---

From source file:pegasus.rwr.RWRNaive.java

License:Apache License

public static Path Saxpy(Configuration conf, int nreducer, Path py, Path px, Path out_path, double a)
        throws Exception {
    //System.out.println("Running Saxpy: py=" + py.getName() + ", px=" + px.getName() + ",
    // a=" +a);//from w w w  .  j ava  2s  .  c o  m

    String[] args = new String[4];
    args[0] = new String("" + nreducer);
    args[1] = new String(py.getName());
    args[2] = new String(px.getName());
    args[3] = new String("" + a);
    int saxpy_result = ToolRunner.run(conf, new Saxpy(), args);

    //Path ret_path = null;
    FileSystem fs = FileSystem.get(conf);
    fs.delete(out_path, true);

    if (saxpy_result == 1)
        fs.rename(new Path("saxpy_output1"), out_path);
    else
        fs.rename(new Path("saxpy_output"), out_path);

    return out_path;
}

---

From source file:pegasus.rwr.RWRNaive.java

License:Apache License

public static Path SaxpyTextoutput(Configuration conf, int nreducer, Path py, Path px, Path out_path, double a)
        throws Exception {
    //System.out.println("Running Saxpy: py=" + py.getName() + ", px=" + px.getName() + ",
    // a=" +a);/*w  w w .j  a v  a 2 s . c om*/

    String[] args = new String[4];
    args[0] = new String("" + nreducer);
    args[1] = new String(py.getName());
    args[2] = new String(px.getName());
    args[3] = new String("" + a);
    int saxpy_result = ToolRunner.run(conf, new SaxpyTextoutput(), args);

    //Path ret_path = null;
    FileSystem fs = FileSystem.get(conf);
    fs.delete(out_path, true);

    if (saxpy_result == 1)
        fs.rename(new Path("saxpy_output1"), out_path);
    else
        fs.rename(new Path("saxpy_output"), out_path);

    return out_path;
}

---

From source file:pegasus.RWRBlock.java

License:Apache License

public static Path SaxpyBlock(Configuration conf, int nreducer, Path py, Path px, Path out_path, double a,
        int block_width) throws Exception {
    //System.out.println("Running Saxpy: py=" + py.getName() + ", px=" + px.getName() + ", a=" +a);

    String[] args = new String[5];
    args[0] = new String("" + nreducer);
    args[1] = new String(py.getName());
    args[2] = new String(px.getName());
    args[3] = new String("" + a);
    args[4] = new String("" + block_width);
    int saxpy_result = ToolRunner.run(conf, new SaxpyBlock(), args);

    //Path ret_path = null;
    FileSystem fs = FileSystem.get(conf);
    fs.delete(out_path, true);/*from w ww .  j a  va2  s .  com*/

    if (saxpy_result == 1)
        fs.rename(new Path("saxpy_output1"), out_path);
    else
        fs.rename(new Path("saxpy_output"), out_path);

    return out_path;
}

---

From source file:pegasus.RWRNaive.java

License:Apache License

public static Path Saxpy(Configuration conf, int nreducer, Path py, Path px, Path out_path, double a)
        throws Exception {
    //System.out.println("Running Saxpy: py=" + py.getName() + ", px=" + px.getName() + ", a=" +a);

    String[] args = new String[4];
    args[0] = new String("" + nreducer);
    args[1] = new String(py.getName());
    args[2] = new String(px.getName());
    args[3] = new String("" + a);
    int saxpy_result = ToolRunner.run(conf, new Saxpy(), args);

    //Path ret_path = null;
    FileSystem fs = FileSystem.get(conf);
    fs.delete(out_path, true);/*from   w w w.  j ava2 s  .  co  m*/

    if (saxpy_result == 1)
        fs.rename(new Path("saxpy_output1"), out_path);
    else
        fs.rename(new Path("saxpy_output"), out_path);

    return out_path;
}

---

From source file:pegasus.RWRNaive.java

License:Apache License

public static Path SaxpyTextoutput(Configuration conf, int nreducer, Path py, Path px, Path out_path, double a)
        throws Exception {
    //System.out.println("Running Saxpy: py=" + py.getName() + ", px=" + px.getName() + ", a=" +a);

    String[] args = new String[4];
    args[0] = new String("" + nreducer);
    args[1] = new String(py.getName());
    args[2] = new String(px.getName());
    args[3] = new String("" + a);
    int saxpy_result = ToolRunner.run(conf, new SaxpyTextoutput(), args);

    //Path ret_path = null;
    FileSystem fs = FileSystem.get(conf);
    fs.delete(out_path, true);//from   w w w. jav  a 2s  . c  om

    if (saxpy_result == 1)
        fs.rename(new Path("saxpy_output1"), out_path);
    else
        fs.rename(new Path("saxpy_output"), out_path);

    return out_path;
}

---

From source file:pegasus.SSSPResultInfo.java

License:Apache License

public int run(final String[] args) throws Exception {
    if (args.length != 8) {
        return printUsage();
    }//from w  w w  . jav a 2  s  .c o  m

    edge_path = new Path(args[0]);
    csr_path = new Path(args[1]);
    tempbm_path = new Path(args[2]);
    nextbm_path = new Path(args[3]);
    output_path = new Path(args[4]);
    summaryout_path = new Path("concmpt_summaryout");
    number_nodes = Integer.parseInt(args[5]);
    nreducers = Integer.parseInt(args[6]);

    if (args[7].compareTo("makesym") == 0)
        make_symmetric = 1;
    else
        make_symmetric = 0;

    System.out.println("\n-----===[PEGASUS: A Peta-Scale Graph Mining System]===-----\n");
    System.out.println(
            "[PEGASUS] Computing breadth-first search. Edge path = " + args[0] + ", Reducers = " + nreducers);

    local_output_path = args[4] + "_temp";

    JobClient.runJob(configStage1());
    FileSystem.get(getConf()).rename(csr_path, tempbm_path);

    // Iteratively calculate neighborhood function. 
    for (int i = cur_iter; i < MAX_ITERATIONS; i++) {
        cur_iter++;

        JobClient.runJob(configStage2());
        JobClient.runJob(configStage3());

        FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path));

        final FileSystem fs = FileSystem.get(getConf());

        // copy neighborhood information from HDFS to local disk, and read it!
        String new_path = local_output_path + "/" + i;
        fs.copyToLocalFile(output_path, new Path(new_path));
        SSSPResultInfo ri = readIterationOutput(new_path);

        white_nodes[iter_counter] = ri.white;
        gray_nodes[iter_counter] = ri.gray;
        black_nodes[iter_counter] = ri.black;

        iter_counter++;

        System.out.println(
                "Hop " + i + " : white = " + ri.white + ", gray = " + ri.gray + ", black = " + ri.black);

        // Stop when the minimum neighborhood doesn't change
        if (ri.gray == 0) {
            System.out.println("All vertices have been reached. Finishing...");
            //fs.delete(csr_path);
            fs.delete(tempbm_path);
            fs.delete(output_path);
            fs.rename(nextbm_path, output_path);

            break;
        }

        // rotate directory
        //fs.delete(csr_path);
        fs.delete(tempbm_path);
        fs.delete(output_path);
        fs.rename(nextbm_path, tempbm_path);

    }

    //   FileUtil.fullyDelete( FileSystem.getLocal(getConf()), new Path(local_output_path));

    // calculate summary information using an additional stage
    //System.out.println("Summarizing connected components information...");
    //JobClient.runJob(configStage4());

    // finishing.
    System.out.println("\n[PEGASUS] Breadth-first search computed.");
    System.out.println("[PEGASUS] Total Iteration = " + iter_counter);
    System.out.println(
            "[PEGASUS] BFS distance labels are saved in the HDFS sssp_output as\n\"vertex   vertex   number   distance\" format.\n");

    return 0;
}

---

From source file:ph.fingra.hadoop.mapred.common.HdfsFileUtil.java

License:Apache License

public static boolean deleteNBackupFile(String srcdir, String srcfile, int maxcount, String runday,
        final String dbfnameprefix) throws IOException {

    Configuration conf = new Configuration();

    FileSystem hdfs = FileSystem.get(conf);

    Path targetPath = null;/*from   www . ja  va  2  s  .  c  o  m*/
    Path rootPath = new Path(srcdir);
    Path sourcePath = new Path(srcfile);
    String target_day = "";
    String target_file = "";
    boolean success = false;

    // if not exist srcfile, stop backup and return true
    if (hdfs.exists(sourcePath) == false) {
        return true;
    }

    // make backup file name as yesterday date
    target_day = DateTimeUtil.addDays(runday, -1, "yyyyMMdd");
    target_file = srcfile + "-" + target_day;
    //System.out.println("target_file - " + target_file);
    targetPath = new Path(target_file);

    // delete backup file if exist same name, then rename source file to backup file
    if (hdfs.exists(new Path(target_file))) {
        hdfs.delete(targetPath, true);
    }
    success = hdfs.rename(sourcePath, targetPath);

    // get bakup file list
    PathFilter resultFileFilter = new PathFilter() {
        @Override
        public boolean accept(Path path) {
            return path.getName().startsWith(dbfnameprefix + "-");
        }
    };

    try {
        FileStatus[] status = hdfs.listStatus(rootPath, resultFileFilter);

        Path[] listedPaths = FileUtil.stat2Paths(status);

        // delete more than maximum number of backup files
        if (listedPaths.length > maxcount) {

            Comparator<Path> c = new Comparator<Path>() {
                public int compare(Path o1, Path o2) {
                    int ret = 0;
                    ret = o1.getName().compareTo(o2.getName());
                    return -(ret); // order by reverse of the period
                }
            };

            Arrays.sort(listedPaths, c);

            for (int i = maxcount; i < listedPaths.length; i++) {
                Path path = listedPaths[i];
                hdfs.delete(path, true);
            }
        }
    } catch (FileNotFoundException ignore) {
    } catch (InvalidInputException ignore) {
        ; // throw not FileNotFoundException but InvalidInputException
          // at Hadoop 1.x version
    }

    return success;
}

---

From source file:pic.MinMaxInfo.java

License:Apache License

public int run(final String[] args) throws Exception {
    if (args.length != 10) {
        return printUsage();
    }/* www .  ja  v a  2 s. co m*/

    int i;
    BufferedWriter out = new BufferedWriter(new FileWriter("./time_" + args[9] + ".log"));
    W_path = new Path(args[0] + "_" + args[9]);
    finalout_path = new Path("./pic_out_" + args[9]); //append taskid
    prediff_path = new Path("./pic_prediff_" + args[9]);
    thresbase = Double.parseDouble(args[1]);
    tempmv_path = new Path("./tempmv_path_" + args[9]);
    tempmv2_path = new Path("./tempmv2_path_" + args[9]);
    output_path = new Path(args[2] + "_" + args[9]);
    number_nodes = Integer.parseInt(args[3]);
    nreducers = Integer.parseInt(args[4]);
    niteration = Integer.parseInt(args[5]);

    if (args[6].compareTo("makesym") == 0)
        make_symmetric = 1;
    else
        make_symmetric = 0;

    int cur_iteration = 1;
    if (args[7].startsWith("cont"))
        cur_iteration = Integer.parseInt(args[7].substring(4));

    if (args[8].compareTo("start0") == 0)
        start1 = false;
    else
        start1 = true;

    FileSystem fs = FileSystem.get(getConf());
    if (fs.exists(output_path))
        fs.delete(output_path, true);
    if (fs.exists(tempmv_path))
        fs.delete(tempmv_path, true);
    if (fs.exists(tempmv2_path))
        fs.delete(tempmv2_path, true);

    local_output_path = args[2] + "_temp";

    converge_threshold = (thresbase / (double) number_nodes);

    System.out.println("\n-----===[PIC]===-----\n");

    long startTime = System.currentTimeMillis();
    //    if (cur_iteration == 1)
    //      gen_initial_vector(number_nodes, finalout_path, start1);

    // Run pagerank until converges.
    for (i = cur_iteration; i <= niteration; i++) {
        System.out.println("[PIC] Computing Pic. Max iteration = " + niteration + ", threshold = "
                + converge_threshold + ", cur_iteration=" + i + "\n");
        out.write(String.format("iter:%d\ttime:%d\n", i, (System.currentTimeMillis() - startTime) / 1000));

        JobClient.runJob(configStage1());
        JobClient.runJob(configStage2());
        RunningJob job = JobClient.runJob(configStage3());

        // The counter is newly created per every iteration.
        Counters c = job.getCounters();
        long changed = c.getCounter(PrCounters.CONVERGE_CHECK);
        System.out.println("[PIC] Iteration = " + i + ", changed reducer = " + changed);

        if (changed == 0) {
            System.out.println("Pic vector converged. Now preparing to finish...");
            fs.delete(finalout_path);
            fs.delete(tempmv_path);
            fs.delete(tempmv2_path);
            fs.rename(output_path, finalout_path); // src, dst
            break;
        }

        // rotate directory
        fs.delete(finalout_path);
        fs.delete(tempmv_path);
        fs.delete(tempmv2_path);
        fs.rename(output_path, finalout_path);
    }

    if (i == niteration) {
        System.out.println("Reached the max iteration. Now preparing to finish...");
    }

    /*
     * // find min/max of pageranks System.out.println("Finding minimum and maximum pageranks...");
     * JobClient.runJob(configStage3());
     * 
     * FileUtil.fullyDelete(FileSystem.getLocal(getConf()), new Path(local_output_path)); String
     * new_path = local_output_path + "/"; fs.copyToLocalFile(minmax_path, new Path(new_path));
     * 
     * MinMaxInfo mmi = readMinMax(new_path); System.out.println("min = " + mmi.min + ", max = " +
     * mmi.max);
     * 
     * // find distribution of pageranks JobClient.runJob(configStage4(mmi.min, mmi.max));
     */
    long endTime = System.currentTimeMillis();
    long elapse = (endTime - startTime) / 1000;
    long h = elapse / 3600;
    long m = (elapse - h * 3600) / 60;
    long s = elapse - h * 3600 - m * 60;
    fs.delete(tempmv2_path);
    System.out.println("\n[PIC] Pic computed.");
    System.out.println("[PIC] The final Pic are in the HDFS ./pic_out.");
    System.out.println(String.format("[PIC] total time runing PIC: %d secs (%d h %d m %d s)", elapse, h, m, s));
    System.out.println(String.format("[PIC] total number of iteration: %d", i));

    out.write(String.format("[PIC] total time runing PIC: %d secs (%d h %d m %d s)\n", elapse, h, m, s));
    out.write(String.format("[PIC] total number of iteration: %d", i));
    out.close();

    return 0;
}

---

From source file:simsql.runtime.SelectionOp.java

License:Apache License

public boolean run(RuntimeParameter params, boolean verbose) {

    // if not chained OR if we need to run a reducer, run normally.
    if (!isChained || runSelectionReducer) {
        return super.run(params, verbose);
    }/*from ww w  . j a va 2  s.c o m*/

    // otherwise, just rename.
    System.out.println("This is a chained map-only selection that does not need to run a job.");
    System.out.println("Renaming the input/output paths and copying the stats.");

    String[] inDirs = getNetworkOnInputSide().getPipelinedInputFiles();
    if (inDirs.length > 1) {
        throw new RuntimeException("Why is this selection using more than one input?");
    }

    // set the size.
    if (getDB() != null) {
        long inSize = getDB().getTableSize(getDB().getTableName(inDirs[0]));
        getDB().setTableSize(getDB().getTableName(getOutput()), inSize);
        getDB().setNumAtts(getDB().getTableName(getOutput()), getOutputAttNames().length);
    }

    // do the renaming
    try {
        Configuration conf = new Configuration();
        FileSystem dfs = FileSystem.get(conf);
        Path src = new Path(inDirs[0]);
        Path dest = new Path(getOutput());

        return dfs.rename(src, dest);
    } catch (Exception e) {
        throw new RuntimeException("Could not rename directories!" + e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »