Example usage for org.apache.hadoop.mapred Counters log

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred Counters log.

Prototype

public void log(Logger log) 

Source Link

Document

Logs the current counter values.

Usage

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CiteRank.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 4) {
        return usage("<input path> <output path> <iterations> <tolerance>");
    }/*from   ww w  .j a v  a 2  s  .c o m*/

    FileSystem fs = FileSystem.get(getConf());
    String inpath = args[0];
    String outpath = args[1];
    String input = outpath + File.separator + CiteRankTool.PREVIOUS_RANKS;
    String output = outpath + File.separator + CiteRankTool.CURRENT_RANKS;
    int iterationLimit = Integer.parseInt(args[2]);
    final double toleranceArg = Double.parseDouble(args[3]);

    //reset the counters. This is an abuse of a singleton and should be replaced
    //with instance values as soon as possible
    resetCounters();
    //clean the data up
    exec("Data cleanup", new CheckingData(), inpath, output);
    //count the data
    String countFile = outpath + File.separator + CiteRankTool.COUNT;
    exec("Page count", new CountPages(), output, countFile);
    String count = read(fs, countFile);
    exec("InitializeRanks", new InitializeRanks(), output, input, count);
    int iterations = 0;
    String sortedRanksDir = outpath + File.separator + CiteRankTool.SORTED_RANKS;
    while (iterations < iterationLimit) {
        String danglingFile = outpath + File.separator + CiteRankTool.DANGLING;
        exec("DanglingPages", new DanglingPages(), input, danglingFile);
        String dangling = read(fs, danglingFile);
        exec("UpdateRanks", new UpdateRanks(), input, output, count, dangling);
        overwrite(fs, new Path(output), new Path(input));
        if ((iterations > CiteRankTool.CHECK_CONVERGENCE_FREQUENCY)
                && (iterations % CiteRankTool.CHECK_CONVERGENCE_FREQUENCY == 0)) {
            String convergenceFile = outpath + File.separator + CiteRankTool.CONVERGENCE;
            exec("CheckConvergence", new CheckConvergence(), input, convergenceFile);
            double tolerance = Double.parseDouble(read(fs, convergenceFile));

            if (tolerance <= toleranceArg) {
                //exit the loop when we are happy
                break;
            }
        }

        if (HTML_TABLE) {
            exec("SortRanks #" + (iterations + 1), new SortRanks(), input, sortedRanksDir);
            Path htmlsource = new Path(sortedRanksDir + File.separator + "part-00000");
            BufferedReader in = null;
            PrintWriter out = null;
            Path outfile = new Path(sortedRanksDir + "-" + iterations + ".dat");
            try {
                in = new BufferedReader(new InputStreamReader(fs.open(htmlsource)));
                out = new PrintWriter(fs.create(outfile).getWrappedStream());
                for (int j = 0; j < HTML_TABLE_ROWS; j++) {
                    StringTokenizer st = new StringTokenizer(in.readLine());
                    out.write(st.nextToken() + "\n");
                }
            } finally {
                close(in);
                close(out);
            }
        }

        iterations++;
    }

    exec("SortRanks", new SortRanks(), input, sortedRanksDir);

    if (HTML_TABLE) {
        exec("HTMLTable", new HTMLTable(), outpath, Integer.toString(HTML_TABLE_ROWS),
                Integer.toString(iterations));
    }

    Counters totals = getCounters();
    totals.log(LOG);
    return 0;
}

---