Example usage for org.apache.hadoop.mapred Counters getCounter

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred Counters getCounter.

Prototype

public synchronized long getCounter(Enum<?> key) 

Source Link

Document

Returns current value of the specified counter, or 0 if the counter does not exist.

Usage

From source file:WikipediaForwardIndexBuilder.java

License:Apache License

@SuppressWarnings("static-access")
@Override/*w  w  w.j a  v  a  2  s .  c o m*/
public int run(String[] args) throws Exception {
    Options options = new Options();
    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input").create(INPUT_OPTION));
    options.addOption(
            OptionBuilder.withArgName("path").hasArg().withDescription("index file").create(INDEX_FILE_OPTION));
    options.addOption(OptionBuilder.withArgName("en|sv|de|cs|es|zh|ar|tr").hasArg()
            .withDescription("two-letter language code").create(LANGUAGE_OPTION));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        return -1;
    }

    if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(INDEX_FILE_OPTION)) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(this.getClass().getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        return -1;
    }

    Path inputPath = new Path(cmdline.getOptionValue(INPUT_OPTION));
    String indexFile = cmdline.getOptionValue(INDEX_FILE_OPTION);

    String tmpPath = "tmp-" + WikipediaForwardIndexBuilder.class.getSimpleName() + "-" + RANDOM.nextInt(10000);

    if (!inputPath.isAbsolute()) {
        System.err.println("Error: " + INPUT_OPTION + " must be an absolute path!");
        return -1;
    }

    String language = null;
    if (cmdline.hasOption(LANGUAGE_OPTION)) {
        language = cmdline.getOptionValue(LANGUAGE_OPTION);
        if (language.length() != 2) {
            System.err.println("Error: \"" + language + "\" unknown language!");
            return -1;
        }
    }

    JobConf conf = new JobConf(getConf(), WikipediaForwardIndexBuilder.class);
    FileSystem fs = FileSystem.get(conf);

    LOG.info("Tool name: " + this.getClass().getName());
    LOG.info(" - input path: " + inputPath);
    LOG.info(" - index file: " + indexFile);
    LOG.info(" - language: " + language);
    LOG.info("Note: This tool only works on block-compressed SequenceFiles!");

    conf.setJobName(String.format("BuildWikipediaForwardIndex[%s: %s, %s: %s, %s: %s]", INPUT_OPTION, inputPath,
            INDEX_FILE_OPTION, indexFile, LANGUAGE_OPTION, language));

    conf.setNumReduceTasks(1);

    FileInputFormat.setInputPaths(conf, inputPath);
    FileOutputFormat.setOutputPath(conf, new Path(tmpPath));
    FileOutputFormat.setCompressOutput(conf, false);

    if (language != null) {
        conf.set("wiki.language", language);
    }

    conf.setInputFormat(NoSplitSequenceFileInputFormat.class);
    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapRunnerClass(MyMapRunner.class);
    conf.setReducerClass(IdentityReducer.class);

    // Delete the output directory if it exists already.
    fs.delete(new Path(tmpPath), true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int blocks = (int) counters.getCounter(Blocks.Total);

    LOG.info("number of blocks: " + blocks);

    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(tmpPath + "/part-00000")));
    FSDataOutputStream out = fs.create(new Path(indexFile), true);

    out.writeUTF(edu.umd.cloud9.collection.wikipedia.WikipediaForwardIndex.class.getCanonicalName());
    out.writeUTF(inputPath.toString());
    out.writeInt(blocks);

    int cnt = 0;
    Text line = new Text();
    while (reader.readLine(line) > 0) {
        String[] arr = line.toString().split("\\s+");

        int docno = Integer.parseInt(arr[0]);
        int offset = Integer.parseInt(arr[1]);
        short fileno = Short.parseShort(arr[2]);

        out.writeInt(docno);
        out.writeInt(offset);
        out.writeShort(fileno);

        cnt++;

        if (cnt % 100000 == 0) {
            LOG.info(cnt + " blocks written");
        }
    }

    reader.close();
    out.close();

    if (cnt != blocks) {
        throw new RuntimeException("Error: mismatch in block count!");
    }

    // Clean up.
    fs.delete(new Path(tmpPath), true);

    return 0;
}

---

From source file:cascading.flow.hadoop.HadoopStepStats.java

License:Open Source License

@Override
public long getCounterValue(Enum counter) {
    try {//from w  w  w.j a  v a 2  s. c  o m
        RunningJob runningJob = getRunningJob();

        if (runningJob == null)
            return 0;

        Counters counters = runningJob.getCounters();

        if (counters == null)
            return 0;

        return counters.getCounter(counter);
    } catch (IOException exception) {
        throw new FlowException("unable to get remote counter values");
    }
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void execute() throws Exception {
    writeFiles(true, true, true);/* w  w w  .j  a v  a2 s. c  om*/

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String[] { "--threshold=0.015", "--max-file-blocks=1", "--verbose",

            "--regex=.+/other",
            "--replacement=${crush.timestamp}-${crush.task.num}-middle-${crush.file.num}-tail",
            "--input-format=" + SequenceFileInputFormat.class.getName(),
            "--output-format=" + TextOutputFormat.class.getName(),

            "--regex=.+/dir",
            "--replacement=secondregex-${crush.timestamp}-${crush.task.num}-${crush.file.num}",
            "--input-format=" + TextInputFormat.class.getName(),
            "--output-format=" + SequenceFileOutputFormat.class.getName(),

            "--regex=.+/dir/([^/]+/)*(.+)",
            "--replacement=thirdregex-$2-${crush.timestamp}-${crush.task.num}-${crush.file.num}",
            "--input-format=" + SequenceFileInputFormat.class.getName(),
            "--output-format=" + SequenceFileOutputFormat.class.getName(),

            "--regex=.+/text",
            "--replacement=fourthregex-${crush.task.num}-${crush.timestamp}-${crush.file.num}",
            "--input-format=" + TextInputFormat.class.getName(),
            "--output-format=" + TextOutputFormat.class.getName(),

            /*
             * This is the default regex and replacement, which we add last so we can exercise the default logic.
             */
            "--regex=.+", "--replacement=crushed_file-${crush.timestamp}-${crush.task.num}-${crush.file.num}",
            "--input-format=" + SequenceFileInputFormat.class.getName(),
            "--output-format=" + TextOutputFormat.class.getName(),

            "--compress=" + CustomCompressionCodec.class.getName(),

            "in", "out", "20101116153015" });

    /*
     * Crushed files.
     */
    verifyOutput(homeDir + "/out/dir", "secondregex-20101116153015-*-*", Format.TEXT, Format.SEQUENCE,
            customCodec, "file10", "file11", "file12", "file13");
    verifyOutput(homeDir + "/out/dir/subdir", "thirdregex-subdir-20101116153015-*-*", Format.SEQUENCE,
            Format.SEQUENCE, customCodec, "file20", "file21", "file22", "file23", "file24");
    verifyOutput(homeDir + "/out/dir/subdir/subsubdir", "thirdregex-subsubdir-20101116153015-*-*",
            Format.SEQUENCE, Format.SEQUENCE, customCodec, "file30", "file31", "file32", "file33", "file34");
    verifyOutput(homeDir + "/out/dir/subdir/other", "20101116153015-*-middle-*-tail", Format.SEQUENCE,
            Format.TEXT, customCodec, "file40", "file41", "file42", "file43");
    verifyOutput(homeDir + "/out/dir/other", "20101116153015-*-middle-*-tail", Format.SEQUENCE, Format.TEXT,
            customCodec, "file50", "file51", "file52", "file53", "file54", "file55");
    verifyOutput(homeDir + "/out/text", "fourthregex-*-20101116153015-*", Format.TEXT, Format.TEXT, customCodec,
            "file60", "file61", "file62", "file63");
    verifyOutput(homeDir + "/out", "crushed_file-20101116153015-*-*", Format.SEQUENCE, Format.TEXT, customCodec,
            "file70", "file71", "file72");

    /*
     * Skipped files should have been moved to the output dir.
     */
    verifyOutput(homeDir + "/out/dir/skipped", "file80", Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
            "file80");
    verifyHugeFile(homeDir + "/out/huge", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);

    /*
     * Crush input files should remain in the input dir.
     */
    for (String file : new String[] { "file10", "file11", "file12", "file13" }) {
        verifyOutput(homeDir + "/in/dir", file, Format.TEXT, Format.TEXT, null, file);
    }

    for (String file : new String[] { "file20", "file21", "file22", "file23", "file24" }) {
        verifyOutput(homeDir + "/in/dir/subdir", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file30", "file31", "file32", "file33", "file34" }) {
        verifyOutput(homeDir + "/in/dir/subdir/subsubdir", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
                file);
    }

    for (String file : new String[] { "file40", "file41", "file42", "file43" }) {
        verifyOutput(homeDir + "/in/dir/subdir/other", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
                file);
    }

    for (String file : new String[] { "file50", "file51", "file52", "file53", "file54", "file55" }) {
        verifyOutput(homeDir + "/in/dir/other", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file60", "file61", "file62", "file63" }) {
        verifyOutput(homeDir + "/in/text", file, Format.TEXT, Format.TEXT, null, file);
    }

    for (String file : new String[] { "file70", "file71", "file72" }) {
        verifyOutput(homeDir + "/in", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(8L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(7L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(1L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(33L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(31L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(2L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(31L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(1256L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeHugeFilesOnly() throws Exception {
    writeHugeFile("in/huge", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    writeHugeFile("in/foo/huge1", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    writeHugeFile("in/foo/huge2", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);

    Crush crush = new Crush();

    ToolRunner.run(job, crush,//ww w  .  ja  v  a2 s  .  com
            new String[] { "--threshold=0.015", "--max-file-blocks=1", "--verbose",
                    "--compress=" + CustomCompressionCodec.class.getName(),

                    "in", "out", "20101116153015" });

    /*
     * Skipped files should have been moved to the output dir.
     */
    verifyHugeFile(homeDir + "/out/huge", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    verifyHugeFile(homeDir + "/out/foo/huge1", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    verifyHugeFile(homeDir + "/out/foo/huge2", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(3L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(3L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(0L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(0L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeNoFiles() throws Exception {
    FileSystem.get(job).mkdirs(new Path("in/foo"));
    FileSystem.get(job).mkdirs(new Path("in/hello/world"));

    Crush crush = new Crush();

    ToolRunner.run(job, crush,/*  w w w .ja v  a 2s .  com*/
            new String[] { "--threshold=0.015", "--max-file-blocks=1", "--verbose",
                    "--compress=" + CustomCompressionCodec.class.getName(),

                    "in", "out", "20101116153015" });

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(4L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(4L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(0L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(0L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(0L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeClone() throws Exception {
    writeFiles(true, true, true);/*from   w  w w  .  j  av a 2 s.  c o  m*/

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String[] { "--threshold=0.015", "--max-file-blocks=1", "--verbose",

            "--regex=.+/other",
            "--replacement=${crush.timestamp}-${crush.task.num}-middle-${crush.file.num}-tail",
            "--input-format=sequence", "--output-format=text",

            "--regex=.+/dir",
            "--replacement=secondregex-${crush.timestamp}-${crush.task.num}-${crush.file.num}",
            "--input-format=text", "--output-format=" + SequenceFileOutputFormat.class.getName(),

            "--regex=.+/dir/([^/]+/)*(.+)",
            "--replacement=thirdregex-$2-${crush.timestamp}-${crush.task.num}-${crush.file.num}",
            "--input-format=" + SequenceFileInputFormat.class.getName(), "--output-format=sequence",

            "--regex=.+/text",
            "--replacement=fourthregex-${crush.task.num}-${crush.timestamp}-${crush.file.num}",
            "--input-format=" + TextInputFormat.class.getName(),
            "--output-format=" + TextOutputFormat.class.getName(),

            /*
             * This is the default regex and replacement, which we add last so we can exercise the default logic.
             */
            "--regex=.+", "--replacement=crushed_file-${crush.timestamp}-${crush.task.num}-${crush.file.num}",
            "--input-format=" + SequenceFileInputFormat.class.getName(),
            "--output-format=" + TextOutputFormat.class.getName(),

            "--compress=" + CustomCompressionCodec.class.getName(),

            "--clone",

            "in", "out", "20101116153015" });

    /*
     * Crushed files.
     */
    verifyOutput(homeDir + "/in/dir", "secondregex-20101116153015-*-*", Format.TEXT, Format.SEQUENCE,
            customCodec, "file10", "file11", "file12", "file13");
    verifyOutput(homeDir + "/in/dir/subdir", "thirdregex-subdir-20101116153015-*-*", Format.SEQUENCE,
            Format.SEQUENCE, customCodec, "file20", "file21", "file22", "file23", "file24");
    verifyOutput(homeDir + "/in/dir/subdir/subsubdir", "thirdregex-subsubdir-20101116153015-*-*",
            Format.SEQUENCE, Format.SEQUENCE, customCodec, "file30", "file31", "file32", "file33", "file34");
    verifyOutput(homeDir + "/in/dir/subdir/other", "20101116153015-*-middle-*-tail", Format.SEQUENCE,
            Format.TEXT, customCodec, "file40", "file41", "file42", "file43");
    verifyOutput(homeDir + "/in/dir/other", "20101116153015-*-middle-*-tail", Format.SEQUENCE, Format.TEXT,
            customCodec, "file50", "file51", "file52", "file53", "file54", "file55");
    verifyOutput(homeDir + "/in/text", "fourthregex-*-20101116153015-*", Format.TEXT, Format.TEXT, customCodec,
            "file60", "file61", "file62", "file63");
    verifyOutput(homeDir + "/in", "crushed_file-20101116153015-*-*", Format.SEQUENCE, Format.TEXT, customCodec,
            "file70", "file71", "file72");

    /*
     * Skipped files should remain in the input dir.
     */
    verifyOutput(homeDir + "/in/dir/skipped", "file80", Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
            "file80");
    verifyHugeFile(homeDir + "/in/huge", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);

    /*
     * Crush input files should be moved to the clone dir.
     */
    for (String file : new String[] { "file10", "file11", "file12", "file13" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir", file, Format.TEXT, Format.TEXT, null, file);
    }

    for (String file : new String[] { "file20", "file21", "file22", "file23", "file24" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir", file, Format.SEQUENCE, Format.SEQUENCE,
                defaultCodec, file);
    }

    for (String file : new String[] { "file30", "file31", "file32", "file33", "file34" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir/subsubdir", file, Format.SEQUENCE,
                Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file40", "file41", "file42", "file43" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir/other", file, Format.SEQUENCE,
                Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file50", "file51", "file52", "file53", "file54", "file55" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/other", file, Format.SEQUENCE, Format.SEQUENCE,
                defaultCodec, file);
    }

    for (String file : new String[] { "file60", "file61", "file62", "file63" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/text", file, Format.TEXT, Format.TEXT, null, file);
    }

    for (String file : new String[] { "file70", "file71", "file72" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
                file);
    }

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(8L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(7L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(1L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(33L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(31L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(2L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(31L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(1256L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeCloneHugeFilesOnly() throws Exception {
    writeHugeFile("in/huge", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    writeHugeFile("in/foo/huge1", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    writeHugeFile("in/foo/huge2", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);

    Crush crush = new Crush();

    ToolRunner.run(job, crush,//from www  .j  a  v  a2 s .  com
            new String[] { "--threshold=0.015", "--max-file-blocks=1", "--verbose",
                    "--compress=" + CustomCompressionCodec.class.getName(), "--clone",

                    "in", "out", "20101116153015" });

    /*
     * Skipped files should remain in the input dir.
     */
    verifyHugeFile(homeDir + "/in/huge", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    verifyHugeFile(homeDir + "/in/foo/huge1", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);
    verifyHugeFile(homeDir + "/in/foo/huge2", (long) (((float) 0.015) * 1024 * 1024 * 64) + 1);

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(3L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(3L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(0L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(0L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeCloneNoFiles() throws Exception {
    FileSystem.get(job).mkdirs(new Path("in/foo"));
    FileSystem.get(job).mkdirs(new Path("in/hello/world"));

    Crush crush = new Crush();

    ToolRunner.run(job, crush,/*from w  ww.j  a v  a  2  s. c  om*/
            new String[] { "--threshold=0.015", "--max-file-blocks=1", "--verbose",
                    "--compress=" + CustomCompressionCodec.class.getName(), "--clone",

                    "in", "out", "20101116153015" });

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(4L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(4L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(0L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(0L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(0L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(0L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeBackwardsCompatibleText() throws Exception {
    writeFiles(false, true, false);//from w  w  w .  ja v  a  2s .  c o m

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String[] { "in", "out", "2", "TEXT" });

    /*
     * Crushed files.
     */
    verifyOutput(homeDir + "/in/dir", "crushed_file-*-*-*", Format.TEXT, Format.TEXT, defaultCodec, "file10",
            "file11", "file12", "file13");
    verifyOutput(homeDir + "/in/text", "crushed_file-*-*-*", Format.TEXT, Format.TEXT, defaultCodec, "file60",
            "file61", "file62", "file63");

    /*
     * Crush input files should be moved to the clone dir.
     */
    for (String file : new String[] { "file10", "file11", "file12", "file13" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir", file, Format.TEXT, Format.TEXT, null, file);
    }

    for (String file : new String[] { "file60", "file61", "file62", "file63" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/text", file, Format.TEXT, Format.TEXT, null, file);
    }

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(3L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(1L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(8L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(8L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(0L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(8L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(292L));
}

---

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Test
public void executeBackwardsCompatibleSequence() throws Exception {
    writeFiles(true, false, false);/*from w ww .ja v  a2 s  .co  m*/

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String[] { "in", "out", "2" });

    /*
     * Crushed files.
     */
    verifyOutput(homeDir + "/in/dir/subdir", "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE,
            defaultCodec, "file20", "file21", "file22", "file23", "file24");
    verifyOutput(homeDir + "/in/dir/subdir/subsubdir", "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE,
            defaultCodec, "file30", "file31", "file32", "file33", "file34");
    verifyOutput(homeDir + "/in/dir/subdir/other", "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE,
            defaultCodec, "file40", "file41", "file42", "file43");
    verifyOutput(homeDir + "/in/dir/other", "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE,
            defaultCodec, "file50", "file51", "file52", "file53", "file54", "file55");
    verifyOutput(homeDir + "/in", "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
            "file70", "file71", "file72");

    /*
     * Skipped files should be copied to the output.
     */
    verifyOutput(homeDir + "/in/dir/skipped", "file80", Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
            "file80");

    /*
     * Crush input files should be moved to the clone dir.
     */
    for (String file : new String[] { "file20", "file21", "file22", "file23", "file24" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir", file, Format.SEQUENCE, Format.SEQUENCE,
                defaultCodec, file);
    }

    for (String file : new String[] { "file30", "file31", "file32", "file33", "file34" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir/subsubdir", file, Format.SEQUENCE,
                Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file40", "file41", "file42", "file43" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir/other", file, Format.SEQUENCE,
                Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file50", "file51", "file52", "file53", "file54", "file55" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in/dir/other", file, Format.SEQUENCE, Format.SEQUENCE,
                defaultCodec, file);
    }

    for (String file : new String[] { "file70", "file71", "file72" }) {
        verifyOutput(homeDir + "/out" + homeDir + "/in", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec,
                file);
    }

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND), equalTo(7L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE), equalTo(5L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED), equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND), equalTo(24L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_ELIGIBLE), equalTo(23L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_SKIPPED), equalTo(1L));

    assertThat(jobCounters.getCounter(ReducerCounter.FILES_CRUSHED), equalTo(23L));
    assertThat(jobCounters.getCounter(ReducerCounter.RECORDS_CRUSHED), equalTo(964L));
}

---

• «
• 1
• 2
• 3
• »