Example usage for org.apache.hadoop.mapred WordCount WordCount

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred WordCount WordCount.

Prototype

WordCount

Source Link

Usage

From source file:pl.edu.icm.coansys.heeut.TestMapReduce.java

License:Apache License

@Test(timeout = 1800000)
public void testWordCount() throws Exception {

    String prefix = getCurrentDateAppended("wordcount");
    String inputDirName = prefix + "-input";
    String outputDirName = prefix + "-output";

    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();
    Path inputDir = new Path(inputDirName);
    Path qualifiedInputDir = dfs.makeQualified(inputDir);

    dfs.copyFromLocalFile(new Path("src/test/resource/input/wordcount/apache_projects.dat"), qualifiedInputDir);
    ToolRunner.run(UTIL.getConfiguration(), new WordCount(), new String[] { inputDirName, outputDirName });

    InputStream contentStream = dfs.open(new Path(outputDirName + "/part-00000"));
    BufferedReader contentReader = new BufferedReader(new InputStreamReader(contentStream));
    Assert.assertEquals("Apache\t3", contentReader.readLine());
    Assert.assertEquals("HBase\t1", contentReader.readLine());
    Assert.assertEquals("Hadoop\t1", contentReader.readLine());
    Assert.assertEquals("Pig\t1", contentReader.readLine());

    Assert.assertNull(contentReader.readLine());
    contentReader.close();/*from ww w.  j  a va2s . com*/
}

---

From source file:pl.edu.icm.coansys.heeut.TestMapReduce.java

License:Apache License

@Test(timeout = 1800000)
public void testWordCountDiff() throws Exception {

    String prefix = getCurrentDateAppended("wordcount");
    String inputDirName = prefix + "-input";
    String outputDirName = prefix + "-output";

    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();
    Path inputDir = new Path(inputDirName);
    Path qualifiedInputDir = dfs.makeQualified(inputDir);

    String inputFileName = "src/test/resource/input/wordcount/apache_projects.dat";
    dfs.copyFromLocalFile(new Path(inputFileName), qualifiedInputDir);
    ToolRunner.run(UTIL.getConfiguration(), new WordCount(), new String[] { inputDirName, outputDirName });

    Path outputDir = new Path(outputDirName);
    Path qualifiedOutputDir = dfs.makeQualified(outputDir);

    String localOutputDir = "src/test/resource/output/wordcount/" + prefix;
    dfs.copyToLocalFile(qualifiedOutputDir, new Path(localOutputDir));

    File outputFile = new File(localOutputDir + "/part-00000");
    File expectedFile = new File("src/test/resource/exp/wordcount/apache_projects.exp");
    boolean isEqual = FileUtils.contentEquals(outputFile, expectedFile);
    Assert.assertTrue(isEqual);/*  w ww. j  av a2 s. com*/
}

---