Example usage for org.apache.hadoop.mapreduce StatusReporter getCounter

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce StatusReporter getCounter.

Prototype

public abstract Counter getCounter(Enum<?> name);

Source Link

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private static Mapper<Text, FileStatus, NullWritable, Text>.Context getMapperContext(CopyMapper copyMapper,
        final StatusReporter reporter, final InMemoryWriter writer) throws IOException, InterruptedException {
    Mapper.Context ctx = Mockito.mock(Mapper.Context.class);
    Mockito.when(ctx.getConfiguration()).thenReturn(getConfiguration());
    Mockito.doAnswer(new Answer() {
        @Override//from www . j a va2  s  . c o m
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            writer.write((NullWritable) invocationOnMock.getArguments()[0],
                    (Text) invocationOnMock.getArguments()[1]);
            return null;
        }
    }).when(ctx).write(Mockito.any(), Mockito.any());

    Mockito.doAnswer(new Answer<Counter>() {
        @Override
        public Counter answer(InvocationOnMock invocationOnMock) throws Throwable {
            return reporter.getCounter((Enum<?>) invocationOnMock.getArguments()[0]);
        }
    }).when(ctx).getCounter(Mockito.any(CopyMapper.Counter.class));

    Mockito.doAnswer(new Answer<Counter>() {
        @Override
        public Counter answer(InvocationOnMock invocationOnMock) throws Throwable {
            return reporter.getCounter((String) invocationOnMock.getArguments()[0],
                    (String) invocationOnMock.getArguments()[1]);
        }
    }).when(ctx).getCounter(Mockito.any(String.class), Mockito.any(String.class));

    final TaskAttemptID id = Mockito.mock(TaskAttemptID.class);
    Mockito.when(id.toString()).thenReturn("attempt1");
    Mockito.doAnswer(new Answer<TaskAttemptID>() {

        @Override
        public TaskAttemptID answer(InvocationOnMock invocationOnMock) throws Throwable {
            return id;
        }
    }).when(ctx).getTaskAttemptID();
    return ctx;
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testRun() {
    try {/*w w  w.  jav  a2s  .  c o  m*/
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);
        copyMapper.setup(context);

        for (Path path : pathList) {
            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
                    fs.getFileStatus(path), context);
        }
        // Check that the maps worked.
        for (Path path : pathList) {
            final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
            Assert.assertTrue(fs.exists(targetPath));
            Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path));
            Assert.assertEquals(fs.getFileStatus(path).getReplication(),
                    fs.getFileStatus(targetPath).getReplication());
            Assert.assertEquals(fs.getFileStatus(path).getBlockSize(),
                    fs.getFileStatus(targetPath).getBlockSize());
            Assert.assertTrue(
                    !fs.isFile(targetPath) || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path)));
        }

        Assert.assertEquals(pathList.size(), reporter.getCounter(CopyMapper.Counter.PATHS_COPIED).getValue());
        // Here file is compressed file. So, we should compare the file length
        // with the number of bytes read
        long totalSize = 0;
        for (Path path : pathList) {
            totalSize += fs.getFileStatus(path).getLen();
        }
        Assert.assertEquals(totalSize, reporter.getCounter(CopyMapper.Counter.BYTES_COPIED).getValue());
        long totalCounterValue = 0;
        for (Text value : writer.values()) {
            String tmp[] = value.toString().split(ConduitConstants.AUDIT_COUNTER_NAME_DELIMITER);
            Assert.assertEquals(4, tmp.length);
            Long numOfMsgs = Long.parseLong(tmp[3]);
            totalCounterValue += numOfMsgs;
        }
        Assert.assertEquals(nFiles * NUMBER_OF_MESSAGES_PER_FILE, totalCounterValue);
        testCopyingExistingFiles(fs, copyMapper, context);
    } catch (Exception e) {
        LOG.error("Unexpected exception: ", e);
        Assert.assertTrue(false);
    }
}

---