Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce JobContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. JobContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

public void testNoCommitAction() {

    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    final String[] statusString = new String[1];
    try {/*from w w  w .java 2  s. c o m*/
        Mockito.doAnswer(new Answer() {
            @Override
            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                statusString[0] = (String) invocationOnMock.getArguments()[0];
                return null; //To change body of implemented methods use File | Settings | File Templates.
            }
        }).when(taskAttemptContext).setStatus(Mockito.anyString());
    } catch (Throwable e) {
    }
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        committer.commitJob(jobContext);
        Assert.assertEquals(statusString[0], "Commit Successful");

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertEquals(statusString[0], "Commit Successful");
    } catch (IOException e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Commit failed");
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

public void testValidationPass() {

    config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 100);
    Counters counters = new Counters();
    CounterGroup grp = counters.getGroup(CopyMapper.Counter.class.getName());
    grp.findCounter(CopyMapper.Counter.BYTES_COPIED.name()).increment(50);
    grp.findCounter(CopyMapper.Counter.BYTES_FAILED.name()).increment(20);
    grp.findCounter(CopyMapper.Counter.BYTES_SKIPPED.name()).increment(30);
    counterProvider.setCounters(counters);

    try {//from   w  w  w.jav a  2 s. c  o m
        TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
        JobContext jobContext = Mockito.mock(JobContext.class);
        Mockito.when(jobContext.getConfiguration()).thenReturn(config);
        JobID jobID = new JobID();
        Mockito.when(jobContext.getJobID()).thenReturn(jobID);
        final String[] statusString = new String[1];
        try {
            Mockito.doAnswer(new Answer() {
                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    LOG.info("XXXX  crap I am called now " + invocationOnMock.getArguments()[0]);
                    statusString[0] = (String) invocationOnMock.getArguments()[0];
                    return null; //To change body of implemented methods use File | Settings | File Templates.
                }
            }).when(taskAttemptContext).setStatus(Mockito.anyString());
        } catch (Throwable e) {
        }
        try {
            OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
            committer.commitJob(jobContext);
            Assert.assertEquals(statusString[0], "Commit Successful");
        } catch (IOException e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Commit failed");
        }
    } finally {
        config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 0);
        counterProvider.setCounters(EMPTY_COUNTERS);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testPreserveStatus() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);

    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String sourceBase;/*from   w ww .j  ava 2 s  .co m*/
    String targetBase;
    FileSystem fs = null;
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        FsPermission sourcePerm = new FsPermission((short) 511);
        FsPermission initialPerm = new FsPermission((short) 448);
        sourceBase = TestDistCpUtils.createTestSetup(fs, sourcePerm);
        targetBase = TestDistCpUtils.createTestSetup(fs, initialPerm);

        DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), new Path("/out"));
        options.preserve(FileAttribute.PERMISSION);
        options.appendToConf(conf);

        CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
        Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
        listing.buildListing(listingFile, options);

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);

        committer.commitJob(jobContext);
        if (!checkDirectoryPermissions(fs, targetBase, sourcePerm)) {
            Assert.fail("Permission don't match");
        }

        //Test for idempotent commit
        committer.commitJob(jobContext);
        if (!checkDirectoryPermissions(fs, targetBase, sourcePerm)) {
            Assert.fail("Permission don't match");
        }

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Preserve status failure");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp1");
        conf.unset(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
    }

}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testDeleteMissing() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String sourceBase;//w ww . ja v a  2  s.  c om
    String targetBase;
    FileSystem fs = null;
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        sourceBase = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
        targetBase = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
        String targetBaseAdd = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
        fs.rename(new Path(targetBaseAdd), new Path(targetBase));

        DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), new Path("/out"));
        options.setSyncFolder(true);
        options.setDeleteMissing(true);
        options.appendToConf(conf);

        CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
        Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
        listing.buildListing(listingFile, options);

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);

        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
            Assert.fail("Source and target folders are not in sync");
        }

        //Test for idempotent commit
        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
    } catch (Throwable e) {
        LOG.error("Exception encountered while testing for delete missing", e);
        Assert.fail("Delete missing failure");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp1");
    }

}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testDeleteMissingFlatInterleavedFiles() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String sourceBase;/*from   w  ww  .j  a v a  2s . co m*/
    String targetBase;
    FileSystem fs = null;
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        sourceBase = "/tmp1/" + String.valueOf(rand.nextLong());
        targetBase = "/tmp1/" + String.valueOf(rand.nextLong());
        TestDistCpUtils.createFile(fs, sourceBase + "/1");
        TestDistCpUtils.createFile(fs, sourceBase + "/3");
        TestDistCpUtils.createFile(fs, sourceBase + "/4");
        TestDistCpUtils.createFile(fs, sourceBase + "/5");
        TestDistCpUtils.createFile(fs, sourceBase + "/7");
        TestDistCpUtils.createFile(fs, sourceBase + "/8");
        TestDistCpUtils.createFile(fs, sourceBase + "/9");

        TestDistCpUtils.createFile(fs, targetBase + "/2");
        TestDistCpUtils.createFile(fs, targetBase + "/4");
        TestDistCpUtils.createFile(fs, targetBase + "/5");
        TestDistCpUtils.createFile(fs, targetBase + "/7");
        TestDistCpUtils.createFile(fs, targetBase + "/9");
        TestDistCpUtils.createFile(fs, targetBase + "/A");

        DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), new Path("/out"));
        options.setSyncFolder(true);
        options.setDeleteMissing(true);
        options.appendToConf(conf);

        CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
        Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
        listing.buildListing(listingFile, options);

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);

        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);

        //Test for idempotent commit
        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);
    } catch (IOException e) {
        LOG.error("Exception encountered while testing for delete missing", e);
        Assert.fail("Delete missing failure");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp1");
    }

}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testAtomicCommitMissingFinal() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
    String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
    FileSystem fs = null;/* ww  w  .j  ava2s.c o m*/
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        fs.mkdirs(new Path(workPath));

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);
        //XXX set label to false explicitly, conf is not mixed up
        conf.setBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false);

        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertFalse(fs.exists(new Path(finalPath)));
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Atomic commit failure");
    } finally {
        TestDistCpUtils.delete(fs, workPath);
        TestDistCpUtils.delete(fs, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testAtomicCommitExistingFinal() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class, Mockito.RETURNS_DEEP_STUBS);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
    String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
    FileSystem fs = null;// w w w . j a  v  a 2s .  c  om
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        fs.mkdirs(new Path(workPath));
        fs.mkdirs(new Path(finalPath));

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);
        //XXX set label to false explicitly, conf is not mixed up
        conf.setBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false);

        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Atomic commit failure");
    } finally {
        TestDistCpUtils.delete(fs, workPath);
        TestDistCpUtils.delete(fs, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyOutputFormat.java

License:Apache License

@Test
public void testCheckOutputSpecs() {
    try {// w w  w.  j a v  a  2  s. c o  m
        OutputFormat outputFormat = new CopyOutputFormat();
        Configuration conf = new Configuration();
        Job job = new Job(conf);
        JobID jobID = new JobID("200707121733", 1);

        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
            Assert.fail("No checking for invalid work/commit path");
        } catch (IllegalStateException ignore) {
        }

        CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
            Assert.fail("No checking for invalid commit path");
        } catch (IllegalStateException ignore) {
        }

        job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
        CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));
        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
            Assert.fail("No checking for invalid work path");
        } catch (IllegalStateException ignore) {
        }

        CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
        CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));
        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
        } catch (IllegalStateException ignore) {
            ignore.printStackTrace();
            Assert.fail("Output spec check failed.");
        }

    } catch (IOException e) {
        LOG.error("Exception encountered while testing checkoutput specs", e);
        Assert.fail("Checkoutput Spec failure");
    } catch (InterruptedException e) {
        LOG.error("Exception encountered while testing checkoutput specs", e);
        Assert.fail("Checkoutput Spec failure");
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestUniformSizeInputFormat.java

License:Apache License

public void testGetSplits(int nMaps) throws Exception {
    DistCpOptions options = getOptions(nMaps);
    Configuration configuration = new Configuration();
    configuration.set("mapred.map.tasks", String.valueOf(options.getMaxMaps()));
    Path listFile = new Path(cluster.getFileSystem().getUri().toString() + "/tmp/testGetSplits_1/fileList.seq");
    CopyListing.getCopyListing(configuration, CREDENTIALS, options).buildListing(listFile, options);

    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(configuration);
    Mockito.when(jobContext.getJobID()).thenReturn(new JobID());
    UniformSizeInputFormat uniformSizeInputFormat = new UniformSizeInputFormat();
    List<InputSplit> splits = uniformSizeInputFormat.getSplits(jobContext);

    //Removing the legacy check - Refer HADOOP-9230
    int sizePerMap = totalFileSize / nMaps;

    checkSplits(listFile, splits);//ww  w  . ja  v  a 2s  . c  om

    int doubleCheckedTotalSize = 0;
    int previousSplitSize = -1;
    for (int i = 0; i < splits.size(); ++i) {
        InputSplit split = splits.get(i);
        int currentSplitSize = 0;
        TaskAttemptID taskId = new TaskAttemptID("", 0, true, 0, 0);
        final TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class);
        Mockito.when(taskAttemptContext.getConfiguration()).thenReturn(configuration);
        Mockito.when(taskAttemptContext.getTaskAttemptID()).thenReturn(taskId);
        RecordReader<Text, FileStatus> recordReader = uniformSizeInputFormat.createRecordReader(split,
                taskAttemptContext);
        recordReader.initialize(split, taskAttemptContext);
        while (recordReader.nextKeyValue()) {
            Path sourcePath = recordReader.getCurrentValue().getPath();
            FileSystem fs = sourcePath.getFileSystem(configuration);
            FileStatus fileStatus[] = fs.listStatus(sourcePath);
            Assert.assertEquals(fileStatus.length, 1);
            currentSplitSize += fileStatus[0].getLen();
        }
        Assert.assertTrue(previousSplitSize == -1
                || Math.abs(currentSplitSize - previousSplitSize) < 0.1 * sizePerMap || i == splits.size() - 1);

        doubleCheckedTotalSize += currentSplitSize;
    }

    Assert.assertEquals(totalFileSize, doubleCheckedTotalSize);
}

From source file:com.intel.hadoop.hbase.dot.KEY.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    TableInputFormat tif = new TableInputFormat();
    tif.setConf(context.getConfiguration());
    return tif.getSplits(context);
}