Example usage for org.apache.hadoop.mapreduce MRJobConfig CLASSPATH_FILES

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce MRJobConfig CLASSPATH_FILES.

Prototype

String CLASSPATH_FILES

To view the source code for org.apache.hadoop.mapreduce MRJobConfig CLASSPATH_FILES.

Click Source Link

Usage

From source file:io.druid.indexer.HdfsClasspathSetupTest.java

License:Apache License

@Test
public void testAddSnapshotJarToClasspath() throws IOException {
    Job job = Job.getInstance(conf, "test-job");
    DistributedFileSystem fs = miniCluster.getFileSystem();
    Path intermediatePath = new Path("/tmp/classpath");
    JobHelper.addSnapshotJarToClassPath(dummyJarFile, intermediatePath, fs, job);
    Path expectedJarPath = new Path(intermediatePath, dummyJarFile.getName());
    // check file gets uploaded to HDFS
    Assert.assertTrue(fs.exists(expectedJarPath));
    // check file gets added to the classpath
    Assert.assertEquals(expectedJarPath.toString(), job.getConfiguration().get(MRJobConfig.CLASSPATH_FILES));
    Assert.assertEquals(dummyJarString, StringUtils.fromUtf8(IOUtils.toByteArray(fs.open(expectedJarPath))));
}

---

From source file:io.druid.indexer.HdfsClasspathSetupTest.java

License:Apache License

@Test
public void testAddNonSnapshotJarToClasspath() throws IOException {
    Job job = Job.getInstance(conf, "test-job");
    DistributedFileSystem fs = miniCluster.getFileSystem();
    JobHelper.addJarToClassPath(dummyJarFile, finalClasspath, intermediatePath, fs, job);
    Path expectedJarPath = new Path(finalClasspath, dummyJarFile.getName());
    // check file gets uploaded to final HDFS path
    Assert.assertTrue(fs.exists(expectedJarPath));
    // check that the intermediate file gets deleted
    Assert.assertFalse(fs.exists(new Path(intermediatePath, dummyJarFile.getName())));
    // check file gets added to the classpath
    Assert.assertEquals(expectedJarPath.toString(), job.getConfiguration().get(MRJobConfig.CLASSPATH_FILES));
    Assert.assertEquals(dummyJarString, StringUtils.fromUtf8(IOUtils.toByteArray(fs.open(expectedJarPath))));
}

---

From source file:io.druid.indexer.HdfsClasspathSetupTest.java

License:Apache License

@Test
public void testConcurrentUpload()
        throws IOException, InterruptedException, ExecutionException, TimeoutException {
    final int concurrency = 10;
    ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrency));
    // barrier ensures that all jobs try to add files to classpath at same time.
    final CyclicBarrier barrier = new CyclicBarrier(concurrency);
    final DistributedFileSystem fs = miniCluster.getFileSystem();
    final Path expectedJarPath = new Path(finalClasspath, dummyJarFile.getName());
    List<ListenableFuture<Boolean>> futures = new ArrayList<>();

    for (int i = 0; i < concurrency; i++) {
        futures.add(pool.submit(new Callable() {
            @Override//from   ww  w  . j ava  2 s.  c om
            public Boolean call() throws Exception {
                int id = barrier.await();
                Job job = Job.getInstance(conf, "test-job-" + id);
                Path intermediatePathForJob = new Path(intermediatePath, "job-" + id);
                JobHelper.addJarToClassPath(dummyJarFile, finalClasspath, intermediatePathForJob, fs, job);
                // check file gets uploaded to final HDFS path
                Assert.assertTrue(fs.exists(expectedJarPath));
                // check that the intermediate file is not present
                Assert.assertFalse(fs.exists(new Path(intermediatePathForJob, dummyJarFile.getName())));
                // check file gets added to the classpath
                Assert.assertEquals(expectedJarPath.toString(),
                        job.getConfiguration().get(MRJobConfig.CLASSPATH_FILES));
                return true;
            }
        }));
    }

    Futures.allAsList(futures).get(30, TimeUnit.SECONDS);

    pool.shutdownNow();
}

---