Example usage for org.apache.hadoop.mapreduce.lib.output FileOutputCommitter SUCCEEDED_FILE

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.output FileOutputCommitter SUCCEEDED_FILE_NAME.

Prototype

String SUCCEEDED_FILE_NAME

To view the source code for org.apache.hadoop.mapreduce.lib.output FileOutputCommitter SUCCEEDED_FILE_NAME.

Click Source Link

Usage

From source file:com.asakusafw.bulkloader.collector.ExportFileSend.java

License:Apache License

/**
 * ?Hadoop??????{@code true}?// w w  w.  j a  va 2 s .co  m
 * @param path ?
 * @return ???????{@code true}????????{@code false}
 */
private boolean isSystemFile(Path path) {
    assert path != null;
    String name = path.getName();
    return name.equals(FileOutputCommitter.SUCCEEDED_FILE_NAME) || name.equals("_logs");
}

From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunnerTest.java

License:Apache License

private Set<String> read(File file) throws IOException {
    return read(file, new FileFilter() {
        @Override//from  www.  ja v  a 2  s. c  o m
        public boolean accept(File pathname) {
            String name = pathname.getName();
            if (name.startsWith(".") || name.equals(FileOutputCommitter.SUCCEEDED_FILE_NAME)) {
                return false;
            }
            return true;
        }
    });
}

From source file:com.asakusafw.testdriver.FlowPartTestDriver.java

License:Apache License

private boolean isSystemFile(Path path) {
    assert path != null;
    String name = path.getName();
    return name.equals(FileOutputCommitter.SUCCEEDED_FILE_NAME) || name.equals("_logs");
}

From source file:org.apache.tez.mapreduce.TestMRRJobs.java

License:Apache License

@Test(timeout = 60000)
public void testRandomWriter() throws IOException, InterruptedException, ClassNotFoundException {

    LOG.info("\n\n\nStarting testRandomWriter().");
    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;/* www.ja va2  s  . co m*/
    }

    RandomTextWriterJob randomWriterJob = new RandomTextWriterJob();
    mrrTezCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
    mrrTezCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
    Job job = randomWriterJob.createJob(mrrTezCluster.getConfig());
    Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
    FileOutputFormat.setOutputPath(job, outputDir);
    job.setSpeculativeExecution(false);
    job.setJarByClass(RandomTextWriterJob.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.submit();
    String trackingUrl = job.getTrackingURL();
    String jobId = job.getJobID().toString();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId,
            trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));

    // Make sure there are three files in the output-dir

    RemoteIterator<FileStatus> iterator = FileContext.getFileContext(mrrTezCluster.getConfig())
            .listStatus(outputDir);
    int count = 0;
    while (iterator.hasNext()) {
        FileStatus file = iterator.next();
        if (!file.getPath().getName().equals(FileOutputCommitter.SUCCEEDED_FILE_NAME)) {
            count++;
        }
    }
    Assert.assertEquals("Number of part files is wrong!", 3, count);

}

Example usage for org.apache.hadoop.mapreduce.lib.output FileOutputCommitter SUCCEEDED_FILE_NAME

Introduction

Prototype

Usage