Example usage for org.apache.hadoop.mapreduce Job submit

List of usage examples for org.apache.hadoop.mapreduce Job submit

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. submit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job submit.

Prototype

public void submit() throws IOException, InterruptedException, ClassNotFoundException

Source Link

Document

Submit the job to the cluster and return immediately.

Usage

From source file:org.apache.bigtop.bigpetstore.generator.TestPetStoreTransactionGeneratorJob.java

License:Apache License

@Test
public void test() throws Exception {
    System.out.println("memory : " + Runtime.getRuntime().freeMemory() / 1000000);
    if (Runtime.getRuntime().freeMemory() / 1000000 < 75) {
        // throw new
        // RuntimeException("need more memory to run this test !");
    }/*from www.  j a  v a2s . c  o m*/
    int records = 20;
    /**
     * Setup configuration with prop.
     */
    Configuration c = new Configuration();
    c.setInt(props.bigpetstore_records.name(), records);

    /**
     * Run the job
     */
    Path output = new Path("petstoredata/" + (new Date()).toString());
    Job createInput = BPSGenerator.getCreateTransactionRecordsJob(output, c);
    createInput.submit();
    System.out.println(createInput);
    createInput.waitForCompletion(true);

    FileSystem fs = FileSystem.getLocal(new Configuration());

    /**
     * Read file output into string.
     */
    DataInputStream f = fs.open(new Path(output, "part-r-00000"));
    BufferedReader br = new BufferedReader(new InputStreamReader(f));
    String s;
    int recordsSeen = 0;
    boolean CTseen = false;
    boolean AZseen = false;

    // confirm that both CT and AZ are seen in the outputs.
    while (br.ready()) {
        s = br.readLine();
        System.out.println("===>" + s);
        recordsSeen++;
        if (s.contains(State.CT.name())) {
            CTseen = true;
        }
        if (s.contains(State.AZ.name())) {
            AZseen = true;
        }
    }

    // records seen should = 20
    assertEquals(records, recordsSeen);
    // Assert that a couple of the states are seen (todo make it
    // comprehensive for all states).
    assertTrue(CTseen);
    assertTrue(AZseen);
    log.info("Created " + records + " , file was " + fs.getFileStatus(new Path(output, "part-r-00000")).getLen()
            + " bytes.");
}

From source file:org.apache.blur.mapreduce.lib.BlurOutputFormatTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testBlurOutputFormatValidateReducerCount()
        throws IOException, InterruptedException, ClassNotFoundException {

    Path input = getInDir();/*from  w w  w.  jav  a  2  s .  co  m*/
    Path output = getOutDir();
    _fileSystem.delete(input, true);
    _fileSystem.delete(output, true);
    writeRecordsFile(new Path(input, "part1"), 1, 1, 1, 1, "cf1");
    writeRecordsFile(new Path(input, "part2"), 1, 1, 2, 1, "cf1");

    Job job = Job.getInstance(_conf, "blur index");
    job.setJarByClass(BlurOutputFormatTest.class);
    job.setMapperClass(CsvBlurMapper.class);
    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.addInputPath(job, input);
    CsvBlurMapper.addColumns(job, "cf1", "col");

    Path tablePath = new Path(new Path(_root, "table"), "test");

    TableDescriptor tableDescriptor = new TableDescriptor();
    tableDescriptor.setShardCount(1);
    tableDescriptor.setTableUri(tablePath.toString());
    tableDescriptor.setName("test");

    createShardDirectories(getOutDir(), 1);

    BlurOutputFormat.setupJob(job, tableDescriptor);
    BlurOutputFormat.setOutputPath(job, output);
    BlurOutputFormat.setReducerMultiplier(job, 2);
    job.setNumReduceTasks(4);
    job.submit();

}

From source file:org.apache.blur.mapreduce.lib.BlurOutputFormatTest.java

License:Apache License

public void testBlurOutputFormatCleanupDuringJobKillTest()
        throws IOException, InterruptedException, ClassNotFoundException {
    Path input = getInDir();/* ww  w .  ja v a 2s.  c om*/
    Path output = getOutDir();
    _fileSystem.delete(input, true);
    _fileSystem.delete(output, true);
    // 1500 * 50 = 75,000
    writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1");
    // 100 * 5000 = 500,000
    writeRecordsFile(new Path(input, "part2"), 1, 5000, 2000, 100, "cf1");

    Job job = Job.getInstance(_conf, "blur index");
    job.setJarByClass(BlurOutputFormatTest.class);
    job.setMapperClass(CsvBlurMapper.class);
    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.addInputPath(job, input);
    CsvBlurMapper.addColumns(job, "cf1", "col");

    Path tablePath = new Path(new Path(_root, "table"), "test");

    TableDescriptor tableDescriptor = new TableDescriptor();
    tableDescriptor.setShardCount(2);
    tableDescriptor.setTableUri(tablePath.toString());
    tableDescriptor.setName("test");

    createShardDirectories(getOutDir(), 2);

    BlurOutputFormat.setupJob(job, tableDescriptor);
    BlurOutputFormat.setOutputPath(job, output);
    BlurOutputFormat.setIndexLocally(job, false);

    job.submit();
    boolean killCalled = false;
    while (!job.isComplete()) {
        Thread.sleep(1000);
        System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100,
                job.reduceProgress() * 100);
        if (job.reduceProgress() > 0.7 && !killCalled) {
            job.killJob();
            killCalled = true;
        }
    }

    assertFalse(job.isSuccessful());

    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
        Path path = new Path(output, ShardUtil.getShardName(i));
        FileSystem fileSystem = path.getFileSystem(job.getConfiguration());
        FileStatus[] listStatus = fileSystem.listStatus(path);
        assertEquals(toString(listStatus), 0, listStatus.length);
    }
}

From source file:org.apache.druid.indexer.DeterminePartitionsJob.java

License:Apache License

@Override
public boolean run() {
    try {//from w ww  .j a v  a  2  s . co  m
        /*
         * Group by (timestamp, dimensions) so we can correctly count dimension values as they would appear
         * in the final segment.
         */

        if (!(config.getPartitionsSpec() instanceof SingleDimensionPartitionsSpec)) {
            throw new ISE(
                    "DeterminePartitionsJob can only be run for SingleDimensionPartitionsSpec, partitionSpec found [%s]",
                    config.getPartitionsSpec());
        }

        final SingleDimensionPartitionsSpec partitionsSpec = (SingleDimensionPartitionsSpec) config
                .getPartitionsSpec();

        if (!partitionsSpec.isAssumeGrouped()) {
            groupByJob = Job.getInstance(new Configuration(), StringUtils.format(
                    "%s-determine_partitions_groupby-%s", config.getDataSource(), config.getIntervals()));

            JobHelper.injectSystemProperties(groupByJob);
            config.addJobProperties(groupByJob);

            groupByJob.setMapperClass(DeterminePartitionsGroupByMapper.class);
            groupByJob.setMapOutputKeyClass(BytesWritable.class);
            groupByJob.setMapOutputValueClass(NullWritable.class);
            groupByJob.setCombinerClass(DeterminePartitionsGroupByReducer.class);
            groupByJob.setReducerClass(DeterminePartitionsGroupByReducer.class);
            groupByJob.setOutputKeyClass(BytesWritable.class);
            groupByJob.setOutputValueClass(NullWritable.class);
            groupByJob.setOutputFormatClass(SequenceFileOutputFormat.class);
            JobHelper.setupClasspath(JobHelper.distributedClassPath(config.getWorkingPath()),
                    JobHelper.distributedClassPath(config.makeIntermediatePath()), groupByJob);

            config.addInputPaths(groupByJob);
            config.intoConfiguration(groupByJob);
            FileOutputFormat.setOutputPath(groupByJob, config.makeGroupedDataDir());

            groupByJob.submit();
            log.info("Job %s submitted, status available at: %s", groupByJob.getJobName(),
                    groupByJob.getTrackingURL());

            // Store the jobId in the file
            if (groupByJob.getJobID() != null) {
                JobHelper.writeJobIdToFile(config.getHadoopJobIdFileName(), groupByJob.getJobID().toString());
            }

            try {
                if (!groupByJob.waitForCompletion(true)) {
                    log.error("Job failed: %s", groupByJob.getJobID());
                    failureCause = Utils.getFailureMessage(groupByJob, config.JSON_MAPPER);
                    return false;
                }
            } catch (IOException ioe) {
                if (!Utils.checkAppSuccessForJobIOException(ioe, groupByJob,
                        config.isUseYarnRMJobStatusFallback())) {
                    throw ioe;
                }
            }
        } else {
            log.info("Skipping group-by job.");
        }

        /*
         * Read grouped data and determine appropriate partitions.
         */
        final Job dimSelectionJob = Job.getInstance(new Configuration(), StringUtils.format(
                "%s-determine_partitions_dimselection-%s", config.getDataSource(), config.getIntervals()));

        dimSelectionJob.getConfiguration().set("io.sort.record.percent", "0.19");

        JobHelper.injectSystemProperties(dimSelectionJob);
        config.addJobProperties(dimSelectionJob);

        if (!partitionsSpec.isAssumeGrouped()) {
            // Read grouped data from the groupByJob.
            dimSelectionJob.setMapperClass(DeterminePartitionsDimSelectionPostGroupByMapper.class);
            dimSelectionJob.setInputFormatClass(SequenceFileInputFormat.class);
            FileInputFormat.addInputPath(dimSelectionJob, config.makeGroupedDataDir());
        } else {
            // Directly read the source data, since we assume it's already grouped.
            dimSelectionJob.setMapperClass(DeterminePartitionsDimSelectionAssumeGroupedMapper.class);
            config.addInputPaths(dimSelectionJob);
        }

        SortableBytes.useSortableBytesAsMapOutputKey(dimSelectionJob,
                DeterminePartitionsDimSelectionPartitioner.class);
        dimSelectionJob.setMapOutputValueClass(Text.class);
        dimSelectionJob.setCombinerClass(DeterminePartitionsDimSelectionCombiner.class);
        dimSelectionJob.setReducerClass(DeterminePartitionsDimSelectionReducer.class);
        dimSelectionJob.setOutputKeyClass(BytesWritable.class);
        dimSelectionJob.setOutputValueClass(Text.class);
        dimSelectionJob.setOutputFormatClass(DeterminePartitionsDimSelectionOutputFormat.class);
        dimSelectionJob.setNumReduceTasks(config.getGranularitySpec().bucketIntervals().get().size());
        JobHelper.setupClasspath(JobHelper.distributedClassPath(config.getWorkingPath()),
                JobHelper.distributedClassPath(config.makeIntermediatePath()), dimSelectionJob);

        config.intoConfiguration(dimSelectionJob);
        FileOutputFormat.setOutputPath(dimSelectionJob, config.makeIntermediatePath());

        dimSelectionJob.submit();
        log.info("Job %s submitted, status available at: %s", dimSelectionJob.getJobName(),
                dimSelectionJob.getTrackingURL());

        // Store the jobId in the file
        if (dimSelectionJob.getJobID() != null) {
            JobHelper.writeJobIdToFile(config.getHadoopJobIdFileName(), dimSelectionJob.getJobID().toString());
        }

        try {
            if (!dimSelectionJob.waitForCompletion(true)) {
                log.error("Job failed: %s", dimSelectionJob.getJobID().toString());
                failureCause = Utils.getFailureMessage(dimSelectionJob, config.JSON_MAPPER);
                return false;
            }
        } catch (IOException ioe) {
            if (!Utils.checkAppSuccessForJobIOException(ioe, dimSelectionJob,
                    config.isUseYarnRMJobStatusFallback())) {
                throw ioe;
            }
        }

        /*
         * Load partitions determined by the previous job.
         */

        log.info("Job completed, loading up partitions for intervals[%s].",
                config.getSegmentGranularIntervals());
        FileSystem fileSystem = null;
        Map<Long, List<HadoopyShardSpec>> shardSpecs = new TreeMap<>();
        int shardCount = 0;
        for (Interval segmentGranularity : config.getSegmentGranularIntervals().get()) {
            final Path partitionInfoPath = config.makeSegmentPartitionInfoPath(segmentGranularity);
            if (fileSystem == null) {
                fileSystem = partitionInfoPath.getFileSystem(dimSelectionJob.getConfiguration());
            }
            if (Utils.exists(dimSelectionJob, fileSystem, partitionInfoPath)) {
                List<ShardSpec> specs = config.JSON_MAPPER.readValue(
                        Utils.openInputStream(dimSelectionJob, partitionInfoPath),
                        new TypeReference<List<ShardSpec>>() {
                        });

                List<HadoopyShardSpec> actualSpecs = Lists.newArrayListWithExpectedSize(specs.size());
                for (int i = 0; i < specs.size(); ++i) {
                    actualSpecs.add(new HadoopyShardSpec(specs.get(i), shardCount++));
                    log.info("DateTime[%s], partition[%d], spec[%s]", segmentGranularity, i,
                            actualSpecs.get(i));
                }

                shardSpecs.put(segmentGranularity.getStartMillis(), actualSpecs);
            } else {
                log.info("Path[%s] didn't exist!?", partitionInfoPath);
            }
        }
        config.setShardSpecs(shardSpecs);

        return true;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

From source file:org.apache.druid.indexer.updater.HadoopConverterJob.java

License:Apache License

public List<DataSegment> run() throws IOException {
    final JobConf jobConf = new JobConf();
    jobConf.setKeepFailedTaskFiles(false);
    for (Map.Entry<String, String> entry : converterConfig.getHadoopProperties().entrySet()) {
        jobConf.set(entry.getKey(), entry.getValue(), "converterConfig.getHadoopProperties()");
    }//from w ww  . j a  v a  2 s.c om
    final List<DataSegment> segments = converterConfig.getSegments();
    if (segments.isEmpty()) {
        throw new IAE("No segments found for datasource [%s]", converterConfig.getDataSource());
    }
    converterConfigIntoConfiguration(converterConfig, segments, jobConf);

    jobConf.setNumReduceTasks(0); // Map only. Number of map tasks determined by input format
    jobConf.setWorkingDirectory(new Path(converterConfig.getDistributedSuccessCache()));

    setJobName(jobConf, segments);

    if (converterConfig.getJobPriority() != null) {
        jobConf.setJobPriority(JobPriority.valueOf(converterConfig.getJobPriority()));
    }

    final Job job = Job.getInstance(jobConf);

    job.setInputFormatClass(ConfigInputFormat.class);
    job.setMapperClass(ConvertingMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setMapSpeculativeExecution(false);
    job.setOutputFormatClass(ConvertingOutputFormat.class);

    JobHelper.setupClasspath(JobHelper.distributedClassPath(jobConf.getWorkingDirectory()),
            JobHelper.distributedClassPath(getJobClassPathDir(job.getJobName(), jobConf.getWorkingDirectory())),
            job);

    Throwable throwable = null;
    try {
        job.submit();
        log.info("Job %s submitted, status available at %s", job.getJobName(), job.getTrackingURL());
        final boolean success = job.waitForCompletion(true);
        if (!success) {
            final TaskReport[] reports = job.getTaskReports(TaskType.MAP);
            if (reports != null) {
                for (final TaskReport report : reports) {
                    log.error("Error in task [%s] : %s", report.getTaskId(),
                            Arrays.toString(report.getDiagnostics()));
                }
            }
            return null;
        }
        try {
            loadedBytes = job.getCounters().findCounter(COUNTER_GROUP, COUNTER_LOADED).getValue();
            writtenBytes = job.getCounters().findCounter(COUNTER_GROUP, COUNTER_WRITTEN).getValue();
        } catch (IOException ex) {
            log.error(ex, "Could not fetch counters");
        }
        final JobID jobID = job.getJobID();

        final Path jobDir = getJobPath(jobID, job.getWorkingDirectory());
        final FileSystem fs = jobDir.getFileSystem(job.getConfiguration());
        final RemoteIterator<LocatedFileStatus> it = fs.listFiles(jobDir, true);
        final List<Path> goodPaths = new ArrayList<>();
        while (it.hasNext()) {
            final LocatedFileStatus locatedFileStatus = it.next();
            if (locatedFileStatus.isFile()) {
                final Path myPath = locatedFileStatus.getPath();
                if (ConvertingOutputFormat.DATA_SUCCESS_KEY.equals(myPath.getName())) {
                    goodPaths.add(new Path(myPath.getParent(), ConvertingOutputFormat.DATA_FILE_KEY));
                }
            }
        }
        if (goodPaths.isEmpty()) {
            log.warn("No good data found at [%s]", jobDir);
            return null;
        }
        final List<DataSegment> returnList = ImmutableList
                .copyOf(Lists.transform(goodPaths, new Function<Path, DataSegment>() {
                    @Nullable
                    @Override
                    public DataSegment apply(final Path input) {
                        try {
                            if (!fs.exists(input)) {
                                throw new ISE("Somehow [%s] was found but [%s] is missing at [%s]",
                                        ConvertingOutputFormat.DATA_SUCCESS_KEY,
                                        ConvertingOutputFormat.DATA_FILE_KEY, jobDir);
                            }
                        } catch (final IOException e) {
                            throw Throwables.propagate(e);
                        }
                        try (final InputStream stream = fs.open(input)) {
                            return HadoopDruidConverterConfig.jsonMapper.readValue(stream, DataSegment.class);
                        } catch (final IOException e) {
                            throw Throwables.propagate(e);
                        }
                    }
                }));
        if (returnList.size() == segments.size()) {
            return returnList;
        } else {
            throw new ISE(
                    "Tasks reported success but result length did not match! Expected %d found %d at path [%s]",
                    segments.size(), returnList.size(), jobDir);
        }
    } catch (InterruptedException | ClassNotFoundException e) {
        RuntimeException exception = Throwables.propagate(e);
        throwable = exception;
        throw exception;
    } catch (Throwable t) {
        throwable = t;
        throw t;
    } finally {
        try {
            cleanup(job);
        } catch (IOException e) {
            if (throwable != null) {
                throwable.addSuppressed(e);
            } else {
                log.error(e, "Could not clean up job [%s]", job.getJobID());
            }
        }
    }
}

From source file:org.apache.falcon.hive.HiveDRTool.java

License:Apache License

public Job execute() throws Exception {
    assert inputOptions != null;
    assert getConf() != null;
    executionStage = inputOptions.getExecutionStage();
    LOG.info("Executing Workflow stage : {}", executionStage);
    if (executionStage.equalsIgnoreCase(HiveDRUtils.ExecutionStage.LASTEVENTS.name())) {
        String lastEventsIdFile = getLastEvents(jobConf);
        LOG.info("Last successfully replicated Event file : {}", lastEventsIdFile);
        return null;
    } else if (executionStage.equalsIgnoreCase(HiveDRUtils.ExecutionStage.EXPORT.name())) {
        createStagingDirectory();/*  w w w .ja  v  a2 s.c  o m*/
        eventsMetaFile = sourceEvents();
        LOG.info("Sourced Events meta file : {}", eventsMetaFile);
        if (StringUtils.isEmpty(eventsMetaFile)) {
            LOG.info("No events to process");
            return null;
        } else {
            /*
             * eventsMetaFile contains the events to be processed by HiveDr. This file should be available
             * for the import action as well. Persist the file at a location common to both export and import.
             */
            persistEventsMetafileLocation(eventsMetaFile);
        }
    } else if (executionStage.equalsIgnoreCase(HiveDRUtils.ExecutionStage.IMPORT.name())) {
        // read the location of eventsMetaFile from hdfs
        eventsMetaFile = getEventsMetaFileLocation();
        if (StringUtils.isEmpty(eventsMetaFile)) {
            LOG.info("No events to process");
            return null;
        }
    } else {
        throw new HiveReplicationException("Invalid Execution stage : " + inputOptions.getExecutionStage());
    }

    Job job = createJob();
    job.submit();

    String jobID = job.getJobID().toString();
    job.getConfiguration().set("HIVEDR_JOB_ID", jobID);

    LOG.info("HiveDR job-id: {}", jobID);
    if (inputOptions.shouldBlock() && !job.waitForCompletion(true)) {
        throw new IOException(
                "HiveDR failure: Job " + jobID + " has failed: " + job.getStatus().getFailureInfo());
    }

    return job;
}

From source file:org.apache.giraph.job.GiraphJob.java

License:Apache License

/**
 * Runs the actual graph application through Hadoop Map-Reduce.
 *
 * @param verbose If true, provide verbose output, false otherwise
 * @return True if success, false otherwise
 * @throws ClassNotFoundException//  w ww  .  j  a v  a 2 s.  c o m
 * @throws InterruptedException
 * @throws IOException
 */
public final boolean run(boolean verbose) throws IOException, InterruptedException, ClassNotFoundException {
    // Most users won't hit this hopefully and can set it higher if desired
    setIntConfIfDefault("mapreduce.job.counters.limit", 512);

    // Capacity scheduler-specific settings.  These should be enough for
    // a reasonable Giraph job
    setIntConfIfDefault("mapred.job.map.memory.mb", 1024);
    setIntConfIfDefault("mapred.job.reduce.memory.mb", 0);

    // Speculative execution doesn't make sense for Giraph
    giraphConfiguration.setBoolean("mapred.map.tasks.speculative.execution", false);

    // Set the ping interval to 5 minutes instead of one minute
    // (DEFAULT_PING_INTERVAL)
    Client.setPingInterval(giraphConfiguration, 60000 * 5);

    // Should work in MAPREDUCE-1938 to let the user jars/classes
    // get loaded first
    giraphConfiguration.setBoolean("mapreduce.user.classpath.first", true);
    giraphConfiguration.setBoolean("mapreduce.job.user.classpath.first", true);

    // If the checkpoint frequency is 0 (no failure handling), set the max
    // tasks attempts to be 0 to encourage faster failure of unrecoverable jobs
    if (giraphConfiguration.getCheckpointFrequency() == 0) {
        int oldMaxTaskAttempts = giraphConfiguration.getMaxTaskAttempts();
        giraphConfiguration.setMaxTaskAttempts(0);
        if (LOG.isInfoEnabled()) {
            LOG.info("run: Since checkpointing is disabled (default), "
                    + "do not allow any task retries (setting " + GiraphConstants.MAX_TASK_ATTEMPTS.getKey()
                    + " = 0, " + "old value = " + oldMaxTaskAttempts + ")");
        }
    }

    // Set the job properties, check them, and submit the job
    ImmutableClassesGiraphConfiguration conf = new ImmutableClassesGiraphConfiguration(giraphConfiguration);
    checkLocalJobRunnerConfiguration(conf);

    int tryCount = 0;
    GiraphJobRetryChecker retryChecker = conf.getJobRetryChecker();
    while (true) {
        JobProgressTrackerService jobProgressTrackerService = JobProgressTrackerService
                .createJobProgressServer(conf);

        tryCount++;
        Job submittedJob = new Job(conf, jobName);
        if (submittedJob.getJar() == null) {
            submittedJob.setJarByClass(getClass());
        }
        submittedJob.setNumReduceTasks(0);
        submittedJob.setMapperClass(GraphMapper.class);
        submittedJob.setInputFormatClass(BspInputFormat.class);
        submittedJob.setOutputFormatClass(BspOutputFormat.class);
        if (jobProgressTrackerService != null) {
            jobProgressTrackerService.setJob(submittedJob);
        }

        GiraphJobObserver jobObserver = conf.getJobObserver();
        jobObserver.launchingJob(submittedJob);
        submittedJob.submit();
        if (LOG.isInfoEnabled()) {
            LOG.info("Tracking URL: " + submittedJob.getTrackingURL());
            LOG.info("Waiting for resources... Job will start only when it gets all "
                    + (conf.getMinWorkers() + 1) + " mappers");
        }
        jobObserver.jobRunning(submittedJob);
        HaltApplicationUtils.printHaltInfo(submittedJob, conf);

        boolean passed = submittedJob.waitForCompletion(verbose);
        if (jobProgressTrackerService != null) {
            jobProgressTrackerService.stop(passed);
        }
        jobObserver.jobFinished(submittedJob, passed);

        if (!passed) {
            String restartFrom = retryChecker.shouldRestartCheckpoint(submittedJob);
            if (restartFrom != null) {
                GiraphConstants.RESTART_JOB_ID.set(conf, restartFrom);
                continue;
            }
        }

        if (passed || !retryChecker.shouldRetry(submittedJob, tryCount)) {
            return passed;
        }
        if (LOG.isInfoEnabled()) {
            LOG.info("run: Retrying job, " + tryCount + " try");
        }
    }
}

From source file:org.apache.gobblin.runtime.mapreduce.MRTask.java

License:Apache License

@Override
public void run() {

    try {/*from   www  .ja v  a  2  s  . co  m*/
        Job job = createJob();

        if (job == null) {
            log.info("No MR job created. Skipping.");
            this.workingState = WorkUnitState.WorkingState.SUCCESSFUL;
            this.eventSubmitter.submit(Events.MR_JOB_SKIPPED);
            onSkippedMRJob();
            return;
        }

        job.submit();

        log.info("MR tracking URL {} for job {}", job.getTrackingURL(), job.getJobName());

        this.eventSubmitter.submit(Events.MR_JOB_STARTED_EVENT, Events.JOB_URL, job.getTrackingURL());
        job.waitForCompletion(false);
        this.mrJob = job;

        if (job.isSuccessful()) {
            this.eventSubmitter.submit(Events.MR_JOB_SUCCESSFUL, Events.JOB_URL, job.getTrackingURL());
            this.onMRTaskComplete(true, null);
        } else {
            this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.JOB_URL, job.getTrackingURL());
            this.onMRTaskComplete(false,
                    new IOException(String.format("MR Job:%s is not successful", job.getTrackingURL())));
        }
    } catch (Throwable t) {
        log.error("Failed to run MR job.", t);
        this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.FAILURE_CONTEXT, t.getMessage());
        this.onMRTaskComplete(false, t);
    }
}

From source file:org.apache.hadoop.examples.pi.Util.java

License:Apache License

/** Run a job. */
static void runJob(String name, Job job, Machine machine, String startmessage, Util.Timer timer) {
    JOB_SEMAPHORE.acquireUninterruptibly();
    Long starttime = null;//from ww w  .  j  a v  a 2 s  . c  o m
    try {
        try {
            starttime = timer.tick("starting " + name + " ...\n  " + startmessage);

            //initialize and submit a job
            machine.init(job);
            job.submit();

            // Separate jobs
            final long sleeptime = 1000L * job.getConfiguration().getInt(JOB_SEPARATION_PROPERTY, 10);
            if (sleeptime > 0) {
                Util.out.println(name + "> sleep(" + Util.millis2String(sleeptime) + ")");
                Thread.sleep(sleeptime);
            }
        } finally {
            JOB_SEMAPHORE.release();
        }

        if (!job.waitForCompletion(false))
            throw new RuntimeException(name + " failed.");
    } catch (Exception e) {
        throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e);
    } finally {
        if (starttime != null)
            timer.tick(name + "> timetaken=" + Util.millis2String(timer.tick() - starttime));
    }
}

From source file:org.apache.hcatalog.templeton.tool.TempletonControllerJob.java

License:Apache License

/**
 * Enqueue the job and print out the job id for later collection.
 *//*from w w w .  ja va2  s  .com*/
@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = getConf();
    conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args));
    conf.set("user.name", UserGroupInformation.getCurrentUser().getShortUserName());
    Job job = new Job(conf);
    job.setJarByClass(TempletonControllerJob.class);
    job.setJobName("TempletonControllerJob");
    job.setMapperClass(LaunchMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setInputFormatClass(SingleInputFormat.class);
    NullOutputFormat<NullWritable, NullWritable> of = new NullOutputFormat<NullWritable, NullWritable>();
    job.setOutputFormatClass(of.getClass());
    job.setNumReduceTasks(0);

    JobClient jc = new JobClient(new JobConf(job.getConfiguration()));

    Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
    job.getCredentials().addToken(new Text("mr token"), mrdt);
    job.submit();

    submittedJobId = job.getJobID();

    return 0;
}