Example usage for org.apache.hadoop.ipc Client setPingInterval

Introduction

In this page you can find the example usage for org.apache.hadoop.ipc Client setPingInterval.

Prototype

public static final void setPingInterval(Configuration conf, int pingInterval) 

Source Link

Document

set the ping interval value in configuration

Usage

From source file:org.apache.giraph.graph.GiraphJob.java

License:Apache License

/**
 * Runs the actual graph application through Hadoop Map-Reduce.
 *
 * @param verbose If true, provide verbose output, false otherwise
 * @return True if success, false otherwise
 * @throws ClassNotFoundException//  w  ww.  jav a  2  s  .co m
 * @throws InterruptedException
 * @throws IOException
 */
public final boolean run(boolean verbose) throws IOException, InterruptedException, ClassNotFoundException {
    checkConfiguration();
    checkLocalJobRunnerConfiguration(conf);
    job.setNumReduceTasks(0);
    // Most users won't hit this hopefully and can set it higher if desired
    setIntConfIfDefault("mapreduce.job.counters.limit", 512);

    // Capacity scheduler-specific settings.  These should be enough for
    // a reasonable Giraph job
    setIntConfIfDefault("mapred.job.map.memory.mb", 1024);
    setIntConfIfDefault("mapred.job.reduce.memory.mb", 1024);

    // Speculative execution doesn't make sense for Giraph
    conf.setBoolean("mapred.map.tasks.speculative.execution", false);

    // Set the ping interval to 5 minutes instead of one minute
    // (DEFAULT_PING_INTERVAL)
    Client.setPingInterval(conf, 60000 * 5);

    if (job.getJar() == null) {
        job.setJarByClass(GiraphJob.class);
    }
    // Should work in MAPREDUCE-1938 to let the user jars/classes
    // get loaded first
    conf.setBoolean("mapreduce.user.classpath.first", true);

    job.setMapperClass(GraphMapper.class);
    job.setInputFormatClass(BspInputFormat.class);
    job.setOutputFormatClass(BspOutputFormat.class);
    return job.waitForCompletion(verbose);
}

---

From source file:org.apache.giraph.job.GiraphJob.java

License:Apache License

/**
 * Runs the actual graph application through Hadoop Map-Reduce.
 *
 * @param verbose If true, provide verbose output, false otherwise
 * @return True if success, false otherwise
 * @throws ClassNotFoundException/*w  w  w .j  a v  a2s .  com*/
 * @throws InterruptedException
 * @throws IOException
 */
public final boolean run(boolean verbose) throws IOException, InterruptedException, ClassNotFoundException {
    // Most users won't hit this hopefully and can set it higher if desired
    setIntConfIfDefault("mapreduce.job.counters.limit", 512);

    // Capacity scheduler-specific settings.  These should be enough for
    // a reasonable Giraph job
    setIntConfIfDefault("mapred.job.map.memory.mb", 1024);
    setIntConfIfDefault("mapred.job.reduce.memory.mb", 0);

    // Speculative execution doesn't make sense for Giraph
    giraphConfiguration.setBoolean("mapred.map.tasks.speculative.execution", false);

    // Set the ping interval to 5 minutes instead of one minute
    // (DEFAULT_PING_INTERVAL)
    Client.setPingInterval(giraphConfiguration, 60000 * 5);

    // Should work in MAPREDUCE-1938 to let the user jars/classes
    // get loaded first
    giraphConfiguration.setBoolean("mapreduce.user.classpath.first", true);
    giraphConfiguration.setBoolean("mapreduce.job.user.classpath.first", true);

    // If the checkpoint frequency is 0 (no failure handling), set the max
    // tasks attempts to be 0 to encourage faster failure of unrecoverable jobs
    if (giraphConfiguration.getCheckpointFrequency() == 0) {
        int oldMaxTaskAttempts = giraphConfiguration.getMaxTaskAttempts();
        giraphConfiguration.setMaxTaskAttempts(0);
        if (LOG.isInfoEnabled()) {
            LOG.info("run: Since checkpointing is disabled (default), "
                    + "do not allow any task retries (setting " + GiraphConstants.MAX_TASK_ATTEMPTS.getKey()
                    + " = 0, " + "old value = " + oldMaxTaskAttempts + ")");
        }
    }

    // Set the job properties, check them, and submit the job
    ImmutableClassesGiraphConfiguration conf = new ImmutableClassesGiraphConfiguration(giraphConfiguration);
    checkLocalJobRunnerConfiguration(conf);

    int tryCount = 0;
    GiraphJobRetryChecker retryChecker = conf.getJobRetryChecker();
    while (true) {
        JobProgressTrackerService jobProgressTrackerService = JobProgressTrackerService
                .createJobProgressServer(conf);

        tryCount++;
        Job submittedJob = new Job(conf, jobName);
        if (submittedJob.getJar() == null) {
            submittedJob.setJarByClass(getClass());
        }
        submittedJob.setNumReduceTasks(0);
        submittedJob.setMapperClass(GraphMapper.class);
        submittedJob.setInputFormatClass(BspInputFormat.class);
        submittedJob.setOutputFormatClass(BspOutputFormat.class);
        if (jobProgressTrackerService != null) {
            jobProgressTrackerService.setJob(submittedJob);
        }

        GiraphJobObserver jobObserver = conf.getJobObserver();
        jobObserver.launchingJob(submittedJob);
        submittedJob.submit();
        if (LOG.isInfoEnabled()) {
            LOG.info("Tracking URL: " + submittedJob.getTrackingURL());
            LOG.info("Waiting for resources... Job will start only when it gets all "
                    + (conf.getMinWorkers() + 1) + " mappers");
        }
        jobObserver.jobRunning(submittedJob);
        HaltApplicationUtils.printHaltInfo(submittedJob, conf);

        boolean passed = submittedJob.waitForCompletion(verbose);
        if (jobProgressTrackerService != null) {
            jobProgressTrackerService.stop(passed);
        }
        jobObserver.jobFinished(submittedJob, passed);

        if (!passed) {
            String restartFrom = retryChecker.shouldRestartCheckpoint(submittedJob);
            if (restartFrom != null) {
                GiraphConstants.RESTART_JOB_ID.set(conf, restartFrom);
                continue;
            }
        }

        if (passed || !retryChecker.shouldRetry(submittedJob, tryCount)) {
            return passed;
        }
        if (LOG.isInfoEnabled()) {
            LOG.info("run: Retrying job, " + tryCount + " try");
        }
    }
}

---