Example usage for org.apache.hadoop.mapreduce Job getTrackingURL

List of usage examples for org.apache.hadoop.mapreduce Job getTrackingURL

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. getTrackingURL

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getTrackingURL.

Prototype

public String getTrackingURL()

Source Link

Document

Get the URL where some job progress information will be displayed.

Usage

From source file:org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine.java

License:Apache License

/**
 * {@inheritDoc}//w  w  w  .  j a v  a  2 s. co m
 */
@Override
public boolean submit(JobRequest mrJobRequest) {
    // We're supporting only map reduce jobs
    MRJobRequest request = (MRJobRequest) mrJobRequest;

    // Clone global configuration
    Configuration configuration = new Configuration(globalConfiguration);

    // Serialize driver context into job configuration
    for (Map.Entry<String, String> entry : request.getDriverContext()) {
        if (entry.getValue() == null) {
            LOG.warn("Ignoring null driver context value for key " + entry.getKey());
            continue;
        }
        configuration.set(entry.getKey(), entry.getValue());
    }

    // Serialize connector context as a sub namespace
    for (Map.Entry<String, String> entry : request.getConnectorContext(Direction.FROM)) {
        if (entry.getValue() == null) {
            LOG.warn("Ignoring null connector context value for key " + entry.getKey());
            continue;
        }
        configuration.set(MRJobConstants.PREFIX_CONNECTOR_FROM_CONTEXT + entry.getKey(), entry.getValue());
    }

    for (Map.Entry<String, String> entry : request.getConnectorContext(Direction.TO)) {
        if (entry.getValue() == null) {
            LOG.warn("Ignoring null connector context value for key " + entry.getKey());
            continue;
        }
        configuration.set(MRJobConstants.PREFIX_CONNECTOR_TO_CONTEXT + entry.getKey(), entry.getValue());
    }

    // Set up notification URL if it's available
    if (request.getNotificationUrl() != null) {
        configuration.set("job.end.notification.url", request.getNotificationUrl());
    }

    // Turn off speculative execution
    configuration.setBoolean("mapred.map.tasks.speculative.execution", false);
    configuration.setBoolean("mapred.reduce.tasks.speculative.execution", false);

    // Promote all required jars to the job
    configuration.set("tmpjars", StringUtils.join(request.getJars(), ","));

    try {
        Job job = new Job(configuration);

        // link configs
        MRConfigurationUtils.setConnectorLinkConfig(Direction.FROM, job,
                request.getConnectorLinkConfig(Direction.FROM));
        MRConfigurationUtils.setConnectorLinkConfig(Direction.TO, job,
                request.getConnectorLinkConfig(Direction.TO));

        // from and to configs
        MRConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getJobConfig(Direction.FROM));
        MRConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getJobConfig(Direction.TO));

        MRConfigurationUtils.setDriverConfig(job, request.getDriverConfig());
        MRConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema());
        MRConfigurationUtils.setConnectorSchema(Direction.TO, job, request.getSummary().getToSchema());

        if (request.getJobName() != null) {
            job.setJobName("Sqoop: " + request.getJobName());
        } else {
            job.setJobName("Sqoop job with id: " + request.getJobId());
        }

        job.setInputFormatClass(request.getInputFormatClass());

        job.setMapperClass(request.getMapperClass());
        job.setMapOutputKeyClass(request.getMapOutputKeyClass());
        job.setMapOutputValueClass(request.getMapOutputValueClass());

        // Set number of reducers as number of configured loaders  or suppress
        // reduce phase entirely if loaders are not set at all.
        if (request.getLoaders() != null) {
            job.setNumReduceTasks(request.getLoaders());
        } else {
            job.setNumReduceTasks(0);
        }

        job.setOutputFormatClass(request.getOutputFormatClass());
        job.setOutputKeyClass(request.getOutputKeyClass());
        job.setOutputValueClass(request.getOutputValueClass());

        // If we're in local mode than wait on completion. Local job runner do not
        // seems to be exposing API to get previously submitted job which makes
        // other methods of the submission engine quite useless.
        if (isLocal()) {
            job.waitForCompletion(true);
        } else {
            job.submit();
        }

        String jobId = job.getJobID().toString();
        request.getSummary().setExternalId(jobId);
        request.getSummary().setExternalLink(job.getTrackingURL());

        LOG.debug("Executed new map-reduce job with id " + jobId);
    } catch (Exception e) {
        request.getSummary().setException(e);
        LOG.error("Error in submitting job", e);
        return false;
    }
    return true;
}

From source file:org.apache.tez.mapreduce.TestMRRJobs.java

License:Apache License

@Test(timeout = 60000)
public void testMRRSleepJob() throws IOException, InterruptedException, ClassNotFoundException {
    LOG.info("\n\n\nStarting testMRRSleepJob().");

    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;/*from  w w w  .  jav  a2  s  . c o  m*/
    }

    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());

    MRRSleepJob sleepJob = new MRRSleepJob();
    sleepJob.setConf(sleepConf);

    Job job = sleepJob.createJob(1, 1, 1, 1, 1, 1, 1, 1, 1, 1);

    job.setJarByClass(MRRSleepJob.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.submit();
    String trackingUrl = job.getTrackingURL();
    String jobId = job.getJobID().toString();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId,
            trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));

    // FIXME once counters and task progress can be obtained properly
    // TODO use dag client to test counters and task progress?
    // what about completed jobs?
}

From source file:org.apache.tez.mapreduce.TestMRRJobs.java

License:Apache License

@Test(timeout = 60000)
public void testRandomWriter() throws IOException, InterruptedException, ClassNotFoundException {

    LOG.info("\n\n\nStarting testRandomWriter().");
    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;/*www.  j  ava 2  s .co  m*/
    }

    RandomTextWriterJob randomWriterJob = new RandomTextWriterJob();
    mrrTezCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
    mrrTezCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
    Job job = randomWriterJob.createJob(mrrTezCluster.getConfig());
    Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
    FileOutputFormat.setOutputPath(job, outputDir);
    job.setSpeculativeExecution(false);
    job.setJarByClass(RandomTextWriterJob.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.submit();
    String trackingUrl = job.getTrackingURL();
    String jobId = job.getJobID().toString();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId,
            trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));

    // Make sure there are three files in the output-dir

    RemoteIterator<FileStatus> iterator = FileContext.getFileContext(mrrTezCluster.getConfig())
            .listStatus(outputDir);
    int count = 0;
    while (iterator.hasNext()) {
        FileStatus file = iterator.next();
        if (!file.getPath().getName().equals(FileOutputCommitter.SUCCEEDED_FILE_NAME)) {
            count++;
        }
    }
    Assert.assertEquals("Number of part files is wrong!", 3, count);

}

From source file:org.apache.tez.mapreduce.TestMRRJobs.java

License:Apache License

@Test(timeout = 60000)
public void testMRRSleepJobWithCompression() throws IOException, InterruptedException, ClassNotFoundException {
    LOG.info("\n\n\nStarting testMRRSleepJobWithCompression().");

    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;//from   w w w.j a va 2s  . c  o  m
    }

    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());

    MRRSleepJob sleepJob = new MRRSleepJob();
    sleepJob.setConf(sleepConf);

    Job job = sleepJob.createJob(1, 1, 2, 1, 1, 1, 1, 1, 1, 1);

    job.setJarByClass(MRRSleepJob.class);
    job.setMaxMapAttempts(1); // speed up failures

    // enable compression
    job.getConfiguration().setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, true);
    job.getConfiguration().set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, DefaultCodec.class.getName());

    job.submit();
    String trackingUrl = job.getTrackingURL();
    String jobId = job.getJobID().toString();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId,
            trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));

    // FIXME once counters and task progress can be obtained properly
    // TODO use dag client to test counters and task progress?
    // what about completed jobs?

}

From source file:org.finra.datagenerator.samples.distributor.hdfs.HDFSDistributor.java

License:Apache License

@Override
public void distribute(List<Frontier> searchProblemList) {
    // We need to write the List out to a file on HDFS
    // That file will be input into the MR job

    // Add variables job configuration
    configuration.set("stateMachineText", stateMachineText);
    configuration.setLong("maxNumberOfLines", maxNumberOfLines);

    // Write input problems
    try {//from   w w w. j  ava  2s  .co  m
        writeProblemsToHDFS(searchProblemList);
    } catch (IOException e) {
        log.error("Problem writing " + mapperInputFilePath + " prior to MR job execution");
        return;
    }

    // Prepare and submit job
    try {
        Job job = prepareJob();
        job.waitForCompletion(true);
        log.info("DataGen MR job can be tracked at " + job.getTrackingURL());
    } catch (IOException e) {
        e.printStackTrace();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
    // Cleanup
}

From source file:org.huahinframework.manager.rest.service.JobService.java

License:Apache License

/**
 * @param jobId//from ww w .  j av a  2 s . c om
 * @return {@link JSONObject}
 * @throws JSONException
 */
@Path("/detail/{" + JOBID + "}")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONObject detail(@PathParam(JOBID) String jobId) throws JSONException {
    JSONObject jsonObject = null;
    JobConf conf = getJobConf();
    try {
        final Map<String, Object> job = getStatus(jobId);
        if (job != null) {
            Cluster cluster = new Cluster(conf);
            Job j = cluster.getJob(JobID.forName(jobId));
            if (j != null) {
                String jobFile = j.getJobFile();
                job.put(Response.JOB_FILE, jobFile);
                job.put(Response.TRACKING_URL, j.getTrackingURL());

                Map<String, String> jobConf = JobUtils.getJobConfiguration(jobFile, conf);
                if (jobConf != null) {
                    job.put(Response.CONFIGURATION, jobConf);
                }
            }
            jsonObject = new JSONObject(job);
        }
    } catch (Exception e) {
        e.printStackTrace();
        log.error(e);
        Map<String, String> status = new HashMap<String, String>();
        status.put(Response.STATUS, e.getMessage());
        jsonObject = new JSONObject(status);
    }

    if (jsonObject == null) {
        Map<String, String> status = new HashMap<String, String>();
        status.put(Response.STATUS, "Could not find job " + jobId);
        jsonObject = new JSONObject(status);
    }

    return jsonObject;
}

From source file:org.lilyproject.indexer.master.IndexerMaster.java

License:Apache License

private void startFullIndexBuild(String indexName) {
    try {//from   w  w  w  .ja v  a  2 s. c  o m
        String lock = indexerModel.lockIndex(indexName);
        try {
            // Read current situation of record and assure it is still actual
            IndexDefinition index = indexerModel.getMutableIndex(indexName);
            if (needsBatchBuildStart(index)) {
                Job job = BatchIndexBuilder.startBatchBuildJob(index, mapReduceJobConf, hbaseConf,
                        zkConnectString, zkSessionTimeout);

                ActiveBatchBuildInfo jobInfo = new ActiveBatchBuildInfo();
                jobInfo.setSubmitTime(System.currentTimeMillis());
                jobInfo.setJobId(job.getJobID().toString());
                jobInfo.setTrackingUrl(job.getTrackingURL());
                index.setActiveBatchBuildInfo(jobInfo);

                index.setBatchBuildState(IndexBatchBuildState.BUILDING);

                indexerModel.updateIndexInternal(index);

                log.info(
                        "Started index build job for index " + indexName + ", job ID =  " + jobInfo.getJobId());
            }
        } finally {
            indexerModel.unlockIndex(lock);
        }
    } catch (Throwable t) {
        log.error("Error trying to start index build job for index " + indexName, t);
    }
}

From source file:org.mrgeo.mapreduce.MapReduceUtils.java

License:Apache License

public static boolean runJob(Job job, Progress progress, JobListener jl)
        throws JobFailedException, JobCancelledException {
    boolean success = false;
    if (jl != null) {
        //append job id to the job name for easy identification
        job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName());
        jl.addJob(job);//w  w w  .j av  a 2 s .co m
    }

    long start = System.currentTimeMillis();
    log.info("Running job {}", job.getJobName());

    try {
        job.submit();
        log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start));
        if (progress == null) {
            job.waitForCompletion(true);
        } else {
            float initP = progress.get();
            float percentP = 100 - initP;
            while (job.isComplete() == false) {
                float p = job.mapProgress() * .9f + job.reduceProgress() * .1f;
                progress.set(p * percentP + initP);
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e) {
                    log.info("Job Cancelled by user");
                    throw new JobCancelledException("Job Cancelled by user.");
                }
            }
        }

        log.info("Job {} time: {}ms", job.getJobName(), (System.currentTimeMillis() - start));

        if (job.isSuccessful() == false) {
            throw new JobFailedException("Job failed: " + job.getTrackingURL());
        }
        success = job.isSuccessful();
    } catch (InterruptedException e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    }
    // when submitting jobs under JBoss, Exception doesn't appear to be caught
    catch (Throwable e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    }
    return success;
}

From source file:org.mrgeo.mapreduce.MapReduceUtils.java

License:Apache License

/**
 * Check on the progress of a job and return true if the job has completed. Note
 * that a return value of true does not mean the job was successful, just that
 * it completed.//w ww  .  jav  a 2 s . co m
 * 
 * @param job
 * @param progress
 * @param jl
 * @return
 * @throws IOException
 * @throws FileNotFoundException
 * @throws JobFailedException
 * @throws JobCancelledException
 */
public static boolean checkJobProgress(Job job, Progress progress, JobListener jl)
        throws IOException, JobFailedException, JobCancelledException {
    boolean result = job.isComplete();
    if (progress != null) {
        float initP = progress.get();
        float percentP = 100 - initP;
        if (!result) {
            float p = job.mapProgress() * .9f + job.reduceProgress() * .1f;
            progress.set(p * percentP + initP);
        }
    }

    if (result) {
        if (!job.isSuccessful()) {
            if (jl != null && jl.isCancelled()) {
                throw new JobCancelledException(job.getJobName() + " - Job Cancelled by user");
            }
            throw new JobFailedException("Job failed: " + job.getTrackingURL());
        }
    }
    return result;
}

From source file:org.springframework.data.hadoop.batch.mapreduce.JobTasklet.java

License:Apache License

private static void saveJobStats(Job job, StepExecution stepExecution) {
    if (stepExecution == null) {
        return;//ww w  .  ja v  a2  s . co m
    }
    ExecutionContext executionContext = stepExecution.getExecutionContext();
    String statusPrefix = "Job Status::";
    executionContext.put(statusPrefix + "ID", JobUtils.getJobId(job).toString());
    executionContext.put(statusPrefix + "Name", job.getJobName());
    executionContext.put(statusPrefix + "Tracking URL", job.getTrackingURL());
    executionContext.put(statusPrefix + "State", JobUtils.getStatus(job).toString());
    try {
        for (String cgName : job.getCounters().getGroupNames()) {
            CounterGroup group = job.getCounters().getGroup(cgName);
            Iterator<Counter> ci = group.iterator();
            while (ci.hasNext()) {
                Counter c = ci.next();
                executionContext.put(group.getDisplayName().trim() + "::" + c.getDisplayName().trim(),
                        c.getValue());
            }
        }
    } catch (Exception ignore) {
    }
}