Example usage for org.apache.hadoop.mapreduce TypeConverter toYarn

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TypeConverter toYarn.

Prototype

public static Counters toYarn(org.apache.hadoop.mapreduce.Counters counters) 

Source Link

Usage

From source file:org.apache.tez.mapreduce.client.YARNRunner.java

License:Apache License

@Override
public JobStatus getJobStatus(JobID jobID) throws IOException, InterruptedException {
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    String jobFile = MRApps.getJobFile(conf, user, jobID);
    DAGStatus dagStatus;//w  ww  .j  a v a  2s. c  o  m
    try {
        if (dagClient == null) {
            dagClient = MRTezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId(), tezConf, null);
        }
        dagStatus = dagClient.getDAGStatus(null);
        return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
    } catch (TezException e) {
        throw new IOException(e);
    }
}

---

From source file:org.apache.tez.mapreduce.client.YARNRunner.java

License:Apache License

@Override
public void killJob(JobID arg0) throws IOException, InterruptedException {
    /* check if the status is not running, if not send kill to RM */
    JobStatus status = getJobStatus(arg0);
    if (status.getState() == JobStatus.State.RUNNING || status.getState() == JobStatus.State.PREP) {
        try {/*w  w  w.j  a va  2  s  . c o m*/
            resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
        } catch (YarnException e) {
            throw new IOException(e);
        }
        return;
    }
}

---

From source file:org.apache.tez.mapreduce.examples.OrderedWordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: wordcount <in> <out>");
        System.exit(2);//from   www.  j  av  a 2 s .  co  m
    }

    // Configure intermediate reduces
    conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);

    // Set reducer class for intermediate reduce
    conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduce.class"),
            IntSumReducer.class, Reducer.class);
    // Set reducer output key class
    conf.setClass(
            MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.map.output.key.class"),
            IntWritable.class, Object.class);
    // Set reducer output value class
    conf.setClass(
            MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.map.output.value.class"),
            Text.class, Object.class);
    conf.setInt(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduces"), 2);

    @SuppressWarnings("deprecation")
    Job job = new Job(conf, "orderedwordcount");
    job.setJarByClass(OrderedWordCount.class);

    // Configure map
    job.setMapperClass(TokenizerMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    // Configure reduce
    job.setReducerClass(MyOrderByNoOpReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    YarnClient yarnClient = new YarnClientImpl();
    yarnClient.init(conf);
    yarnClient.start();

    TezClient tezClient = new TezClient(new TezConfiguration(conf));

    job.submit();
    JobID jobId = job.getJobID();
    ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();

    DAGClient dagClient = tezClient.getDAGClient(appId);
    DAGStatus dagStatus = null;
    while (true) {
        dagStatus = dagClient.getDAGStatus();
        if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED
                || dagStatus.getState() == DAGStatus.State.FAILED
                || dagStatus.getState() == DAGStatus.State.KILLED
                || dagStatus.getState() == DAGStatus.State.ERROR) {
            break;
        }
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            // continue;
        }
    }

    while (dagStatus.getState() == DAGStatus.State.RUNNING) {
        try {
            ExampleDriver.printMRRDAGStatus(dagStatus);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                // continue;
            }
            dagStatus = dagClient.getDAGStatus();
        } catch (TezException e) {
            LOG.fatal("Failed to get application progress. Exiting");
            System.exit(-1);
        }
    }

    ExampleDriver.printMRRDAGStatus(dagStatus);
    LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
    System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
}

---

From source file:org.apache.tez.mapreduce.hadoop.TezTypeConverters.java

License:Apache License

public static TaskAttemptId toYarn(TezTaskAttemptID taskAttemptId) {
    TaskAttemptID mrTaskAttemptId = IDConverter.toMRTaskAttemptId(taskAttemptId);
    TaskAttemptId mrv2TaskAttemptId = TypeConverter.toYarn(mrTaskAttemptId);
    return mrv2TaskAttemptId;
}

---

From source file:org.apache.tez.mapreduce.YARNRunner.java

License:Apache License

@Override
public JobStatus getJobStatus(JobID jobID) throws IOException, InterruptedException {
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    String jobFile = MRApps.getJobFile(conf, user, jobID);
    DAGStatus dagStatus;//from   ww  w  . j av a 2  s  . c  om
    try {
        if (dagClient == null) {
            dagClient = tezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId());
        }
        dagStatus = dagClient.getDAGStatus();
        return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
    } catch (TezException e) {
        throw new IOException(e);
    }
}

---