Example usage for org.apache.hadoop.mapreduce TypeConverter fromYarn

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TypeConverter fromYarn.

Prototype

public static TaskTrackerInfo fromYarn(NodeReport node) 

Source Link

Usage

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

public AppStatusChecker(HadoopJobMonitorConfiguration conf, ApplicationReport appReport,
        ClientCache clientCache, ResourceMgrDelegate rmDelegate, AppCheckerProgress appCheckerProgress) {
    this.appReport = appReport;
    this.appId = appReport.getApplicationId();
    JobID jobId = TypeConverter.fromYarn(appReport.getApplicationId());
    this.jobId = jobId;
    this.clientCache = clientCache;
    LOG.debug("getting a client connection for " + appReport.getApplicationId());
    this.vConf = conf;
    this.rmDelegate = rmDelegate;
    this.appCheckerProgress = appCheckerProgress;
}

---

From source file:org.apache.tez.auxservices.TestShuffleHandler.java

License:Apache License

@Test
public void testRecovery() throws IOException {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(12345, 1);
    final JobID jobId = JobID.downgrade(TypeConverter.fromYarn(appId));
    final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")),
            TestShuffleHandler.class.getName());
    Configuration conf = new Configuration();
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
    ShuffleHandler shuffle = new ShuffleHandler();
    // emulate aux services startup with recovery enabled
    shuffle.setRecoveryPath(new Path(tmpDir.toString()));
    tmpDir.mkdirs();//from w ww. j  a  v  a 2  s  .c o  m
    try {
        shuffle.init(conf);
        shuffle.start();

        // setup a shuffle token for an application
        DataOutputBuffer outputBuffer = new DataOutputBuffer();
        outputBuffer.reset();
        Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(),
                "password".getBytes(), new Text(user), new Text("shuffleService"));
        jt.write(outputBuffer);
        shuffle.initializeApplication(new ApplicationInitializationContext(user, appId,
                ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength())));

        // verify we are authorized to shuffle
        int rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);

        // emulate shuffle handler restart
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        shuffle.start();

        // verify we are still authorized to shuffle to the old application
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);

        // shutdown app and verify access is lost
        shuffle.stopApplication(new ApplicationTerminationContext(appId));
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc);

        // emulate shuffle handler restart
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        shuffle.start();

        // verify we still don't have access
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc);
    } finally {
        if (shuffle != null) {
            shuffle.close();
        }
        FileUtil.fullyDelete(tmpDir);
    }
}

---

From source file:org.apache.tez.mapreduce.client.DAGJobStatus.java

License:Apache License

@Override
public JobID getJobID() {
    return TypeConverter.fromYarn(report.getApplicationId());
}

---

From source file:org.apache.tez.mapreduce.client.ResourceMgrDelegate.java

License:Apache License

public JobID getNewJobID() throws IOException, InterruptedException {
    try {//w  w  w . j  a va  2 s  . co  m
        this.application = client.createApplication().getNewApplicationResponse();
    } catch (YarnException e) {
        throw new IOException(e);
    }
    this.applicationId = this.application.getApplicationId();
    return TypeConverter.fromYarn(applicationId);
}

---

From source file:org.apache.tez.mapreduce.committer.MROutputCommitter.java

License:Apache License

private JobContext getJobContextFromVertexContext(OutputCommitterContext context) throws IOException {
    JobID jobId = TypeConverter.fromYarn(context.getApplicationId());
    return new MRJobContextImpl(jobConf, jobId);
}

---

From source file:org.apache.tez.mapreduce.hadoop.TezTypeConverters.java

License:Apache License

public static TezTaskAttemptID toTez(TaskAttemptId taskAttemptId) {
    TaskAttemptID mrTaskAttemptId = TypeConverter.fromYarn(taskAttemptId);
    TezTaskAttemptID tezTaskAttemptId = IDConverter.fromMRTaskAttemptId(mrTaskAttemptId);
    return tezTaskAttemptId;
}

---