Example usage for org.apache.hadoop.yarn.client.api YarnClient init

List of usage examples for org.apache.hadoop.yarn.client.api YarnClient init

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.client.api.*
  5. YarnClient
  6. init

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api YarnClient init.

Prototype

@Override
public void init(Configuration conf)

Source Link

Document

This invokes #serviceInit

Usage

From source file:org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample.java

License:Apache License

private DAG createDAG(FileSystem fs, TezConfiguration tezConf, Path stagingDir, boolean doLocalityCheck)
        throws IOException, YarnException {

    int numBroadcastTasks = 2;
    int numOneToOneTasks = 3;
    if (doLocalityCheck) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(tezConf);
        yarnClient.start();//from   ww w.j av  a 2 s  .  c o m
        int numNMs = yarnClient.getNodeReports(NodeState.RUNNING).size();
        yarnClient.stop();
        // create enough 1-1 tasks to run in parallel
        numOneToOneTasks = numNMs - numBroadcastTasks - 1;// 1 AM
        if (numOneToOneTasks < 1) {
            numOneToOneTasks = 1;
        }
    }
    byte[] procByte = { (byte) (doLocalityCheck ? 1 : 0), 1 };
    UserPayload procPayload = UserPayload.create(ByteBuffer.wrap(procByte));

    System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");

    Vertex broadcastVertex = Vertex.create("Broadcast",
            ProcessorDescriptor.create(InputProcessor.class.getName()), numBroadcastTasks);

    Vertex inputVertex = Vertex.create("Input",
            ProcessorDescriptor.create(InputProcessor.class.getName()).setUserPayload(procPayload),
            numOneToOneTasks);

    Vertex oneToOneVertex = Vertex.create("OneToOne",
            ProcessorDescriptor.create(OneToOneProcessor.class.getName()).setUserPayload(procPayload));
    oneToOneVertex.setVertexManagerPlugin(
            VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));

    UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
            .newBuilder(Text.class.getName(), IntWritable.class.getName()).setFromConfiguration(tezConf)
            .build();

    DAG dag = DAG.create("BroadcastAndOneToOneExample");
    dag.addVertex(inputVertex).addVertex(broadcastVertex).addVertex(oneToOneVertex)
            .addEdge(Edge.create(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
            .addEdge(Edge.create(broadcastVertex, oneToOneVertex,
                    edgeConf.createDefaultBroadcastEdgeProperty()));
    return dag;
}

From source file:org.apache.tez.mapreduce.examples.OrderedWordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: wordcount <in> <out>");
        System.exit(2);//  w  ww .j  a v a 2s  .  c om
    }

    // Configure intermediate reduces
    conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);

    // Set reducer class for intermediate reduce
    conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduce.class"),
            IntSumReducer.class, Reducer.class);
    // Set reducer output key class
    conf.setClass(
            MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.map.output.key.class"),
            IntWritable.class, Object.class);
    // Set reducer output value class
    conf.setClass(
            MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.map.output.value.class"),
            Text.class, Object.class);
    conf.setInt(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduces"), 2);

    @SuppressWarnings("deprecation")
    Job job = new Job(conf, "orderedwordcount");
    job.setJarByClass(OrderedWordCount.class);

    // Configure map
    job.setMapperClass(TokenizerMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    // Configure reduce
    job.setReducerClass(MyOrderByNoOpReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    YarnClient yarnClient = new YarnClientImpl();
    yarnClient.init(conf);
    yarnClient.start();

    TezClient tezClient = new TezClient(new TezConfiguration(conf));

    job.submit();
    JobID jobId = job.getJobID();
    ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();

    DAGClient dagClient = tezClient.getDAGClient(appId);
    DAGStatus dagStatus = null;
    while (true) {
        dagStatus = dagClient.getDAGStatus();
        if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED
                || dagStatus.getState() == DAGStatus.State.FAILED
                || dagStatus.getState() == DAGStatus.State.KILLED
                || dagStatus.getState() == DAGStatus.State.ERROR) {
            break;
        }
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            // continue;
        }
    }

    while (dagStatus.getState() == DAGStatus.State.RUNNING) {
        try {
            ExampleDriver.printMRRDAGStatus(dagStatus);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                // continue;
            }
            dagStatus = dagClient.getDAGStatus();
        } catch (TezException e) {
            LOG.fatal("Failed to get application progress. Exiting");
            System.exit(-1);
        }
    }

    ExampleDriver.printMRRDAGStatus(dagStatus);
    LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
    System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
}

From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java

License:Apache License

private void stopAndVerifyYarnApp(TezClient tezSession) throws TezException, IOException, YarnException {
    ApplicationId appId = tezSession.getAppMasterApplicationId();
    tezSession.stop();/*  www .  ja va 2  s  . c  o m*/
    Assert.assertEquals(TezAppMasterStatus.SHUTDOWN, tezSession.getAppMasterStatus());

    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    while (true) {
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        if (appReport.getYarnApplicationState().equals(YarnApplicationState.FINISHED)
                || appReport.getYarnApplicationState().equals(YarnApplicationState.FAILED)
                || appReport.getYarnApplicationState().equals(YarnApplicationState.KILLED)) {
            break;
        }
    }

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    Assert.assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState());
    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus());
}

From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java

License:Apache License

public State testMRRSleepJobDagSubmitCore(boolean dagViaRPC, boolean killDagWhileRunning,
        boolean closeSessionBeforeSubmit, TezClient reUseTezSession, boolean genSplitsInAM,
        Class<? extends InputInitializer> initializerClass, Map<String, LocalResource> additionalLocalResources)
        throws IOException, InterruptedException, TezException, ClassNotFoundException, YarnException {
    LOG.info("\n\n\nStarting testMRRSleepJobDagSubmit().");

    JobConf stage1Conf = new JobConf(mrrTezCluster.getConfig());
    JobConf stage2Conf = new JobConf(mrrTezCluster.getConfig());
    JobConf stage3Conf = new JobConf(mrrTezCluster.getConfig());

    stage1Conf.setLong(MRRSleepJob.MAP_SLEEP_TIME, 1);
    stage1Conf.setInt(MRRSleepJob.MAP_SLEEP_COUNT, 1);
    stage1Conf.setInt(MRJobConfig.NUM_MAPS, 1);
    stage1Conf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName());
    stage1Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
    stage1Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
    stage1Conf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, SleepInputFormat.class.getName());
    stage1Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());

    stage2Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1);
    stage2Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1);
    stage2Conf.setInt(MRJobConfig.NUM_REDUCES, 1);
    stage2Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName());
    stage2Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
    stage2Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
    stage2Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());

    stage3Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1);
    stage3Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1);
    stage3Conf.setInt(MRJobConfig.NUM_REDUCES, 1);
    stage3Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName());
    stage3Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
    stage3Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());

    MRHelpers.translateMRConfToTez(stage1Conf);
    MRHelpers.translateMRConfToTez(stage2Conf);
    MRHelpers.translateMRConfToTez(stage3Conf);
    MRHelpers.configureMRApiUsage(stage1Conf);
    MRHelpers.configureMRApiUsage(stage2Conf);
    MRHelpers.configureMRApiUsage(stage3Conf);

    Path remoteStagingDir = remoteFs
            .makeQualified(new Path("/tmp", String.valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf);
    UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf);

    DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000));

    Class<? extends InputInitializer> inputInitializerClazz = genSplitsInAM
            ? (initializerClass == null ? MRInputAMSplitGenerator.class : initializerClass)
            : null;//from w w  w  . j av  a 2s  . co  m
    LOG.info("Using initializer class: " + initializerClass);

    DataSourceDescriptor dsd;
    if (!genSplitsInAM) {
        dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, remoteStagingDir, true);
    } else {
        if (initializerClass == null) {
            dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).build();
        } else {
            InputInitializerDescriptor iid = InputInitializerDescriptor.create(inputInitializerClazz.getName());
            dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class)
                    .setCustomInitializerDescriptor(iid).build();
        }
    }

    Vertex stage1Vertex = Vertex.create("map",
            ProcessorDescriptor.create(MapProcessor.class.getName()).setUserPayload(stage1Payload),
            dsd.getNumberOfShards(), Resource.newInstance(256, 1));
    stage1Vertex.addDataSource("MRInput", dsd);
    Vertex stage2Vertex = Vertex.create("ireduce",
            ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage2Payload), 1,
            Resource.newInstance(256, 1));
    Vertex stage3Vertex = Vertex.create("reduce",
            ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage3Payload), 1,
            Resource.newInstance(256, 1));
    stage3Conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true);
    DataSinkDescriptor dataSinkDescriptor = MROutputLegacy
            .createConfigBuilder(stage3Conf, NullOutputFormat.class).build();
    Assert.assertFalse(dataSinkDescriptor.getOutputDescriptor().getHistoryText().isEmpty());
    stage3Vertex.addDataSink("MROutput", dataSinkDescriptor);

    // TODO env, resources

    dag.addVertex(stage1Vertex);
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);

    Edge edge1 = Edge.create(stage1Vertex, stage2Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload),
            InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload)));
    Edge edge2 = Edge.create(stage2Vertex, stage3Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload),
            InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload)));

    dag.addEdge(edge1);
    dag.addEdge(edge2);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

    DAGClient dagClient = null;
    boolean reuseSession = reUseTezSession != null;
    TezClient tezSession = null;
    if (!dagViaRPC) {
        Preconditions.checkArgument(reuseSession == false);
    }
    if (!reuseSession) {
        TezConfiguration tempTezconf = new TezConfiguration(tezConf);
        if (!dagViaRPC) {
            tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
        } else {
            tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
        }
        tezSession = TezClient.create("testsession", tempTezconf);
        tezSession.start();
    } else {
        tezSession = reUseTezSession;
    }
    if (!dagViaRPC) {
        // TODO Use utility method post TEZ-205 to figure out AM arguments etc.
        dagClient = tezSession.submitDAG(dag);
    }

    if (dagViaRPC && closeSessionBeforeSubmit) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        boolean sentKillSession = false;
        while (true) {
            Thread.sleep(500l);
            ApplicationReport appReport = yarnClient
                    .getApplicationReport(tezSession.getAppMasterApplicationId());
            if (appReport == null) {
                continue;
            }
            YarnApplicationState appState = appReport.getYarnApplicationState();
            if (!sentKillSession) {
                if (appState == YarnApplicationState.RUNNING) {
                    tezSession.stop();
                    sentKillSession = true;
                }
            } else {
                if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED
                        || appState == YarnApplicationState.FAILED) {
                    LOG.info("Application completed after sending session shutdown" + ", yarnApplicationState="
                            + appState + ", finalAppStatus=" + appReport.getFinalApplicationStatus());
                    Assert.assertEquals(YarnApplicationState.FINISHED, appState);
                    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED,
                            appReport.getFinalApplicationStatus());
                    break;
                }
            }
        }
        yarnClient.stop();
        return null;
    }

    if (dagViaRPC) {
        LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId()
                + " and Dag Name=" + dag.getName());
        if (additionalLocalResources != null) {
            tezSession.addAppMasterLocalFiles(additionalLocalResources);
        }
        dagClient = tezSession.submitDAG(dag);
        Assert.assertEquals(TezAppMasterStatus.RUNNING, tezSession.getAppMasterStatus());
    }
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info(
                "Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        if (killDagWhileRunning && dagStatus.getState() == DAGStatus.State.RUNNING) {
            LOG.info("Killing running dag/session");
            if (dagViaRPC) {
                tezSession.stop();
            } else {
                dagClient.tryKillDAG();
            }
        }
        dagStatus = dagClient.getDAGStatus(null);
    }
    if (!reuseSession) {
        tezSession.stop();
    }
    return dagStatus.getState();
}

From source file:org.apache.tez.test.MiniTezCluster.java

License:Apache License

private void waitForAppsToFinish() {
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(getConfig());
    yarnClient.start();/*  w w  w.ja v  a2s.c  o m*/
    try {
        while (true) {
            List<ApplicationReport> appReports = yarnClient.getApplications();
            Collection<ApplicationReport> unCompletedApps = Collections2.filter(appReports,
                    new Predicate<ApplicationReport>() {
                        @Override
                        public boolean apply(ApplicationReport appReport) {
                            return EnumSet
                                    .of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING,
                                            YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED,
                                            YarnApplicationState.RUNNING)
                                    .contains(appReport.getYarnApplicationState());
                        }
                    });
            if (unCompletedApps.size() == 0) {
                break;
            }
            LOG.info("wait for applications to finish in MiniTezCluster");
            Thread.sleep(1000);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        yarnClient.stop();
    }
}

From source file:org.apache.tez.test.TestExceptionPropagation.java

License:Apache License

/**
 * verify the diagnostics in {@link DAGStatus} is correct in non-session mode,
 * and also verify that diagnostics from {@link DAGStatus} should match that
 * from {@link ApplicationReport}/*  w ww  .  ja v  a 2 s  .c  o m*/
 * 
 * @throws Exception
 */
@Test(timeout = 120000)
public void testExceptionPropagationNonSession() throws Exception {
    try {
        startMiniTezCluster();
        startNonSessionClient();

        ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs;
        LOG.info("NonSession mode, Test for Exception from:" + exLocation.name());
        DAG dag = createDAG(exLocation);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.waitForCompletion();
        String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
        LOG.info("Diagnostics:" + diagnostics);
        assertTrue(diagnostics.contains(exLocation.name()));

        // wait for app complete (unregisterApplicationMaster is done)
        ApplicationId appId = tezClient.getAppMasterApplicationId();
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(tezConf);
        yarnClient.start();
        Set<YarnApplicationState> FINAL_APPLICATION_STATES = EnumSet.of(YarnApplicationState.KILLED,
                YarnApplicationState.FAILED, YarnApplicationState.FINISHED);
        ApplicationReport appReport = null;
        while (true) {
            appReport = yarnClient.getApplicationReport(appId);
            Thread.sleep(1000);
            LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
            LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
            if (FINAL_APPLICATION_STATES.contains(appReport.getYarnApplicationState())) {
                break;
            }
        }
        // wait for 1 second and call getApplicationReport again to ensure get the
        // diagnostics
        // TODO remove it after YARN-2560
        Thread.sleep(1000);
        appReport = yarnClient.getApplicationReport(appId);

        LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
        LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
        assertTrue(appReport.getDiagnostics().contains(exLocation.name()));
        // use "\n" as separator, because we also use it in Tez internally when
        // assembling the application diagnostics.
        assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(),
                appReport.getDiagnostics().trim());
    } finally {
        stopNonSessionClient();
        Thread.sleep(10 * 1000);
        stopTezMiniCluster();
    }
}

From source file:org.apache.tez.test.TestTezJobs.java

License:Apache License

@Test(timeout = 60000)
public void testSimpleSessionExample() throws Exception {
    Path stagingDirPath = new Path("/tmp/owc-staging-dir");
    remoteFs.mkdirs(stagingDirPath);/*from  ww  w  . j  a  va  2 s  . c  o m*/

    int numIterations = 2;
    String[] inputPaths = new String[numIterations];
    String[] outputPaths = new String[numIterations];
    Path[] outputDirs = new Path[numIterations];
    for (int i = 0; i < numIterations; ++i) {
        String inputDirStr = "/tmp/owc-input-" + i + "/";
        inputPaths[i] = inputDirStr;
        Path inputDir = new Path(inputDirStr);
        remoteFs.mkdirs(inputDir);
        generateOrderedWordCountInput(inputDir, remoteFs);
        String outputDirStr = "/tmp/owc-output-" + i + "/";
        outputPaths[i] = outputDirStr;
        Path outputDir = new Path(outputDirStr);
        outputDirs[i] = outputDir;
    }

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    YarnClient yarnClient = YarnClient.createYarnClient();

    try {

        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();

        List<ApplicationReport> apps = yarnClient.getApplications();
        int appsBeforeCount = apps != null ? apps.size() : 0;

        SimpleSessionExample job = new SimpleSessionExample();
        tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
        Assert.assertTrue("SimpleSessionExample failed", job.run(tezConf,
                new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" },
                null) == 0);

        for (int i = 0; i < numIterations; ++i) {
            verifyOutput(outputDirs[i], remoteFs);
        }

        apps = yarnClient.getApplications();
        int appsAfterCount = apps != null ? apps.size() : 0;

        // Running in session mode. So should only create 1 more app.
        Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
    } finally {
        remoteFs.delete(stagingDirPath, true);
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }

}

From source file:org.apache.tez.test.TestTezJobs.java

License:Apache License

@Test(timeout = 60000)
public void testInvalidQueueSubmission() throws Exception {

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    YarnClient yarnClient = YarnClient.createYarnClient();
    try {/*from www . j a v a 2  s  . c  o m*/

        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();

        SimpleSessionExample job = new SimpleSessionExample();
        tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
        tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");

        String[] inputPaths = new String[1];
        String[] outputPaths = new String[1];
        String inputDirStr = "/tmp/owc-input";
        inputPaths[0] = inputDirStr;
        Path inputDir = new Path(inputDirStr);
        remoteFs.mkdirs(inputDir);
        String outputDirStr = "/tmp/owc-output";
        outputPaths[0] = outputDirStr;
        job.run(tezConf,
                new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" },
                null);
        fail("Job submission should have thrown an exception");
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains("Failed to submit application to YARN"));
    } finally {
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }

}

From source file:org.apache.tez.tests.MiniTezClusterWithTimeline.java

License:Apache License

private void waitForAppsToFinish() {
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(getConfig());
    yarnClient.start();/*w  w w  .j a  v  a  2 s  .c  o  m*/
    try {
        while (true) {
            List<ApplicationReport> appReports = yarnClient.getApplications();
            Collection<ApplicationReport> unCompletedApps = Collections2.filter(appReports,
                    new Predicate<ApplicationReport>() {
                        @Override
                        public boolean apply(ApplicationReport appReport) {
                            return EnumSet
                                    .of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING,
                                            YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED,
                                            YarnApplicationState.RUNNING)
                                    .contains(appReport.getYarnApplicationState());
                        }
                    });
            if (unCompletedApps.size() == 0) {
                break;
            }
            LOG.info("wait for applications to finish in MiniTezClusterWithTimeline");
            Thread.sleep(1000);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        yarnClient.stop();
    }
}

From source file:org.apache.tez.tests.TestExternalTezServicesErrors.java

License:Apache License

private void testFatalError(String methodName, Vertex.VertexExecutionContext lhsExecutionContext,
        String dagNameSuffix, List<String> expectedDiagMessages)
        throws IOException, TezException, YarnException, InterruptedException {
    TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    TezClient tezClient = TezClient// w  ww . j av  a2  s . c o  m
            .newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session",
                    tezClientConf)
            .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();

    ApplicationId appId = null;
    try {
        tezClient.start();
        LOG.info("TezSessionStarted for " + methodName);
        tezClient.waitTillReady();
        LOG.info("TezSession ready for submission for " + methodName);

        JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT,
                lhsExecutionContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH,
                dagNameSuffix);

        DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()),
                HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3);

        DAGClient dagClient = tezClient.submitDAG(dag);

        DAGStatus dagStatus = dagClient
                .waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
        assertEquals(DAGStatus.State.ERROR, dagStatus.getState());
        boolean foundDiag = false;
        for (String diag : dagStatus.getDiagnostics()) {
            foundDiag = checkDiag(diag, expectedDiagMessages);
            if (foundDiag) {
                break;
            }
        }
        appId = tezClient.getAppMasterApplicationId();
        assertTrue(foundDiag);
    } catch (InterruptedException e) {
        e.printStackTrace();
    } finally {
        tezClient.stop();
    }
    // Verify the state of the application.
    if (appId != null) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        try {
            yarnClient.init(tezClientConf);
            yarnClient.start();

            ApplicationReport appReport = yarnClient.getApplicationReport(appId);
            YarnApplicationState appState = appReport.getYarnApplicationState();
            while (!EnumSet
                    .of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED, YarnApplicationState.KILLED)
                    .contains(appState)) {
                Thread.sleep(200L);
                appReport = yarnClient.getApplicationReport(appId);
                appState = appReport.getYarnApplicationState();
            }

            // TODO Workaround for YARN-4554. AppReport does not provide diagnostics - need to fetch them from ApplicationAttemptReport
            ApplicationAttemptId appAttemptId = appReport.getCurrentApplicationAttemptId();
            ApplicationAttemptReport appAttemptReport = yarnClient.getApplicationAttemptReport(appAttemptId);
            String diag = appAttemptReport.getDiagnostics();
            assertEquals(FinalApplicationStatus.FAILED, appReport.getFinalApplicationStatus());
            assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState());
            checkDiag(diag, expectedDiagMessages);
        } finally {
            yarnClient.stop();
        }
    }
}