Example usage for org.apache.hadoop.mapred RunningJob isComplete

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred RunningJob isComplete.

Prototype

public boolean isComplete() throws IOException;

Source Link

Document

Check if the job is finished or not.

Usage

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

private void _testSubmitWithCredentials(String name, String actionXml) throws Exception {

    Context context = createContextWithCredentials("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }/*from   w w w  .java 2s  . c  o m*/
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    assertTrue(MapperReducerCredentialsForTest.hasCredentials(mrJob));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testMapReduceWithConfigClassNotFound() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();//from   w  ww .j a  v a  2s. c o  m

    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getMapReduceConfig(inputDir.toString(), outputDir.toString()).toXmlString(false)
            + "<config-class>org.apache.oozie.does.not.exist</config-class>" + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    waitFor(120 * 2000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(launcherJob));

    final Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, context.getActionDir(),
            context.getProtoActionConf());
    Properties errorProps = PropertiesUtils
            .stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
    assertEquals("An Exception occured while instantiating the action config class",
            errorProps.getProperty("exception.message"));
    assertTrue(errorProps.getProperty("exception.stacktrace")
            .startsWith(OozieActionConfiguratorException.class.getName()));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testMapReduceWithConfigClassThrowException() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();//ww w.j a  v a 2s  .c  o m

    XConfiguration conf = getMapReduceConfig(inputDir.toString(), outputDir.toString());
    conf.setBoolean("oozie.test.throw.exception", true); // causes OozieActionConfiguratorForTest to throw an exception
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + conf.toXmlString(false) + "<config-class>"
            + OozieActionConfiguratorForTest.class.getName() + "</config-class>" + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    waitFor(120 * 2000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(launcherJob));

    final Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, context.getActionDir(),
            context.getProtoActionConf());
    Properties errorProps = PropertiesUtils
            .stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
    assertEquals("doh", errorProps.getProperty("exception.message"));
    assertTrue(errorProps.getProperty("exception.stacktrace")
            .startsWith(OozieActionConfiguratorException.class.getName()));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testSetExecutionStats_when_user_has_specified_stats_write_TRUE() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*  w  w  w  .  j a  v  a 2s  .  c  om*/

    // set user stats write property as true explicitly in the
    // configuration.
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getOozieActionExternalStatsWriteProperty(inputDir.toString(), outputDir.toString(), "true")
                    .toXmlString(false)
            + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));

    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    // Assert for stats info stored in the context.
    assertNotNull(context.getExecutionStats());
    assertTrue(context.getExecutionStats().contains("ACTION_TYPE"));
    assertTrue(context.getExecutionStats().contains("Counter"));

    // External Child IDs used to be null, but after 4.0, become Non-Null in case of MR action.
    assertNotNull(context.getExternalChildIDs());

    // hadoop.counters will always be set in case of MR action.
    assertNotNull(context.getVar("hadoop.counters"));
    String counters = context.getVar("hadoop.counters");
    assertTrue(counters.contains("Counter"));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testSetExecutionStats_when_user_has_specified_stats_write_FALSE() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*from   w ww .  j a  v  a  2  s . c  o  m*/

    // set user stats write property as false explicitly in the
    // configuration.
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getOozieActionExternalStatsWriteProperty(inputDir.toString(), outputDir.toString(), "false")
                    .toXmlString(false)
            + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 2000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    // Assert for stats info stored in the context.
    assertNull(context.getExecutionStats());

    // External Child IDs used to be null, but after 4.0, become Non-Null in case of MR action.
    assertNotNull(context.getExternalChildIDs());

    // hadoop.counters will always be set in case of MR action.
    assertNotNull(context.getVar("hadoop.counters"));
    String counters = context.getVar("hadoop.counters");
    assertTrue(counters.contains("Counter"));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testEndWithoutConfiguration() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*from  w  w w . j ava  2 s  . com*/

    // set user stats write property as false explicitly in the
    // configuration.
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getOozieActionExternalStatsWriteProperty(inputDir.toString(), outputDir.toString(), "false")
                    .toXmlString(false)
            + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 2000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node></map-reduce>";

    WorkflowActionBean action = (WorkflowActionBean) context.getAction();
    action.setConf(actionXml);

    try {
        ae.end(context, context.getAction());
    } catch (Exception e) {
        fail("unexpected exception throwing " + e);
    }
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

/**
 * Test "oozie.launcher.mapred.job.name" and "mapred.job.name" can be set in
 * the action configuration and not overridden by the action executor
 *
 * @throws Exception/*from w ww.j a  va2  s .co m*/
 */
public void testSetMapredJobName() throws Exception {
    final String launcherJobName = "MapReduceLauncherTest";
    final String mapredJobName = "MapReduceTest";

    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();

    XConfiguration mrConfig = getMapReduceConfig(inputDir.toString(), outputDir.toString());
    mrConfig.set("oozie.launcher.mapred.job.name", launcherJobName);
    mrConfig.set("mapred.job.name", mapredJobName);

    StringBuilder sb = new StringBuilder("<map-reduce>").append("<job-tracker>").append(getJobTrackerUri())
            .append("</job-tracker>").append("<name-node>").append(getNameNodeUri()).append("</name-node>")
            .append(mrConfig.toXmlString(false)).append("</map-reduce>");
    String actionXml = sb.toString();

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 2000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });

    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
    // Assert launcher job name has been set
    System.out.println("Launcher job name: " + launcherJob.getJobName());
    assertTrue(launcherJob.getJobName().equals(launcherJobName));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");

    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    // Assert Mapred job name has been set
    System.out.println("Mapred job name: " + mrJob.getJobName());
    assertTrue(mrJob.getJobName().equals(mapredJobName));

    // Assert for stats info stored in the context.
    assertNull(context.getExecutionStats());

    // External Child IDs used to be null, but after 4.0, become Non-Null in case of MR action.
    assertNotNull(context.getExternalChildIDs());

    // hadoop.counters will always be set in case of MR action.
    assertNotNull(context.getVar("hadoop.counters"));
    String counters = context.getVar("hadoop.counters");
    assertTrue(counters.contains("Counter"));
}

---

From source file:org.apache.oozie.action.hadoop.TestPigActionExecutor.java

License:Apache License

private void evaluateLauncherJob(final RunningJob launcherJob) throws Exception {
    waitFor(180 * 1000, new Predicate() {
        @Override//www .j a va  2 s.  c  o  m
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    sleep(2000);
}

---

From source file:org.apache.oozie.action.hadoop.TestPyspark.java

License:Apache License

private void testPysparkHelper(String sparkOpts, WorkflowJobBean wf, String externalStatus,
        WorkflowAction.Status wfStatus) throws Exception {
    Context context = createContext(getActionXml(sparkOpts), wf);
    final RunningJob launcherJob = submitAction(context);
    waitFor(200 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }//w w  w. jav  a 2  s . co m
    });
    assertTrue(launcherJob.isSuccessful());
    SparkActionExecutor ae = new SparkActionExecutor();
    ae.check(context, context.getAction());
    assertEquals(externalStatus, context.getAction().getExternalStatus());
    ae.end(context, context.getAction());
    assertEquals(wfStatus, context.getAction().getStatus());
}

---

From source file:org.apache.oozie.action.hadoop.TestShellActionExecutor.java

License:Apache License

/**
 * Test that env variable can contain '=' symbol within value
 *
 * @throws Exception/*  ww w .  j av  a2s .c o m*/
 */
public void testEnvVar() throws Exception {
    Services.get().destroy();
    Services services = new Services();
    services.getConf().setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 8 * 1042);
    services.init();

    FileSystem fs = getFileSystem();
    // Create the script file with canned shell command
    Path script = new Path(getAppPath(), SHELL_SCRIPTNAME);
    Writer w = new OutputStreamWriter(fs.create(script));
    w.write(SHELL_SCRIPT_CONTENT_ENVVAR);
    w.close();

    String envValueHavingEqualSign = "a=b;c=d";
    // Create sample shell action xml
    String actionXml = "<shell>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<exec>" + SHELL_EXEC + "</exec>" + "<argument>" + SHELL_PARAM
            + "</argument>" + "<argument>" + SHELL_SCRIPTNAME + "</argument>" + "<argument>A</argument>"
            + "<argument>B</argument>" + "<env-var>var1=val1</env-var>" + "<env-var>var2="
            + envValueHavingEqualSign + "</env-var>" + "<file>" + script.toString() + "#" + script.getName()
            + "</file>" + "<capture-output />" + "</shell>";

    Context context = createContext(actionXml);
    // Submit the action
    final RunningJob launcherJob = submitAction(context);
    waitFor(180 * 1000, new Predicate() { // Wait for the external job to
        // finish
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });

    ShellActionExecutor ae = new ShellActionExecutor();
    WorkflowAction action = context.getAction();
    ae.check(context, action);
    ae.end(context, action);

    // Checking action data from shell script output
    assertEquals(envValueHavingEqualSign,
            PropertiesUtils.stringToProperties(action.getData()).getProperty("var2"));

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »