Example usage for org.apache.hadoop.mapred RunningJob isSuccessful

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred RunningJob isSuccessful.

Prototype

public boolean isSuccessful() throws IOException;

Source Link

Document

Check if the job completed successfully.

Usage

From source file:org.apache.oozie.action.hadoop.TestLauncher.java

License:Apache License

public void testOutput() throws Exception {
    Path actionDir = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    final RunningJob runningJob = _test("out");
    waitFor(2000, new Predicate() {
        @Override/* w w  w.  ja v  a 2s.  c  om*/
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());

    Configuration conf = new XConfiguration();
    conf.set("user.name", getTestUser());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, actionDir, conf);
    assertTrue(fs.exists(LauncherMapperHelper.getActionDataSequenceFilePath(actionDir)));
    assertTrue(LauncherMapperHelper.isMainDone(runningJob));
    assertTrue(LauncherMapperHelper.isMainSuccessful(runningJob));
    assertTrue(LauncherMapperHelper.hasOutputData(actionData));
    assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
    assertTrue(LauncherMapperHelper.isMainDone(runningJob));
}

---

From source file:org.apache.oozie.action.hadoop.TestLauncher.java

License:Apache License

public void testNewId() throws Exception {
    Path actionDir = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    final RunningJob runningJob = _test("id");
    waitFor(2000, new Predicate() {
        @Override/*w  w  w .  ja  v a2 s  .  c om*/
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());

    Configuration conf = new XConfiguration();
    conf.set("user.name", getTestUser());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, actionDir, conf);
    assertTrue(fs.exists(LauncherMapperHelper.getActionDataSequenceFilePath(actionDir)));
    assertTrue(LauncherMapperHelper.isMainDone(runningJob));
    assertTrue(LauncherMapperHelper.isMainSuccessful(runningJob));
    assertFalse(LauncherMapperHelper.hasOutputData(actionData));
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
    assertTrue(LauncherMapperHelper.isMainDone(runningJob));
}

---

From source file:org.apache.oozie.action.hadoop.TestLauncher.java

License:Apache License

public void testSecurityManager() throws Exception {
    Path actionDir = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    final RunningJob runningJob = _test("securityManager");
    waitFor(2000, new Predicate() {
        @Override//from  w ww . j  a  va2 s .  c o m
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());

    Configuration conf = new XConfiguration();
    conf.set("user.name", getTestUser());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, actionDir, conf);
    assertFalse(fs.exists(LauncherMapperHelper.getActionDataSequenceFilePath(actionDir)));
    assertTrue(LauncherMapperHelper.isMainDone(runningJob));
    assertTrue(LauncherMapperHelper.isMainSuccessful(runningJob));
    assertFalse(LauncherMapperHelper.hasOutputData(actionData));
    assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
    assertTrue(LauncherMapperHelper.isMainDone(runningJob));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

private String _testSubmit(String name, String actionXml) throws Exception {

    Context context = createContext(name, actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 2000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }//from   ww  w.  j a  v a2s. c o  m
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    //hadoop.counters will always be set in case of MR action.
    assertNotNull(context.getVar("hadoop.counters"));
    String counters = context.getVar("hadoop.counters");
    assertTrue(counters.contains("Counter"));
    assertTrue(counters.contains("\"MAP_OUTPUT_RECORDS\":2"));

    //External Child IDs used to be null, but after 4.0, become Non-Null in case of MR action.
    assertNotNull(context.getExternalChildIDs());

    return mrJob.getID().toString();
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

private void _testSubmitWithCredentials(String name, String actionXml) throws Exception {

    Context context = createContextWithCredentials("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }/*  ww w  .j a  v a  2  s.  co  m*/
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    assertTrue(MapperReducerCredentialsForTest.hasCredentials(mrJob));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testMapReduceWithConfigClassNotFound() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*from   www  .  j  a va 2s.  c om*/

    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getMapReduceConfig(inputDir.toString(), outputDir.toString()).toXmlString(false)
            + "<config-class>org.apache.oozie.does.not.exist</config-class>" + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    waitFor(120 * 2000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(launcherJob));

    final Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, context.getActionDir(),
            context.getProtoActionConf());
    Properties errorProps = PropertiesUtils
            .stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
    assertEquals("An Exception occured while instantiating the action config class",
            errorProps.getProperty("exception.message"));
    assertTrue(errorProps.getProperty("exception.stacktrace")
            .startsWith(OozieActionConfiguratorException.class.getName()));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testMapReduceWithConfigClassThrowException() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*from   www  . ja  va2 s  .  c o  m*/

    XConfiguration conf = getMapReduceConfig(inputDir.toString(), outputDir.toString());
    conf.setBoolean("oozie.test.throw.exception", true); // causes OozieActionConfiguratorForTest to throw an exception
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + conf.toXmlString(false) + "<config-class>"
            + OozieActionConfiguratorForTest.class.getName() + "</config-class>" + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    waitFor(120 * 2000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(launcherJob));

    final Map<String, String> actionData = LauncherMapperHelper.getActionData(fs, context.getActionDir(),
            context.getProtoActionConf());
    Properties errorProps = PropertiesUtils
            .stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
    assertEquals("doh", errorProps.getProperty("exception.message"));
    assertTrue(errorProps.getProperty("exception.stacktrace")
            .startsWith(OozieActionConfiguratorException.class.getName()));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testSetExecutionStats_when_user_has_specified_stats_write_TRUE() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*from  w  w  w. ja v a2s . c o  m*/

    // set user stats write property as true explicitly in the
    // configuration.
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getOozieActionExternalStatsWriteProperty(inputDir.toString(), outputDir.toString(), "true")
                    .toXmlString(false)
            + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));

    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    // Assert for stats info stored in the context.
    assertNotNull(context.getExecutionStats());
    assertTrue(context.getExecutionStats().contains("ACTION_TYPE"));
    assertTrue(context.getExecutionStats().contains("Counter"));

    // External Child IDs used to be null, but after 4.0, become Non-Null in case of MR action.
    assertNotNull(context.getExternalChildIDs());

    // hadoop.counters will always be set in case of MR action.
    assertNotNull(context.getVar("hadoop.counters"));
    String counters = context.getVar("hadoop.counters");
    assertTrue(counters.contains("Counter"));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testSetExecutionStats_when_user_has_specified_stats_write_FALSE() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*from   w w  w  . ja  v a2 s.co m*/

    // set user stats write property as false explicitly in the
    // configuration.
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getOozieActionExternalStatsWriteProperty(inputDir.toString(), outputDir.toString(), "false")
                    .toXmlString(false)
            + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 2000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    // Assert for stats info stored in the context.
    assertNull(context.getExecutionStats());

    // External Child IDs used to be null, but after 4.0, become Non-Null in case of MR action.
    assertNotNull(context.getExternalChildIDs());

    // hadoop.counters will always be set in case of MR action.
    assertNotNull(context.getVar("hadoop.counters"));
    String counters = context.getVar("hadoop.counters");
    assertTrue(counters.contains("Counter"));
}

---

From source file:org.apache.oozie.action.hadoop.TestMapReduceActionExecutor.java

License:Apache License

public void testEndWithoutConfiguration() throws Exception {
    FileSystem fs = getFileSystem();

    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();//from  w w w .ja v  a  2 s  .  c o m

    // set user stats write property as false explicitly in the
    // configuration.
    String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>"
            + getOozieActionExternalStatsWriteProperty(inputDir.toString(), outputDir.toString(), "false")
                    .toXmlString(false)
            + "</map-reduce>";

    Context context = createContext("map-reduce", actionXml);
    final RunningJob launcherJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();
    waitFor(120 * 2000, new Predicate() {
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
            context.getProtoActionConf());
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    MapReduceActionExecutor ae = new MapReduceActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(launcherId.equals(context.getAction().getExternalId()));

    JobConf conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    String user = conf.get("user.name");
    String group = conf.get("group.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob mrJob = jobClient.getJob(JobID.forName(context.getAction().getExternalChildIDs()));

    waitFor(120 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    ae.check(context, context.getAction());

    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node></map-reduce>";

    WorkflowActionBean action = (WorkflowActionBean) context.getAction();
    action.setConf(actionXml);

    try {
        ae.end(context, context.getAction());
    } catch (Exception e) {
        fail("unexpected exception throwing " + e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• »