Example usage for org.apache.hadoop.mapred RunningJob isComplete

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred RunningJob isComplete.

Prototype

public boolean isComplete() throws IOException;

Source Link

Document

Check if the job is finished or not.

Usage

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testExit0SubmitOK() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "<arg>exit0</arg>" + "</java>";

    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override//  w w w  .  j a  v  a  2 s .c  om
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testExit1SubmitError() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "<arg>exit1</arg>" + "</java>";

    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override//  ww w .  j a v  a  2 s. c  om
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(runningJob));
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
    assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
    assertEquals("1", context.getAction().getErrorCode());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testExceptionSubmitException() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "<arg>exception</arg>" + "</java>";

    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override/*from   ww w .j  a  v  a2 s . c  o  m*/
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(runningJob));
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
    assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testExceptionSubmitThrowable() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "<arg>throwable</arg>" + "</java>";

    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override//from  w  ww  . jav a  2  s.  c  o  m
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    assertFalse(LauncherMapperHelper.isMainSuccessful(runningJob));
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
    assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testKill() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "</java>";
    final Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    assertFalse(runningJob.isComplete());
    ActionExecutor ae = new JavaActionExecutor();
    ae.kill(context, context.getAction());
    assertEquals(WorkflowAction.Status.DONE, context.getAction().getStatus());
    assertEquals("KILLED", context.getAction().getExternalStatus());
    assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));

    waitFor(60 * 1000, new Predicate() {
        @Override// w  ww. jav  a  2 s .  c  o  m
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertFalse(runningJob.isSuccessful());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testRecovery() throws Exception {
    final String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "</java>";
    final Context context = createContext(actionXml, null);
    RunningJob runningJob = submitAction(context);
    String launcherId = context.getAction().getExternalId();

    waitFor(60 * 1000, new Predicate() {
        @Override//from   ww w  .j  a v a 2s. c  om
        public boolean evaluate() throws Exception {
            JavaActionExecutor ae = new JavaActionExecutor();
            Configuration conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
            return LauncherMapperHelper.getRecoveryId(conf, context.getActionDir(),
                    context.getRecoveryId()) != null;
        }
    });

    final RunningJob runningJob2 = submitAction(context);

    assertEquals(launcherId, runningJob2.getJobID().toString());
    assertEquals(launcherId, context.getAction().getExternalId());

    waitFor(60 * 1000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return runningJob2.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testPrepare() throws Exception {
    FileSystem fs = getFileSystem();
    Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
    Path delete = new Path(getFsTestCaseDir(), "delete");
    fs.mkdirs(delete);//from  w  w w.  ja v a 2 s  . com

    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<prepare>" + "<mkdir path='" + mkdir + "'/>"
            + "<delete path='" + delete + "'/>" + "</prepare>" + "<configuration>" + "<property>"
            + "<name>dfs.umaskmode</name>" + "<value>026</value>" + "</property>" + "<property>"
            + "<name>fs.hdfs.impl.disable.cache</name>" + "<value>true</value>" + "</property>"
            + "</configuration>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>"
            + "</java>";
    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    assertTrue(fs.exists(mkdir));
    // Check if the action configuration is applied in the prepare block
    assertEquals("rwxr-x--x", fs.getFileStatus(mkdir).getPermission().toString());
    assertFalse(fs.exists(delete));
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testActionShareLibWithNonDefaultNamenode() throws Exception {

    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);

    Path systemLibPath = new Path(wps.getSystemLibPath(), ShareLibService.SHARE_LIB_PREFIX
            + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()).toString());

    File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "sourcejar.jar", LauncherMainTester.class);
    InputStream is = new FileInputStream(jarFile);
    Path javaShareLibPath = new Path(systemLibPath, "java");
    getFileSystem().mkdirs(javaShareLibPath);
    Path jar1Path = new Path(javaShareLibPath, "jar1.jar");
    OutputStream os1 = getFileSystem().create(jar1Path);
    IOUtils.copyStream(is, os1);/*from   www  .  java 2 s .  c om*/
    Path jar2Path = new Path(javaShareLibPath, "jar2.jar");
    OutputStream os2 = getFileSystem().create(jar2Path);
    is = new FileInputStream(jarFile); // is not resetable
    IOUtils.copyStream(is, os2);
    Path launcherPath = new Path(systemLibPath, "oozie");
    getFileSystem().mkdirs(launcherPath);
    Path jar3Path = new Path(launcherPath, "jar3.jar");
    OutputStream os3 = getFileSystem().create(jar3Path);
    is = new FileInputStream(jarFile);
    IOUtils.copyStream(is, os3);

    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNode2Uri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<main-class>"
            + LauncherMainTester.class.getName() + "</main-class>" + "</java>";

    XConfiguration jConf = new XConfiguration();
    jConf.set("p", "v");
    OutputStream os = getFileSystem().create(new Path(getAppPath(), "job.xml"));
    jConf.writeXml(os);
    os.close();

    Context context = createContext(actionXml, null);

    Services.get().setService(ShareLibService.class);

    // Test oozie server action sharelib setting
    WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
    XConfiguration wfConf = new XConfiguration();
    wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
    wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(), "workflow.xml").toString());
    wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
    workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());

    ConfigurationService.set("oozie.action.sharelib.for.java", "java");

    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
}

---

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testJobSubmissionWithoutYarnKill() throws Exception {
    Path inputDir = new Path(getFsTestCaseDir(), "input");
    Path outputDir = new Path(getFsTestCaseDir(), "output");

    Writer w = new OutputStreamWriter(getFileSystem().create(new Path(inputDir, "data.txt")));
    w.write("dummy\n");
    w.write("dummy\n");
    w.close();/*w  w w. jav a  2  s.co m*/

    w = new OutputStreamWriter(getFileSystem().create(new Path(inputDir, "id.pig")));
    w.write("A = load '$INPUT' using PigStorage(':');\n");
    w.write("store B into '$OUTPUT' USING PigStorage();\n");
    w.close();
    String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<prepare>" + "<delete path='outputdir' />" + "</prepare>"
            + "<configuration>" + "<property>" + "<name>mapred.compress.map.output</name>"
            + "<value>true</value>" + "</property>" + "<property>" + "<name>mapred.job.queue.name</name>"
            + "<value>default</value>" + "</property>" + "</configuration>" + "<script>" + inputDir.toString()
            + "/id.pig" + "</script>" + "<param>INPUT=" + inputDir.toUri().getPath() + "</param>"
            + "<param>OUTPUT=" + outputDir.toUri().getPath() + "/pig-output</param>" + "</pig>";

    PigActionExecutor ae = new PigActionExecutor();
    WorkflowJobBean wfBean = addRecordToWfJobTable("test1", actionXml);
    WorkflowActionBean action = (WorkflowActionBean) wfBean.getActions().get(0);
    action.setType(ae.getType());
    action.setConf(actionXml);
    Context context = new Context(wfBean, action);

    ConfigurationService.setBoolean(JavaActionExecutor.HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART, false);

    final RunningJob runningJob = submitAction(context, ae);
    waitFor(60 * 1000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
}

---

From source file:org.apache.oozie.action.hadoop.TestLauncher.java

License:Apache License

private RunningJob _test(String... arg) throws Exception {
    Path actionDir = getFsTestCaseDir();

    File jar = IOUtils.createJar(new File(getTestCaseDir()), "launcher.jar", LauncherMapper.class,
            LauncherMainException.class, LauncherSecurityManager.class, LauncherException.class,
            LauncherMainTester.class);

    FileSystem fs = getFileSystem();

    Path launcherJar = new Path(actionDir, "launcher.jar");
    fs.copyFromLocalFile(new Path(jar.toString()), launcherJar);

    JobConf jobConf = Services.get().get(HadoopAccessorService.class)
            .createJobConf(new URI(getNameNodeUri()).getAuthority());
    //        jobConf.setJar(jar.getAbsolutePath());
    jobConf.set("user.name", getTestUser());
    jobConf.setInt("mapred.map.tasks", 1);
    jobConf.setInt("mapred.map.max.attempts", 1);
    jobConf.setInt("mapred.reduce.max.attempts", 1);

    jobConf.set("mapreduce.framework.name", "yarn");
    jobConf.set("mapred.job.tracker", getJobTrackerUri());
    jobConf.set("fs.default.name", getNameNodeUri());

    LauncherMapperHelper.setupMainClass(jobConf, LauncherMainTester.class.getName());
    LauncherMapperHelper.setupMainArguments(jobConf, arg);

    Configuration actionConf = new XConfiguration();
    LauncherMapperHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
    LauncherMapperHelper.setupYarnRestartHandling(jobConf, jobConf, "1@a", System.currentTimeMillis());

    assertEquals("1", actionConf.get("oozie.job.id"));
    assertEquals("1@a", actionConf.get("oozie.action.id"));

    DistributedCache.addFileToClassPath(new Path(launcherJar.toUri().getPath()), jobConf);

    JobClient jobClient = createJobClient();

    final RunningJob runningJob = jobClient.submitJob(jobConf);

    System.out.println("Action Dir: " + actionDir);
    System.out.println("LauncherMapper ID: " + runningJob.getJobID().toString());

    waitFor(180 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }/*from  www .  j  av  a2 s  .c o m*/
    });

    assertTrue(jobConf.get("oozie.action.prepare.xml").equals(""));
    return runningJob;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »