Example usage for org.apache.hadoop.mapred RunningJob isComplete

List of usage examples for org.apache.hadoop.mapred RunningJob isComplete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. RunningJob
  6. isComplete

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred RunningJob isComplete.

Prototype

public boolean isComplete() throws IOException;

Source Link

Document

Check if the job is finished or not.

Usage

From source file:org.apache.mahout.avro.text.mapred.AvroDocumentProcessor.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf();
    if (args.length != 2) {
        System.err.println("Usage: wordcount <in> <out>");
        return 0;
    }//from ww  w. jav a  2  s.c o m

    conf.setStrings("io.serializations",
            new String[] { WritableSerialization.class.getName(), AvroSpecificSerialization.class.getName(),
                    AvroReflectSerialization.class.getName(), AvroGenericSerialization.class.getName() });

    AvroComparator.setSchema(AvroDocument._SCHEMA); //TODO: must be done in mapper, reducer configure method.

    conf.setClass("mapred.output.key.comparator.class", AvroComparator.class, RawComparator.class);

    conf.setJarByClass(AvroDocumentProcessor.class);
    conf.setMapperClass(ProcessorMapper.class);
    conf.setReducerClass(IdentityReducer.class);
    conf.setOutputKeyClass(AvroDocument.class);
    conf.setOutputValueClass(NullWritable.class);

    conf.setInputFormat(AvroInputFormat.class);
    conf.setOutputFormat(AvroOutputFormat.class);

    AvroInputFormat.setAvroInputClass(conf, AvroDocument.class);
    AvroOutputFormat.setAvroOutputClass(conf, AvroDocument.class);

    Path input = new Path(args[0]);
    Path output = new Path(args[1]);

    FileSystem fs = FileSystem.get(conf);
    fs.delete(output, true);

    FileInputFormat.addInputPath(conf, input);
    FileOutputFormat.setOutputPath(conf, output);

    RunningJob job = JobClient.runJob(conf);
    job.waitForCompletion();

    return job.isComplete() ? 0 : 1;
}

From source file:org.apache.oozie.action.hadoop.JavaActionExecutor.java

License:Apache License

@Override
public void check(Context context, WorkflowAction action) throws ActionExecutorException {
    JobClient jobClient = null;//from  w  ww  . j a v  a  2s . co m
    boolean exception = false;
    LogUtils.setLogInfo(action);
    try {
        Element actionXml = XmlUtils.parseXml(action.getConf());
        FileSystem actionFs = context.getAppFileSystem();
        JobConf jobConf = createBaseHadoopConf(context, actionXml);
        jobClient = createJobClient(context, jobConf);
        RunningJob runningJob = getRunningJob(context, action, jobClient);
        if (runningJob == null) {
            context.setExecutionData(FAILED, null);
            throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017",
                    "Could not lookup launched hadoop Job ID [{0}] which was associated with "
                            + " action [{1}].  Failing this action!",
                    getActualExternalId(action), action.getId());
        }
        if (runningJob.isComplete()) {
            Path actionDir = context.getActionDir();
            String newId = null;
            // load sequence file into object
            Map<String, String> actionData = LauncherMapperHelper.getActionData(actionFs, actionDir, jobConf);
            if (actionData.containsKey(LauncherMapper.ACTION_DATA_NEW_ID)) {
                newId = actionData.get(LauncherMapper.ACTION_DATA_NEW_ID);
                String launcherId = action.getExternalId();
                runningJob = jobClient.getJob(JobID.forName(newId));
                if (runningJob == null) {
                    context.setExternalStatus(FAILED);
                    throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017",
                            "Unknown hadoop job [{0}] associated with action [{1}].  Failing this action!",
                            newId, action.getId());
                }
                context.setExternalChildIDs(newId);
                LOG.info(XLog.STD, "External ID swap, old ID [{0}] new ID [{1}]", launcherId, newId);
            } else {
                String externalIDs = actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS);
                if (externalIDs != null) {
                    context.setExternalChildIDs(externalIDs);
                    LOG.info(XLog.STD, "Hadoop Jobs launched : [{0}]", externalIDs);
                }
            }
            if (runningJob.isComplete()) {
                // fetching action output and stats for the Map-Reduce action.
                if (newId != null) {
                    actionData = LauncherMapperHelper.getActionData(actionFs, context.getActionDir(), jobConf);
                }
                LOG.info(XLog.STD, "action completed, external ID [{0}]", action.getExternalId());
                if (LauncherMapperHelper.isMainSuccessful(runningJob)) {
                    if (getCaptureOutput(action) && LauncherMapperHelper.hasOutputData(actionData)) {
                        context.setExecutionData(SUCCEEDED, PropertiesUtils
                                .stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_OUTPUT_PROPS)));
                        LOG.info(XLog.STD, "action produced output");
                    } else {
                        context.setExecutionData(SUCCEEDED, null);
                    }
                    if (LauncherMapperHelper.hasStatsData(actionData)) {
                        context.setExecutionStats(actionData.get(LauncherMapper.ACTION_DATA_STATS));
                        LOG.info(XLog.STD, "action produced stats");
                    }
                    getActionData(actionFs, runningJob, action, context);
                } else {
                    String errorReason;
                    if (actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS)) {
                        Properties props = PropertiesUtils
                                .stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
                        String errorCode = props.getProperty("error.code");
                        if ("0".equals(errorCode)) {
                            errorCode = "JA018";
                        }
                        if ("-1".equals(errorCode)) {
                            errorCode = "JA019";
                        }
                        errorReason = props.getProperty("error.reason");
                        LOG.warn("Launcher ERROR, reason: {0}", errorReason);
                        String exMsg = props.getProperty("exception.message");
                        String errorInfo = (exMsg != null) ? exMsg : errorReason;
                        context.setErrorInfo(errorCode, errorInfo);
                        String exStackTrace = props.getProperty("exception.stacktrace");
                        if (exMsg != null) {
                            LOG.warn("Launcher exception: {0}{E}{1}", exMsg, exStackTrace);
                        }
                    } else {
                        errorReason = XLog.format("LauncherMapper died, check Hadoop LOG for job [{0}:{1}]",
                                action.getTrackerUri(), action.getExternalId());
                        LOG.warn(errorReason);
                    }
                    context.setExecutionData(FAILED_KILLED, null);
                }
            } else {
                context.setExternalStatus("RUNNING");
                LOG.info(XLog.STD, "checking action, hadoop job ID [{0}] status [RUNNING]", runningJob.getID());
            }
        } else {
            context.setExternalStatus("RUNNING");
            LOG.info(XLog.STD, "checking action, hadoop job ID [{0}] status [RUNNING]", runningJob.getID());
        }
    } catch (Exception ex) {
        LOG.warn("Exception in check(). Message[{0}]", ex.getMessage(), ex);
        exception = true;
        throw convertException(ex);
    } finally {
        if (jobClient != null) {
            try {
                jobClient.close();
            } catch (Exception e) {
                if (exception) {
                    LOG.error("JobClient error: ", e);
                } else {
                    throw convertException(e);
                }
            }
        }
    }
}

From source file:org.apache.oozie.action.hadoop.LauncherHelper.java

License:Apache License

public static boolean isMainDone(RunningJob runningJob) throws IOException {
    return runningJob.isComplete();
}

From source file:org.apache.oozie.action.hadoop.TestDistCpActionExecutor.java

License:Apache License

public void testDistCpFile() throws Exception {
    Path inputPath = new Path(getFsTestCaseDir(), "input.txt");
    final Path outputPath = new Path(getFsTestCaseDir(), "output.txt");
    byte[] content = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes();

    OutputStream os = getFileSystem().create(inputPath);
    os.write(content);/* w w w . j  ava2  s. c  om*/
    os.close();

    String actionXml = "<distcp>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<arg>" + inputPath + "</arg>" + "<arg>" + outputPath
            + "</arg>" + "</distcp>";
    Context context = createContext(actionXml);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());

    waitFor(60 * 1000, new Predicate() {
        public boolean evaluate() throws Exception {
            return getFileSystem().exists(outputPath);
        }
    });
    assertTrue(getFileSystem().exists(outputPath));

    byte[] readContent = new byte[content.length];
    InputStream is = getFileSystem().open(outputPath);
    int offset = 0;
    while (offset < readContent.length) {
        int numRead = is.read(readContent, offset, readContent.length);
        if (numRead == -1) {
            break;
        }
        offset += numRead;
    }
    assertEquals(is.read(), -1);
    is.close();
    offset = 0;
    while (offset < readContent.length) {
        assertEquals(readContent[offset], content[offset]);
        offset++;
    }
}

From source file:org.apache.oozie.action.hadoop.TestHive2ActionExecutor.java

License:Apache License

@SuppressWarnings("deprecation")
public void testHive2Action() throws Exception {
    setupHiveServer2();/*from  w  w w . ja v a 2s.c om*/
    Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME);
    Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME);
    FileSystem fs = getFileSystem();

    {
        String query = getHive2Script(inputDir.toString(), outputDir.toString());
        Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
        dataWriter.write(SAMPLE_DATA_TEXT);
        dataWriter.close();
        Context context = createContext(getQueryActionXml(query));
        final RunningJob launcherJob = submitAction(context,
                Namespace.getNamespace("uri:oozie:hive2-action:0.2"));
        String launcherId = context.getAction().getExternalId();
        waitFor(200 * 1000, new Predicate() {
            @Override
            public boolean evaluate() throws Exception {
                return launcherJob.isComplete();
            }
        });
        assertTrue(launcherJob.isSuccessful());
        Configuration conf = new XConfiguration();
        conf.set("user.name", getTestUser());
        Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(),
                context.getActionDir(), conf);
        assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
        Hive2ActionExecutor ae = new Hive2ActionExecutor();
        ae.check(context, context.getAction());
        assertTrue(launcherId.equals(context.getAction().getExternalId()));
        assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
        ae.end(context, context.getAction());
        assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
        assertNotNull(context.getAction().getData());
        Properties outputData = new Properties();
        outputData.load(new StringReader(context.getAction().getData()));
        assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
        assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs());
        assertTrue(fs.exists(outputDir));
        assertTrue(fs.isDirectory(outputDir));
    }
    {
        Path script = new Path(getAppPath(), HIVE_SCRIPT_FILENAME);
        Writer scriptWriter = new OutputStreamWriter(fs.create(script));
        scriptWriter.write(getHive2Script(inputDir.toString(), outputDir.toString()));
        scriptWriter.close();

        Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
        dataWriter.write(SAMPLE_DATA_TEXT);
        dataWriter.close();
        Context context = createContext(getScriptActionXml());
        final RunningJob launcherJob = submitAction(context,
                Namespace.getNamespace("uri:oozie:hive2-action:0.1"));
        String launcherId = context.getAction().getExternalId();
        waitFor(200 * 1000, new Predicate() {
            @Override
            public boolean evaluate() throws Exception {
                return launcherJob.isComplete();
            }
        });
        assertTrue(launcherJob.isSuccessful());
        Configuration conf = new XConfiguration();
        conf.set("user.name", getTestUser());
        Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(),
                context.getActionDir(), conf);
        assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
        Hive2ActionExecutor ae = new Hive2ActionExecutor();
        ae.check(context, context.getAction());
        assertTrue(launcherId.equals(context.getAction().getExternalId()));
        assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
        ae.end(context, context.getAction());
        assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
        assertNotNull(context.getAction().getData());
        Properties outputData = new Properties();
        outputData.load(new StringReader(context.getAction().getData()));
        assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
        assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs());
        assertTrue(fs.exists(outputDir));
        assertTrue(fs.isDirectory(outputDir));
    }
}

From source file:org.apache.oozie.action.hadoop.TestHiveActionExecutor.java

License:Apache License

public void testHiveAction() throws Exception {
    Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME);
    Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME);
    String hiveScript = getHiveScript(inputDir.toString(), outputDir.toString());
    FileSystem fs = getFileSystem();

    {/* w  w  w .  ja v  a 2 s.  c  om*/
        Path script = new Path(getAppPath(), HIVE_SCRIPT_FILENAME);
        Writer scriptWriter = new OutputStreamWriter(fs.create(script));
        scriptWriter.write(hiveScript);
        scriptWriter.close();
        Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
        dataWriter.write(SAMPLE_DATA_TEXT);
        dataWriter.close();
        Context context = createContext(getActionScriptXml());
        Namespace ns = Namespace.getNamespace("uri:oozie:hive-action:0.2");
        final RunningJob launcherJob = submitAction(context, ns);
        String launcherId = context.getAction().getExternalId();
        waitFor(200 * 1000, new Predicate() {
            public boolean evaluate() throws Exception {
                return launcherJob.isComplete();
            }
        });
        assertTrue(launcherJob.isSuccessful());
        Configuration conf = new XConfiguration();
        conf.set("user.name", getTestUser());
        Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(),
                context.getActionDir(), conf);
        assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
        HiveActionExecutor ae = new HiveActionExecutor();
        ae.check(context, context.getAction());
        assertTrue(launcherId.equals(context.getAction().getExternalId()));
        assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
        assertNotNull(context.getAction().getData());
        ae.end(context, context.getAction());
        assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
        assertNotNull(context.getAction().getData());
        Properties outputData = new Properties();
        outputData.load(new StringReader(context.getAction().getData()));
        assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
        assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs());
        //while this works in a real cluster, it does not with miniMR
        //assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0);
        //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
        assertTrue(fs.exists(outputDir));
        assertTrue(fs.isDirectory(outputDir));
    }
    {
        Context context = createContext(getActionQueryXml(hiveScript));
        Namespace ns = Namespace.getNamespace("uri:oozie:hive-action:0.6");
        final RunningJob launcherJob = submitAction(context, ns);
        String launcherId = context.getAction().getExternalId();
        waitFor(200 * 1000, new Predicate() {
            public boolean evaluate() throws Exception {
                return launcherJob.isComplete();
            }
        });
        assertTrue(launcherJob.isSuccessful());
        Configuration conf = new XConfiguration();
        conf.set("user.name", getTestUser());
        Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(),
                context.getActionDir(), conf);
        assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
        HiveActionExecutor ae = new HiveActionExecutor();
        ae.check(context, context.getAction());
        assertTrue(launcherId.equals(context.getAction().getExternalId()));
        assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
        assertNotNull(context.getAction().getData());
        ae.end(context, context.getAction());
        assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
        assertNotNull(context.getAction().getData());
        Properties outputData = new Properties();
        outputData.load(new StringReader(context.getAction().getData()));
        assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
        assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs());
        //while this works in a real cluster, it does not with miniMR
        //assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0);
        //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
        assertTrue(fs.exists(outputDir));
        assertTrue(fs.isDirectory(outputDir));
    }
}

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testSimpestSleSubmitOK() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "</java>";
    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override//from w  w w  .jav a2  s . c o m
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
}

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testOutputSubmitOK() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "<arg>out</arg>" + "<capture-output/>" + "</java>";
    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override/*  w ww .j a  va  2  s  . co m*/
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ActionExecutor ae = new JavaActionExecutor();
    ae.check(context, context.getAction());
    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNotNull(context.getAction().getData());
    StringReader sr = new StringReader(context.getAction().getData());
    Properties props = new Properties();
    props.load(sr);
    assertEquals("A", props.get("a"));

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
}

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testIdSwapSubmitOK() throws Exception {
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
            + "</main-class>" + "<arg>id</arg>" + "<capture-output/>" + "</java>";
    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    waitFor(60 * 1000, new Predicate() {
        @Override//  w ww . j  av  a2  s  .c  o  m
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ActionExecutor ae = new JavaActionExecutor();
    try {
        ae.check(context, context.getAction());
    } catch (ActionExecutorException ex) {
        if (!ex.getMessage().contains("IDSWAP")) {
            fail();
        }
    }
}

From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java

License:Apache License

public void testAdditionalJarSubmitOK() throws Exception {
    Path appJarPath = new Path("test-extra.jar");

    File jarFile = IOUtils.createJar(new File(getTestCaseDir()), appJarPath.getName(),
            LauncherMainTester2.class);
    InputStream is = new FileInputStream(jarFile);
    OutputStream os = getFileSystem().create(new Path(getAppPath(), appJarPath.toString()));
    IOUtils.copyStream(is, os);/*  w w  w.ja v  a  2 s .  c o  m*/

    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
            + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester2.class.getName()
            + "</main-class>" + "<file>" + appJarPath.toString() + "</file>" + "</java>";

    Context context = createContext(actionXml, null);
    final RunningJob runningJob = submitAction(context);
    ActionExecutor ae = new JavaActionExecutor();
    assertFalse(ae.isCompleted(context.getAction().getExternalStatus()));
    waitFor(60 * 1000, new Predicate() {
        @Override
        public boolean evaluate() throws Exception {
            return runningJob.isComplete();
        }
    });
    assertTrue(runningJob.isSuccessful());
    ae.check(context, context.getAction());
    assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());

    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
}