Example usage for org.apache.commons.exec DefaultExecutor setStreamHandler

List of usage examples for org.apache.commons.exec DefaultExecutor setStreamHandler

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.exec.*
  5. DefaultExecutor
  6. setStreamHandler

Introduction

In this page you can find the example usage for org.apache.commons.exec DefaultExecutor setStreamHandler.

Prototype

public void setStreamHandler(final ExecuteStreamHandler streamHandler)

Source Link

Usage

From source file:org.apache.zeppelin.submarine.job.thread.JobRunThread.java

public void run() {
    boolean tryLock = lockRunning.tryLock();
    if (false == tryLock) {
        LOGGER.warn("Can not get JobRunThread lockRunning!");
        return;//from   www.  j  ava2s  .c om
    }

    SubmarineUI submarineUI = submarineJob.getSubmarineUI();
    try {
        InterpreterContext intpContext = submarineJob.getIntpContext();
        String noteId = intpContext.getNoteId();
        String userName = intpContext.getAuthenticationInfo().getUser();
        String jobName = SubmarineUtils.getJobName(userName, noteId);

        if (true == running.get()) {
            String message = String.format("Job %s already running.", jobName);
            submarineUI.outputLog("WARN", message);
            LOGGER.warn(message);
            return;
        }
        running.set(true);

        Properties properties = submarineJob.getProperties();
        HdfsClient hdfsClient = submarineJob.getHdfsClient();
        File pythonWorkDir = submarineJob.getPythonWorkDir();

        submarineJob.setCurrentJobState(EXECUTE_SUBMARINE);

        String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, "");
        if (!algorithmPath.startsWith("hdfs://")) {
            String message = "Algorithm file upload HDFS path, " + "Must be `hdfs://` prefix. now setting "
                    + algorithmPath;
            submarineUI.outputLog("Configuration error", message);
            return;
        }

        List<ParagraphInfo> paragraphInfos = intpContext.getIntpEventClient().getParagraphList(userName,
                noteId);
        String outputMsg = hdfsClient.saveParagraphToFiles(noteId, paragraphInfos,
                pythonWorkDir == null ? "" : pythonWorkDir.getAbsolutePath(), properties);
        if (!StringUtils.isEmpty(outputMsg)) {
            submarineUI.outputLog("Save algorithm file", outputMsg);
        }

        HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, true);

        URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_JOBRUN_TF_JINJA);
        String template = Resources.toString(urlTemplate, Charsets.UTF_8);
        Jinjava jinjava = new Jinjava();
        String submarineCmd = jinjava.render(template, jinjaParams);
        // If the first line is a newline, delete the newline
        int firstLineIsNewline = submarineCmd.indexOf("\n");
        if (firstLineIsNewline == 0) {
            submarineCmd = submarineCmd.replaceFirst("\n", "");
        }

        StringBuffer sbLogs = new StringBuffer(submarineCmd);
        submarineUI.outputLog("Submarine submit command", sbLogs.toString());

        long timeout = Long
                .valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
        CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
        cmdLine.addArgument(submarineCmd, false);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
        executor.setWatchdog(watchDog);
        StringBuffer sbLogOutput = new StringBuffer();
        executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
            @Override
            protected void processLine(String line, int level) {
                line = line.trim();
                if (!StringUtils.isEmpty(line)) {
                    sbLogOutput.append(line + "\n");
                }
            }
        }));

        if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }

        Map<String, String> env = new HashMap<>();
        String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
        if (StringUtils.equals(launchMode, "yarn")) {
            // Set environment variables in the submarine interpreter container run on yarn
            String javaHome, hadoopHome, hadoopConf;
            javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
            hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
            hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
            env.put("JAVA_HOME", javaHome);
            env.put("HADOOP_HOME", hadoopHome);
            env.put("HADOOP_HDFS_HOME", hadoopHome);
            env.put("HADOOP_CONF_DIR", hadoopConf);
            env.put("YARN_CONF_DIR", hadoopConf);
            env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
            env.put("ZEPPELIN_FORCE_STOP", "true");
        }

        LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);
        AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
        executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
            @Override
            public void onProcessComplete(int exitValue) {
                String message = String.format("jobName %s ProcessComplete exit value is : %d", jobName,
                        exitValue);
                LOGGER.info(message);
                submarineUI.outputLog("JOR RUN COMPLETE", message);
                cmdLineRunning.set(false);
                submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_FINISHED);
            }

            @Override
            public void onProcessFailed(ExecuteException e) {
                String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s",
                        jobName, e.getExitValue(), e.getMessage());
                LOGGER.error(message);
                submarineUI.outputLog("JOR RUN FAILED", message);
                cmdLineRunning.set(false);
                submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
            }
        });
        int loopCount = 100;
        while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.destroyProcess();
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.killedProcess();
        }

        // Check if it has been submitted to YARN
        Map<String, Object> jobState = submarineJob.getJobStateByYarn(jobName);
        loopCount = 50;
        while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
            Thread.sleep(3000);
            jobState = submarineJob.getJobStateByYarn(jobName);
        }

        if (!jobState.containsKey("state")) {
            String message = String.format("JOB %s was not submitted to YARN!", jobName);
            LOGGER.error(message);
            submarineUI.outputLog("JOR RUN FAILED", message);
            submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR);
        submarineUI.outputLog("Exception", e.getMessage());
    } finally {
        running.set(false);
        lockRunning.unlock();
    }
}

From source file:org.apache.zeppelin.submarine.job.thread.TensorboardRunThread.java

public void run() {
    SubmarineUI submarineUI = submarineJob.getSubmarineUI();

    boolean tryLock = lockRunning.tryLock();

    try {/*ww w . j av  a2 s.c  om*/
        Properties properties = submarineJob.getProperties();
        String tensorboardName = SubmarineUtils.getTensorboardName(submarineJob.getUserName());
        if (true == running.get()) {
            String message = String.format("tensorboard %s already running.", tensorboardName);
            submarineUI.outputLog("WARN", message);
            LOGGER.warn(message);
            return;
        }
        running.set(true);

        HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, false);
        // update jobName -> tensorboardName
        jinjaParams.put(SubmarineConstants.JOB_NAME, tensorboardName);

        URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_TENSORBOARD_JINJA);
        String template = Resources.toString(urlTemplate, Charsets.UTF_8);
        Jinjava jinjava = new Jinjava();
        String submarineCmd = jinjava.render(template, jinjaParams);
        // If the first line is a newline, delete the newline
        int firstLineIsNewline = submarineCmd.indexOf("\n");
        if (firstLineIsNewline == 0) {
            submarineCmd = submarineCmd.replaceFirst("\n", "");
        }
        StringBuffer sbLogs = new StringBuffer(submarineCmd);
        submarineUI.outputLog("Submarine submit command", sbLogs.toString());

        long timeout = Long
                .valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout));
        CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell);
        cmdLine.addArgument(submarineCmd, false);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout);
        executor.setWatchdog(watchDog);
        StringBuffer sbLogOutput = new StringBuffer();
        executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
            @Override
            protected void processLine(String line, int level) {
                line = line.trim();
                if (!StringUtils.isEmpty(line)) {
                    sbLogOutput.append(line + "\n");
                }
            }
        }));

        if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) {
            executor.setWorkingDirectory(new File(System.getProperty("user.home")));
        }

        Map<String, String> env = new HashMap<>();
        String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE);
        if (StringUtils.equals(launchMode, "yarn")) {
            // Set environment variables in the container
            String javaHome, hadoopHome, hadoopConf;
            javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME);
            hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME);
            hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR);
            env.put("JAVA_HOME", javaHome);
            env.put("HADOOP_HOME", hadoopHome);
            env.put("HADOOP_HDFS_HOME", hadoopHome);
            env.put("HADOOP_CONF_DIR", hadoopConf);
            env.put("YARN_CONF_DIR", hadoopConf);
            env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
        }

        LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd);

        AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
        executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
            @Override
            public void onProcessComplete(int exitValue) {
                String message = String.format("jobName %s ProcessComplete exit value is : %d", tensorboardName,
                        exitValue);
                LOGGER.info(message);
                submarineUI.outputLog("TENSORBOARD RUN COMPLETE", message);
                cmdLineRunning.set(false);
            }

            @Override
            public void onProcessFailed(ExecuteException e) {
                String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s",
                        tensorboardName, e.getExitValue(), e.getMessage());
                LOGGER.error(message);
                submarineUI.outputLog("TENSORBOARD RUN FAILED", message);
                cmdLineRunning.set(false);
            }
        });
        int loopCount = 100;
        while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) {
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.destroyProcess();
            Thread.sleep(1000);
        }
        if (watchDog.isWatching()) {
            watchDog.killedProcess();
        }

        // Check if it has been submitted to YARN
        Map<String, Object> jobState = submarineJob.getJobStateByYarn(tensorboardName);
        loopCount = 50;
        while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) {
            Thread.sleep(3000);
            jobState = submarineJob.getJobStateByYarn(tensorboardName);
        }

        if (!jobState.containsKey("state")) {
            String message = String.format("tensorboard %s was not submitted to YARN!", tensorboardName);
            LOGGER.error(message);
            submarineUI.outputLog("JOR RUN FAILED", message);
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        submarineUI.outputLog("Exception", e.getMessage());
    } finally {
        running.set(false);
        lockRunning.unlock();
    }
}

From source file:org.apache.zeppelin.submarine.SubmarineJobTest.java

@Test
public void defaultExecutorTest() throws IOException {
    DefaultExecutor executor = new DefaultExecutor();
    CommandLine cmdLine = CommandLine.parse(shell);

    URL urlTemplate = Resources.getResource(DEFAULT_EXECUTOR_TEST);

    cmdLine.addArgument(urlTemplate.getFile(), false);

    Map<String, String> env = new HashMap<>();
    env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`");
    env.put("EVN", "test");

    AtomicBoolean cmdLineRunning = new AtomicBoolean(true);
    StringBuffer sbLogOutput = new StringBuffer();
    executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() {
        @Override//from  w w  w . j  a  v  a  2 s . co m
        protected void processLine(String line, int level) {
            //LOGGER.info(line);
            sbLogOutput.append(line + "\n");
        }
    }));

    executor.execute(cmdLine, env, new DefaultExecuteResultHandler() {
        @Override
        public void onProcessComplete(int exitValue) {
            cmdLineRunning.set(false);
        }

        @Override
        public void onProcessFailed(ExecuteException e) {
            cmdLineRunning.set(false);
            LOGGER.error(e.getMessage());
        }
    });
    int loopCount = 100;
    while ((loopCount-- > 0) && cmdLineRunning.get()) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    LOGGER.info(sbLogOutput.toString());
}

From source file:org.apache.zeppelin.util.ProcessLauncher.java

public void launch() {
    DefaultExecutor executor = new DefaultExecutor();
    this.processOutput = new ProcessLogOutputStream();
    executor.setStreamHandler(new PumpStreamHandler(processOutput));
    this.watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
    executor.setWatchdog(watchdog);//from   w w w  . jav  a2s .  c om
    try {
        executor.execute(commandLine, envs, this);
        transition(State.LAUNCHED);
        LOGGER.info("Process is launched: {}", commandLine);
    } catch (IOException e) {
        this.processOutput.stopCatchLaunchOutput();
        LOGGER.error("Fail to launch process: " + commandLine, e);
        transition(State.TERMINATED);
        errorMessage = e.getMessage();
    }
}

From source file:org.bonitasoft.platform.setup.PlatformSetupDistributionIT.java

@Test
public void setupSh_should_work_with_init_on_h2_and_prevent_pushing_deletion() throws Exception {
    //given//from   ww  w.j  ava2 s .c om
    CommandLine oCmdLine = PlatformSetupTestUtils.createCommandLine();
    oCmdLine.addArgument("init");
    DefaultExecutor executor = PlatformSetupTestUtils.createExecutor(distFolder);
    executor.setStreamHandler(PlatformSetupTestUtils.getExecuteStreamHandler("yes"));

    //when
    int iExitValue = executor.execute(oCmdLine);

    //then
    assertThat(iExitValue).isEqualTo(0);
    Connection jdbcConnection = PlatformSetupTestUtils.getJdbcConnection(distFolder);
    Statement statement = jdbcConnection.createStatement();
    ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) AS nb FROM CONFIGURATION");
    resultSet.next();
    assertThat(resultSet.getInt("nb")).isGreaterThan(0);

    oCmdLine = PlatformSetupTestUtils.createCommandLine();
    oCmdLine.addArgument("pull");
    iExitValue = executor.execute(oCmdLine);
    assertThat(iExitValue).isEqualTo(0);

    final Path platform_engine = distFolder.toPath().resolve("platform_conf").resolve("current")
            .resolve("platform_engine");
    FileUtils.deleteDirectory(platform_engine.toFile());

    oCmdLine = PlatformSetupTestUtils.createCommandLine();
    oCmdLine.addArgument("push");
    executor.setExitValue(1);
    iExitValue = executor.execute(oCmdLine);
    assertThat(iExitValue).isEqualTo(1);

    oCmdLine.addArgument("--force");
    executor.setExitValue(0);
    iExitValue = executor.execute(oCmdLine);
    assertThat(iExitValue).isEqualTo(0);
}

From source file:org.bonitasoft.platform.setup.PlatformSetupDistributionIT.java

@Test
public void setupSh_should_work_with_init_on_h2_with_overridden_system_property() throws Exception {
    //given/*w w  w  . j a  v  a 2 s .c om*/
    CommandLine oCmdLine = PlatformSetupTestUtils.createCommandLine();
    oCmdLine.addArguments("init -Ddb.user=myUser");
    DefaultExecutor executor = PlatformSetupTestUtils.createExecutor(distFolder);
    executor.setStreamHandler(PlatformSetupTestUtils.getExecuteStreamHandler("Y"));
    //when
    int iExitValue = executor.execute(oCmdLine);
    //then
    assertThat(iExitValue).isEqualTo(0);
    Connection jdbcConnection = PlatformSetupTestUtils.getJdbcConnection(distFolder, "myUser");
    Statement statement = jdbcConnection.createStatement();
    ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) AS nb FROM CONFIGURATION");
    resultSet.next();
    assertThat(resultSet.getInt("nb")).isGreaterThan(0);
}

From source file:org.cloudifysource.shell.commands.TestRecipe.java

/**
 * Execute a command line in with a given map of environment settings. The execution outupt is filtered unless
 * verbose is set to true.//from  ww w  . j a v a  2 s .c  o  m
 *
 * @param cmdLine
 *            The command to execute
 * @param env
 *            Environment settings available for the command execution
 * @return the command's execution exit code, or -2 if the command failed to execute
 */
private int executeRecipe(final CommandLine cmdLine, final Map<Object, Object> env) {
    final DefaultExecutor executor = new DefaultExecutor();

    // The watchdog will terminate the process if it does not end within the
    // specified timeout
    final int externalProcessTimeout = (this.timeout + EXTERNAL_PROCESS_WATCHDOG_ADDITIONAL_TIMEOUT) * 1000;
    final ExecuteWatchdog watchdog = new TestRecipeWatchdog(externalProcessTimeout);
    executor.setWatchdog(watchdog);

    executor.setExitValue(0);
    int result = -1;

    PipedInputStream in = null;
    PipedOutputStream out = null;
    BufferedReader reader = null;
    try {
        in = new PipedInputStream();
        out = new PipedOutputStream(in);
        reader = new BufferedReader(new InputStreamReader(in));

        final Thread thread = new Thread(new FilteredOutputHandler(reader, this.verbose));
        thread.setDaemon(true);
        thread.start();

        final PumpStreamHandler psh = new PumpStreamHandler(out, out);
        executor.setStreamHandler(psh);
        result = executor.execute(cmdLine, env);
    } catch (final ExecuteException e) {
        logger.log(Level.SEVERE, "A problem was encountered while executing the recipe: " + e.getMessage(), e);
    } catch (final IOException e) {
        logger.log(Level.SEVERE,
                "An IO Exception was encountered while executing the recipe: " + e.getMessage(), e);
        result = UNEXPECTED_ERROR_EXIT_CODE;
    }

    return result;
}

From source file:org.codice.git.GitIntegrationTest.java

private int executeGitCommand(String[] args) throws IOException {
    int exitValue = 0;
    List<String> outputLines = null;
    CommandLine cmdLine = new CommandLine("git");
    for (String arg : args) {
        cmdLine.addArgument(arg);/*from  ww w  .java  2s .c o m*/
    }
    DefaultExecutor executor = new DefaultExecutor();
    executor.setExitValue(0);
    executor.setWorkingDirectory(trash);
    CollectingLogOutputStream output = new CollectingLogOutputStream();
    PumpStreamHandler pump = new PumpStreamHandler(output, output);
    executor.setStreamHandler(pump);
    try {
        exitValue = executor.execute(cmdLine);
        outputLines = output.getLines();
    } catch (IOException e) {
        boolean hookRan = false;
        String errorMessage = "";
        // Check if we got the aborted message from the hook - implies it ran successfully
        outputLines = output.getLines();
        if ((outputLines != null) && (outputLines.size() > 0)) {
            errorMessage = outputLines.get(0);
            for (String line : outputLines) {
                if (line.contains("HOOK ABORTED")) {
                    hookRan = true;
                    break;
                }
            }
        }
        if (hookRan) {
            LOGGER.debug("Hook ran successfully - returning an error to abort the git command");
        } else {
            LOGGER.warn("Unexpected error during hook processing - first line of output: {}", errorMessage, e);
            throw e;
        }
        exitValue = 1;
    }

    for (String line : outputLines) {
        System.err.println(line);
    }
    return exitValue;
}

From source file:org.dataconservancy.dcs.access.server.util.VivoUtil.java

public static String cmdExec(String query, String sparqlEndpoint) {
    String cmd = "curl -s -S -X POST --data-binary \"" + query + "\" " + sparqlEndpoint;
    CommandLine cmdLine = CommandLine.parse(cmd);
    DefaultExecutor executor = new DefaultExecutor();
    ByteArrayOutputStream stdout = new ByteArrayOutputStream();
    PumpStreamHandler psh = new PumpStreamHandler(stdout);

    executor.setStreamHandler(psh);

    int exitValue;
    try {/*ww  w.  j  a  v  a 2  s. c o m*/
        exitValue = executor.execute(cmdLine);
    } catch (ExecuteException e) {
        //logger.log(Level.SEVERE, e.getMessage());
        e.printStackTrace();
    } catch (IOException e) {
        //.log(Level.SEVERE, e.getMessage());
        e.printStackTrace();
    }

    //String str = "";
    BufferedReader br = null;
    try {
        br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(stdout.toByteArray())));
    } catch (Exception e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    String output_line = null;
    String xml = "";
    try {
        int flag = 0;

        while ((output_line = br.readLine()) != null) {
            if (output_line.contains("--:--")) {
                if (output_line.contains("<?xml version"))
                    output_line = output_line.substring(output_line.indexOf("<?xml version"));
                else
                    continue;
            }
            xml += output_line;

        }

    } catch (IOException e) {
        //logger.log(Level.SEVERE, e.getMessage());
        e.printStackTrace();
    }
    return xml;
}

From source file:org.dataconservancy.dcs.id.impl.DataciteIdService.java

ByteArrayOutputStream executeCommand(String command) throws IOException {
    CommandLine cmdLine = CommandLine.parse(command);
    DefaultExecutor executor = new DefaultExecutor();
    ByteArrayOutputStream stdout = new ByteArrayOutputStream();
    PumpStreamHandler psh = new PumpStreamHandler(stdout);

    executor.setStreamHandler(psh);

    int exitValue = executor.execute(cmdLine);
    return stdout;
}