List of usage examples for org.apache.commons.exec LogOutputStream LogOutputStream
public LogOutputStream()
From source file:com.codeabovelab.dm.common.utils.ProcessUtils.java
public static int executeCommand(String command, ExecuteWatchdog watchdog, OutputStream outputStream, OutputStream errorStream, InputStream inputStream, Map<String, String> env) { CommandLine cmdLine = CommandLine.parse(command); DefaultExecutor executor = new DefaultExecutor(); if (outputStream == null) { outputStream = new LogOutputStream() { @Override// w w w.j a v a2 s.c om protected void processLine(String s, int i) { log.error(s); } }; } if (errorStream == null) { errorStream = new LogOutputStream() { @Override protected void processLine(String s, int i) { log.error(s); } }; } executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream, inputStream)); executor.setExitValues(new int[] { 0, 1 }); if (watchdog != null) { executor.setWatchdog(watchdog); } int exitValue; try { exitValue = executor.execute(cmdLine, env); } catch (IOException e) { exitValue = 1; LOGGER.error("error executing command", e); } return exitValue; }
From source file:com.zxy.commons.exec.CmdExecutor.java
/** * /*w ww . j a v a 2s.c o m*/ * * @param workHome workHome * @param command command * @throws InterruptedException InterruptedException * @throws IOException IOException */ public static void execAsyc(String workHome, String command) throws InterruptedException, IOException { CommandLine cmdLine = CommandLine.parse(PRE_CMD + command); Executor executor = new DefaultExecutor(); executor.setWorkingDirectory(new File(workHome)); executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() { @Override protected void processLine(String line, int level) { LOGGER.debug(line); } }, new LogOutputStream() { @Override protected void processLine(String line, int level) { LOGGER.debug(line); } })); ExecuteResultHandler resultHandler = new DefaultExecuteResultHandler(); executor.execute(cmdLine, EnvironmentUtils.getProcEnvironment(), resultHandler); // resultHandler.waitFor(); }
From source file:com.mobilecashout.osprey.remote.RemoteClient.java
public synchronized void execute(String command, DeploymentContext context, String[] roles) { executeInParallel((remoteTarget, innerContext) -> { try {/* www . j a v a2 s . co m*/ final ChannelExec exec = (ChannelExec) remoteTarget.getSession().openChannel("exec"); String commandSubstituted = remoteTarget.getSubstitutor() .replace(String.format("cd %s;%s", remoteTarget.getRelativeCurrentReleaseRoot(), command)); exec.setErrStream(new LogOutputStream() { @Override protected void processLine(String s, int i) { logger.error(s); } }); exec.setOutputStream(new LogOutputStream() { @Override protected void processLine(String s, int i) { logger.info(s); } }); exec.setCommand(commandSubstituted); exec.connect(); exec.run(); while (!exec.isClosed()) { Thread.sleep(10); } exec.disconnect(); if (0 != exec.getExitStatus()) { throw new DeploymentActionError(String.format("Failed to execute %s, exit code %d", commandSubstituted, exec.getExitStatus())); } logger.info("Executed {} on {}", commandSubstituted, remoteTarget.getTarget().host()); } catch (JSchException | InterruptedException e) { throw new DeploymentActionError(e); } }, context, roles); }
From source file:io.uengine.mail.MailAsyncService.java
private Session setMailProperties(final String toUser) { Properties props = new Properties(); props.put("mail.smtp.auth", auth); props.put("mail.smtp.starttls.enable", starttls); props.put("mail.smtp.host", host); props.put("mail.smtp.port", port); LogOutputStream loggerToStdOut = new LogOutputStream() { @Override/*from www.jav a 2 s . co m*/ protected void processLine(String line, int level) { logger.debug("[JavaMail] [{}] {}", toUser, line); } }; Session session = Session.getInstance(props, new Authenticator() { protected PasswordAuthentication getPasswordAuthentication() { return new PasswordAuthentication(account, password); } }); session.setDebug(true); session.setDebugOut(new PrintStream(loggerToStdOut)); return session; }
From source file:com.datastax.driver.core.CCMBridge.java
private String execute(String command, Object... args) { String fullCommand = String.format(command, args) + " --config-dir=" + ccmDir; Closer closer = Closer.create();//from w ww. java 2s. c o m // 10 minutes timeout ExecuteWatchdog watchDog = new ExecuteWatchdog(TimeUnit.MINUTES.toMillis(10)); StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw); closer.register(pw); try { logger.trace("Executing: " + fullCommand); CommandLine cli = CommandLine.parse(fullCommand); Executor executor = new DefaultExecutor(); LogOutputStream outStream = new LogOutputStream() { @Override protected void processLine(String line, int logLevel) { String out = "ccmout> " + line; logger.debug(out); pw.println(out); } }; LogOutputStream errStream = new LogOutputStream() { @Override protected void processLine(String line, int logLevel) { String err = "ccmerr> " + line; logger.error(err); pw.println(err); } }; closer.register(outStream); closer.register(errStream); ExecuteStreamHandler streamHandler = new PumpStreamHandler(outStream, errStream); executor.setStreamHandler(streamHandler); executor.setWatchdog(watchDog); int retValue = executor.execute(cli, ENVIRONMENT_MAP); if (retValue != 0) { logger.error("Non-zero exit code ({}) returned from executing ccm command: {}", retValue, fullCommand); pw.flush(); throw new CCMException( String.format("Non-zero exit code (%s) returned from executing ccm command: %s", retValue, fullCommand), sw.toString()); } } catch (IOException e) { if (watchDog.killedProcess()) logger.error("The command {} was killed after 10 minutes", fullCommand); pw.flush(); throw new CCMException(String.format("The command %s failed to execute", fullCommand), sw.toString(), e); } finally { try { closer.close(); } catch (IOException e) { Throwables.propagate(e); } } return sw.toString(); }
From source file:org.apache.karaf.decanter.kibana6.KibanaController.java
public KibanaController(File workingDirectory) { this.workingDirectory = workingDirectory; this.workingDirectory.mkdirs(); this.executor = new DaemonExecutor(); PumpStreamHandler pumpStreamHandler = new PumpStreamHandler(new LogOutputStream() { @Override//from w w w . j a va 2s .c o m protected void processLine(String line, int logLevel) { KIBANA_LOGGER.info(line); } }); executor.setStreamHandler(pumpStreamHandler); executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT)); executeResultHandler = new DefaultExecuteResultHandler(); }
From source file:org.apache.zeppelin.submarine.job.thread.JobRunThread.java
public void run() { boolean tryLock = lockRunning.tryLock(); if (false == tryLock) { LOGGER.warn("Can not get JobRunThread lockRunning!"); return;/*from ww w. jav a 2 s .c o m*/ } SubmarineUI submarineUI = submarineJob.getSubmarineUI(); try { InterpreterContext intpContext = submarineJob.getIntpContext(); String noteId = intpContext.getNoteId(); String userName = intpContext.getAuthenticationInfo().getUser(); String jobName = SubmarineUtils.getJobName(userName, noteId); if (true == running.get()) { String message = String.format("Job %s already running.", jobName); submarineUI.outputLog("WARN", message); LOGGER.warn(message); return; } running.set(true); Properties properties = submarineJob.getProperties(); HdfsClient hdfsClient = submarineJob.getHdfsClient(); File pythonWorkDir = submarineJob.getPythonWorkDir(); submarineJob.setCurrentJobState(EXECUTE_SUBMARINE); String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, ""); if (!algorithmPath.startsWith("hdfs://")) { String message = "Algorithm file upload HDFS path, " + "Must be `hdfs://` prefix. now setting " + algorithmPath; submarineUI.outputLog("Configuration error", message); return; } List<ParagraphInfo> paragraphInfos = intpContext.getIntpEventClient().getParagraphList(userName, noteId); String outputMsg = hdfsClient.saveParagraphToFiles(noteId, paragraphInfos, pythonWorkDir == null ? "" : pythonWorkDir.getAbsolutePath(), properties); if (!StringUtils.isEmpty(outputMsg)) { submarineUI.outputLog("Save algorithm file", outputMsg); } HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, true); URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_JOBRUN_TF_JINJA); String template = Resources.toString(urlTemplate, Charsets.UTF_8); Jinjava jinjava = new Jinjava(); String submarineCmd = jinjava.render(template, jinjaParams); // If the first line is a newline, delete the newline int firstLineIsNewline = submarineCmd.indexOf("\n"); if (firstLineIsNewline == 0) { submarineCmd = submarineCmd.replaceFirst("\n", ""); } StringBuffer sbLogs = new StringBuffer(submarineCmd); submarineUI.outputLog("Submarine submit command", sbLogs.toString()); long timeout = Long .valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout)); CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell); cmdLine.addArgument(submarineCmd, false); DefaultExecutor executor = new DefaultExecutor(); ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout); executor.setWatchdog(watchDog); StringBuffer sbLogOutput = new StringBuffer(); executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() { @Override protected void processLine(String line, int level) { line = line.trim(); if (!StringUtils.isEmpty(line)) { sbLogOutput.append(line + "\n"); } } })); if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) { executor.setWorkingDirectory(new File(System.getProperty("user.home"))); } Map<String, String> env = new HashMap<>(); String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE); if (StringUtils.equals(launchMode, "yarn")) { // Set environment variables in the submarine interpreter container run on yarn String javaHome, hadoopHome, hadoopConf; javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME); hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME); hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR); env.put("JAVA_HOME", javaHome); env.put("HADOOP_HOME", hadoopHome); env.put("HADOOP_HDFS_HOME", hadoopHome); env.put("HADOOP_CONF_DIR", hadoopConf); env.put("YARN_CONF_DIR", hadoopConf); env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`"); env.put("ZEPPELIN_FORCE_STOP", "true"); } LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd); AtomicBoolean cmdLineRunning = new AtomicBoolean(true); executor.execute(cmdLine, env, new DefaultExecuteResultHandler() { @Override public void onProcessComplete(int exitValue) { String message = String.format("jobName %s ProcessComplete exit value is : %d", jobName, exitValue); LOGGER.info(message); submarineUI.outputLog("JOR RUN COMPLETE", message); cmdLineRunning.set(false); submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_FINISHED); } @Override public void onProcessFailed(ExecuteException e) { String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", jobName, e.getExitValue(), e.getMessage()); LOGGER.error(message); submarineUI.outputLog("JOR RUN FAILED", message); cmdLineRunning.set(false); submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR); } }); int loopCount = 100; while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) { Thread.sleep(1000); } if (watchDog.isWatching()) { watchDog.destroyProcess(); Thread.sleep(1000); } if (watchDog.isWatching()) { watchDog.killedProcess(); } // Check if it has been submitted to YARN Map<String, Object> jobState = submarineJob.getJobStateByYarn(jobName); loopCount = 50; while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) { Thread.sleep(3000); jobState = submarineJob.getJobStateByYarn(jobName); } if (!jobState.containsKey("state")) { String message = String.format("JOB %s was not submitted to YARN!", jobName); LOGGER.error(message); submarineUI.outputLog("JOR RUN FAILED", message); submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); submarineJob.setCurrentJobState(EXECUTE_SUBMARINE_ERROR); submarineUI.outputLog("Exception", e.getMessage()); } finally { running.set(false); lockRunning.unlock(); } }
From source file:org.apache.zeppelin.submarine.job.thread.TensorboardRunThread.java
public void run() { SubmarineUI submarineUI = submarineJob.getSubmarineUI(); boolean tryLock = lockRunning.tryLock(); try {/*from w ww . j a va2 s. com*/ Properties properties = submarineJob.getProperties(); String tensorboardName = SubmarineUtils.getTensorboardName(submarineJob.getUserName()); if (true == running.get()) { String message = String.format("tensorboard %s already running.", tensorboardName); submarineUI.outputLog("WARN", message); LOGGER.warn(message); return; } running.set(true); HashMap jinjaParams = SubmarineUtils.propertiesToJinjaParams(properties, submarineJob, false); // update jobName -> tensorboardName jinjaParams.put(SubmarineConstants.JOB_NAME, tensorboardName); URL urlTemplate = Resources.getResource(SubmarineJob.SUBMARINE_TENSORBOARD_JINJA); String template = Resources.toString(urlTemplate, Charsets.UTF_8); Jinjava jinjava = new Jinjava(); String submarineCmd = jinjava.render(template, jinjaParams); // If the first line is a newline, delete the newline int firstLineIsNewline = submarineCmd.indexOf("\n"); if (firstLineIsNewline == 0) { submarineCmd = submarineCmd.replaceFirst("\n", ""); } StringBuffer sbLogs = new StringBuffer(submarineCmd); submarineUI.outputLog("Submarine submit command", sbLogs.toString()); long timeout = Long .valueOf(properties.getProperty(SubmarineJob.TIMEOUT_PROPERTY, SubmarineJob.defaultTimeout)); CommandLine cmdLine = CommandLine.parse(SubmarineJob.shell); cmdLine.addArgument(submarineCmd, false); DefaultExecutor executor = new DefaultExecutor(); ExecuteWatchdog watchDog = new ExecuteWatchdog(timeout); executor.setWatchdog(watchDog); StringBuffer sbLogOutput = new StringBuffer(); executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() { @Override protected void processLine(String line, int level) { line = line.trim(); if (!StringUtils.isEmpty(line)) { sbLogOutput.append(line + "\n"); } } })); if (Boolean.valueOf(properties.getProperty(SubmarineJob.DIRECTORY_USER_HOME))) { executor.setWorkingDirectory(new File(System.getProperty("user.home"))); } Map<String, String> env = new HashMap<>(); String launchMode = (String) jinjaParams.get(SubmarineConstants.INTERPRETER_LAUNCH_MODE); if (StringUtils.equals(launchMode, "yarn")) { // Set environment variables in the container String javaHome, hadoopHome, hadoopConf; javaHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_JAVA_HOME); hadoopHome = (String) jinjaParams.get(SubmarineConstants.DOCKER_HADOOP_HDFS_HOME); hadoopConf = (String) jinjaParams.get(SubmarineConstants.SUBMARINE_HADOOP_CONF_DIR); env.put("JAVA_HOME", javaHome); env.put("HADOOP_HOME", hadoopHome); env.put("HADOOP_HDFS_HOME", hadoopHome); env.put("HADOOP_CONF_DIR", hadoopConf); env.put("YARN_CONF_DIR", hadoopConf); env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`"); } LOGGER.info("Execute EVN: {}, Command: {} ", env.toString(), submarineCmd); AtomicBoolean cmdLineRunning = new AtomicBoolean(true); executor.execute(cmdLine, env, new DefaultExecuteResultHandler() { @Override public void onProcessComplete(int exitValue) { String message = String.format("jobName %s ProcessComplete exit value is : %d", tensorboardName, exitValue); LOGGER.info(message); submarineUI.outputLog("TENSORBOARD RUN COMPLETE", message); cmdLineRunning.set(false); } @Override public void onProcessFailed(ExecuteException e) { String message = String.format("jobName %s ProcessFailed exit value is : %d, exception is : %s", tensorboardName, e.getExitValue(), e.getMessage()); LOGGER.error(message); submarineUI.outputLog("TENSORBOARD RUN FAILED", message); cmdLineRunning.set(false); } }); int loopCount = 100; while ((loopCount-- > 0) && cmdLineRunning.get() && running.get()) { Thread.sleep(1000); } if (watchDog.isWatching()) { watchDog.destroyProcess(); Thread.sleep(1000); } if (watchDog.isWatching()) { watchDog.killedProcess(); } // Check if it has been submitted to YARN Map<String, Object> jobState = submarineJob.getJobStateByYarn(tensorboardName); loopCount = 50; while ((loopCount-- > 0) && !jobState.containsKey("state") && running.get()) { Thread.sleep(3000); jobState = submarineJob.getJobStateByYarn(tensorboardName); } if (!jobState.containsKey("state")) { String message = String.format("tensorboard %s was not submitted to YARN!", tensorboardName); LOGGER.error(message); submarineUI.outputLog("JOR RUN FAILED", message); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); submarineUI.outputLog("Exception", e.getMessage()); } finally { running.set(false); lockRunning.unlock(); } }
From source file:org.apache.zeppelin.submarine.SubmarineJobTest.java
@Test public void defaultExecutorTest() throws IOException { DefaultExecutor executor = new DefaultExecutor(); CommandLine cmdLine = CommandLine.parse(shell); URL urlTemplate = Resources.getResource(DEFAULT_EXECUTOR_TEST); cmdLine.addArgument(urlTemplate.getFile(), false); Map<String, String> env = new HashMap<>(); env.put("CLASSPATH", "`$HADOOP_HDFS_HOME/bin/hadoop classpath --glob`"); env.put("EVN", "test"); AtomicBoolean cmdLineRunning = new AtomicBoolean(true); StringBuffer sbLogOutput = new StringBuffer(); executor.setStreamHandler(new PumpStreamHandler(new LogOutputStream() { @Override//from w ww . j a va 2 s. c om protected void processLine(String line, int level) { //LOGGER.info(line); sbLogOutput.append(line + "\n"); } })); executor.execute(cmdLine, env, new DefaultExecuteResultHandler() { @Override public void onProcessComplete(int exitValue) { cmdLineRunning.set(false); } @Override public void onProcessFailed(ExecuteException e) { cmdLineRunning.set(false); LOGGER.error(e.getMessage()); } }); int loopCount = 100; while ((loopCount-- > 0) && cmdLineRunning.get()) { try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } } LOGGER.info(sbLogOutput.toString()); }
From source file:org.eclipse.ecf.python.AbstractPythonLauncher.java
@Override public void launch(String[] args, OutputStream output) throws Exception { synchronized (this.launchLock) { if (isLaunched()) throw new IllegalStateException("Already started"); this.shuttingDown = false; if (enabled) { String pythonLaunchCommand = createPythonLaunchCommand(); if (pythonLaunchCommand == null) throw new NullPointerException("pythonLaunchCommand must not be null"); logger.debug("pythonLaunchCommand=" + pythonLaunchCommand); this.executor = createExecutor(); if (this.pythonWorkingDirectory != null) this.executor.setWorkingDirectory(pythonWorkingDirectory); if (output == null) { output = new LogOutputStream() { @Override/*from w ww . ja v a 2 s .c o m*/ protected void processLine(String line, int level) { logger.debug("PYTHON: " + line); } }; } executor.setStreamHandler(new PumpStreamHandler(output)); this.executor.setProcessDestroyer(new PythonProcessDestroyer()); ExecuteResultHandler executeHandler = new DefaultExecuteResultHandler() { @Override public void onProcessComplete(int exitValue) { logger.debug("PYTHON EXIT=" + exitValue); } @Override public void onProcessFailed(ExecuteException e) { if (!shuttingDown) logger.debug("PYTHON EXCEPTION", e); } }; CommandLine commandLine = new CommandLine(pythonExec).addArgument(PYTHON_LAUNCH_COMMAND_OPTION); commandLine.addArgument(pythonLaunchCommand, true); List<String> argsList = (args == null) ? Collections.emptyList() : Arrays.asList(args); if (this.javaPort != null && !argsList.contains(JAVA_PORT_OPTION)) { commandLine.addArgument(JAVA_PORT_OPTION); commandLine.addArgument(String.valueOf(this.javaPort)); } if (this.pythonPort != null && !argsList.contains(PYTHON_PORT_OPTION)) { commandLine.addArgument(PYTHON_PORT_OPTION); commandLine.addArgument(String.valueOf(this.pythonPort)); } if (args != null) commandLine.addArguments(args); logger.debug("PythonLauncher.launch: " + commandLine); try { executor.execute(commandLine, executeHandler); } catch (Exception e) { this.executor = null; throw e; } } else logger.debug("PythonLauncher DISABLED. Python process must be started manually"); } }