Java tutorial
/** * Copyright (c) 2015 Genome Research Ltd. * * Author: Cancer Genome Project cgpit@sanger.ac.uk * * This file is part of WwDocker. * * WwDocker is free software: you can redistribute it and/or modify it under * the terms of the GNU Affero General Public License as published by the Free * Software Foundation; either version 3 of the License, or (at your option) any * later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more * details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * 1. The usage of a range of years within a copyright statement contained within * this distribution should be interpreted as being equivalent to a list of years * including the first and last year specified and all consecutive years between * them. For example, a copyright statement that reads 'Copyright (c) 2005, 2007- * 2009, 2011-2012' should be interpreted as being identical to a statement that * reads 'Copyright (c) 2005, 2007, 2008, 2009, 2011, 2012' and a copyright * statement that reads "Copyright (c) 2005-2012' should be interpreted as being * identical to a statement that reads 'Copyright (c) 2005, 2006, 2007, 2008, * 2009, 2010, 2011, 2012'." */ package uk.ac.sanger.cgp.wwdocker.actions; import com.jcraft.jsch.Channel; import com.jcraft.jsch.ChannelExec; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.List; import java.util.Properties; import org.apache.commons.configuration.BaseConfiguration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; /** * * @author kr2 */ public class Remote { private static final Logger logger = LogManager.getLogger(); private static final int SSH_TIMEOUT = 20000; // 20 seconds private static Session addNewHost(BaseConfiguration config, String host) { Session session = null; logger.warn("Host '" + host + "' is not known attempting to resolve"); try { Session thisSess = unconnectedSession(config, host); Properties props = new Properties(); props.put("StrictHostKeyChecking", "no"); thisSess.setConfig(props); thisSess.connect(SSH_TIMEOUT); session = thisSess; } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection: " + e.getMessage(), e); } logger.warn("Successfully added new key for '" + host + "'"); return session; } private static File privateKeyFile() { String fSep = System.getProperty("file.separator"); File key = new File(System.getProperty("user.home") + fSep + ".ssh" + fSep + "id_dsa"); if (!key.exists()) { key = new File(System.getProperty("user.home") + fSep + ".ssh" + fSep + "id_rsa"); } if (!key.exists()) { throw new RuntimeException( "Unable to find any identity key files, e.g. ~/.ssh/id_dsa or ~/.ssh/id_rsa"); } return key; } private static Session unconnectedSession(BaseConfiguration config, String host) { Session session; String fSep = System.getProperty("file.separator"); String userKnownHosts = System.getProperty("user.home") + fSep + ".ssh" + fSep + "known_hosts"; try { JSch jsch = new JSch(); jsch.setKnownHosts(userKnownHosts); jsch.addIdentity(privateKeyFile().getAbsolutePath()); Session thisSess = jsch.getSession(config.getString("ssh_user"), host, 22); thisSess.setServerAliveInterval(1000); thisSess.setPassword(config.getString("ssh_pw")); session = thisSess; } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection: " + e.getMessage(), e); } return session; } public static Session getSession(BaseConfiguration config, String host) { Session session; try { Session thisSess = unconnectedSession(config, host); thisSess.connect(SSH_TIMEOUT); logger.info("Host '" + host + "' is known"); session = thisSess; } catch (JSchException e) { if (e.getMessage().startsWith("UnknownHostKey")) { session = addNewHost(config, host); } else { // still falls over if the host key is changed throw new RuntimeException("Failure in SSH connection: " + e.getMessage(), e); } } return session; } public static boolean dockerRunning(Session session, String user) { boolean isRunning = true; // assume least destructive state String command = "ps -fu " + user + " | grep docker | grep -cv grep"; int exitCode = 0; try { exitCode = execCommand(session, command); } catch (JSchException e) { if (e.getMessage().contains("session is down")) { logger.warn("Session terminated mid query, abort check"); } else { throw new RuntimeException("Failure in SSH connection", e); } } if (exitCode == 1) { isRunning = false; } return isRunning; } public static boolean workerRunning(Session session, String user) { boolean isRunning = true; // assume least destructive state String command = "ps -fu " + user + " | grep -E \"WwDocker-.*.jar\" | grep -cv grep"; int exitCode = 0; try { exitCode = execCommand(session, command); } catch (JSchException e) { if (e.getMessage().contains("session is down")) { logger.warn("Session terminated mid query, abort check"); } else { throw new RuntimeException("Failure in SSH connection", e); } } if (exitCode == 1) { isRunning = false; } return isRunning; } public static int dockerLoad(Session session, String[] images, String workspace) { int exitCode = -1; if (images.length == 1 && images[0].length() == 0) { return 0; } try { for (String i : images) { String destFile; if (i.startsWith("/")) { destFile = i; } else { destFile = curl(session, i, workspace); if (destFile == null) { logger.error("Failed to retrieve file via curl: " + i); return 1; } } String command = "docker load -i " + destFile; exitCode = execCommand(session, command); if (exitCode != 0) { break; } } } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection", e); } return exitCode; } public static int dockerPull(Session session, String[] images) { int exitCode = -1; try { for (String i : images) { String command = "docker pull " + i; exitCode = execCommand(session, command); if (exitCode != 0) { break; } } } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection", e); } return exitCode; } public static void cleanupOldImages(Session session) { String command = "docker images | grep \"<none>\" | awk '{print $3}' | xargs docker rmi"; try { execCommand(session, command); } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection", e); } } public static void cleanHost(Session session, String[] paths) { if (paths.length == 0) { throw new RuntimeException("Potentially deleting root of storage, aborting"); } for (String p : paths) { String command = "rm -rf"; if (p.length() == 0) { throw new RuntimeException("Potentially deleting root of storage, aborting"); } command = command.concat(" ").concat(p).concat("/*"); paramExec(session, command); } } public static int cleanFiles(Session session, String[] files) { String command = "rm -f"; for (String p : files) { if (p.equals("/*")) { throw new RuntimeException("Potentially deleting root of storage, aborting"); } command = command.concat(" ").concat(p); } return paramExec(session, command); } public static int expandJre(Session session, File localJre) { String command = "tar --strip-components=1 -C /opt/wwdocker/jre -zxf /opt/wwdocker/" .concat(localJre.getName()); return paramExec(session, command); } public static int expandWorkflow(Session session, File localWorkflow, File seqwareJar, String workflowBase) { //java -cp seqware-distribution-1.1.0-alpha.6-full.jar net.sourceforge.seqware.pipeline.tools.UnZip --input-zip String workflowDir = workflowBase.concat("/").concat(localWorkflow.getName()); workflowDir = workflowDir.replaceFirst("[.]zip$", ""); String command = "/opt/wwdocker/jre/bin/java -Xmx128m -cp "; command = command.concat(seqwareJar.getAbsolutePath()); command = command.concat(" net.sourceforge.seqware.pipeline.tools.UnZip --input-zip "); command = command.concat(localWorkflow.getAbsolutePath()); command = command.concat(" --output-dir "); command = command.concat(workflowDir); return paramExec(session, command); } public static int createPaths(Session session, List<String> paths) { String[] array = paths.toArray(new String[paths.size()]); return createPaths(session, array); } public static int createPaths(Session session, String[] paths) { String command = "mkdir -p"; return paramExec(session, command, paths); } public static int chmodPath(Session session, String mode, String path, boolean recursive) { String[] paths = { path }; return chmodPaths(session, mode, paths, recursive); } public static int chmodPaths(Session session, String mode, List<String> paths, boolean recursive) { String[] array = paths.toArray(new String[paths.size()]); return chmodPaths(session, mode, array, recursive); } public static int chmodPaths(Session session, String mode, String[] paths, boolean recursive) { String command = "chmod "; if (recursive) { command = command.concat("-R "); } command = command.concat(mode); return paramExec(session, command, paths); } private static int paramExec(Session session, String command) { return paramExec(session, command, new String[0]); } private static int paramExec(Session session, String command, String[] params) { int exitCode = 0; for (String param : params) { command = command.concat(" ").concat(param); } try { exitCode = execCommand(session, command); } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection", e); } return exitCode; } public static void listDir(Session session, String path) { String command = "ls -l " + path; try { execCommand(session, command); } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection", e); } } public static int startWorkerDaemon(Session session, String jarName, String confName, String mode) { int exitCode = -1; //java -Dlog4j.configurationFile="config/log4j.properties.xml" -jar target/WwDocker-0.1.jar Primary config/default.cfg String command = "/opt/wwdocker/jre/bin/java -Xmx256m -Dlog4j.configurationFile=\"/opt/wwdocker/log4j.properties_worker.xml\" -jar /opt/wwdocker/" .concat(jarName).concat(" /opt/wwdocker/").concat(confName).concat(" Worker"); if (mode != null) { command = command.concat(" ").concat(mode); } command = command.concat(" >& /dev/null &"); try { exitCode = execCommand(session, command); } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection while starting daemon", e); } return exitCode; } public static String curl(Session session, String source, String destPath) { String finalPath = null; try { String[] elements = source.split("/"); if (!destPath.endsWith(System.getProperty("file.separator"))) { destPath = destPath.concat(System.getProperty("file.separator")); } destPath = destPath.concat(elements[elements.length - 1]); // -z only transfer if modified String getCommand = "curl -RLsS".concat(" -z ").concat(destPath).concat(" -o ").concat(destPath) .concat(" ").concat(source); if (execCommand(session, getCommand) == 0) { finalPath = destPath; } } catch (JSchException e) { throw new RuntimeException("Failure in SSH connection", e); } return finalPath; } private static int execCommand(Session session, String command) throws JSchException { int exitCode = -1; Channel channel = session.openChannel("exec"); ((ChannelExec) channel).setCommand(command); logger.trace("Remote exection of command: [".concat(session.getHost()).concat("]: ").concat(command)); String fullOut = new String(); try { InputStream in = channel.getInputStream(); channel.connect(); byte[] tmp = new byte[1024]; while (true) { while (in.available() > 0) { int i = in.read(tmp, 0, 1024); if (i < 0) break; fullOut = fullOut.concat(new String(tmp, 0, i)); fullOut = Utils.logOutput(fullOut); } if (channel.isClosed()) { if (in.available() > 0) continue; exitCode = channel.getExitStatus(); Utils.logOutput(fullOut + System.lineSeparator()); logger.info("Exit code: " + exitCode); break; } try { Thread.sleep(1000); } catch (Exception ee) { } } } catch (IOException e) { throw new JSchException("IOException during ssh action: " + command, e); } channel.disconnect(); return exitCode; } private static boolean remoteIsEqual(Session session, String localPath, String remotePath) throws JSchException { boolean isEqual = false; File lFile = new File(localPath); if (remotePath.endsWith("/.")) { remotePath = remotePath.substring(0, remotePath.length() - 1).concat(lFile.getName()); } String rawString = remoteExecStdout(session, "date --utc --reference=" + remotePath + " +%s"); if (rawString == null) { logger.debug("no existing remote file: " + remotePath); } else { long rMtime = Long.valueOf(rawString).longValue() * 1000; long rSize = Long.valueOf(remoteExecStdout(session, "ls -l --full-time " + remotePath).split("\\s+")[4]) .longValue(); logger.debug("lFile size: " + lFile.length()); logger.debug("rFile size: " + rSize); logger.debug("lFile mtime: " + lFile.lastModified()); logger.debug("rFile mtime: " + rMtime); if (rMtime == lFile.lastModified() && rSize == lFile.length()) { isEqual = true; } } return isEqual; } public static boolean processExists(Session session, Long pid) throws JSchException { boolean exists = false; int exitCode = execCommand(session, "ps " + pid); if (exitCode == 0) { exists = true; } return exists; } private static String remoteExecStdout(Session session, String command) throws JSchException { int exitCode = -1; Channel channel = session.openChannel("exec"); ((ChannelExec) channel).setCommand(command); logger.trace("Remote exection of command: [".concat(session.getHost()).concat("]: ").concat(command)); String fullOut = new String(); try { InputStream in = channel.getInputStream(); channel.connect(); byte[] tmp = new byte[1024]; while (true) { while (in.available() > 0) { int i = in.read(tmp, 0, 1024); if (i < 0) break; fullOut = fullOut.concat(new String(tmp, 0, i)); } if (channel.isClosed()) { if (in.available() > 0) continue; exitCode = channel.getExitStatus(); logger.info("Exit code: " + exitCode); break; } try { Thread.sleep(1000); } catch (Exception ee) { } } } catch (IOException e) { throw new JSchException("IOException during ssh action: " + command, e); } channel.disconnect(); if (exitCode != 0) { fullOut = null; } else { fullOut = fullOut.trim(); } return fullOut; } public static int fileTo(Session session, String localFile, String remoteFile) throws JSchException { if (remoteIsEqual(session, localFile, remoteFile)) { logger.info("Files are equal, not performing SCP"); return 0; } logger.info("Sending file to remote [".concat(session.getHost()).concat("]: ").concat(localFile)); boolean ptimestamp = true; // exec 'scp -t rfile' remotely String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteFile; logger.trace(command); Channel channel = session.openChannel("exec"); ((ChannelExec) channel).setCommand(command); try { // get I/O streams for remote scp OutputStream out = channel.getOutputStream(); InputStream in = channel.getInputStream(); channel.connect(); FileInputStream fis = null; if (checkAck(in) != 0) { logger.error("scp failed"); return 1; } File _lfile = new File(localFile); if (ptimestamp) { command = "T " + (_lfile.lastModified() / 1000) + " 0"; // The access time should be sent here, // but it is not accessible with JavaAPI ;-< command += (" " + (_lfile.lastModified() / 1000) + " 0\n"); out.write(command.getBytes()); out.flush(); if (checkAck(in) != 0) { logger.error("scp failed"); return 1; } } // send "C0644 filesize filename", where filename should not include '/' long filesize = _lfile.length(); command = "C0644 " + filesize + " "; if (localFile.lastIndexOf('/') > 0) { command += localFile.substring(localFile.lastIndexOf('/') + 1); } else { command += localFile; } command += "\n"; out.write(command.getBytes()); out.flush(); if (checkAck(in) != 0) { logger.error("scp failed"); return 1; } // send a content of lfile fis = new FileInputStream(localFile); byte[] buf = new byte[1024]; while (true) { int len = fis.read(buf, 0, buf.length); if (len <= 0) { break; } out.write(buf, 0, len); //out.flush(); } fis.close(); fis = null; // send '\0' buf[0] = 0; out.write(buf, 0, 1); out.flush(); if (checkAck(in) != 0) { logger.error("scp failed"); return 1; } out.close(); } catch (IOException e) { throw new JSchException("IOException during ssh action: " + command, e); } channel.disconnect(); logger.info("Send complete"); return 0; } private static int checkAck(InputStream in) throws IOException { int b = in.read(); // b may be 0 for success, // 1 for error, // 2 for fatal error, // -1 if (b == 0) { return b; } if (b == -1) { return b; } if (b == 1 || b == 2) { StringBuffer sb = new StringBuffer(); int c; do { c = in.read(); sb.append((char) c); } while (c != '\n'); if (b == 1) { // error logger.error(sb.toString()); } if (b == 2) { // fatal error logger.fatal(sb.toString()); } } return b; } public static void closeSsh(Session ssh) { ssh.disconnect(); try { while (ssh.isConnected()) { Thread.sleep(10); } } catch (InterruptedException e) { logger.warn("Issue during closing ssh session... continuing", e); } return; } }