Java tutorial
/** * @UNCC Fodor Lab * @author Michael Sioda * @email msioda@uncc.edu * @date Feb 9, 2017 * @disclaimer This code is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version, * provided that any use properly credits the author. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details at http://www.gnu.org */ package bioLockJ; import java.io.BufferedReader; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.InputStreamReader; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.GregorianCalendar; import java.util.List; import java.util.Locale; import java.util.StringTokenizer; import java.util.zip.GZIPInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.commons.io.filefilter.WildcardFileFilter; import bioLockJ.module.agent.MailAgent; import bioLockJ.util.MetadataUtil; import bioLockJ.util.ProcessUtil; /** * This is the main program used to control top level execution. */ public class AppController { /** * Get a BufferedReader for standard text file or gzipped file. * @param file * @return * @throws Exception */ public static BufferedReader getFileReader(final File file) throws Exception { return file.getName().toLowerCase().endsWith(".gz") ? new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)))) : new BufferedReader(new FileReader(file)); } public static Module getModule(final String name) throws Exception { for (final Module m : Config.getModules()) { if (m.getClass().getName().equals(name)) { return m; } } return null; } public static Module getRequiredModule(final String name) throws Exception { final Module module = getModule(name); if (module != null) { return module; } throw new Exception("Unable to find module: " + name); } /** * Get runtime message based on startTime passed. * @return */ public static String getRunTime() { return getRunTime(APP_START_TIME, System.currentTimeMillis()); } /** * Get runtime message based on startTime passed. * @return */ public static String getRunTime(final long start, final long end) { final String format = String.format("%%0%dd", 2); final long elapsedTime = (end - start) / 1000; final String seconds = String.format(format, elapsedTime % 60); final String minutes = String.format(format, (elapsedTime % 3600) / 60); final String hours = String.format(format, elapsedTime / 3600); return hours + " hours : " + minutes + " minutes : " + seconds + " seconds"; } /** * The main method is the first method called when BioLockJ is run. Here we * read property file, copy it to project directory, initialize Config * and call runProgram(). * * If the password param is given, the password is encrypted & stored to the * prop file. * * @param args - args[0] path to property file - args[1] clear-text admin * email password */ public static void main(String[] args) { try { args = validateJavaParameters(args); System.out.println("args[CONFIG_PARAM]: " + args[CONFIG_PARAM]); System.out.println("args[OPTIONAL_PARAM]: " + args[OPTIONAL_PARAM]); if (changePassword(args[OPTIONAL_PARAM])) { System.out.println("Encrypting and storing new admin email password!"); MailAgent.encryptAndStoreEmailPassword(args[CONFIG_PARAM], args[OPTIONAL_PARAM]); System.exit(0); } Config.loadProperties(args[CONFIG_PARAM]); setProjectDir(args[OPTIONAL_PARAM]); if (!doRestart(args[OPTIONAL_PARAM])) { logWelcomeMsg(); Config.copyConfig(); } Log.initialize(); initializeModules(); runProgram(); } catch (final Exception ex) { if (Log.out != null) { Log.out.error("Error occurred running program! ", ex); } else { System.out.println("FATAL APPLICATION ERROR - Log file = null: args: " + args); ex.printStackTrace(); } } } /** * Utility method to remove quotes from String. * @param inString * @return */ public static String stripQuotes(final String inString) { final StringBuffer buff = new StringBuffer(); for (int x = 0; x < inString.length(); x++) { final char c = inString.charAt(x); if (c != '\"') { buff.append(c); } } return buff.toString().trim(); } protected static void initializeModules() throws Exception { File metadata = Config.getExistingFile(Config.INPUT_METADATA); Log.out.debug("===> Initial metadata: " + metadata); Module.ignoreFiles(Config.getSet(Config.INPUT_IGNORE_FILES)); Module.ignoreFile(Constants.BLJ_STARTED); Module.ignoreFile(Constants.BLJ_COMPLETE); for (final Module module : Config.getModules()) { final int index = Config.getModules().indexOf(module); module.setExecutorDir(index); if (module.hasFailed()) { recursiveFileDelete(module.getExecutorDir()); } if (Config.getModules().indexOf(module) == 0) { module.initInputFiles(null); } else { final Module previousModule = Config.getModules().get(index - 1); module.setPreviousModule(previousModule); if (previousModule.passThroughInputSeqs()) { module.initInputFiles(null); } else { module.setInputDir(previousModule.getOutputDir()); } } if (module.isComplete() && (module.getOutputMetadata() != null)) { metadata = module.getOutputMetadata(); Log.out.debug(" ====> New metadata: " + metadata.getAbsolutePath()); } else { module.checkDependencies(); } } if (metadata != null) { MetadataUtil.setMetadata(metadata); MetadataUtil.refresh(); } } /** * Output welcome message to the output file with BioLockJ version, lab citation, * and freeware msg. */ protected static void logWelcomeMsg() { Log.addMsg(Constants.RETURN); Log.addMsg(Constants.LOG_SPACER); Log.addMsg("Launching BioLockJ " + Constants.BLJ_VERSION + " ~ Distributed by UNCC Fodor Lab @2017"); Log.addMsg(Constants.LOG_SPACER); Log.addMsg("This code is free software; you can redistribute and/or modify it"); Log.addMsg("under the terms of the GNU General Public License as published by"); Log.addMsg("the Free Software Foundation; either version 2 of the License, or"); Log.addMsg("any later version, provided proper credit is given to the authors."); Log.addMsg("This program is distributed in the hope that it will be useful,"); Log.addMsg("but WITHOUT ANY WARRANTY; without even the implied warranty of"); Log.addMsg("MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the"); Log.addMsg("GNU General Public License for more details at http://www.gnu.org"); Log.addMsg(Constants.LOG_SPACER); } /** * Called by main(args[]) to check all of the executor dependencies, execute scripts, * and then clean up by deleting temp dirs if needed. * @throws Exception */ protected static void runProgram() throws Exception { try { if (Config.getBoolean(PROJECT_COPY_FILES)) { copyInputDirs(new File(Config.requireExistingDirectory(Config.PROJECT_DIR).getAbsolutePath() + File.separator + "input")); } for (final Module module : Config.getModules()) { if (!module.isComplete()) { executeAndWaitForScriptsIfAny(module); } } markProjectComplete(); } catch (final Exception ex) { if (Log.out != null) { Log.out.error("Error occurred during module execution! ", ex); for (final Module module : Config.getModules()) { if (module instanceof MailAgent) { executeAndWaitForScriptsIfAny(module); } } } else { System.out.println("Error occurred during module execution!"); ex.printStackTrace(); } } } protected static String[] validateJavaParameters(final String[] args) throws Exception { final String[] params = new String[2]; if ((args == null) || (args.length < 1) || (args.length > 2)) { throw new Exception("BioLockJ accepts only 1-2 Java application parameters" + Constants.RETURN + "Required Arg = path to config file" + Constants.RETURN + "Optional Arg = [ new_email_password ] or [ restart_flag (\"restart\" or \"r\") ]" + Constants.RETURN + Constants.RETURN + "PROGRAM TERMINATED!"); } File configFile = new File(args[0]); String optionalParam = null; if ((args.length == 1) && (!configFile.exists() || configFile.isDirectory())) { throw new Exception(configFile.getAbsolutePath() + " is not a valid file!"); } if (args.length == 2) { optionalParam = args[1].toLowerCase(); if (!configFile.exists() || configFile.isDirectory()) { optionalParam = args[0].toLowerCase(); configFile = new File(args[1]); params[0] = configFile.getAbsolutePath(); if (!configFile.exists() || configFile.isDirectory()) { throw new Exception("Neither parameter is a valid file path [ " + args[0] + " / " + args[1] + " ] does not exist!"); } } if (restartFlags.contains(optionalParam)) { params[1] = Constants.TRUE; } else { params[1] = optionalParam; } } params[0] = configFile.getAbsolutePath(); params[1] = optionalParam; return params; } private static boolean changePassword(final String val) { return (val != null) && !restartFlags.contains(val); } /** * If user prop indicates they need a copy of the input files, copy them to the project dir. * * @throws Exception */ private static void copyInputDirs(final File targetDir) throws Exception { if ((targetDir == null) || !targetDir.isDirectory()) { throw new Exception("Unable to copy input files. Parameter \"targetDir\" is not a directory: " + ((targetDir == null) ? "null" : targetDir.getAbsolutePath())); } final File startedFlag = new File(targetDir.getAbsolutePath() + File.separator + Constants.BLJ_STARTED); if (startedFlag.exists()) { recursiveFileDelete(targetDir); } if (!targetDir.exists()) { targetDir.mkdirs(); } for (final File dir : Config.requireExistingDirectories(Config.INPUT_DIRS)) { Log.out.info("Copying input files from " + dir + " to " + targetDir); FileUtils.copyDirectory(dir, targetDir); } final File completeFlag = new File(targetDir.getAbsolutePath() + File.separator + Constants.BLJ_COMPLETE); final FileWriter writer = new FileWriter(completeFlag); writer.close(); if (!completeFlag.exists()) { throw new Exception("Unable to create " + completeFlag.getAbsolutePath()); } } private static boolean doRestart(final String val) { return (val != null) && restartFlags.contains(val); } /** * Execute the Module scripts (if any). * * @param module * @throws Exception */ private static void executeAndWaitForScriptsIfAny(final Module module) throws Exception { module.markStarted(); module.executeProjectFile(); if (module.hasScripts()) { executeCHMOD(module.getScriptDir()); executeScript(module.executeScriptCommand()); pollAndSpin(module); } module.markComplete(); } /** * Execute the chmod param to make the new bash scripts executable. * @param scriptDir * @throws Exception */ private static void executeCHMOD(final File scriptDir) throws Exception { final File[] listOfFiles = scriptDir.listFiles(); for (final File file : listOfFiles) { if (!file.getName().startsWith(".")) { ProcessUtil .submit(getArgs(Config.requireString(Config.SCRIPT_CHMOD_COMMAND), file.getAbsolutePath())); } } } /** * Execute the given script via ProcessUtil. * @param script * @throws Exception */ private static void executeScript(final String[] cmd) throws Exception { if (cmd == null) { return; } ProcessUtil.submit(cmd); } /** * Populate args to pass to ProcessUtil. * @param command * @param filePath * @return */ private static String[] getArgs(final String command, final String filePath) { final StringTokenizer sToken = new StringTokenizer(command + " " + filePath); final List<String> list = new ArrayList<>(); while (sToken.hasMoreTokens()) { list.add(sToken.nextToken()); } final String[] args = new String[list.size()]; for (int x = 0; x < list.size(); x++) { args[x] = list.get(x); } return args; } private static File getRestartDir() throws Exception { File restartDir = null; GregorianCalendar mostRecent = null; final FileFilter ff = new WildcardFileFilter(Config.requireString(Config.PROJECT_NAME) + "*"); final File[] dirs = Config.requireExistingDirectory(PROJECTS_DIR).listFiles(ff); for (final File d : dirs) { if (!d.isDirectory() || (d.getName().length() != (Config.requireString(Config.PROJECT_NAME).length() + 10))) { continue; } final String name = d.getName(); final int len = name.length(); final String year = name.substring(len - 9, len - 5); final String mon = name.substring(len - 5, len - 2); final String day = name.substring(len - 2); final Date date = new SimpleDateFormat("yyyyMMMdd").parse(year + mon + day); final GregorianCalendar projectDate = new GregorianCalendar(); projectDate.setTime(date); // Value > 0 if projectDate has a more recent date than mostRecent if ((mostRecent == null) || (projectDate.compareTo(mostRecent) > 0)) { Log.addMsg("Found previous run = " + d.getAbsolutePath()); restartDir = d; mostRecent = projectDate; } } if (restartDir == null) { throw new Exception( "Unalbe to locate restart directory in --> " + Config.requireExistingDirectory(PROJECTS_DIR)); } if (isProjectComplete(restartDir)) { throw new Exception("RESTART FAILED! Project ran successfully: " + restartDir.getAbsolutePath()); } Log.addMsg(Constants.RETURN); Log.addMsg(Constants.RETURN); Log.addMsg(Constants.LOG_SPACER); Log.addMsg(Constants.LOG_SPACER); Log.addMsg(Constants.RETURN); Log.addMsg("RESTART PROJECT DIR --> " + restartDir.getAbsolutePath()); Log.addMsg(Constants.RETURN); Log.addMsg(Constants.LOG_SPACER); Log.addMsg(Constants.LOG_SPACER); Log.addMsg(Constants.RETURN); return restartDir; } private static boolean isProjectComplete(final File projDir) throws Exception { final File f = new File(projDir.getAbsolutePath() + File.separator + Constants.BLJ_COMPLETE); return f.exists(); } private static void markProjectComplete() throws Exception { final File f = new File(Config.requireExistingDirectory(Config.PROJECT_DIR).getAbsolutePath() + File.separator + Constants.BLJ_COMPLETE); final FileWriter writer = new FileWriter(f); writer.close(); if (!f.exists()) { throw new Exception("Unable to create " + f.getAbsolutePath()); } } /** * Poll checks the Module's script dir for flag files indicating either * SUCCESS or FAILURE. Output message to output indicating num pass/fail. * Exit if failures found and exitOnFailure flag set to Y. * * @param scriptFiles * @param mainScript * @return * @throws Exception */ private static boolean poll(final List<File> scriptFiles, final File mainScript) throws Exception { File failure = null; int numSuccess = 0; int numFailed = 0; for (final File f : scriptFiles) { final File testSuccess = new File(f.getAbsolutePath() + "_" + Constants.SUCCESS); if (testSuccess.exists()) { numSuccess++; } else { final File testFailure = new File(f.getAbsolutePath() + "_" + Constants.FAILED); if (testFailure.exists()) { failure = testFailure; numFailed++; } } } final int numScripts = scriptFiles.size(); final File mainFailed = new File(mainScript.getAbsolutePath() + "_" + Constants.FAILED); if (mainFailed.exists()) { failure = mainFailed; } final String logMsg = mainScript.getName() + " Status (Total=" + numScripts + "): Success=" + numSuccess + "; Failure=" + numFailed; if (!statusMsg.equals(logMsg)) { statusMsg = logMsg; Log.out.info(logMsg); } else if ((pollUpdateMeter++ % 10) == 0) { Log.out.info(logMsg); } if (mainFailed.exists() || (Config.getBoolean(Config.SCRIPT_EXIT_ON_ERROR) && (failure != null) && failure.exists())) { throw new Exception("SCRIPT FAILED: " + failure.getAbsolutePath()); } return (numSuccess + numFailed) == numScripts; } /** * This method calls poll to check status of scripts and then sleeps for pollTime seconds. * @param scripts * @param mainScript * @throws Exception */ private static void pollAndSpin(final Module module) throws Exception { int numMinutes = 0; boolean finished = false; while (!finished) { finished = poll(module.getScriptFiles(), module.getMainScript()); if (!finished) { final int timeOut = module.getScriptTimeout(); if ((timeOut > 0) && (numMinutes++ >= timeOut)) { throw new Exception(module.getMainScript().getAbsolutePath() + " timed out after " + numMinutes + " minutes."); } Thread.sleep(POLL_TIME * 1000); } } pollUpdateMeter = 0; } private static void recursiveFileDelete(final File dir) { final Collection<File> files = FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE); for (final File f : files) { Log.addMsg("Delete: " + f.getAbsolutePath()); f.delete(); } } /** * This method creates the sub-dir under projects by attaching date-string to the project name * unless restarting, in which case we send the most recent project with a matching name. * * @param String 2nd Java param - if restartFlag, set to existing project dir, * otherwise, build new projectDir * @throws Exception if projectDir already exists */ private static void setProjectDir(final String optionalParam) throws Exception { if (doRestart(optionalParam)) { Config.setProperty(Config.PROJECT_DIR, getRestartDir().getAbsolutePath()); } else { final String year = String.valueOf(new GregorianCalendar().get(Calendar.YEAR)); final String month = new GregorianCalendar().getDisplayName(Calendar.MONTH, Calendar.SHORT, Locale.ENGLISH); final String day = twoDigitVal(new GregorianCalendar().get(Calendar.DATE)); final File projectDir = new File(Config.requireExistingDirectory(PROJECTS_DIR).getAbsolutePath() + File.separator + Config.requireString(Config.PROJECT_NAME) + "_" + year + month + day); if (projectDir.exists()) { throw new Exception("Project already exists with today's date: " + projectDir.getAbsolutePath() + ". Set restart flag to continue failed pipeline or provide a unique value for: " + Config.PROJECT_NAME); } projectDir.mkdirs(); Config.setProperty(Config.PROJECT_DIR, projectDir.getAbsolutePath()); } } private static String twoDigitVal(final Integer input) { String val = input.toString(); if (val.length() == 1) { val = "0" + val; } return val; } protected static final String PROJECT_COPY_FILES = "control.copyInput"; protected static final String PROJECT_DELETE_TEMP_FILES = "control.deleteTempFiles"; protected static final String PROJECTS_DIR = "project.rootDir"; private static final long APP_START_TIME = System.currentTimeMillis(); private static final int CONFIG_PARAM = 0; private static final int OPTIONAL_PARAM = 1; private static final int POLL_TIME = 60; private static int pollUpdateMeter = 0; private static final List<String> restartFlags = new ArrayList<>(); private static String statusMsg = ""; static { restartFlags.add("restart"); restartFlags.add("r"); } }