List of usage examples for java.io File exists
public boolean exists()
From source file:com.moss.veracity.core.Veracity.java
public static void main(String[] args) throws Exception { File log4jConfigFile = new File("log4j.xml"); if (log4jConfigFile.exists()) { DOMConfigurator.configureAndWatch(log4jConfigFile.getAbsolutePath(), 1000); }//w w w .ja va 2 s .c o m final Log log = LogFactory.getLog(Veracity.class); File homeDir = new File(System.getProperty("user.home")); File currentDir = new File(System.getProperty("user.dir")); List<File> configLocations = new LinkedList<File>(); configLocations.addAll(Arrays.asList(new File[] { new File("/etc/veracity.config"), new File(homeDir, ".veracity.config"), new File(currentDir, "config.xml") })); String customConfigFileProperty = System.getProperty("veracity.configFile"); if (customConfigFileProperty != null) { configLocations.clear(); configLocations.add(new File(customConfigFileProperty)); } File configFile = null; Iterator<File> i = configLocations.iterator(); while ((configFile == null || !configFile.exists()) && i.hasNext()) { configFile = i.next(); } LaunchParameters parameters; if (!configFile.exists()) { if (log.isDebugEnabled()) { log.debug("Creating default config file at " + configFile.getAbsolutePath()); } parameters = new LaunchParameters(); parameters.save(configFile); } else { if (log.isDebugEnabled()) { log.debug("Loading parameters from config file at " + configFile.getAbsolutePath()); } parameters = LaunchParameters.load(configFile); } parameters.readSystemProperties(); new Veracity(parameters); }
From source file:eu.fbk.dkm.sectionextractor.pantheon.WikipediaGoodTextExtractor.java
public static void main(String args[]) throws IOException { CommandLineWithLogger commandLineWithLogger = new CommandLineWithLogger(); commandLineWithLogger.addOption(OptionBuilder.withArgName("file").hasArg() .withDescription("wikipedia xml dump file").isRequired().withLongOpt("wikipedia-dump").create("d")); commandLineWithLogger.addOption(OptionBuilder.withArgName("dir").hasArg() .withDescription("output directory in which to store output files").isRequired() .withLongOpt("output-dir").create("o")); commandLineWithLogger// www. j a v a 2 s .c o m .addOption(OptionBuilder.withDescription("use NAF format").withLongOpt("naf").create("n")); commandLineWithLogger.addOption(OptionBuilder.withDescription("tokenize and ssplit with Stanford") .withLongOpt("stanford").create("s")); commandLineWithLogger.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("Filter file") .withLongOpt("filter").create("f")); commandLineWithLogger.addOption(OptionBuilder.withArgName("file").hasArg() .withDescription("ID and category file").withLongOpt("idcat").create("i")); commandLineWithLogger.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("Redirect file") .withLongOpt("redirect").create("r")); commandLineWithLogger.addOption(OptionBuilder.withArgName("int").hasArg() .withDescription( "number of threads (default " + AbstractWikipediaXmlDumpParser.DEFAULT_THREADS_NUMBER + ")") .withLongOpt("num-threads").create("t")); commandLineWithLogger.addOption(OptionBuilder.withArgName("int").hasArg() .withDescription("number of pages to process (default all)").withLongOpt("num-pages").create("p")); commandLineWithLogger.addOption(OptionBuilder.withArgName("int").hasArg() .withDescription("receive notification every n pages (default " + AbstractWikipediaExtractor.DEFAULT_NOTIFICATION_POINT + ")") .withLongOpt("notification-point").create("b")); commandLineWithLogger.addOption(new Option("n", "NAF format")); CommandLine commandLine = null; try { commandLine = commandLineWithLogger.getCommandLine(args); PropertyConfigurator.configure(commandLineWithLogger.getLoggerProps()); } catch (Exception e) { System.exit(1); } int numThreads = Integer.parseInt(commandLine.getOptionValue("num-threads", Integer.toString(AbstractWikipediaXmlDumpParser.DEFAULT_THREADS_NUMBER))); int numPages = Integer.parseInt(commandLine.getOptionValue("num-pages", Integer.toString(AbstractWikipediaExtractor.DEFAULT_NUM_PAGES))); int notificationPoint = Integer.parseInt(commandLine.getOptionValue("notification-point", Integer.toString(AbstractWikipediaExtractor.DEFAULT_NOTIFICATION_POINT))); boolean nafFormat = commandLine.hasOption("n"); boolean useStanford = commandLine.hasOption("s"); HashMap<Integer, String> idCategory = new HashMap<>(); String idcatFileName = commandLine.getOptionValue("idcat"); if (idcatFileName != null) { logger.info("Loading categories"); File idcatFile = new File(idcatFileName); if (idcatFile.exists()) { List<String> lines = Files.readLines(idcatFile, Charsets.UTF_8); for (String line : lines) { line = line.trim(); if (line.length() == 0) { continue; } String[] parts = line.split("\\s+"); if (parts.length < 3) { continue; } idCategory.put(Integer.parseInt(parts[1]), parts[2]); } } } HashMap<String, String> redirects = new HashMap<>(); String redirectFileName = commandLine.getOptionValue("redirect"); if (redirectFileName != null) { logger.info("Loading redirects"); File redirectFile = new File(redirectFileName); if (redirectFile.exists()) { List<String> lines = Files.readLines(redirectFile, Charsets.UTF_8); for (String line : lines) { line = line.trim(); if (line.length() == 0) { continue; } String[] parts = line.split("\\t+"); if (parts.length < 2) { continue; } redirects.put(parts[0], parts[1]); } } } HashSet<String> pagesToConsider = null; String filterFileName = commandLine.getOptionValue("filter"); if (filterFileName != null) { logger.info("Loading file list"); File filterFile = new File(filterFileName); if (filterFile.exists()) { pagesToConsider = new HashSet<>(); List<String> lines = Files.readLines(filterFile, Charsets.UTF_8); for (String line : lines) { line = line.trim(); if (line.length() == 0) { continue; } line = line.replaceAll("\\s+", "_"); pagesToConsider.add(line); addRedirects(pagesToConsider, redirects, line, 0); } } } ExtractorParameters extractorParameters = new ExtractorParameters( commandLine.getOptionValue("wikipedia-dump"), commandLine.getOptionValue("output-dir")); File outputFolder = new File(commandLine.getOptionValue("output-dir")); if (!outputFolder.exists()) { boolean mkdirs = outputFolder.mkdirs(); if (!mkdirs) { throw new IOException("Unable to create folder " + outputFolder.getAbsolutePath()); } } WikipediaExtractor wikipediaPageParser = new WikipediaGoodTextExtractor(numThreads, numPages, extractorParameters.getLocale(), outputFolder, nafFormat, pagesToConsider, useStanford, idCategory); wikipediaPageParser.setNotificationPoint(notificationPoint); wikipediaPageParser.start(extractorParameters); logger.info("extraction ended " + new Date()); }
From source file:cn.z.Ocr5.java
public static void main(String[] args) throws Exception { // ---step1 downloadImage // String url = "http://reg.keepc.com/getcode/getCode.php"; // //w ww. j av a 2s.c o m // CommonUtil.downloadImage(url, clazz); File file = new File("img/" + clazz); if (!file.exists()) { file.mkdirs(); } new File("train/" + clazz).mkdirs(); new File("result/" + clazz).mkdirs(); File[] files = file.listFiles(); // result/ocr for (int i = 0; i < files.length; ++i) { final String text = getAllOcr(files[i], files[i].getName()); System.out.println(i + ".jpg = " + text); } // CommonUtil.scaleTraindata(clazz, whiteThreshold); // svm_train train = new svm_train(); // train.run(new String[] { new File("train/" + clazz + "/data.txt").getAbsolutePath(), new File("train/" + clazz + "/data.txt.model").getAbsolutePath() }); }
From source file:com.mgmtp.jfunk.core.ui.JFunkFrame.java
public static void main(final String[] args) { List<File> roots = new ArrayList<File>(Math.max(1, args.length)); if (args.length > 0) { for (String arg : args) { File file = new File(arg); if (file.exists()) { roots.add(new File(arg)); }/*from w w w . j a v a 2 s .c o m*/ } } else { roots.add(new File("scripts")); } JFunkFrame.createAndShow(roots); }
From source file:etomica.virial.MCMoveClusterRingRegrowOrientation.java
public static void main(String[] args) { ISpace space = Space3D.getInstance(); ClusterWeight cluster = new ClusterWeight() { @Override// w w w . j a va 2s . c o m public double value(BoxCluster box) { // TODO Auto-generated method stub return 1; } @Override public void setTemperature(double temperature) { // TODO Auto-generated method stub } @Override public int pointCount() { // TODO Auto-generated method stub return 1; } @Override public ClusterAbstract makeCopy() { // TODO Auto-generated method stub return null; } }; BoxCluster box = new BoxCluster(cluster, space); Simulation sim = new Simulation(space); sim.addBox(box); IAtomTypeOriented atype = new AtomTypeOrientedSphere(Hydrogen.INSTANCE, space); SpeciesSpheresHetero species = new SpeciesSpheresHetero(space, new IAtomTypeOriented[] { atype }); sim.addSpecies(species); File file1 = new File("acceptance.dat"); if (file1.exists()) { file1.delete(); } for (int p = 2; p <= 512; p *= 2) { box.setNMolecules(species, 0); species.setChildCount(new int[] { p }); box.setNMolecules(species, 1); IntegratorMC integrator = new IntegratorMC(sim, null); integrator.setBox(box); MCMoveClusterRingRegrowOrientation move = new MCMoveClusterRingRegrowOrientation(sim.getRandom(), space, p); for (int iTemp = 40; iTemp <= 40; iTemp += 2) { move.foo = 0; move.setStiffness(Kelvin.UNIT.toSim(iTemp), species.getAtomType(0).getMass()); integrator.getMoveManager().addMCMove(move); integrator.reset(); int total = 100; for (int i = 0; i < total; i++) { integrator.doStep(); } try { FileWriter Temp = new FileWriter("acceptance.dat", true); Temp.write( iTemp + " " + p + " " + move.getStiffness() + " " + ((double) move.foo) / total + "\n"); Temp.close(); } catch (IOException ex1) { throw new RuntimeException(ex1); } System.out.println( "p = " + p + " ,Temp = " + iTemp + " ,acceptance ratio = " + ((double) move.foo) / total); } } }
From source file:de.prozesskraft.pkraft.Commitit.java
public static void main(String[] args) throws org.apache.commons.cli.ParseException, IOException { // try//from ww w . ja v a 2 s.co m // { // if (args.length != 3) // { // System.out.println("Please specify processdefinition file (xml) and an outputfilename"); // } // // } // catch (ArrayIndexOutOfBoundsException e) // { // System.out.println("***ArrayIndexOutOfBoundsException: Please specify processdefinition.xml, openoffice_template.od*, newfile_for_processdefinitions.odt\n" + e.toString()); // } /*---------------------------- get options from ini-file ----------------------------*/ java.io.File inifile = new java.io.File( WhereAmI.getInstallDirectoryAbsolutePath(Commitit.class) + "/" + "../etc/pkraft-commitit.ini"); if (inifile.exists()) { try { ini = new Ini(inifile); } catch (InvalidFileFormatException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } else { System.err.println("ini file does not exist: " + inifile.getAbsolutePath()); System.exit(1); } /*---------------------------- create boolean options ----------------------------*/ Option ohelp = new Option("help", "print this message"); /*---------------------------- create argument options ----------------------------*/ Option oinstance = OptionBuilder.withArgName("FILE").hasArg() .withDescription("[mandatory] process instance file") // .isRequired() .create("instance"); Option ostep = OptionBuilder.withArgName("STEPNAME").hasArg() .withDescription("[optional, default: root] process step to commit to") // .isRequired() .create("step"); Option ofile = OptionBuilder.withArgName("FILE").hasArg() .withDescription("[optional] this file will be committed as file. key will be set to 'default'") // .isRequired() .create("file"); Option okey = OptionBuilder.withArgName("KEY").hasArg() .withDescription( "[optional, default: default] this string will be considered as the key for the commit.") // .isRequired() .create("key"); Option ovariable = OptionBuilder.withArgName("VALUE").hasArg() .withDescription("[optional] this string will be committed as a variable.") // .isRequired() .create("variable"); /*---------------------------- create options object ----------------------------*/ Options options = new Options(); options.addOption(ohelp); options.addOption(oinstance); options.addOption(ostep); options.addOption(ofile); options.addOption(okey); options.addOption(ovariable); /*---------------------------- create the parser ----------------------------*/ CommandLineParser parser = new GnuParser(); try { // parse the command line arguments commandline = parser.parse(options, args); } catch (Exception exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); exiter(); } /*---------------------------- usage/help ----------------------------*/ if (commandline.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("commit", options); System.exit(0); } /*---------------------------- ueberpruefen ob eine schlechte kombination von parametern angegeben wurde ----------------------------*/ if (!(commandline.hasOption("instance"))) { System.out.println("option -instance is mandatory."); exiter(); } else if (!(commandline.hasOption("dir")) && !(commandline.hasOption("file")) && !(commandline.hasOption("varfile")) && !(commandline.hasOption("varname")) && !(commandline.hasOption("varvalue")) && !(commandline.hasOption("variable"))) { System.out.println( "at least one of these options needed. -dir -file -varfile -variable -varname -varvalue."); exiter(); } else if ((commandline.hasOption("varname") && !(commandline.hasOption("varvalue"))) || (!(commandline.hasOption("varname")) && commandline.hasOption("varvalue"))) { System.out.println("use options -varname and -varvalue only in combination with each other."); exiter(); } /*---------------------------- die lizenz ueberpruefen und ggf abbrechen ----------------------------*/ // check for valid license ArrayList<String> allPortAtHost = new ArrayList<String>(); allPortAtHost.add(ini.get("license-server", "license-server-1")); allPortAtHost.add(ini.get("license-server", "license-server-2")); allPortAtHost.add(ini.get("license-server", "license-server-3")); MyLicense lic = new MyLicense(allPortAtHost, "1", "user-edition", "0.1"); // lizenz-logging ausgeben for (String actLine : (ArrayList<String>) lic.getLog()) { System.err.println(actLine); } // abbruch, wenn lizenz nicht valide if (!lic.isValid()) { System.exit(1); } /*---------------------------- die eigentliche business logic ----------------------------*/ // setzen des steps String stepname = "root"; if (commandline.hasOption("step")) { stepname = commandline.getOptionValue("step"); } // setzen des key String key = "default"; if (commandline.hasOption("key")) { key = commandline.getOptionValue("key"); } Process p1 = new Process(); p1.setInfilebinary(commandline.getOptionValue("instance")); System.out.println("info: reading process instance " + commandline.getOptionValue("instance")); Process p2 = p1.readBinary(); p2.setOutfilebinary(commandline.getOptionValue("instance")); // step ueber den namen heraussuchen Step step = p2.getStep(stepname); if (step == null) { System.err.println("step not found: " + stepname); exiter(); } // den Commit 'by-process-commitit' heraussuchen oder einen neuen Commit dieses Namens erstellen Commit commit = step.getCommit("by-hand"); if (commit == null) { commit = new Commit(step); commit.setName("by-process-commitit"); } // committen if (commandline.hasOption("file")) { File file = new File(); file.setKey(key); file.setGlob(commandline.getOptionValue("file")); commit.addFile(file); commit.doIt(); } if (commandline.hasOption("variable")) { Variable variable = new Variable(); variable.setKey(key); variable.setValue(commandline.getOptionValue("variable")); commit.addVariable(variable); commit.doIt(); } p2.writeBinary(); System.out.println("info: writing process instance " + p2.getOutfilebinary()); }
From source file:edu.kit.dama.util.release.GenerateSourceRelease.java
public static void main(String[] args) throws Exception { if (args.length != 3) { System.err.println("Usage: GenerateSourceRelease TYPE SOURCE DESTINATION"); System.err.println(""); System.err.println("TYPE\tThe release type. Must be one of KITDM, GENERIC_CLIENT or BARE_DEMO"); System.err.println("SOURCE\tThe source folder containing all sources for the selected release type."); System.err.println("DESTINATION\tThe destination folder where all sources of the release are placed."); System.exit(1);/*from w w w . j a va2s .c o m*/ } String releaseType = args[0]; RELEASE_TYPE type = RELEASE_TYPE.KITDM; try { type = RELEASE_TYPE.valueOf(releaseType); } catch (IllegalArgumentException ex) { System.err.println( "Invalid release type. Valid relase types arguments are KITDM, GENERIC_CLIENT or BARE_DEMO"); System.exit(1); } String source = args[1]; String destination = args[2]; File sourceFile = new File(source); File destinationFile = new File(destination); if ((sourceFile.exists() && !sourceFile.isDirectory()) || (destinationFile.exists() && !destinationFile.isDirectory())) { System.err.println("Either source or destination are no directories."); System.exit(1); } if (!sourceFile.exists() || !sourceFile.canRead()) { System.err.println("Source either does not exist or is not readable."); System.exit(1); } if ((destinationFile.exists() && !sourceFile.canWrite()) || (!destinationFile.exists() && !destinationFile.mkdirs())) { System.err.println("Destination is either not writable or cannot be created."); System.exit(1); } ReleaseConfiguration config = null; switch (type) { case KITDM: config = getKITDMSourceReleaseConfig(source, destination); break; case GENERIC_CLIENT: config = getGenericRepoClientSourceReleaseConfig(source, destination); break; case BARE_DEMO: config = getBaReDemoSourceReleaseConfig(source, destination); break; } generateSourceRelease(config); System.out.println("Generating Release finished."); System.out.println("Please manually check pom.xml:"); System.out.println(" - Remove profiles"); System.out.println(" - Update links to SCM, ciManagement and internal repositories"); }
From source file:ValidateLicenseHeaders.java
/** * ValidateLicenseHeaders jboss-src-root * //from www.j a v a2 s. com * @param args */ public static void main(String[] args) throws Exception { if (args.length == 0 || args[0].startsWith("-h")) { log.info("Usage: ValidateLicenseHeaders [-addheader] jboss-src-root"); System.exit(1); } int rootArg = 0; if (args.length == 2) { if (args[0].startsWith("-add")) addDefaultHeader = true; else { log.severe("Uknown argument: " + args[0]); log.info("Usage: ValidateLicenseHeaders [-addheader] jboss-src-root"); System.exit(1); } rootArg = 1; } File jbossSrcRoot = new File(args[rootArg]); if (jbossSrcRoot.exists() == false) { log.info("Src root does not exist, check " + jbossSrcRoot.getAbsolutePath()); System.exit(1); } URL u = Thread.currentThread().getContextClassLoader() .getResource("META-INF/services/javax.xml.parsers.DocumentBuilderFactory"); System.err.println(u); // Load the valid copyright statements for the licenses File licenseInfo = new File(jbossSrcRoot, "varia/src/etc/license-info.xml"); if (licenseInfo.exists() == false) { log.severe("Failed to find the varia/src/etc/license-info.xml under the src root"); System.exit(1); } DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder db = factory.newDocumentBuilder(); Document doc = db.parse(licenseInfo); NodeList licenses = doc.getElementsByTagName("license"); for (int i = 0; i < licenses.getLength(); i++) { Element license = (Element) licenses.item(i); String key = license.getAttribute("id"); ArrayList headers = new ArrayList(); licenseHeaders.put(key, headers); NodeList copyrights = license.getElementsByTagName("terms-header"); for (int j = 0; j < copyrights.getLength(); j++) { Element copyright = (Element) copyrights.item(j); copyright.normalize(); String id = copyright.getAttribute("id"); // The id will be blank if there is no id attribute if (id.length() == 0) continue; String text = getElementContent(copyright); if (text == null) continue; // Replace all duplicate whitespace and '*' with a single space text = text.replaceAll("[\\s*]+", " "); if (text.length() == 1) continue; text = text.toLowerCase().trim(); // Replace any copyright date0-date1,date2 with copyright ... text = text.replaceAll(COPYRIGHT_REGEX, "..."); LicenseHeader lh = new LicenseHeader(id, text); headers.add(lh); } } log.fine(licenseHeaders.toString()); File[] files = jbossSrcRoot.listFiles(dotJavaFilter); log.info("Root files count: " + files.length); processSourceFiles(files, 0); log.info("Processed " + totalCount); log.info("Updated jboss headers: " + jbossCount); // Files with no headers details log.info("Files with no headers: " + noheaders.size()); FileWriter fw = new FileWriter("NoHeaders.txt"); for (Iterator iter = noheaders.iterator(); iter.hasNext();) { File f = (File) iter.next(); fw.write(f.getAbsolutePath()); fw.write('\n'); } fw.close(); // Files with unknown headers details log.info("Files with invalid headers: " + invalidheaders.size()); fw = new FileWriter("InvalidHeaders.txt"); for (Iterator iter = invalidheaders.iterator(); iter.hasNext();) { File f = (File) iter.next(); fw.write(f.getAbsolutePath()); fw.write('\n'); } fw.close(); // License usage summary log.info("Creating HeadersSummary.txt"); fw = new FileWriter("HeadersSummary.txt"); for (Iterator iter = licenseHeaders.entrySet().iterator(); iter.hasNext();) { Map.Entry entry = (Map.Entry) iter.next(); String key = (String) entry.getKey(); fw.write("+++ License type=" + key); fw.write('\n'); List list = (List) entry.getValue(); Iterator jiter = list.iterator(); while (jiter.hasNext()) { LicenseHeader lh = (LicenseHeader) jiter.next(); fw.write('\t'); fw.write(lh.id); fw.write(", count="); fw.write("" + lh.count); fw.write('\n'); } } fw.close(); }
From source file:DIA_Umpire_Quant.DIA_Umpire_IntLibSearch.java
/** * @param args the command line arguments *///from www. j a v a 2 s .c o m public static void main(String[] args) throws FileNotFoundException, IOException, Exception { System.out.println( "================================================================================================="); System.out.println("DIA-Umpire targeted re-extraction analysis using internal library (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length != 1) { System.out.println( "command format error, the correct format should be : java -jar -Xmx10G DIA_Umpire_IntLibSearch.jar diaumpire_module.params"); return; } try { ConsoleLogger.SetConsoleLogger(Level.INFO); ConsoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "diaumpire_intlibsearch.log"); } catch (Exception e) { } Logger.getRootLogger().info("Version: " + UmpireInfo.GetInstance().Version); Logger.getRootLogger().info("Parameter file:" + args[0]); BufferedReader reader = new BufferedReader(new FileReader(args[0])); String line = ""; String WorkFolder = ""; int NoCPUs = 2; String InternalLibID = ""; float ProbThreshold = 0.99f; float RTWindow_Int = -1f; float Freq = 0f; int TopNFrag = 6; TandemParam tandemPara = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); HashMap<String, File> AssignFiles = new HashMap<>(); //<editor-fold defaultstate="collapsed" desc="Reading parameter file"> while ((line = reader.readLine()) != null) { line = line.trim(); Logger.getRootLogger().info(line); if (!"".equals(line) && !line.startsWith("#")) { //System.out.println(line); if (line.equals("==File list begin")) { do { line = reader.readLine(); line = line.trim(); if (line.equals("==File list end")) { continue; } else if (!"".equals(line)) { File newfile = new File(line); if (newfile.exists()) { AssignFiles.put(newfile.getAbsolutePath(), newfile); } else { Logger.getRootLogger().info("File: " + newfile + " does not exist."); } } } while (!line.equals("==File list end")); } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); String value = line.split("=")[1].trim(); switch (type) { case "Path": { WorkFolder = value; break; } case "path": { WorkFolder = value; break; } case "Thread": { NoCPUs = Integer.parseInt(value); break; } case "InternalLibID": { InternalLibID = value; break; } case "RTWindow_Int": { RTWindow_Int = Float.parseFloat(value); break; } case "ProbThreshold": { ProbThreshold = Float.parseFloat(value); break; } case "TopNFrag": { TopNFrag = Integer.parseInt(value); break; } case "Freq": { Freq = Float.parseFloat(value); break; } case "Fasta": { tandemPara.FastaPath = value; break; } } } } //</editor-fold> //Initialize PTM manager using compomics library PTMManager.GetInstance(); //Check if the fasta file can be found if (!new File(tandemPara.FastaPath).exists()) { Logger.getRootLogger().info("Fasta file :" + tandemPara.FastaPath + " cannot be found, the process will be terminated, please check."); System.exit(1); } //Generate DIA file list ArrayList<DIAPack> FileList = new ArrayList<>(); try { File folder = new File(WorkFolder); if (!folder.exists()) { Logger.getRootLogger().info("The path : " + WorkFolder + " cannot be found."); System.exit(1); } for (final File fileEntry : folder.listFiles()) { if (fileEntry.isFile() && (fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry.getAbsolutePath(), fileEntry); } if (fileEntry.isDirectory()) { for (final File fileEntry2 : fileEntry.listFiles()) { if (fileEntry2.isFile() && (fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry2.getAbsolutePath(), fileEntry2); } } } } Logger.getRootLogger().info("No. of files assigned :" + AssignFiles.size()); for (File fileEntry : AssignFiles.values()) { Logger.getRootLogger().info(fileEntry.getAbsolutePath()); } for (File fileEntry : AssignFiles.values()) { String mzXMLFile = fileEntry.getAbsolutePath(); if (mzXMLFile.toLowerCase().endsWith(".mzxml") | mzXMLFile.toLowerCase().endsWith(".mzml")) { DIAPack DiaFile = new DIAPack(mzXMLFile, NoCPUs); Logger.getRootLogger().info( "================================================================================================="); Logger.getRootLogger().info("Processing " + mzXMLFile); if (!DiaFile.LoadDIASetting()) { Logger.getRootLogger().info("Loading DIA setting failed, job is incomplete"); System.exit(1); } if (!DiaFile.LoadParams()) { Logger.getRootLogger().info("Loading parameters failed, job is incomplete"); System.exit(1); } Logger.getRootLogger().info("Loading identification results " + mzXMLFile + "...."); //If the serialization file for ID file existed if (DiaFile.ReadSerializedLCMSID()) { DiaFile.IDsummary.ReduceMemoryUsage(); DiaFile.IDsummary.FastaPath = tandemPara.FastaPath; FileList.add(DiaFile); } } } //<editor-fold defaultstate="collapsed" desc="Targete re-extraction using internal library"> Logger.getRootLogger().info( "================================================================================================="); if (FileList.size() > 1) { Logger.getRootLogger().info("Targeted re-extraction using internal library"); FragmentLibManager libManager = FragmentLibManager.ReadFragmentLibSerialization(WorkFolder, InternalLibID); if (libManager == null) { Logger.getRootLogger().info("Building internal spectral library"); libManager = new FragmentLibManager(InternalLibID); ArrayList<LCMSID> LCMSIDList = new ArrayList<>(); for (DIAPack dia : FileList) { LCMSIDList.add(dia.IDsummary); } libManager.ImportFragLibTopFrag(LCMSIDList, Freq, TopNFrag); libManager.WriteFragmentLibSerialization(WorkFolder); } libManager.ReduceMemoryUsage(); Logger.getRootLogger() .info("Building retention time prediction model and generate candidate peptide list"); for (int i = 0; i < FileList.size(); i++) { FileList.get(i).IDsummary.ClearMappedPep(); } for (int i = 0; i < FileList.size(); i++) { for (int j = i + 1; j < FileList.size(); j++) { RTAlignedPepIonMapping alignment = new RTAlignedPepIonMapping(WorkFolder, FileList.get(i).GetParameter(), FileList.get(i).IDsummary, FileList.get(j).IDsummary); alignment.GenerateModel(); alignment.GenerateMappedPepIon(); } FileList.get(i).ExportID(); FileList.get(i).IDsummary = null; } Logger.getRootLogger().info("Targeted matching........"); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); } if (!diafile.IDsummary.GetMappedPepIonList().isEmpty()) { diafile.UseMappedIon = true; diafile.FilterMappedIonByProb = false; diafile.BuildStructure(); diafile.MS1FeatureMap.ReadPeakCluster(); diafile.MS1FeatureMap.ClearMonoisotopicPeakOfCluster(); diafile.GenerateMassCalibrationRTMap(); diafile.TargetedExtractionQuant(false, libManager, ProbThreshold, RTWindow_Int); diafile.MS1FeatureMap.ClearAllPeaks(); diafile.IDsummary.ReduceMemoryUsage(); diafile.IDsummary.RemoveLowProbMappedIon(ProbThreshold); diafile.ExportID(); Logger.getRootLogger().info("Peptide ions: " + diafile.IDsummary.GetPepIonList().size() + " Mapped ions: " + diafile.IDsummary.GetMappedPepIonList().size()); diafile.ClearStructure(); } diafile.IDsummary = null; System.gc(); } Logger.getRootLogger().info( "================================================================================================="); } //</editor-fold> Logger.getRootLogger().info("Job done"); Logger.getRootLogger().info( "================================================================================================="); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step4MTurkOutputCollector.java
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { String inputDirWithArgumentPairs = args[0]; File[] resultFiles;/*from ww w . java2 s . c o m*/ if (args[1].contains("*")) { File path = new File(args[1]); File directory = path.getParentFile(); String regex = path.getName().replaceAll("\\*", ""); List<File> files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { regex }, false)); resultFiles = new File[files.size()]; for (int i = 0; i < files.size(); i++) { resultFiles[i] = files.get(i); } } else { // result file is a comma-separated list of CSV files from MTurk String[] split = args[1].split(","); resultFiles = new File[split.length]; for (int i = 0; i < split.length; i++) { resultFiles[i] = new File(split[i]); } } File outputDir = new File(args[2]); if (!outputDir.exists()) { if (!outputDir.mkdirs()) { throw new IOException("Cannot create directory " + outputDir); } } // error if output folder not empty to prevent any confusion by mixing files if (!FileUtils.listFiles(outputDir, null, false).isEmpty()) { throw new IllegalArgumentException("Output dir " + outputDir + " is not empty"); } // collected assignments with empty reason for rejections Set<String> assignmentsWithEmptyReason = new HashSet<>(); // parse with first line as header MTurkOutputReader mTurkOutputReader = new MTurkOutputReader(resultFiles); Collection<File> files = FileUtils.listFiles(new File(inputDirWithArgumentPairs), new String[] { "xml" }, false); if (files.isEmpty()) { throw new IOException("No xml files found in " + inputDirWithArgumentPairs); } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Map<String, Integer>> assignmentsPerHits = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); String hitTypeId = record.get("hittypeid"); if (!wasRejected) { // update statistics if (!assignmentsPerHits.containsKey(hitTypeId)) { assignmentsPerHits.put(hitTypeId, new HashMap<String, Integer>()); } if (!assignmentsPerHits.get(hitTypeId).containsKey(hitID)) { assignmentsPerHits.get(hitTypeId).put(hitID, 0); } assignmentsPerHits.get(hitTypeId).put(hitID, assignmentsPerHits.get(hitTypeId).get(hitID) + 1); } } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Integer> approvedAssignmentsPerHit = new HashMap<>(); Map<String, Integer> rejectedAssignmentsPerHit = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean approved = "Approved".equals(record.get("assignmentstatus")); boolean rejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); if (approved) { // update statistics if (!approvedAssignmentsPerHit.containsKey(hitID)) { approvedAssignmentsPerHit.put(hitID, 0); } approvedAssignmentsPerHit.put(hitID, approvedAssignmentsPerHit.get(hitID) + 1); } else if (rejected) { // update statistics if (!rejectedAssignmentsPerHit.containsKey(hitID)) { rejectedAssignmentsPerHit.put(hitID, 0); } rejectedAssignmentsPerHit.put(hitID, rejectedAssignmentsPerHit.get(hitID) + 1); } else { throw new IllegalStateException( "Unknown state: " + record.get("assignmentstatus") + " HITID: " + hitID); } } // System.out.println("Approved: " + approvedAssignmentsPerHit); // System.out.println("Rejected: " + rejectedAssignmentsPerHit); System.out.println("Approved (values): " + new HashSet<>(approvedAssignmentsPerHit.values())); System.out.println("Rejected (values): " + new HashSet<>(rejectedAssignmentsPerHit.values())); // rejection statistics int totalRejected = 0; for (Map.Entry<String, Integer> rejectionEntry : rejectedAssignmentsPerHit.entrySet()) { totalRejected += rejectionEntry.getValue(); } System.out.println("Total rejections: " + totalRejected); /* // generate .success files for adding more annotations for (File resultFile : resultFiles) { String hitTypeID = mTurkOutputReader.getHitTypeIdForFile().get(resultFile); // assignments for that hittypeid (= file) Map<String, Integer> assignments = assignmentsPerHits.get(hitTypeID); prepareUpdateHITsFiles(assignments, hitTypeID, resultFile); } */ int totalSavedPairs = 0; // load all previously prepared argument pairs for (File file : files) { List<ArgumentPair> argumentPairs = (List<ArgumentPair>) XStreamTools.getXStream().fromXML(file); List<AnnotatedArgumentPair> annotatedArgumentPairs = new ArrayList<>(); for (ArgumentPair argumentPair : argumentPairs) { AnnotatedArgumentPair annotatedArgumentPair = new AnnotatedArgumentPair(argumentPair); // is there such an answer? String key = "Answer." + argumentPair.getId(); // iterate only if there is such column to save time if (mTurkOutputReader.getColumnNames().contains(key)) { // now find the results for (Map<String, String> record : mTurkOutputReader) { if (record.containsKey(key)) { // extract the values AnnotatedArgumentPair.MTurkAssignment assignment = new AnnotatedArgumentPair.MTurkAssignment(); boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); // only non-rejected (if required) if (!wasRejected) { String hitID = record.get("hitid"); String workerID = record.get("workerid"); String assignmentId = record.get("assignmentid"); try { assignment.setAssignmentAcceptTime( DATE_FORMAT.parse(record.get("assignmentaccepttime"))); assignment.setAssignmentSubmitTime( DATE_FORMAT.parse(record.get("assignmentsubmittime"))); assignment.setHitComment(record.get("Answer.feedback")); assignment.setHitID(hitID); assignment.setTurkID(workerID); assignment.setAssignmentId(assignmentId); // and answer specific fields String valueRaw = record.get(key); // so far the label has had format aXXX_aYYY_a1, aXXX_aYYY_a2, or aXXX_aYYY_equal // strip now only true label String label = valueRaw.split("_")[2]; assignment.setValue(label); String reason = record.get(key + "_reason"); // missing reason if (reason == null) { assignmentsWithEmptyReason.add(assignmentId); } else { assignment.setReason(reason); // get worker's stance String stanceRaw = record.get(key + "_stance"); if (stanceRaw != null) { // parse stance String stance = stanceRaw.split("_stance_")[1]; assignment.setWorkerStance(stance); } // we take maximal 5 assignments Collections.sort(annotatedArgumentPair.mTurkAssignments, new Comparator<AnnotatedArgumentPair.MTurkAssignment>() { @Override public int compare(AnnotatedArgumentPair.MTurkAssignment o1, AnnotatedArgumentPair.MTurkAssignment o2) { return o1.getAssignmentAcceptTime() .compareTo(o2.getAssignmentAcceptTime()); } }); if (annotatedArgumentPair.mTurkAssignments .size() < MAXIMUM_ASSIGNMENTS_PER_HIT) { annotatedArgumentPair.mTurkAssignments.add(assignment); } } } catch (IllegalArgumentException | NullPointerException ex) { System.err.println("Malformed annotations for HIT " + hitID + ", worker " + workerID + ", assignment " + assignmentId + "; " + ex.getMessage() + ", full record: " + record); } } } } } // and if there are some annotations, add it to the result set if (!annotatedArgumentPair.mTurkAssignments.isEmpty()) { annotatedArgumentPairs.add(annotatedArgumentPair); } } if (!annotatedArgumentPairs.isEmpty()) { File outputFile = new File(outputDir, file.getName()); XStreamTools.toXML(annotatedArgumentPairs, outputFile); System.out.println("Saved " + annotatedArgumentPairs.size() + " annotated pairs to " + outputFile); totalSavedPairs += annotatedArgumentPairs.size(); } } System.out.println("Total saved " + totalSavedPairs + " pairs"); // print assignments with empty reasons if (!assignmentsWithEmptyReason.isEmpty()) { System.out.println( "== Assignments with empty reason:\nassignmentIdToReject\tassignmentIdToRejectComment"); for (String assignmentId : assignmentsWithEmptyReason) { System.out.println( assignmentId + "\t\"Dear worker, you did not fill the required field with a reason.\""); } } }