List of usage examples for java.lang String format
public static String format(String format, Object... args)
From source file:com.act.biointerpretation.BiointerpretationDriver.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());/*from www.ja va 2s .c om*/ } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(ReactionDesalter.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } if (cl.hasOption(OPTION_CONFIGURATION_FILE)) { List<BiointerpretationStep> steps; File configFile = new File(cl.getOptionValue(OPTION_CONFIGURATION_FILE)); if (!configFile.exists()) { String msg = String.format("Cannot find configuration file at %s", configFile.getAbsolutePath()); LOGGER.error(msg); throw new RuntimeException(msg); } // Read the whole config file. try (InputStream is = new FileInputStream(configFile)) { steps = OBJECT_MAPPER.readValue(is, new TypeReference<List<BiointerpretationStep>>() { }); } catch (IOException e) { LOGGER.error("Caught IO exception when attempting to read configuration file: %s", e.getMessage()); throw e; // Crash after logging if the config file can't be read. } // Ask for explicit confirmation before dropping databases. LOGGER.info("Biointerpretation plan:"); for (BiointerpretationStep step : steps) { crashIfInvalidDBName(step.getReadDBName()); crashIfInvalidDBName(step.getWriteDBName()); LOGGER.info("%s: %s -> %s", step.getOperation(), step.getReadDBName(), step.getWriteDBName()); } LOGGER.warn("WARNING: each DB to be written will be dropped before the writing step commences"); LOGGER.info("Proceed? [y/n]"); String readLine; try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) { readLine = reader.readLine(); } readLine.trim(); if ("y".equalsIgnoreCase(readLine) || "yes".equalsIgnoreCase(readLine)) { LOGGER.info("Biointerpretation plan confirmed, commencing"); for (BiointerpretationStep step : steps) { performOperation(step, true); } LOGGER.info("Biointerpretation plan completed"); } else { LOGGER.info("Biointerpretation plan not confirmed, exiting"); } } else if (cl.hasOption(OPTION_SINGLE_OPERATION)) { if (!cl.hasOption(OPTION_SINGLE_READ_DB) || !cl.hasOption(OPTION_SINGLE_WRITE_DB)) { String msg = "Must specify read and write DB names when performing a single operation"; LOGGER.error(msg); throw new RuntimeException(msg); } BiointerpretationOperation operation; try { operation = BiointerpretationOperation.valueOf(cl.getOptionValue(OPTION_SINGLE_OPERATION)); } catch (IllegalArgumentException e) { LOGGER.error("Caught IllegalArgumentException when trying to parse operation '%s': %s", cl.getOptionValue(OPTION_SINGLE_OPERATION), e.getMessage()); throw e; // Crash if we can't interpret the operation. } String readDB = crashIfInvalidDBName(cl.getOptionValue(OPTION_SINGLE_READ_DB)); String writeDB = crashIfInvalidDBName(cl.getOptionValue(OPTION_SINGLE_WRITE_DB)); performOperation(new BiointerpretationStep(operation, readDB, writeDB), false); } else { String msg = "Must specify either a config file or a single operation to perform."; LOGGER.error(msg); throw new RuntimeException(msg); } }
From source file:edu.cmu.lti.oaqa.apps.Client.java
public static void main(String args[]) { BufferedReader inp = new BufferedReader(new InputStreamReader(System.in)); Options opt = new Options(); Option o = new Option(PORT_SHORT_PARAM, PORT_LONG_PARAM, true, PORT_DESC); o.setRequired(true);/* w w w.ja v a 2 s . c o m*/ opt.addOption(o); o = new Option(HOST_SHORT_PARAM, HOST_LONG_PARAM, true, HOST_DESC); o.setRequired(true); opt.addOption(o); opt.addOption(K_SHORT_PARAM, K_LONG_PARAM, true, K_DESC); opt.addOption(R_SHORT_PARAM, R_LONG_PARAM, true, R_DESC); opt.addOption(QUERY_TIME_SHORT_PARAM, QUERY_TIME_LONG_PARAM, true, QUERY_TIME_DESC); opt.addOption(RET_OBJ_SHORT_PARAM, RET_OBJ_LONG_PARAM, false, RET_OBJ_DESC); opt.addOption(RET_EXTERN_ID_SHORT_PARAM, RET_EXTERN_ID_LONG_PARAM, false, RET_EXTERN_ID_DESC); CommandLineParser parser = new org.apache.commons.cli.GnuParser(); try { CommandLine cmd = parser.parse(opt, args); String host = cmd.getOptionValue(HOST_SHORT_PARAM); String tmp = null; tmp = cmd.getOptionValue(PORT_SHORT_PARAM); int port = -1; try { port = Integer.parseInt(tmp); } catch (NumberFormatException e) { Usage("Port should be integer!"); } boolean retObj = cmd.hasOption(RET_OBJ_SHORT_PARAM); boolean retExternId = cmd.hasOption(RET_EXTERN_ID_SHORT_PARAM); String queryTimeParams = cmd.getOptionValue(QUERY_TIME_SHORT_PARAM); if (null == queryTimeParams) queryTimeParams = ""; SearchType searchType = SearchType.kKNNSearch; int k = 0; double r = 0; if (cmd.hasOption(K_SHORT_PARAM)) { if (cmd.hasOption(R_SHORT_PARAM)) { Usage("Range search is not allowed if the KNN search is specified!"); } tmp = cmd.getOptionValue(K_SHORT_PARAM); try { k = Integer.parseInt(tmp); } catch (NumberFormatException e) { Usage("K should be integer!"); } searchType = SearchType.kKNNSearch; } else if (cmd.hasOption(R_SHORT_PARAM)) { if (cmd.hasOption(K_SHORT_PARAM)) { Usage("KNN search is not allowed if the range search is specified!"); } searchType = SearchType.kRangeSearch; tmp = cmd.getOptionValue(R_SHORT_PARAM); try { r = Double.parseDouble(tmp); } catch (NumberFormatException e) { Usage("The range value should be numeric!"); } } else { Usage("One has to specify either range or KNN-search parameter"); } String separator = System.getProperty("line.separator"); StringBuffer sb = new StringBuffer(); String s; while ((s = inp.readLine()) != null) { sb.append(s); sb.append(separator); } String queryObj = sb.toString(); try { TTransport transport = new TSocket(host, port); transport.open(); TProtocol protocol = new TBinaryProtocol(transport); QueryService.Client client = new QueryService.Client(protocol); if (!queryTimeParams.isEmpty()) client.setQueryTimeParams(queryTimeParams); List<ReplyEntry> res = null; long t1 = System.nanoTime(); if (searchType == SearchType.kKNNSearch) { System.out.println(String.format("Running a %d-NN search", k)); res = client.knnQuery(k, queryObj, retExternId, retObj); } else { System.out.println(String.format("Running a range search (r=%g)", r)); res = client.rangeQuery(r, queryObj, retExternId, retObj); } long t2 = System.nanoTime(); System.out.println(String.format("Finished in %g ms", (t2 - t1) / 1e6)); for (ReplyEntry e : res) { System.out.println(String.format("id=%d dist=%g %s", e.getId(), e.getDist(), retExternId ? "externId=" + e.getExternId() : "")); if (retObj) System.out.println(e.getObj()); } transport.close(); // Close transport/socket ! } catch (TException te) { System.err.println("Apache Thrift exception: " + te); te.printStackTrace(); } } catch (ParseException e) { Usage("Cannot parse arguments"); } catch (Exception e) { e.printStackTrace(); System.exit(1); } }
From source file:com.bigdata.dastor.tools.SSTableImport.java
/** * Converts JSON to an SSTable file. JSON input can either be a file specified * using an optional command line argument, or supplied on standard in. * /* www . j av a 2s. c om*/ * @param args command line arguments * @throws IOException on failure to open/read/write files or output streams * @throws ParseException on failure to parse JSON input */ public static void main(String[] args) throws IOException, ParseException { String usage = String.format("Usage: %s -K keyspace -c column_family <json> <sstable>%n", SSTableImport.class.getName()); CommandLineParser parser = new PosixParser(); try { cmd = parser.parse(options, args); } catch (org.apache.commons.cli.ParseException e1) { System.err.println(e1.getMessage()); System.err.println(usage); System.exit(1); } if (cmd.getArgs().length != 2) { System.err.println(usage); System.exit(1); } String json = cmd.getArgs()[0]; String ssTable = cmd.getArgs()[1]; String keyspace = cmd.getOptionValue(KEYSPACE_OPTION); String cfamily = cmd.getOptionValue(COLFAM_OPTION); importJson(json, keyspace, cfamily, ssTable); System.exit(0); }
From source file:it.units.malelab.ege.util.DUMapper.java
/** * @param args the command line arguments *//* w w w. j a v a2 s. c om*/ public static void main(String[] args) throws IOException, InterruptedException, ExecutionException { modifyMap("/home/eric/experiments/dumapper/modified/whge-nguyen7-3_du.png", 3); modifyMap("/home/eric/experiments/dumapper/modified/whge-nguyen7-3_du.png", 5); modifyMap("/home/eric/experiments/dumapper/modified/LT_E6-14_du.png", 3); modifyMap("/home/eric/experiments/dumapper/modified/LT_E6-14_du.png", 5); modifyMap("/home/eric/experiments/dumapper/modified/legend-colors.png", 3); modifyMap("/home/eric/experiments/dumapper/modified/legend-colors.png", 5); System.exit(0); //saveImages("/home/eric/experiments/dumapper/sge6-nguyen7-1_%s.png", false, 4, buildSGEData(100, 6, new Nguyen7(0))); //saveImages("/home/eric/experiments/dumapper/whge-nguyen7-1_%s.png", false, 1, buildGEData("whge", 100, 256, new Nguyen7(0), 0, 5)); //saveImages("/home/eric/experiments/dumapper/ge-nguyen7-1_%s.png", false, 1, buildGEData("ge", 100, 256, new Nguyen7(0), 0, 5)); /*double[][][] neatData = getNeatData("/home/eric/experiments/dumapper/neat/NEATPopulations", "targetANDcollision_100.0w1_0.1w2_(phase1_from1to300)_NEATPopulationEvolved(%sof300)_100pop_300gen_10cars_3x30.0sec_Run1.eg", 300); saveImages("/home/eric/experiments/dumapper/neat-1_%s.png", false, 4, neatData);*/ /*double[][][] gomeaData = getGomeaData("/home/eric/experiments/dumapper/gomea-1/LT_nguyen7/30393", "population_%d.dat", 100, 127); saveImages("/home/eric/experiments/dumapper/gomea-lt-nguyen7-30393_%s.png", false, 4, gomeaData); System.exit(0);*/ /*double[][][] gsgpData = getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_4/1/", "blocks.txt", 101, 100, 100); saveImages("/home/eric/experiments/dumapper/gsgp-same-t4-nguyen7-1_%s.png", false, 2, gsgpData);*/ /* //many gomea runs String baseDir = "/home/eric/experiments/dumapper/gomea-1/LT_nguyen7"; File dir = new File(baseDir); int i = 0; int runs = 10; double[][][][] datas = new double[runs][][][]; for (File runDir : dir.listFiles()) { if (runDir.isDirectory()) { System.out.println(runDir); datas[i] = getGomeaData(runDir.toString(), "population_%d.dat", 100, 127); saveImages(String.format("/home/eric/experiments/dumapper/gomea-lt-nguyen7-%d_%%s.png", i), false, 4, datas[i]); i = i+1; if (i>=runs) { break; } } } saveImages("/home/eric/experiments/dumapper/gomea-lt-nguyen7-10runs_%s.png", false, 4, merge(datas));*/ /*//many gsgp runs String baseDir = "/home/eric/experiments/dumapper/gsgp/nguyen7/Different_Init_Pop/Tournament_size_4"; File dir = new File(baseDir); int i = 0; int runs = 10; double[][][][] datas = new double[runs][][][]; for (File runDir : dir.listFiles()) { if (runDir.isDirectory()) { System.out.println(runDir); datas[i] = getGsgpData(runDir.toString(), "blocks.txt", 101, 100, 100); saveImages(String.format("/home/eric/experiments/dumapper/gsgp-diff-t4-nguyen7-%d_%%s.png", i), false, 4, datas[i]); i = i+1; if (i>=runs) { break; } } } saveImages("/home/eric/experiments/dumapper/gsgp-diff-t4-nguyen7-10runs_%s.png", false, 4, merge(datas));*/ //many runs whge double[][][][] datas = new double[10][][][]; for (int i = 0; i < 10; i++) { datas[i] = buildGEData("whge", 100, 256, new Nguyen7(0), i, 5); saveImages(String.format("/home/eric/experiments/dumapper/whge-nguyen7-%d_%%s.png", i), false, 1, datas[i]); } saveImages("/home/eric/experiments/dumapper/whge-nguyen7-10runs_%s.png", false, 1, merge(datas)); /*//gsgp diff tournament size saveImages("/home/eric/experiments/dumapper/gsgp-same-random-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/random_selection/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-same-t2-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_2/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-same-t4-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_4/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-same-t6-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_6/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-same-t8-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_8/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-same-t10-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_10/1/", "blocks.txt", 51, 100, 100));*/ /*//different whge problems saveImages("/home/eric/experiments/dumapper/whge-nguyen7-1_%s.png", false, 2, buildGEData("whge", 100, 256, new Nguyen7(0), 1, 5)); saveImages("/home/eric/experiments/dumapper/whge-pagie1-1_%s.png", false, 2, buildGEData("whge", 100, 256, new Pagie1(), 1, 5)); saveImages("/home/eric/experiments/dumapper/whge-text-1_%s.png", false, 2, buildGEData("whge", 100, 256, new Text(), 1, 5)); saveImages("/home/eric/experiments/dumapper/whge-kland4-1_%s.png", false, 2, buildGEData("whge", 100, 256, new KLandscapes(4), 1, 5)); //saveImages("/home/eric/experiments/dumapper/whge200-kland8-1_%s.png", false, 2, buildGEData("whge", 50, 200, new KLandscapes(8), 1, 5)); saveImages("/home/eric/experiments/dumapper/whge-mopm2-1_%s.png", false, 2, buildGEData("whge", 100, 256, new MultipleOutputParallelMultiplier(2), 1, 5)); //saveImages("/home/eric/experiments/dumapper/whge200-mopm4-1_%s.png", false, 2, buildGEData("whge", 50, 200, new MultipleOutputParallelMultiplier(4), 1, 5));*/ //diff selective pressure whge for (int i = 2; i <= 10; i = i + 2) { saveImages(String.format("/home/eric/experiments/dumapper/whge-kland5-t%d-_%%s.png", i), false, 1, buildGEData("whge", 100, 256, new KLandscapes(5), 1, i)); } /*saveImages("/home/eric/experiments/dumapper/gsgp-nguyen7-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/nguyen7/Same_Init_Pop/Tournament_size_4/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-airfoil-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/airfoil/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-concrete-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/concrete/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-slump-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/slump/1/", "blocks.txt", 51, 100, 100)); saveImages("/home/eric/experiments/dumapper/gsgp-yacht-1_%s.png", false, 2, getGsgpData("/home/eric/experiments/dumapper/gsgp/yacht/1/", "blocks.txt", 51, 100, 100));*/ //saveImages("/home/eric/experiments/dumapper/neat-1_%s.png", false, 4, getNeatData2("/home/eric/experiments/dumapper/neat/1-150T_151-400T+C", "pop-%s.eg", 300)); }
From source file:com.act.analysis.surfactant.AnalysisDriver.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());/*from w w w.j a v a 2 s . co m*/ } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } Set<String> seenOutputIds = new HashSet<>(); TSVWriter<String, String> tsvWriter = null; if (cl.hasOption(OPTION_OUTPUT_FILE)) { File outputFile = new File(cl.getOptionValue(OPTION_OUTPUT_FILE)); List<Map<String, String>> oldResults = null; if (outputFile.exists()) { System.err.format( "Output file already exists, reading old results and skipping processed molecules.\n"); TSVParser outputParser = new TSVParser(); outputParser.parse(outputFile); oldResults = outputParser.getResults(); for (Map<String, String> row : oldResults) { // TODO: verify that the last row was written cleanly/completely. seenOutputIds.add(row.get("id")); } } List<String> header = new ArrayList<>(); header.add("name"); header.add("id"); header.add("inchi"); header.add("label"); for (SurfactantAnalysis.FEATURES f : SurfactantAnalysis.FEATURES.values()) { header.add(f.toString()); } // TODO: make this API more auto-closable friendly. tsvWriter = new TSVWriter<>(header); tsvWriter.open(outputFile); if (oldResults != null) { System.out.format("Re-writing %d existing result rows\n", oldResults.size()); tsvWriter.append(oldResults); } } try { Map<SurfactantAnalysis.FEATURES, Double> analysisFeatures; LicenseManager.setLicenseFile(cl.getOptionValue(OPTION_LICENSE_FILE)); if (cl.hasOption(OPTION_INCHI)) { analysisFeatures = SurfactantAnalysis.performAnalysis(cl.getOptionValue(OPTION_INCHI), cl.hasOption(OPTION_DISPLAY)); Map<String, String> tsvFeatures = new HashMap<>(); // Convert features to strings to avoid some weird formatting issues. It's ugly, but it works. for (Map.Entry<SurfactantAnalysis.FEATURES, Double> entry : analysisFeatures.entrySet()) { tsvFeatures.put(entry.getKey().toString(), String.format("%.6f", entry.getValue())); } tsvFeatures.put("name", "direct-inchi-input"); if (tsvWriter != null) { tsvWriter.append(tsvFeatures); } } else if (cl.hasOption(OPTION_INPUT_FILE)) { TSVParser parser = new TSVParser(); parser.parse(new File(cl.getOptionValue(OPTION_INPUT_FILE))); int i = 0; List<Map<String, String>> inputRows = parser.getResults(); for (Map<String, String> row : inputRows) { i++; // Just for warning messages. if (!row.containsKey("name") || !row.containsKey("id") || !row.containsKey("inchi")) { System.err.format( "WARNING: TSV rows must contain at least name, id, and inchi, skipping row %d\n", i); continue; } if (seenOutputIds.contains(row.get("id"))) { System.out.format("Skipping input row with id already in output: %s\n", row.get("id")); continue; } System.out.format("Analysis for chemical %s\n", row.get("name")); try { analysisFeatures = SurfactantAnalysis.performAnalysis(row.get("inchi"), false); } catch (Exception e) { // Ignore exceptions for now. Sometimes the regression analysis or Chemaxon processing chokes unexpectedly. System.err.format("ERROR caught exception while processing '%s':\n", row.get("name")); System.err.format("%s\n", e.getMessage()); e.printStackTrace(System.err); System.err.println("Skipping..."); continue; } System.out.format("--- Done analysis for chemical %s\n", row.get("name")); // This is a duplicate of the OPTION_INCHI block code, but it's inside of a tight loop, so... Map<String, String> tsvFeatures = new HashMap<>(); for (Map.Entry<SurfactantAnalysis.FEATURES, Double> entry : analysisFeatures.entrySet()) { tsvFeatures.put(entry.getKey().toString(), String.format("%.6f", entry.getValue())); } tsvFeatures.put("name", row.get("name")); tsvFeatures.put("id", row.get("id")); tsvFeatures.put("inchi", row.get("inchi")); tsvFeatures.put("label", row.containsKey("label") ? row.get("label") : "?"); if (tsvWriter != null) { tsvWriter.append(tsvFeatures); // Flush every time in case we crash or get interrupted. The features must flow! tsvWriter.flush(); } } } else { throw new RuntimeException("Must specify inchi or input file"); } } finally { if (tsvWriter != null) { tsvWriter.close(); } } }
From source file:io.fluo.stress.trie.NumberIngest.java
public static void main(String[] args) throws IOException, ConfigurationException { // Parse arguments if (args.length != 4) { log.error("Usage: NumberIngest <numMappers> <numbersPerMapper> <nodeSize> <fluoProps>"); System.exit(-1);/*from w w w. j a v a2s . com*/ } int numMappers = Integer.parseInt(args[0]); int numPerMapper = Integer.parseInt(args[1]); int nodeSize = Integer.parseInt(args[2]); String fluoPropsPath = args[3]; String hadoopPrefix = System.getenv("HADOOP_PREFIX"); if (hadoopPrefix == null) { hadoopPrefix = System.getenv("HADOOP_HOME"); if (hadoopPrefix == null) { log.error("HADOOP_PREFIX or HADOOP_HOME needs to be set!"); System.exit(-1); } } // create test name String testId = String.format("test-%d", (new Date().getTime() / 1000)); String testDir = "/trie-stress/" + testId; setupHdfs(hadoopPrefix, testDir, numMappers, numPerMapper); JobConf ingestConf = new JobConf(NumberIngest.class); ingestConf.setJobName("NumberIngest"); FluoConfiguration config = new FluoConfiguration(new File(fluoPropsPath)); loadConfig(ingestConf, ConfigurationConverter.getProperties(config)); ingestConf.setInt(TRIE_NODE_SIZE_PROP, nodeSize); ingestConf.setOutputKeyClass(LongWritable.class); ingestConf.setOutputValueClass(IntWritable.class); ingestConf.setMapperClass(NumberIngest.IngestMapper.class); ingestConf.setReducerClass(NumberIngest.UniqueReducer.class); FileInputFormat.setInputPaths(ingestConf, new Path(testDir + "/input/")); FileOutputFormat.setOutputPath(ingestConf, new Path(testDir + "/unique/")); RunningJob ingestJob = JobClient.runJob(ingestConf); ingestJob.waitForCompletion(); if (ingestJob.isSuccessful()) { JobConf countConf = new JobConf(NumberIngest.class); countConf.setJobName("NumberCount"); countConf.setOutputKeyClass(Text.class); countConf.setOutputValueClass(LongWritable.class); countConf.setMapperClass(NumberIngest.CountMapper.class); countConf.setReducerClass(NumberIngest.CountReducer.class); FileInputFormat.setInputPaths(countConf, new Path(testDir + "/unique/")); FileOutputFormat.setOutputPath(countConf, new Path(testDir + "/output/")); RunningJob countJob = JobClient.runJob(countConf); countJob.waitForCompletion(); if (countJob.isSuccessful()) { log.info("Ingest and count jobs were successful"); log.info("Output can be viewed @ " + testDir); System.exit(0); } else { log.error("Count job failed for " + testId); } } else { log.error("Ingest job failed. Skipping count job for " + testId); } System.exit(-1); }
From source file:general.Main.java
/** * Selects the files to be processed and specifies the files to write to. * * @param args Arguments to specify runtime behavior. *//* ww w. j a va 2 s. co m*/ public static void main(String[] args) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { Options options = new Options(); options.addOption("l", "logging", false, "enables file logging"); options.addOption("j", "jena", false, "uses the Jena SPARQL Parser"); options.addOption("o", "openrdf", false, "uses the OpenRDF SPARQL Parser"); options.addOption("f", "file", true, "defines the input file prefix"); options.addOption("h", "help", false, "displays this help"); options.addOption("t", "tsv", false, "reads from .tsv-files"); // options.addOption("p", "parquet", false, "read from .parquet-files"); options.addOption("n", "numberOfThreads", true, "number of used threads, default 1"); options.addOption("b", "withBots", false, "enables metric calculation for bot queries+"); options.addOption("p", "readPreprocessed", false, "enables reading of preprocessed files"); //some parameters which can be changed through parameters //QueryHandler queryHandler = new OpenRDFQueryHandler(); String inputFilePrefix; String inputFileSuffix = ".tsv"; String queryParserName = "OpenRDF"; Class inputHandlerClass = null; Class queryHandlerClass = null; int numberOfThreads = 1; CommandLineParser parser = new DefaultParser(); CommandLine cmd; try { cmd = parser.parse(options, args); if (cmd.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("help", options); return; } if (cmd.hasOption("openrdf")) { queryHandlerClass = OpenRDFQueryHandler.class; } if (cmd.hasOption("tsv")) { inputFileSuffix = ".tsv"; inputHandlerClass = InputHandlerTSV.class; } if (cmd.hasOption("parquet")) { inputFileSuffix = ".parquet"; Logger.getLogger("org").setLevel(Level.WARN); Logger.getLogger("akka").setLevel(Level.WARN); SparkConf conf = new SparkConf().setAppName("SPARQLQueryAnalyzer").setMaster("local"); JavaSparkContext sc = new JavaSparkContext(conf); inputHandlerClass = InputHandlerParquet.class; } if (inputHandlerClass == null) { System.out.println("Please specify which parser to use, either -t for TSV or -p for parquet."); } if (cmd.hasOption("file")) { inputFilePrefix = cmd.getOptionValue("file").trim(); } else { System.out.println( "Please specify at least the file which we should work on using the option '--file PREFIX' or 'f PREFIX'"); return; } if (cmd.hasOption("logging")) { LoggingHandler.initFileLog(queryParserName, inputFilePrefix); } if (cmd.hasOption("numberOfThreads")) { numberOfThreads = Integer.parseInt(cmd.getOptionValue("numberOfThreads")); } if (cmd.hasOption("withBots")) { withBots = true; } if (cmd.hasOption("readPreprocessed")) { readPreprocessed = true; } } catch (UnrecognizedOptionException e) { System.out.println("Unrecognized commandline option: " + e.getOption()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("help", options); return; } catch (ParseException e) { System.out.println( "There was an error while parsing your command line input. Did you rechecked your syntax before running?"); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("help", options); return; } LoggingHandler.initConsoleLog(); loadPreBuildQueryTypes(); long startTime = System.nanoTime(); ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads); for (int day = 1; day <= 31; day++) { String inputFile = inputFilePrefix + String.format("%02d", day) + inputFileSuffix; Runnable parseOneMonthWorker = new ParseOneMonthWorker(inputFile, inputFilePrefix, inputHandlerClass, queryParserName, queryHandlerClass, day); executor.execute(parseOneMonthWorker); } executor.shutdown(); while (!executor.isTerminated()) { //wait until all workers are finished } writeQueryTypes(inputFilePrefix); long stopTime = System.nanoTime(); long millis = TimeUnit.MILLISECONDS.convert(stopTime - startTime, TimeUnit.NANOSECONDS); Date date = new Date(millis); System.out.println("Finished executing with all threads: " + new SimpleDateFormat("mm-dd HH:mm:ss:SSSSSSS").format(date)); }
From source file:apps.Source2XML.java
public static void main(String[] args) { Options options = new Options(); options.addOption("i", null, true, "input file"); options.addOption("o", null, true, "output file"); options.addOption("reparse_xml", null, false, "reparse each XML entry to ensure the parser doesn't fail"); Joiner commaJoin = Joiner.on(','); options.addOption("source_type", null, true, "document source type: " + commaJoin.join(SourceFactory.getDocSourceList())); Joiner spaceJoin = Joiner.on(' '); CommandLineParser parser = new org.apache.commons.cli.GnuParser(); BufferedWriter outputFile = null; int docNum = 0; if (USE_LEMMATIZER && USE_STEMMER) { System.err.println("Bug/inconsistent code: cann't use the stemmer and lemmatizer at the same time!"); System.exit(1);// www . jav a 2s . c om } //Stemmer stemmer = new Stemmer(); KrovetzStemmer stemmer = new KrovetzStemmer(); System.out.println("Using Stanford NLP? " + USE_STANFORD); System.out.println("Using Stanford lemmatizer? " + USE_LEMMATIZER); System.out.println("Using stemmer? " + USE_STEMMER + (USE_STEMMER ? " (class: " + stemmer.getClass().getCanonicalName() + ")" : "")); try { CommandLine cmd = parser.parse(options, args); String inputFileName = null, outputFileName = null; if (cmd.hasOption("i")) { inputFileName = cmd.getOptionValue("i"); } else { Usage("Specify 'input file'", options); } if (cmd.hasOption("o")) { outputFileName = cmd.getOptionValue("o"); } else { Usage("Specify 'output file'", options); } outputFile = new BufferedWriter( new OutputStreamWriter(CompressUtils.createOutputStream(outputFileName))); String sourceName = cmd.getOptionValue("source_type"); if (sourceName == null) Usage("Specify document source type", options); boolean reparseXML = options.hasOption("reparse_xml"); DocumentSource inpDocSource = SourceFactory.createDocumentSource(sourceName, inputFileName); DocumentEntry inpDoc = null; TextCleaner textCleaner = new TextCleaner( new DictNoComments(new File("data/stopwords.txt"), true /* lower case */), USE_STANFORD, USE_LEMMATIZER); Map<String, String> outputMap = new HashMap<String, String>(); outputMap.put(UtilConst.XML_FIELD_DOCNO, null); outputMap.put(UtilConst.XML_FIELD_TEXT, null); XmlHelper xmlHlp = new XmlHelper(); if (reparseXML) System.out.println("Will reparse every XML entry to verify correctness!"); while ((inpDoc = inpDocSource.next()) != null) { ++docNum; ArrayList<String> toks = textCleaner.cleanUp(inpDoc.mDocText); ArrayList<String> goodToks = new ArrayList<String>(); for (String s : toks) if (s.length() <= MAX_WORD_LEN && // Exclude long and short words s.length() >= MIN_WORD_LEN && isGoodWord(s)) goodToks.add(USE_STEMMER ? stemmer.stem(s) : s); String partlyCleanedText = spaceJoin.join(goodToks); String cleanText = XmlHelper.removeInvaildXMLChars(partlyCleanedText); // isGoodWord combiend with Stanford tokenizer should be quite restrictive already //cleanText = replaceSomePunct(cleanText); outputMap.replace(UtilConst.XML_FIELD_DOCNO, inpDoc.mDocId); outputMap.replace(UtilConst.XML_FIELD_TEXT, cleanText); String xml = xmlHlp.genXMLIndexEntry(outputMap); if (reparseXML) { try { XmlHelper.parseDocWithoutXMLDecl(xml); } catch (Exception e) { System.err.println("Error re-parsing xml for document ID: " + inpDoc.mDocId); System.exit(1); } } /* { System.out.println(inpDoc.mDocId); System.out.println("====================="); System.out.println(partlyCleanedText); System.out.println("====================="); System.out.println(cleanText); } */ try { outputFile.write(xml); outputFile.write(NL); } catch (Exception e) { e.printStackTrace(); System.err.println("Error processing/saving a document!"); } if (docNum % 1000 == 0) System.out.println(String.format("Processed %d documents", docNum)); } } catch (ParseException e) { e.printStackTrace(); Usage("Cannot parse arguments" + e, options); } catch (Exception e) { System.err.println("Terminating due to an exception: " + e); System.exit(1); } finally { System.out.println(String.format("Processed %d documents", docNum)); try { if (null != outputFile) { outputFile.close(); System.out.println("Output file is closed! all seems to be fine..."); } } catch (IOException e) { System.err.println("IO exception: " + e); e.printStackTrace(); } } }
From source file:com.incapture.rapgen.GenApi.java
public static void main(String[] args) { Options options = new Options(); options.addOption("l", true, "Language to generate"); options.addOption("o", true, "Output root folder for kernel files"); options.addOption("a", true, "Output root folder for api files"); options.addOption("w", true, "Output root folder for web files"); options.addOption("d", true, "Template dir to use (use either this or 't')"); options.addOption("t", true, "Template file to use (use either this or 'd')"); options.addOption("g", true, "The type of grammar to generate, current options are 'SDK' or 'API'"); options.addOption("mainApiFile", true, "FileName specifying the api"); options.addOption("codeSamplesJava", true, "A path to search for files that have Java code samples"); options.addOption("codeSamplesPython", true, "A path to search for files that have Python code samples"); CommandLineParser cparser = new PosixParser(); try {/* w w w .j a va 2 s . c o m*/ CommandLine cmd = cparser.parse(options, args); String mainApiFile = cmd.getOptionValue("mainApiFile"); String outputKernelFolder = cmd.getOptionValue('o'); String outputApiFolder = cmd.getOptionValue('a'); String outputWebFolder = cmd.getOptionValue('w'); String codeSamplesJava = cmd.getOptionValue("codeSamplesJava"); String codeSamplesPython = cmd.getOptionValue("codeSamplesPython"); GenType genType = GenType.valueOf(cmd.getOptionValue('g')); // The language will ultimately choose the walker class String language = cmd.getOptionValue('l'); if (cmd.hasOption('d') && cmd.hasOption('t')) { throw new IllegalArgumentException( "Cannot define both a template folder ('d') and file ('t'). Please use one OR the other."); } // And off we go TLexer lexer = new TLexer(); ResourceBasedApiReader apiReader = new ResourceBasedApiReader(); lexer.setApiReader(apiReader); lexer.setCharStream(apiReader.read(mainApiFile)); // Using the lexer as the token source, we create a token // stream to be consumed by the parser // CommonTokenStream tokens = new CommonTokenStream(lexer); // Now we need an instance of our parser // TParser parser = new TParser(tokens); hmxdef_return psrReturn = parser.hmxdef(); // load in T.stg template group, put in templates variable StringTemplateGroup templates = null; if (!isSlateMd(language)) { templates = TemplateRepo.getTemplates(language, genType); } Tree t = psrReturn.getTree(); CommonTreeNodeStream ns = new CommonTreeNodeStream(t); ns.setTokenStream(tokens); if (templates != null) { templates.registerRenderer(String.class, new UpCaseRenderer()); } AbstractTTree walker = TreeFactory.createTreeWalker(ns, templates, language); System.out.println("Generating files with a " + walker.getClass().getName()); if (walker instanceof TTree) { if (genType.equals(GenType.API)) { ((TTree) walker).apiGen(); } else { ((TTree) walker).sdkGen(); } } else if (walker instanceof TTreeRuby) { System.out.println("Running for Ruby"); /* TTreeRuby.hmxdef_return out = */ ((TTreeRuby) walker).hmxdef(); } else if (walker instanceof TTreeJS) { System.out.println("Running for JavaScript"); /* TTreeJS.hmxdef_return out = */ ((TTreeJS) walker).hmxdef(); } else if (walker instanceof TTreeDoc) { System.out.println("Running for Documentation"); /* TTreeDoc.hmxdef_return out = */ ((TTreeDoc) walker).hmxdef(); } else if (walker instanceof TTreeVB) { System.out.println("Running for VB"); /* TTreeVB.hmxdef_return out = */ ((TTreeVB) walker).hmxdef(); } else if (walker instanceof TTreeGo) { System.out.println("Running for Go"); /* TTreeGo.hmxdef_return out = */ ((TTreeGo) walker).hmxdef(); } else if (walker instanceof TTreeCpp) { System.out.println("Running for Cpp"); /* TTreeGo.hmxdef_return out = */ ((TTreeCpp) walker).hmxdef(); } else if (walker instanceof TTreePython) { System.out.println("Running for Python"); /* TTreePython.hmxdef_return out = */ ((TTreePython) walker).hmxdef(); } else if (walker instanceof TTreeSlateMd) { System.out.println("Running for Slate Markdown"); TTreeSlateMd slateMdWalker = (TTreeSlateMd) walker; slateMdWalker.setupCodeParser(codeSamplesJava, codeSamplesPython); slateMdWalker.hmxdef(); } else if (walker instanceof TTreeDotNet) { System.out.println("Running for DotNet"); ((TTreeDotNet) walker).apiGen(); } else if (walker instanceof TTreeCurtisDoc) { System.out.println("Running for CurtisDoc"); ((TTreeCurtisDoc) walker).apiGen(); } // Now dump the files out System.out.println("Genereated source output locations:"); System.out.println(String.format("kernel: [%s]", outputKernelFolder)); System.out.println(String.format("api: [%s]", outputApiFolder)); System.out.println(String.format("web: [%s]", outputWebFolder)); walker.dumpFiles(outputKernelFolder, outputApiFolder, outputWebFolder); } catch (ParseException e) { System.err.println("Error parsing command line - " + e.getMessage()); System.out.println("Usage: " + options.toString()); } catch (IOException | RecognitionException e) { System.err.println("Error running GenApi: " + ExceptionToString.format(e)); } }
From source file:com.betfair.cougar.test.socket.app.SocketCompatibilityTestingApp.java
public static void main(String[] args) throws Exception { Parser parser = new PosixParser(); Options options = new Options(); options.addOption("r", "repo", true, "Repository type to search: local|central"); options.addOption("c", "client-concurrency", true, "Max threads to allow each client tester to run tests, defaults to 10"); options.addOption("t", "test-concurrency", true, "Max client testers to run concurrently, defaults to 5"); options.addOption("m", "max-time", true, "Max time (in minutes) to allow tests to complete, defaults to 10"); options.addOption("v", "version", false, "Print version and exit"); options.addOption("h", "help", false, "This help text"); CommandLine commandLine = parser.parse(options, args); if (commandLine.hasOption("h")) { System.out.println(options); System.exit(0);//from w w w . j a va2 s . c om } if (commandLine.hasOption("v")) { System.out.println("How the hell should I know?"); System.exit(0); } // 1. Find all testers in given repos List<RepoSearcher> repoSearchers = new ArrayList<>(); for (String repo : commandLine.getOptionValues("r")) { if ("local".equals(repo.toLowerCase())) { repoSearchers.add(new LocalRepoSearcher()); } else if ("central".equals(repo.toLowerCase())) { repoSearchers.add(new CentralRepoSearcher()); } else { System.err.println("Unrecognized repo: " + repo); System.err.println(options); System.exit(1); } } int clientConcurrency = 10; if (commandLine.hasOption("c")) { try { clientConcurrency = Integer.parseInt(commandLine.getOptionValue("c")); } catch (NumberFormatException nfe) { System.err.println( "client-concurrency is not a valid integer: '" + commandLine.getOptionValue("c") + "'"); System.exit(1); } } int testConcurrency = 5; if (commandLine.hasOption("t")) { try { testConcurrency = Integer.parseInt(commandLine.getOptionValue("t")); } catch (NumberFormatException nfe) { System.err.println( "test-concurrency is not a valid integer: '" + commandLine.getOptionValue("t") + "'"); System.exit(1); } } int maxMinutes = 10; if (commandLine.hasOption("m")) { try { maxMinutes = Integer.parseInt(commandLine.getOptionValue("m")); } catch (NumberFormatException nfe) { System.err.println("max-time is not a valid integer: '" + commandLine.getOptionValue("m") + "'"); System.exit(1); } } Properties clientProps = new Properties(); clientProps.setProperty("client.concurrency", String.valueOf(clientConcurrency)); File baseRunDir = new File(System.getProperty("user.dir") + "/run"); baseRunDir.mkdirs(); File tmpDir = new File(baseRunDir, "jars"); tmpDir.mkdirs(); List<ServerRunner> serverRunners = new ArrayList<>(); List<ClientRunner> clientRunners = new ArrayList<>(); for (RepoSearcher searcher : repoSearchers) { List<File> jars = searcher.findAndCache(tmpDir); for (File f : jars) { ServerRunner serverRunner = new ServerRunner(f, baseRunDir); System.out.println("Found tester: " + serverRunner.getVersion()); serverRunners.add(serverRunner); clientRunners.add(new ClientRunner(f, baseRunDir, clientProps)); } } // 2. Start servers and collect ports System.out.println(); System.out.println("Starting " + serverRunners.size() + " servers..."); for (ServerRunner server : serverRunners) { server.startServer(); } System.out.println(); List<TestCombo> tests = new ArrayList<>(serverRunners.size() * clientRunners.size()); for (ServerRunner server : serverRunners) { for (ClientRunner client : clientRunners) { tests.add(new TestCombo(server, client)); } } System.out.println("Enqueued " + tests.size() + " test combos to run..."); long startTime = System.currentTimeMillis(); // 3. Run every client against every server, collecting results BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue(serverRunners.size() * clientRunners.size()); ThreadPoolExecutor service = new ThreadPoolExecutor(testConcurrency, testConcurrency, 5000, TimeUnit.MILLISECONDS, workQueue); service.prestartAllCoreThreads(); workQueue.addAll(tests); while (!workQueue.isEmpty()) { Thread.sleep(1000); } service.shutdown(); service.awaitTermination(maxMinutes, TimeUnit.MINUTES); long endTime = System.currentTimeMillis(); long totalTimeSecs = Math.round((endTime - startTime) / 1000.0); for (ServerRunner server : serverRunners) { server.shutdownServer(); } System.out.println(); System.out.println("======="); System.out.println("Results"); System.out.println("-------"); // print a summary int totalTests = 0; int totalSuccess = 0; for (TestCombo combo : tests) { String clientVer = combo.getClientVersion(); String serverVer = combo.getServerVersion(); String results = combo.getClientResults(); ObjectMapper mapper = new ObjectMapper(new JsonFactory()); JsonNode node = mapper.reader().readTree(results); JsonNode resultsArray = node.get("results"); int numTests = resultsArray.size(); int numSuccess = 0; for (int i = 0; i < numTests; i++) { if ("success".equals(resultsArray.get(i).get("result").asText())) { numSuccess++; } } totalSuccess += numSuccess; totalTests += numTests; System.out.println(clientVer + "/" + serverVer + ": " + numSuccess + "/" + numTests + " succeeded - took " + String.format("%2f", combo.getRunningTime()) + " seconds"); } System.out.println("-------"); System.out.println( "Overall: " + totalSuccess + "/" + totalTests + " succeeded - took " + totalTimeSecs + " seconds"); FileWriter out = new FileWriter("results.json"); PrintWriter pw = new PrintWriter(out); // 4. Output full results pw.println("{\n \"results\": ["); for (TestCombo combo : tests) { combo.emitResults(pw, " "); } pw.println(" ],"); pw.println(" \"servers\": ["); for (ServerRunner server : serverRunners) { server.emitInfo(pw, " "); } pw.println(" ],"); pw.close(); }