List of usage examples for java.lang String charAt
public char charAt(int index)
From source file:edu.nyu.tandon.tool.BinnedRawHits.java
@SuppressWarnings("unchecked") public static void main(final String[] arg) throws Exception { SimpleJSAP jsap = new SimpleJSAP(BinnedRawHits.class.getName(), "Loads indices relative to a collection, possibly loads the collection, and answers to queries.", new Parameter[] { new FlaggedOption("collection", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'c', "collection", "The collection of documents indexed by the given indices."), new FlaggedOption("objectCollection", new ObjectParser(DocumentCollection.class, MG4JClassParser.PACKAGE), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'o', "object-collection", "An object specification describing a document collection."), new FlaggedOption("titleList", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 't', "title-list", "A serialized big list of titles (will override collection titles if specified)."), new FlaggedOption("titleFile", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'T', "title-file", "A file of newline-separated, UTF-8 titles (will override collection titles if specified)."), new FlaggedOption("input", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'I', "input", "A file containing the input."), new Switch("noSizes", 'n', "no-sizes", "Disable loading document sizes (they are necessary for BM25 scoring)."), new Switch("http", 'h', "http", "Starts an HTTP query server."), new Switch("verbose", 'v', "verbose", "Print full exception stack traces."), new FlaggedOption("itemClass", MG4JClassParser.getParser(), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'i', "item-class", "The class that will handle item display in the HTTP server."), new FlaggedOption("itemMimeType", JSAP.STRING_PARSER, "text/html", JSAP.NOT_REQUIRED, 'm', "item-mime-type", "A MIME type suggested to the class handling item display in the HTTP server."), new FlaggedOption("port", JSAP.INTEGER_PARSER, "4242", JSAP.NOT_REQUIRED, 'p', "port", "The port on localhost where the server will appear."), new UnflaggedOption("basenameWeight", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.GREEDY, "The indices that the servlet will use. Indices are specified using their basename, optionally followed by a colon and a double representing the weight used to score results from that index. Indices without a specified weight are weighted 1."), new Switch("noMplex", 'P', "noMplex", "Starts with multiplex disabled."), new FlaggedOption("results", JSAP.INTEGER_PARSER, "1000", JSAP.NOT_REQUIRED, 'r', "results", "The # of results to display"), new FlaggedOption("mode", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'M', "time", "The results display mode"), new FlaggedOption("divert", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'd', "divert", "output file"), new FlaggedOption("dumpsize", JSAP.INTEGER_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'D', "dumpsize", "number of queries before dumping") });//w ww .ja v a 2 s. co m final JSAPResult jsapResult = jsap.parse(arg); if (jsap.messagePrinted()) return; final DocumentCollection documentCollection = (DocumentCollection) (jsapResult.userSpecified("collection") ? AbstractDocumentSequence.load(jsapResult.getString("collection")) : jsapResult.userSpecified("objectCollection") ? jsapResult.getObject("objectCollection") : null); final BigList<? extends CharSequence> titleList = (BigList<? extends CharSequence>) (jsapResult .userSpecified("titleList") ? BinIO.loadObject(jsapResult.getString("titleList")) : jsapResult.userSpecified("titleFile") ? new FileLinesBigList(jsapResult.getString("titleFile"), "UTF-8") : null); final String[] basenameWeight = jsapResult.getStringArray("basenameWeight"); final Object2ReferenceLinkedOpenHashMap<String, Index> indexMap = new Object2ReferenceLinkedOpenHashMap<String, Index>( Hash.DEFAULT_INITIAL_SIZE, .5f); final Reference2DoubleOpenHashMap<Index> index2Weight = new Reference2DoubleOpenHashMap<Index>(); final boolean verbose = jsapResult.getBoolean("verbose"); final boolean loadSizes = !jsapResult.getBoolean("noSizes"); BinnedRawHits.loadIndicesFromSpec(basenameWeight, loadSizes, documentCollection, indexMap, index2Weight); final long numberOfDocuments = indexMap.values().iterator().next().numberOfDocuments; if (titleList != null && titleList.size64() != numberOfDocuments) throw new IllegalArgumentException("The number of titles (" + titleList.size64() + " and the number of documents (" + numberOfDocuments + ") do not match"); final Object2ObjectOpenHashMap<String, TermProcessor> termProcessors = new Object2ObjectOpenHashMap<String, TermProcessor>( indexMap.size()); for (String alias : indexMap.keySet()) termProcessors.put(alias, indexMap.get(alias).termProcessor); final SimpleParser simpleParser = new SimpleParser(indexMap.keySet(), indexMap.firstKey(), termProcessors); final Reference2ReferenceMap<Index, Object> index2Parser = new Reference2ReferenceOpenHashMap<Index, Object>(); /* // Fetch parsers for payload-based fields. for( Index index: indexMap.values() ) if ( index.hasPayloads ) { if ( index.payload.getClass() == DatePayload.class ) index2Parser.put( index, DateFormat.getDateInstance( DateFormat.SHORT, Locale.UK ) ); } */ final HitsQueryEngine queryEngine = new HitsQueryEngine(simpleParser, new DocumentIteratorBuilderVisitor( indexMap, index2Parser, indexMap.get(indexMap.firstKey()), MAX_STEMMING), indexMap); queryEngine.setWeights(index2Weight); queryEngine.score(new Scorer[] { new BM25Scorer(), new VignaScorer() }, new double[] { 1, 1 }); queryEngine.multiplex = !jsapResult.userSpecified("moPlex") || jsapResult.getBoolean("noMplex"); queryEngine.intervalSelector = null; queryEngine.equalize(1000); BinnedRawHits query = new BinnedRawHits(queryEngine); // start docHits with at least 10K results query.interpretCommand("$score BM25Scorer"); query.interpretCommand("$mode time"); if (jsapResult.userSpecified("divert")) query.interpretCommand("$divert " + jsapResult.getObject("divert")); query.displayMode = OutputType.DOCHHITS; query.maxOutput = jsapResult.getInt("results", 10000); String q; int n = 0; int dumpsize = jsapResult.userSpecified("dumpsize") ? jsapResult.getInt("dumpsize", 10000) : 10000; buildBins(query.maxOutput, (int) numberOfDocuments); String lastQ = ""; try { final BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(jsapResult.getString("input")))); final ObjectArrayList<DocumentScoreInfo<ObjectArrayList<Byte>>> results = new ObjectArrayList<DocumentScoreInfo<ObjectArrayList<Byte>>>(); for (;;) { q = br.readLine(); if (q == null) { System.err.println(); break; // CTRL-D } if (q.length() == 0) continue; if (q.charAt(0) == '$') { if (!query.interpretCommand(q)) break; continue; } queryCount++; long time = -System.nanoTime(); if (q.compareTo(lastQ) != 0) { try { n = queryEngine.process(q, 0, query.maxOutput, results); } catch (QueryParserException e) { if (verbose) e.getCause().printStackTrace(System.err); else System.err.println(e.getCause()); continue; } catch (Exception e) { if (verbose) e.printStackTrace(System.err); else System.err.println(e); continue; } lastQ = q; } time += System.nanoTime(); query.output(results, documentCollection, titleList, TextMarker.TEXT_BOLDFACE); // dump batch if (queryCount % dumpsize == 0) { dumpBatch(query, numberOfDocuments, false); } // check postHits if (query.postHits.size() > 100000000) dumpPosthits(query, numberOfDocuments, false); } } finally { dumpBatch(query, numberOfDocuments, true); dumpPosthits(query, numberOfDocuments, true); if (query.outputDH != System.out) query.outputDH.close(); } }
From source file:edu.nyu.tandon.tool.RawDocHits_deprecated.java
@SuppressWarnings("unchecked") public static void main(final String[] arg) throws Exception { SimpleJSAP jsap = new SimpleJSAP(RawDocHits_deprecated.class.getName(), "Loads indices relative to a collection, possibly loads the collection, and answers to queries.", new Parameter[] { new FlaggedOption("collection", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'c', "collection", "The collection of documents indexed by the given indices."), new FlaggedOption("objectCollection", new ObjectParser(DocumentCollection.class, MG4JClassParser.PACKAGE), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'o', "object-collection", "An object specification describing a document collection."), new FlaggedOption("titleList", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 't', "title-list", "A serialized big list of titles (will override collection titles if specified)."), new FlaggedOption("titleFile", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'T', "title-file", "A file of newline-separated, UTF-8 titles (will override collection titles if specified)."), new FlaggedOption("input", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'I', "input", "A file containing the input."), new Switch("noSizes", 'n', "no-sizes", "Disable loading document sizes (they are necessary for BM25 scoring)."), new Switch("http", 'h', "http", "Starts an HTTP query server."), new Switch("verbose", 'v', "verbose", "Print full exception stack traces."), new FlaggedOption("itemClass", MG4JClassParser.getParser(), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'i', "item-class", "The class that will handle item display in the HTTP server."), new FlaggedOption("itemMimeType", JSAP.STRING_PARSER, "text/html", JSAP.NOT_REQUIRED, 'm', "item-mime-type", "A MIME type suggested to the class handling item display in the HTTP server."), new FlaggedOption("port", JSAP.INTEGER_PARSER, "4242", JSAP.NOT_REQUIRED, 'p', "port", "The port on localhost where the server will appear."), new UnflaggedOption("basenameWeight", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.GREEDY, "The indices that the servlet will use. Indices are specified using their basename, optionally followed by a colon and a double representing the weight used to score results from that index. Indices without a specified weight are weighted 1."), new Switch("noMplex", 'P', "noMplex", "Starts with multiplex disabled."), new FlaggedOption("results", JSAP.INTEGER_PARSER, "1000", JSAP.NOT_REQUIRED, 'r', "results", "The # of results to display"), new FlaggedOption("mode", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'M', "time", "The results display mode"), new FlaggedOption("divert", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'd', "divert", "output file"), new FlaggedOption("dumpsize", JSAP.INTEGER_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'D', "dumpsize", "number of queries before dumping") });//from w w w . j a v a2 s .c o m final JSAPResult jsapResult = jsap.parse(arg); if (jsap.messagePrinted()) return; final DocumentCollection documentCollection = (DocumentCollection) (jsapResult.userSpecified("collection") ? AbstractDocumentSequence.load(jsapResult.getString("collection")) : jsapResult.userSpecified("objectCollection") ? jsapResult.getObject("objectCollection") : null); final BigList<? extends CharSequence> titleList = (BigList<? extends CharSequence>) (jsapResult .userSpecified("titleList") ? BinIO.loadObject(jsapResult.getString("titleList")) : jsapResult.userSpecified("titleFile") ? new FileLinesBigList(jsapResult.getString("titleFile"), "UTF-8") : null); final String[] basenameWeight = jsapResult.getStringArray("basenameWeight"); final Object2ReferenceLinkedOpenHashMap<String, Index> indexMap = new Object2ReferenceLinkedOpenHashMap<String, Index>( Hash.DEFAULT_INITIAL_SIZE, .5f); final Reference2DoubleOpenHashMap<Index> index2Weight = new Reference2DoubleOpenHashMap<Index>(); final boolean verbose = jsapResult.getBoolean("verbose"); final boolean loadSizes = !jsapResult.getBoolean("noSizes"); RawDocHits_deprecated.loadIndicesFromSpec(basenameWeight, loadSizes, documentCollection, indexMap, index2Weight); final long numberOfDocuments = indexMap.values().iterator().next().numberOfDocuments; if (titleList != null && titleList.size64() != numberOfDocuments) throw new IllegalArgumentException("The number of titles (" + titleList.size64() + " and the number of documents (" + numberOfDocuments + ") do not match"); final Object2ObjectOpenHashMap<String, TermProcessor> termProcessors = new Object2ObjectOpenHashMap<String, TermProcessor>( indexMap.size()); for (String alias : indexMap.keySet()) termProcessors.put(alias, indexMap.get(alias).termProcessor); final SimpleParser simpleParser = new SimpleParser(indexMap.keySet(), indexMap.firstKey(), termProcessors); final Reference2ReferenceMap<Index, Object> index2Parser = new Reference2ReferenceOpenHashMap<Index, Object>(); /* // Fetch parsers for payload-based fields. for( Index index: indexMap.values() ) if ( index.hasPayloads ) { if ( index.payload.getClass() == DatePayload.class ) index2Parser.put( index, DateFormat.getDateInstance( DateFormat.SHORT, Locale.UK ) ); } */ final QueryEngine queryEngine = new QueryEngine(simpleParser, new DocumentIteratorBuilderVisitor(indexMap, index2Parser, indexMap.get(indexMap.firstKey()), MAX_STEMMING), indexMap); queryEngine.setWeights(index2Weight); queryEngine.score(new Scorer[] { new BM25Scorer(), new VignaScorer() }, new double[] { 1, 1 }); // We set up an interval selector only if there is a collection for snippeting queryEngine.intervalSelector = documentCollection != null ? new IntervalSelector(4, 40) : new IntervalSelector(); queryEngine.multiplex = !jsapResult.userSpecified("moPlex") || jsapResult.getBoolean("noMplex"); queryEngine.equalize(1000); RawDocHits_deprecated query = new RawDocHits_deprecated(queryEngine); // start docHits with at least 10K results query.interpretCommand("$score BM25Scorer"); query.interpretCommand("$mode time"); query.interpretCommand("$select"); if (jsapResult.userSpecified("divert")) query.interpretCommand("$divert " + jsapResult.getObject("divert")); query.displayMode = OutputType.DOCHHITS; query.maxOutput = jsapResult.getInt("results", 10000); String q; int n = 0; int dumpsize = jsapResult.userSpecified("dumpsize") ? jsapResult.getInt("dumpsize", 10000) : 1000; buildBins(query.maxOutput, (int) numberOfDocuments); String lastQ = ""; try { final BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(jsapResult.getString("input")))); final ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>> results = new ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>>(); for (;;) { q = br.readLine(); if (q == null) { System.err.println(); break; // CTRL-D } if (q.length() == 0) continue; if (q.charAt(0) == '$') { if (!query.interpretCommand(q)) break; continue; } queryCount++; long time = -System.nanoTime(); if (q.compareTo(lastQ) != 0) { try { n = queryEngine.process(q, 0, query.maxOutput, results); } catch (QueryParserException e) { if (verbose) e.getCause().printStackTrace(System.err); else System.err.println(e.getCause()); continue; } catch (Exception e) { if (verbose) e.printStackTrace(System.err); else System.err.println(e); continue; } lastQ = q; time += System.nanoTime(); query.output(results, documentCollection, titleList, TextMarker.TEXT_BOLDFACE); } else { // repeat last query results time += System.nanoTime(); for (int j = 0; j < results.size(); j++) docHits[lastResults[j]]++; } // dump batch if (queryCount % dumpsize == 0) { dumpBatch(query, numberOfDocuments); } } } finally { dumpBatch(query, numberOfDocuments); if (query.output != System.out) query.output.close(); } }
From source file:com.aerospike.load.AerospikeLoad.java
public static void main(String[] args) throws IOException { Thread statPrinter = new Thread(new PrintStat(counters)); try {// w w w.j ava2 s. c om log.info("Aerospike loader started"); Options options = new Options(); options.addOption("h", "host", true, "Server hostname (default: localhost)"); options.addOption("p", "port", true, "Server port (default: 3000)"); options.addOption("n", "namespace", true, "Namespace (default: test)"); options.addOption("s", "set", true, "Set name. (default: null)"); options.addOption("c", "config", true, "Column definition file name"); options.addOption("wt", "write-threads", true, "Number of writer threads (default: Number of cores * 5)"); options.addOption("rt", "read-threads", true, "Number of reader threads (default: Number of cores * 1)"); options.addOption("l", "rw-throttle", true, "Throttling of reader to writer(default: 10k) "); options.addOption("tt", "transaction-timeout", true, "write transaction timeout in miliseconds(default: No timeout)"); options.addOption("et", "expiration-time", true, "Expiration time of records in seconds (default: never expire)"); options.addOption("T", "timezone", true, "Timezone of source where data dump is taken (default: local timezone)"); options.addOption("ec", "abort-error-count", true, "Error count to abort (default: 0)"); options.addOption("wa", "write-action", true, "Write action if key already exists (default: update)"); options.addOption("v", "verbose", false, "Logging all"); options.addOption("u", "usage", false, "Print usage."); CommandLineParser parser = new PosixParser(); CommandLine cl = parser.parse(options, args, false); if (args.length == 0 || cl.hasOption("u")) { logUsage(options); return; } if (cl.hasOption("l")) { rwThrottle = Integer.parseInt(cl.getOptionValue("l")); } else { rwThrottle = Constants.READLOAD; } // Get all command line options params = Utils.parseParameters(cl); //Get client instance AerospikeClient client = new AerospikeClient(params.host, params.port); if (!client.isConnected()) { log.error("Client is not able to connect:" + params.host + ":" + params.port); return; } if (params.verbose) { log.setLevel(Level.DEBUG); } // Get available processors to calculate default number of threads int cpus = Runtime.getRuntime().availableProcessors(); nWriterThreads = cpus * scaleFactor; nReaderThreads = cpus; // Get writer thread count if (cl.hasOption("wt")) { nWriterThreads = Integer.parseInt(cl.getOptionValue("wt")); nWriterThreads = (nWriterThreads > 0 ? (nWriterThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nWriterThreads) : 1); log.debug("Using writer Threads: " + nWriterThreads); } writerPool = Executors.newFixedThreadPool(nWriterThreads); // Get reader thread count if (cl.hasOption("rt")) { nReaderThreads = Integer.parseInt(cl.getOptionValue("rt")); nReaderThreads = (nReaderThreads > 0 ? (nReaderThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nReaderThreads) : 1); log.debug("Using reader Threads: " + nReaderThreads); } String columnDefinitionFileName = cl.getOptionValue("c", null); log.debug("Column definition files/directory: " + columnDefinitionFileName); if (columnDefinitionFileName == null) { log.error("Column definition files/directory not specified. use -c <file name>"); return; } File columnDefinitionFile = new File(columnDefinitionFileName); if (!columnDefinitionFile.exists()) { log.error("Column definition files/directory does not exist: " + Utils.getFileName(columnDefinitionFileName)); return; } // Get data file list String[] files = cl.getArgs(); if (files.length == 0) { log.error("No data file Specified: add <file/dir name> to end of the command "); return; } List<String> allFileNames = new ArrayList<String>(); allFileNames = Utils.getFileNames(files); if (allFileNames.size() == 0) { log.error("Given datafiles/directory does not exist"); return; } for (int i = 0; i < allFileNames.size(); i++) { log.debug("File names:" + Utils.getFileName(allFileNames.get(i))); File file = new File(allFileNames.get(i)); counters.write.recordTotal = counters.write.recordTotal + file.length(); } //remove column definition file from list allFileNames.remove(columnDefinitionFileName); log.info("Number of data files:" + allFileNames.size()); /** * Process column definition file to get meta data and bin mapping. */ metadataColumnDefs = new ArrayList<ColumnDefinition>(); binColumnDefs = new ArrayList<ColumnDefinition>(); metadataConfigs = new HashMap<String, String>(); if (Parser.processJSONColumnDefinitions(columnDefinitionFile, metadataConfigs, metadataColumnDefs, binColumnDefs, params)) { log.info("Config file processed."); } else { throw new Exception("Config file parsing Error"); } // Add metadata of config to parameters String metadata; if ((metadata = metadataConfigs.get(Constants.INPUT_TYPE)) != null) { params.fileType = metadata; if (params.fileType.equals(Constants.CSV_FILE)) { // Version check metadata = metadataConfigs.get(Constants.VERSION); String[] vNumber = metadata.split("\\."); int v1 = Integer.parseInt(vNumber[0]); int v2 = Integer.parseInt(vNumber[1]); if ((v1 <= Constants.MajorV) && (v2 <= Constants.MinorV)) { log.debug("Config version used:" + metadata); } else throw new Exception("\"" + Constants.VERSION + ":" + metadata + "\" is not Supported"); // Set delimiter if ((metadata = metadataConfigs.get(Constants.DELIMITER)) != null && metadata.length() == 1) { params.delimiter = metadata.charAt(0); } else { log.warn("\"" + Constants.DELIMITER + ":" + metadata + "\" is not properly specified in config file. Default is ','"); } if ((metadata = metadataConfigs.get(Constants.IGNORE_FIRST_LINE)) != null) { params.ignoreFirstLine = metadata.equals("true"); } else { log.warn("\"" + Constants.IGNORE_FIRST_LINE + ":" + metadata + "\" is not properly specified in config file. Default is false"); } if ((metadata = metadataConfigs.get(Constants.COLUMNS)) != null) { counters.write.colTotal = Integer.parseInt(metadata); } else { throw new Exception("\"" + Constants.COLUMNS + ":" + metadata + "\" is not properly specified in config file"); } } else { throw new Exception("\"" + params.fileType + "\" is not supported in config file"); } } else { throw new Exception("\"" + Constants.INPUT_TYPE + "\" is not specified in config file"); } // add config input to column definitions if (params.fileType.equals(Constants.CSV_FILE)) { List<String> binName = null; if (params.ignoreFirstLine) { String line; BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(allFileNames.get(0)), "UTF8")); if ((line = br.readLine()) != null) { binName = Parser.getCSVRawColumns(line, params.delimiter); br.close(); if (binName.size() != counters.write.colTotal) { throw new Exception("Number of column in config file and datafile are mismatch." + " Datafile: " + Utils.getFileName(allFileNames.get(0)) + " Configfile: " + Utils.getFileName(columnDefinitionFileName)); } } } //update columndefs for metadata for (int i = 0; i < metadataColumnDefs.size(); i++) { if (metadataColumnDefs.get(i).staticValue) { } else { if (metadataColumnDefs.get(i).binValuePos < 0) { if (metadataColumnDefs.get(i).columnName == null) { if (metadataColumnDefs.get(i).jsonPath == null) { log.error("dynamic metadata having improper info" + metadataColumnDefs.toString()); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(metadataColumnDefs.get(i).binValueHeader) != -1) { metadataColumnDefs.get(i).binValuePos = binName .indexOf(metadataColumnDefs.get(i).binValueHeader); } else { throw new Exception("binName missing in data file:" + metadataColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) metadataColumnDefs.get(i).binValueHeader = binName .get(metadataColumnDefs.get(i).binValuePos); } } if ((!metadataColumnDefs.get(i).staticValue) && (metadataColumnDefs.get(i).binValuePos < 0)) { throw new Exception("Information for bin mapping is missing in config file:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader == Constants.SET && !metadataColumnDefs.get(i).srcType.equals(SrcColumnType.STRING)) { throw new Exception("Set name should be string type:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader.equalsIgnoreCase(Constants.SET) && params.set != null) { throw new Exception( "Set name is given both in config file and commandline. Provide only once."); } } //update columndefs for bins for (int i = 0; i < binColumnDefs.size(); i++) { if (binColumnDefs.get(i).staticName) { } else { if (binColumnDefs.get(i).binNamePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(binColumnDefs.get(i).binNameHeader) != -1) { binColumnDefs.get(i).binNamePos = binName .indexOf(binColumnDefs.get(i).binNameHeader); } else { throw new Exception("binName missing in data file:" + binColumnDefs.get(i).binNameHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binNameHeader = binName.get(binColumnDefs.get(i).binNamePos); } } if (binColumnDefs.get(i).staticValue) { } else { if (binColumnDefs.get(i).binValuePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.contains(binColumnDefs.get(i).binValueHeader)) { binColumnDefs.get(i).binValuePos = binName .indexOf(binColumnDefs.get(i).binValueHeader); } else if (!binColumnDefs.get(i).binValueHeader.toLowerCase() .equals(Constants.SYSTEM_TIME)) { throw new Exception("Wrong column name mentioned in config file:" + binColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binValueHeader = binName.get(binColumnDefs.get(i).binValuePos); } //check for missing entries in config file if (binColumnDefs.get(i).binValuePos < 0 && binColumnDefs.get(i).binValueHeader == null) { throw new Exception("Information missing(Value header or bin mapping) in config file:" + binColumnDefs.get(i)); } //check for proper data type in config file. if (binColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + binColumnDefs.get(i)); } //check for valid destination type if ((binColumnDefs.get(i).srcType.equals(SrcColumnType.TIMESTAMP) || binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB)) && binColumnDefs.get(i).dstType == null) { throw new Exception("Destination type is not mentioned: " + binColumnDefs.get(i)); } //check for encoding if (binColumnDefs.get(i).dstType != null && binColumnDefs.get(i).encoding == null) { throw new Exception( "Encoding is not given for src-dst type conversion:" + binColumnDefs.get(i)); } //check for valid encoding if (binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB) && !binColumnDefs.get(i).encoding.equals(Constants.HEX_ENCODING)) { throw new Exception("Wrong encoding for blob data:" + binColumnDefs.get(i)); } } //Check static bin name mapped to dynamic bin value if ((binColumnDefs.get(i).binNamePos == binColumnDefs.get(i).binValuePos) && (binColumnDefs.get(i).binNamePos != -1)) { throw new Exception("Static bin name mapped to dynamic bin value:" + binColumnDefs.get(i)); } //check for missing entries in config file if (binColumnDefs.get(i).binNameHeader == null && binColumnDefs.get(i).binNameHeader.length() > Constants.BIN_NAME_LENGTH) { throw new Exception("Information missing binName or large binName in config file:" + binColumnDefs.get(i)); } } } log.info(params.toString()); log.debug("MetadataConfig:" + metadataColumnDefs); log.debug("BinColumnDefs:" + binColumnDefs); // Start PrintStat thread statPrinter.start(); // Reader pool size ExecutorService readerPool = Executors.newFixedThreadPool( nReaderThreads > allFileNames.size() ? allFileNames.size() : nReaderThreads); log.info("Reader pool size : " + nReaderThreads); // Submit all tasks to writer threadpool. for (String aFile : allFileNames) { log.debug("Submitting task for: " + aFile); readerPool.submit(new AerospikeLoad(aFile, client, params)); } // Wait for reader pool to complete readerPool.shutdown(); log.info("Shutdown down reader thread pool"); while (!readerPool.isTerminated()) ; //readerPool.awaitTermination(20, TimeUnit.MINUTES); log.info("Reader thread pool terminated"); // Wait for writer pool to complete after getting all tasks from reader pool writerPool.shutdown(); log.info("Shutdown down writer thread pool"); while (!writerPool.isTerminated()) ; log.info("Writer thread pool terminated"); // Print final statistic of aerospike-loader. log.info("Final Statistics of importer: (Succesfull Writes = " + counters.write.writeCount.get() + ", " + "Errors=" + (counters.write.writeErrors.get() + counters.write.readErrors.get() + counters.write.processingErrors.get()) + "(" + (counters.write.writeErrors.get()) + "-Write," + counters.write.readErrors.get() + "-Read," + counters.write.processingErrors.get() + "-Processing)"); } catch (Exception e) { log.error(e); if (log.isDebugEnabled()) { e.printStackTrace(); } } finally { // Stop statistic printer thread. statPrinter.interrupt(); log.info("Aerospike loader completed"); } }
From source file:edu.nyu.tandon.query.Query.java
@SuppressWarnings("unchecked") public static void main(final String[] arg) throws Exception { SimpleJSAP jsap = new SimpleJSAP(Query.class.getName(), "Loads indices relative to a collection, possibly loads the collection, and answers to queries.", new Parameter[] { new FlaggedOption("collection", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'c', "collection", "The collection of documents indexed by the given indices."), new FlaggedOption("objectCollection", new ObjectParser(DocumentCollection.class, MG4JClassParser.PACKAGE), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'o', "object-collection", "An object specification describing a document collection."), new FlaggedOption("titleList", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 't', "title-list", "A serialized big list of titles (will override collection titles if specified)."), new FlaggedOption("titleFile", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'T', "title-file", "A file of newline-separated, UTF-8 titles (will override collection titles if specified)."), new FlaggedOption("input", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'I', "input", "A file containing the input."), new Switch("noSizes", 'n', "no-sizes", "Disable loading document sizes (they are necessary for BM25 scoring)."), new Switch("http", 'h', "http", "Starts an HTTP query server."), new Switch("verbose", 'v', "verbose", "Print full exception stack traces."), new FlaggedOption("itemClass", MG4JClassParser.getParser(), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'i', "item-class", "The class that will handle item display in the HTTP server."), new FlaggedOption("itemMimeType", JSAP.STRING_PARSER, "text/html", JSAP.NOT_REQUIRED, 'm', "item-mime-type", "A MIME type suggested to the class handling item display in the HTTP server."), new FlaggedOption("port", JSAP.INTEGER_PARSER, "4242", JSAP.NOT_REQUIRED, 'p', "port", "The port on localhost where the server will appear."), new UnflaggedOption("basenameWeight", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.GREEDY, "The indices that the servlet will use. Indices are specified using their basename, optionally followed by a colon and a double representing the weight used to score results from that index. Indices without a specified weight are weighted 1."), new Switch("noMplex", 'P', "noMplex", "Starts with multiplex disabled."), new FlaggedOption("results", JSAP.INTEGER_PARSER, "1000", JSAP.NOT_REQUIRED, 'r', "results", "The # of results to display"), new FlaggedOption("mode", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'M', "time", "The results display mode") }); final JSAPResult jsapResult = jsap.parse(arg); if (jsap.messagePrinted()) return;/*w ww.j a v a 2 s . co m*/ final DocumentCollection documentCollection = (DocumentCollection) (jsapResult.userSpecified("collection") ? AbstractDocumentSequence.load(jsapResult.getString("collection")) : jsapResult.userSpecified("objectCollection") ? jsapResult.getObject("objectCollection") : null); final BigList<? extends CharSequence> titleList = (BigList<? extends CharSequence>) (jsapResult .userSpecified("titleList") ? BinIO.loadObject(jsapResult.getString("titleList")) : jsapResult.userSpecified("titleFile") ? new FileLinesBigList(jsapResult.getString("titleFile"), "UTF-8") : null); final String[] basenameWeight = jsapResult.getStringArray("basenameWeight"); final Object2ReferenceLinkedOpenHashMap<String, Index> indexMap = new Object2ReferenceLinkedOpenHashMap<String, Index>( Hash.DEFAULT_INITIAL_SIZE, .5f); final Reference2DoubleOpenHashMap<Index> index2Weight = new Reference2DoubleOpenHashMap<Index>(); final boolean verbose = jsapResult.getBoolean("verbose"); final boolean loadSizes = !jsapResult.getBoolean("noSizes"); Query.loadIndicesFromSpec(basenameWeight, loadSizes, documentCollection, indexMap, index2Weight); final long numberOfDocuments = indexMap.values().iterator().next().numberOfDocuments; if (titleList != null && titleList.size64() != numberOfDocuments) throw new IllegalArgumentException("The number of titles (" + titleList.size64() + " and the number of documents (" + numberOfDocuments + ") do not match"); final Object2ObjectOpenHashMap<String, TermProcessor> termProcessors = new Object2ObjectOpenHashMap<String, TermProcessor>( indexMap.size()); for (String alias : indexMap.keySet()) termProcessors.put(alias, indexMap.get(alias).termProcessor); final SimpleParser simpleParser = new SimpleParser(indexMap.keySet(), indexMap.firstKey(), termProcessors); final Reference2ReferenceMap<Index, Object> index2Parser = new Reference2ReferenceOpenHashMap<Index, Object>(); /* // Fetch parsers for payload-based fields. for( Index index: indexMap.values() ) if ( index.hasPayloads ) { if ( index.payload.getClass() == DatePayload.class ) index2Parser.put( index, DateFormat.getDateInstance( DateFormat.SHORT, Locale.UK ) ); } */ final QueryEngine queryEngine = new QueryEngine(simpleParser, new DocumentIteratorBuilderVisitor(indexMap, index2Parser, indexMap.get(indexMap.firstKey()), MAX_STEMMING), indexMap); queryEngine.setWeights(index2Weight); queryEngine.score(new Scorer[] { new BM25Scorer(), new VignaScorer() }, new double[] { 1, 1 }); // We set up an interval selector only if there is a collection for snippeting queryEngine.intervalSelector = documentCollection != null ? new IntervalSelector(4, 40) : new IntervalSelector(); queryEngine.multiplex = !jsapResult.userSpecified("moPlex") || jsapResult.getBoolean("noMplex"); queryEngine.equalize(1000); Query query = new Query(queryEngine); query.displayMode = OutputType.TIME; query.maxOutput = jsapResult.getInt("results", 1000); query.interpretCommand("$score BM25Scorer"); String q; System.err.println( "Welcome to the MG4J query class (setup with $mode snippet, $score BM25Scorer VignaScorer, $mplex on, $equalize 1000, $select " + (documentCollection != null ? "4 40" : "all") + ")"); System.err.println("Please type $ for help."); String prompt = indexMap.keySet().toString() + ">"; int n; try { final BufferedReader br = new BufferedReader(new InputStreamReader( jsapResult.userSpecified("input") ? new FileInputStream(jsapResult.getString("input")) : System.in)); final ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>> results = new ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>>(); for (;;) { if (query.displayMode != OutputType.TREC) System.out.print(prompt); q = br.readLine(); if (q == null) { System.err.println(); break; // CTRL-D } if (q.length() == 0) continue; if (q.charAt(0) == '$') { if (!query.interpretCommand(q)) break; continue; } long time = -System.nanoTime(); try { n = queryEngine.process(q, 0, query.maxOutput, results); } catch (QueryParserException e) { if (verbose) e.getCause().printStackTrace(System.err); else System.err.println(e.getCause()); continue; } catch (Exception e) { if (verbose) e.printStackTrace(System.err); else System.err.println(e); continue; } time += System.nanoTime(); query.output(results, documentCollection, titleList, TextMarker.TEXT_BOLDFACE); System.err.println(results.size() + " results; " + n + " documents examined; " + time / 1000000. + " ms; " + Util.format((n * 1000000000.0) / time) + " documents/s, " + Util.format(time / (double) n) + " ns/document"); } } finally { if (query.output != System.out) query.output.close(); } }
From source file:com.adito.server.Main.java
/** * Entry point//from w w w . ja v a 2 s.co m * * @param args * @throws Throwable */ public static void main(String[] args) throws Throwable { // This is a hack to allow the Install4J installer to get the java // runtime that will be used if (args.length > 0 && args[0].equals("--jvmdir")) { System.out.println(SystemProperties.get("java.home")); System.exit(0); } useWrapper = System.getProperty("wrapper.key") != null; final Main main = new Main(); ContextHolder.setContext(main); if (useWrapper) { WrapperManager.start(main, args); } else { Integer returnCode = main.start(args); if (returnCode != null) { if (main.gui) { if (main.startupException == null) { main.startupException = new Exception("An exit code of " + returnCode + " was returned."); } try { if (SystemProperties.get("os.name").toLowerCase().startsWith("windows")) { UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); } } catch (Exception e) { } String mesg = main.startupException.getMessage() == null ? "No message supplied." : main.startupException.getMessage(); StringBuffer buf = new StringBuffer(); int l = 0; char ch = ' '; for (int i = 0; i < mesg.length(); i++) { ch = mesg.charAt(i); if (l > 50 && ch == ' ') { buf.append("\n"); l = 0; } else { if (ch == '\n') { l = 0; } else { l++; } buf.append(ch); } } mesg = buf.toString(); final String fMesg = mesg; SwingUtilities.invokeAndWait(new Runnable() { public void run() { JOptionPane.showMessageDialog(null, fMesg, "Startup Error", JOptionPane.ERROR_MESSAGE); } }); } System.exit(returnCode.intValue()); } else { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { if (!main.shuttingDown) { main.stop(0); } } }); } } }
From source file:it.unimi.di.big.mg4j.query.Query.java
@SuppressWarnings("unchecked") public static void main(final String[] arg) throws Exception { SimpleJSAP jsap = new SimpleJSAP(Query.class.getName(), "Loads indices relative to a collection, possibly loads the collection, and answers to queries.", new Parameter[] { new FlaggedOption("collection", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'c', "collection", "The collection of documents indexed by the given indices."), new FlaggedOption("objectCollection", new ObjectParser(DocumentCollection.class, MG4JClassParser.PACKAGE), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'o', "object-collection", "An object specification describing a document collection."), new FlaggedOption("titleList", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 't', "title-list", "A serialized big list of titles (will override collection titles if specified)."), new FlaggedOption("titleFile", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'T', "title-file", "A file of newline-separated, UTF-8 titles (will override collection titles if specified)."), new FlaggedOption("input", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'I', "input", "A file containing the input."), new Switch("noSizes", 'n', "no-sizes", "Disable loading document sizes (they are necessary for BM25 scoring)."), new Switch("http", 'h', "http", "Starts an HTTP query server."), new Switch("verbose", 'v', "verbose", "Print full exception stack traces."), new FlaggedOption("itemClass", MG4JClassParser.getParser(), JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'i', "item-class", "The class that will handle item display in the HTTP server."), new FlaggedOption("itemMimeType", JSAP.STRING_PARSER, "text/html", JSAP.NOT_REQUIRED, 'm', "item-mime-type", "A MIME type suggested to the class handling item display in the HTTP server."), new FlaggedOption("port", JSAP.INTEGER_PARSER, "4242", JSAP.NOT_REQUIRED, 'p', "port", "The port on localhost where the server will appear."), new UnflaggedOption("basenameWeight", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.GREEDY, "The indices that the servlet will use. Indices are specified using their basename, optionally followed by a colon and a double representing the weight used to score results from that index. Indices without a specified weight are weighted 1.") }); final JSAPResult jsapResult = jsap.parse(arg); if (jsap.messagePrinted()) return;/*w w w . ja va2 s . c o m*/ final DocumentCollection documentCollection = (DocumentCollection) (jsapResult.userSpecified("collection") ? AbstractDocumentSequence.load(jsapResult.getString("collection")) : jsapResult.userSpecified("objectCollection") ? jsapResult.getObject("objectCollection") : null); final BigList<? extends CharSequence> titleList = (BigList<? extends CharSequence>) (jsapResult .userSpecified("titleList") ? BinIO.loadObject(jsapResult.getString("titleList")) : jsapResult.userSpecified("titleFile") ? new FileLinesBigList(jsapResult.getString("titleFile"), "UTF-8") : null); final String[] basenameWeight = jsapResult.getStringArray("basenameWeight"); final Object2ReferenceLinkedOpenHashMap<String, Index> indexMap = new Object2ReferenceLinkedOpenHashMap<String, Index>( Hash.DEFAULT_INITIAL_SIZE, .5f); final Reference2DoubleOpenHashMap<Index> index2Weight = new Reference2DoubleOpenHashMap<Index>(); final boolean verbose = jsapResult.getBoolean("verbose"); final boolean loadSizes = !jsapResult.getBoolean("noSizes"); Query.loadIndicesFromSpec(basenameWeight, loadSizes, documentCollection, indexMap, index2Weight); final long numberOfDocuments = indexMap.values().iterator().next().numberOfDocuments; if (titleList != null && titleList.size64() != numberOfDocuments) throw new IllegalArgumentException("The number of titles (" + titleList.size64() + " and the number of documents (" + numberOfDocuments + ") do not match"); final Object2ObjectOpenHashMap<String, TermProcessor> termProcessors = new Object2ObjectOpenHashMap<String, TermProcessor>( indexMap.size()); for (String alias : indexMap.keySet()) termProcessors.put(alias, indexMap.get(alias).termProcessor); final SimpleParser simpleParser = new SimpleParser(indexMap.keySet(), indexMap.firstKey(), termProcessors); final Reference2ReferenceMap<Index, Object> index2Parser = new Reference2ReferenceOpenHashMap<Index, Object>(); /* // Fetch parsers for payload-based fields. for( Index index: indexMap.values() ) if ( index.hasPayloads ) { if ( index.payload.getClass() == DatePayload.class ) index2Parser.put( index, DateFormat.getDateInstance( DateFormat.SHORT, Locale.UK ) ); } */ final QueryEngine queryEngine = new QueryEngine(simpleParser, new DocumentIteratorBuilderVisitor(indexMap, index2Parser, indexMap.get(indexMap.firstKey()), MAX_STEMMING), indexMap); queryEngine.setWeights(index2Weight); queryEngine.score(new Scorer[] { new BM25Scorer(), new VignaScorer() }, new double[] { 1, 1 }); // We set up an interval selector only if there is a collection for snippeting queryEngine.intervalSelector = documentCollection != null ? new IntervalSelector(4, 40) : new IntervalSelector(); queryEngine.multiplex = true; queryEngine.equalize(1000); Query query = new Query(queryEngine); query.displayMode = OutputType.SNIPPET; String q; System.err.println( "Welcome to the MG4J query class (setup with $mode snippet, $score BM25Scorer VignaScorer, $mplex on, $equalize 1000, $select " + (documentCollection != null ? "4 40" : "all") + ")"); System.err.println("Please type $ for help."); String prompt = indexMap.keySet().toString() + ">"; int n; HttpQueryServer httpQueryServer = null; if (jsapResult.getBoolean("http")) httpQueryServer = new HttpQueryServer(queryEngine, documentCollection, jsapResult.getClass("itemClass"), jsapResult.getString("itemMimeType"), jsapResult.getInt("port"), titleList); try { final BufferedReader br = new BufferedReader(new InputStreamReader( jsapResult.userSpecified("input") ? new FileInputStream(jsapResult.getString("input")) : System.in)); final ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>> results = new ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index, SelectedInterval[]>>>(); for (;;) { System.out.print(prompt); q = br.readLine(); if (q == null) { System.err.println(); break; // CTRL-D } if (q.length() == 0) continue; if (q.charAt(0) == '$') { if (!query.interpretCommand(q)) break; continue; } long time = -System.nanoTime(); try { n = queryEngine.process(q, 0, query.maxOutput, results); } catch (QueryParserException e) { if (verbose) e.getCause().printStackTrace(System.err); else System.err.println(e.getCause()); continue; } catch (Exception e) { if (verbose) e.printStackTrace(System.err); else System.err.println(e); continue; } time += System.nanoTime(); query.output(results, documentCollection, titleList, TextMarker.TEXT_BOLDFACE); System.err.println(results.size() + " results; " + n + " documents examined; " + time / 1000000. + " ms; " + Util.format((n * 1000000000.0) / time) + " documents/s, " + Util.format(time / (double) n) + " ns/document"); } } finally { if (httpQueryServer != null) httpQueryServer.server.stop(); if (query.output != System.out) query.output.close(); } }
From source file:com.kactech.otj.examples.App_otj.java
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { String command = null;//from www .j av a2 s .co m String hisacct = null; String hisacctName = null; String hisacctAsset = null; String asset = null; String assetName = null; List<String> argList = null; boolean newAccount = false; File dir = null; ConnectionInfo connection = null; List<ScriptFilter> filters = null; CommandLineParser parser = new GnuParser(); CommandLine cmd = null; try { cmd = parser.parse(options, args); } catch (Exception e) { System.err.println("Command-line parsing error: " + e.getMessage()); help(); System.exit(-1); } if (cmd.hasOption('h')) { help(); System.exit(0); } @SuppressWarnings("unchecked") List<String> list = cmd.getArgList(); if (list.size() > 1) { System.err.println("only one command is supported, you've typed " + list); help(); System.exit(-1); } if (list.size() > 0) command = list.get(0).trim(); List<SampleAccount> accounts = ExamplesUtils.getSampleAccounts(); if (cmd.hasOption('s')) { String v = cmd.getOptionValue('s').trim(); connection = ExamplesUtils.findServer(v); if (connection == null) { System.err.println("unknown server: " + v); System.exit(-1); } } else { connection = ExamplesUtils.findServer(DEF_SERVER_NAME); if (connection == null) { System.err.println("default server not found server: " + DEF_SERVER_NAME); System.exit(-1); } } if (cmd.hasOption('t')) { String v = cmd.getOptionValue('t'); for (SampleAccount ac : accounts) if (ac.accountName.startsWith(v)) { hisacct = ac.accountID; hisacctName = ac.accountName; hisacctAsset = ac.assetID; break; } if (hisacct == null) if (mayBeValid(v)) hisacct = v; else { System.err.println("invalid hisacct: " + v); System.exit(-1); } } if (cmd.hasOption('p')) { String v = cmd.getOptionValue('p'); for (SampleAccount ac : accounts) if (ac.assetName.startsWith(v)) { asset = ac.assetID; assetName = ac.assetName; break; } if (asset == null) if (mayBeValid(v)) asset = v; else { System.err.println("invalid asset: " + v); System.exit(-1); } } if (cmd.hasOption('a')) { String v = cmd.getOptionValue('a'); argList = new ArrayList<String>(); boolean q = false; StringBuilder b = new StringBuilder(); for (int i = 0; i < v.length(); i++) { char c = v.charAt(i); if (c == '"') { if (q) { argList.add(b.toString()); b = null; q = false; continue; } if (b != null) argList.add(b.toString()); b = new StringBuilder(); q = true; continue; } if (c == ' ' || c == '\t') { if (q) { b.append(c); continue; } if (b != null) argList.add(b.toString()); b = null; continue; } if (b == null) b = new StringBuilder(); b.append(c); } if (b != null) argList.add(b.toString()); if (q) { System.err.println("unclosed quote in args: " + v); System.exit(-1); } } dir = new File(cmd.hasOption('d') ? cmd.getOptionValue('d') : DEF_CLIENT_DIR); if (cmd.hasOption('x')) del(dir); newAccount = cmd.hasOption('n'); if (cmd.hasOption('f')) { filters = new ArrayList<ScriptFilter>(); ScriptEngineManager manager = new ScriptEngineManager(); ScriptEngine engine = manager.getEngineByName("JavaScript"); Compilable compilingEngine = (Compilable) engine; for (String fn : cmd.getOptionValue('f').split(",")) { fn = fn.trim(); if (fn.isEmpty()) continue; fn += ".js"; Reader r = null; try { r = new InputStreamReader(new FileInputStream(new File("filters", fn)), Utils.UTF8); } catch (Exception e) { try { r = new InputStreamReader( App_otj.class.getResourceAsStream("/com/kactech/otj/examples/filters/" + fn)); } catch (Exception e2) { } } if (r == null) { System.err.println("filter not found: " + fn); System.exit(-1); } else try { CompiledScript compiled = compilingEngine.compile(r); ScriptFilter sf = new ScriptFilter(compiled); filters.add(sf); } catch (Exception ex) { System.err.println("error while loading " + fn + ": " + ex); System.exit(-1); } } } System.out.println("server: " + connection.getEndpoint() + " " + connection.getID()); System.out.println("command: '" + command + "'"); System.out.println("args: " + argList); System.out.println("hisacct: " + hisacct); System.out.println("hisacctName: " + hisacctName); System.out.println("hisacctAsset: " + hisacctAsset); System.out.println("asset: " + asset); System.out.println("assetName: " + assetName); if (asset != null && hisacctAsset != null && !asset.equals(hisacctAsset)) { System.err.println("asset differs from hisacctAsset"); System.exit(-1); } EClient client = new EClient(dir, connection); client.setAssetType(asset != null ? asset : hisacctAsset); client.setCreateNewAccount(newAccount); if (filters != null) client.setFilters(filters); try { Utils.init(); Client.DEBUG_JSON = true; client.init(); if ("balance".equals(command)) System.out.println("Balance: " + client.getAccount().getBalance().getAmount()); else if ("acceptall".equals(command)) client.processInbox(); else if ("transfer".equals(command)) { if (hisacct == null) System.err.println("please specify --hisacct"); else { int idx = argList != null ? argList.indexOf("amount") : -1; if (idx < 0) System.err.println("please specify amount"); else if (idx == argList.size()) System.err.println("amount argument needs value"); else { Long amount = -1l; try { amount = new Long(argList.get(idx + 1)); } catch (Exception e) { } if (amount <= 0) System.err.println("invalid amount"); else { client.notarizeTransaction(hisacct, amount); } } } } else if ("reload".equals(command)) client.reloadState(); else if ("procnym".equals(command)) client.processNymbox(); } finally { client.saveState(); client.close(); } }
From source file:com.era7.bioinfo.annotation.AutomaticQualityControl.java
public static void main(String[] args) { if (args.length != 4) { System.out.println("This program expects four parameters: \n" + "1. Gene annotation XML filename \n" + "2. Reference protein set (.fasta)\n" + "3. Output TXT filename\n" + "4. Initial Blast XML results filename (the one used at the very beginning of the semiautomatic annotation process)\n"); } else {// ww w .j av a2 s.c om BufferedWriter outBuff = null; try { File inFile = new File(args[0]); File fastaFile = new File(args[1]); File outFile = new File(args[2]); File blastFile = new File(args[3]); //Primero cargo todos los datos del archivo xml del blast BufferedReader buffReader = new BufferedReader(new FileReader(blastFile)); StringBuilder stBuilder = new StringBuilder(); String line = null; while ((line = buffReader.readLine()) != null) { stBuilder.append(line); } buffReader.close(); System.out.println("Creating blastoutput..."); BlastOutput blastOutput = new BlastOutput(stBuilder.toString()); System.out.println("BlastOutput created! :)"); stBuilder.delete(0, stBuilder.length()); HashMap<String, String> blastProteinsMap = new HashMap<String, String>(); ArrayList<Iteration> iterations = blastOutput.getBlastOutputIterations(); for (Iteration iteration : iterations) { blastProteinsMap.put(iteration.getQueryDef().split("\\|")[1].trim(), iteration.toString()); } //freeing some memory blastOutput = null; //------------------------------------------------------------------------ //Initializing writer for output file outBuff = new BufferedWriter(new FileWriter(outFile)); //reading gene annotation xml file..... buffReader = new BufferedReader(new FileReader(inFile)); stBuilder = new StringBuilder(); line = null; while ((line = buffReader.readLine()) != null) { stBuilder.append(line); } buffReader.close(); XMLElement genesXML = new XMLElement(stBuilder.toString()); //freeing some memory I don't need anymore stBuilder.delete(0, stBuilder.length()); //reading file with the reference proteins set ArrayList<String> proteinsReferenceSet = new ArrayList<String>(); buffReader = new BufferedReader(new FileReader(fastaFile)); while ((line = buffReader.readLine()) != null) { if (line.charAt(0) == '>') { proteinsReferenceSet.add(line.split("\\|")[1]); } } buffReader.close(); Element pGenes = genesXML.asJDomElement().getChild(PredictedGenes.TAG_NAME); List<Element> contigs = pGenes.getChildren(ContigXML.TAG_NAME); System.out.println("There are " + contigs.size() + " contigs to be checked... "); outBuff.write("There are " + contigs.size() + " contigs to be checked... \n"); outBuff.write("Proteins reference set: \n"); for (String st : proteinsReferenceSet) { outBuff.write(st + ","); } outBuff.write("\n"); for (Element elem : contigs) { ContigXML contig = new ContigXML(elem); //escribo el id del contig en el que estoy outBuff.write("Checking contig: " + contig.getId() + "\n"); outBuff.flush(); List<XMLElement> geneList = contig.getChildrenWith(PredictedGene.TAG_NAME); System.out.println("geneList.size() = " + geneList.size()); int numeroDeGenesParaAnalizar = geneList.size() / FACTOR; if (numeroDeGenesParaAnalizar == 0) { numeroDeGenesParaAnalizar++; } ArrayList<Integer> indicesUtilizados = new ArrayList<Integer>(); outBuff.write("\nThe contig has " + geneList.size() + " predicted genes, let's analyze: " + numeroDeGenesParaAnalizar + "\n"); for (int j = 0; j < numeroDeGenesParaAnalizar; j++) { int geneIndex; boolean geneIsDismissed = false; do { geneIsDismissed = false; geneIndex = (int) Math.round(Math.floor(Math.random() * geneList.size())); PredictedGene tempGene = new PredictedGene(geneList.get(geneIndex).asJDomElement()); if (tempGene.getStatus().equals(PredictedGene.STATUS_DISMISSED)) { geneIsDismissed = true; } } while (indicesUtilizados.contains(new Integer(geneIndex)) && geneIsDismissed); indicesUtilizados.add(geneIndex); System.out.println("geneIndex = " + geneIndex); //Ahora hay que sacar el gen correspondiente al indice y hacer el control de calidad PredictedGene gene = new PredictedGene(geneList.get(geneIndex).asJDomElement()); outBuff.write("\nAnalyzing gene with id: " + gene.getId() + " , annotation uniprot id: " + gene.getAnnotationUniprotId() + "\n"); outBuff.write("eValue: " + gene.getEvalue() + "\n"); //--------------PETICION POST HTTP BLAST---------------------- PostMethod post = new PostMethod(BLAST_URL); post.addParameter("program", "blastx"); post.addParameter("sequence", gene.getSequence()); post.addParameter("database", "uniprotkb"); post.addParameter("email", "ppareja@era7.com"); post.addParameter("exp", "1e-10"); post.addParameter("stype", "dna"); // execute the POST HttpClient client = new HttpClient(); int status = client.executeMethod(post); System.out.println("status post = " + status); InputStream inStream = post.getResponseBodyAsStream(); String fileName = "jobid.txt"; FileOutputStream outStream = new FileOutputStream(new File(fileName)); byte[] buffer = new byte[1024]; int len; while ((len = inStream.read(buffer)) != -1) { outStream.write(buffer, 0, len); } outStream.close(); //Once the file is created I just have to read one line in order to extract the job id buffReader = new BufferedReader(new FileReader(new File(fileName))); String jobId = buffReader.readLine(); buffReader.close(); System.out.println("jobId = " + jobId); //--------------HTTP CHECK JOB STATUS REQUEST---------------------- GetMethod get = new GetMethod(CHECK_JOB_STATUS_URL + jobId); String jobStatus = ""; do { try { Thread.sleep(1000);//sleep for 1000 ms } catch (InterruptedException ie) { //If this thread was intrrupted by nother thread } status = client.executeMethod(get); //System.out.println("status get = " + status); inStream = get.getResponseBodyAsStream(); fileName = "jobStatus.txt"; outStream = new FileOutputStream(new File(fileName)); while ((len = inStream.read(buffer)) != -1) { outStream.write(buffer, 0, len); } outStream.close(); //Once the file is created I just have to read one line in order to extract the job id buffReader = new BufferedReader(new FileReader(new File(fileName))); jobStatus = buffReader.readLine(); //System.out.println("jobStatus = " + jobStatus); buffReader.close(); } while (!jobStatus.equals(FINISHED_JOB_STATUS)); //Once I'm here the blast should've already finished //--------------JOB RESULTS HTTP REQUEST---------------------- get = new GetMethod(JOB_RESULT_URL + jobId + "/out"); status = client.executeMethod(get); System.out.println("status get = " + status); inStream = get.getResponseBodyAsStream(); fileName = "jobResults.txt"; outStream = new FileOutputStream(new File(fileName)); while ((len = inStream.read(buffer)) != -1) { outStream.write(buffer, 0, len); } outStream.close(); //--------parsing the blast results file----- TreeSet<GeneEValuePair> featuresBlast = new TreeSet<GeneEValuePair>(); buffReader = new BufferedReader(new FileReader(new File(fileName))); while ((line = buffReader.readLine()) != null) { if (line.length() > 3) { String prefix = line.substring(0, 3); if (prefix.equals("TR:") || prefix.equals("SP:")) { String[] columns = line.split(" "); String id = columns[1]; //System.out.println("id = " + id); String e = ""; String[] arraySt = line.split("\\.\\.\\."); if (arraySt.length > 1) { arraySt = arraySt[1].trim().split(" "); int contador = 0; for (int k = 0; k < arraySt.length && contador <= 2; k++) { String string = arraySt[k]; if (!string.equals("")) { contador++; if (contador == 2) { e = string; } } } } else { //Number before e- String[] arr = arraySt[0].split("e-")[0].split(" "); String numeroAntesE = arr[arr.length - 1]; String numeroDespuesE = arraySt[0].split("e-")[1].split(" ")[0]; e = numeroAntesE + "e-" + numeroDespuesE; } double eValue = Double.parseDouble(e); //System.out.println("eValue = " + eValue); GeneEValuePair g = new GeneEValuePair(id, eValue); featuresBlast.add(g); } } } GeneEValuePair currentGeneEValuePair = new GeneEValuePair(gene.getAnnotationUniprotId(), gene.getEvalue()); System.out.println("currentGeneEValuePair.id = " + currentGeneEValuePair.id); System.out.println("currentGeneEValuePair.eValue = " + currentGeneEValuePair.eValue); boolean blastContainsGene = false; for (GeneEValuePair geneEValuePair : featuresBlast) { if (geneEValuePair.id.equals(currentGeneEValuePair.id)) { blastContainsGene = true; //le pongo la e que tiene en el wu-blast para poder comparar currentGeneEValuePair.eValue = geneEValuePair.eValue; break; } } if (blastContainsGene) { outBuff.write("The protein was found in the WU-BLAST result.. \n"); //Una vez que se que esta en el blast tengo que ver que sea la mejor GeneEValuePair first = featuresBlast.first(); outBuff.write("Protein with best eValue according to the WU-BLAST result: " + first.id + " , " + first.eValue + "\n"); if (first.id.equals(currentGeneEValuePair.id)) { outBuff.write("Proteins with best eValue match up \n"); } else { if (first.eValue == currentGeneEValuePair.eValue) { outBuff.write( "The one with best eValue is not the same protein but has the same eValue \n"); } else if (first.eValue > currentGeneEValuePair.eValue) { outBuff.write( "The one with best eValue is not the same protein but has a worse eValue :) \n"); } else { outBuff.write( "The best protein from BLAST has an eValue smaller than ours, checking if it's part of the reference set...\n"); //System.exit(-1); if (proteinsReferenceSet.contains(first.id)) { //The protein is in the reference set and that shouldn't happen outBuff.write( "The protein was found on the reference set, checking if it belongs to the same contig...\n"); String iterationSt = blastProteinsMap.get(gene.getAnnotationUniprotId()); if (iterationSt != null) { outBuff.write( "The protein was found in the BLAST used at the beginning of the annotation process.\n"); Iteration iteration = new Iteration(iterationSt); ArrayList<Hit> hits = iteration.getIterationHits(); boolean contigFound = false; Hit errorHit = null; for (Hit hit : hits) { if (hit.getHitDef().indexOf(contig.getId()) >= 0) { contigFound = true; errorHit = hit; break; } } if (contigFound) { outBuff.write( "ERROR: A hit from the same contig was find in the Blast file: \n" + errorHit.toString() + "\n"); } else { outBuff.write("There is no hit with the same contig! :)\n"); } } else { outBuff.write( "The protein is NOT in the BLAST used at the beginning of the annotation process.\n"); } } else { //The protein was not found on the reference set so everything's ok outBuff.write( "The protein was not found on the reference, everything's ok :)\n"); } } } } else { outBuff.write("The protein was NOT found on the WU-BLAST !! :( \n"); //System.exit(-1); } } } } catch (Exception ex) { ex.printStackTrace(); } finally { try { //closing outputfile outBuff.close(); } catch (IOException ex) { Logger.getLogger(AutomaticQualityControl.class.getName()).log(Level.SEVERE, null, ex); } } } }
From source file:net.sf.extjwnl.cli.ewn.java
public static void main(String[] args) throws IOException, JWNLException { if (args.length < 1) { System.out.println(USAGE); System.exit(0);// ww w .ja va 2 s . co m } //find dictionary Dictionary d = null; File config = new File(defaultConfig); if (!config.exists()) { if (System.getenv().containsKey("WNHOME")) { String wnHomePath = System.getenv().get("WNHOME"); File wnHome = new File(wnHomePath); if (wnHome.exists()) { d = Dictionary.getFileBackedInstance(wnHomePath); } else { log.error("Cannot find dictionary. Make sure " + defaultConfig + " is available or WNHOME variable is set."); } } } else { d = Dictionary.getInstance(new FileInputStream(config)); } if (null != d) { //parse and execute command line if ((-1 < args[0].indexOf('%') && -1 < args[0].indexOf(':')) || "-script".equals(args[0]) || (-1 < args[0].indexOf('#'))) { d.edit(); //edit if ("-script".equals(args[0])) { if (args.length < 2) { log.error("Filename missing for -script command"); System.exit(1); } else { File script = new File(args[1]); if (script.exists()) { //load into args ArrayList<String> newArgs = new ArrayList<String>(); BufferedReader in = new BufferedReader( new InputStreamReader(new FileInputStream(script), "UTF-8")); try { String str; while ((str = in.readLine()) != null) { String[] bits = str.split(" "); StringBuilder tempArg = null; for (String bit : bits) { int quoteCnt = 0; for (int j = 0; j < bit.length(); j++) { if ('"' == bit.charAt(j)) { quoteCnt++; } } if (null != tempArg) { if (0 == quoteCnt) { tempArg.append(" ").append(bit); } else { tempArg.append(" ").append(bit.replaceAll("\"\"", "\"")); if (1 == (quoteCnt % 2)) { newArgs.add( tempArg.toString().substring(1, tempArg.length() - 1)); tempArg = null; } } } else { if (0 == quoteCnt) { newArgs.add(bit); } else { if (1 == (quoteCnt % 2)) { tempArg = new StringBuilder(bit.replaceAll("\"\"", "\"")); } else { newArgs.add(bit.replaceAll("\"\"", "\"")); } } } } if (null != tempArg) { newArgs.add(tempArg.toString()); } } } finally { try { in.close(); } catch (IOException e) { //nop } } args = newArgs.toArray(args); } } } Word workWord = null; String key = null; String lemma = null; int lexFileNum = -1; int lexId = -1; // String headLemma = null; // int headLexId = -1; POS pos = null; String derivation = null; for (int i = 0; i < args.length; i++) { if (null == key && '-' != args[i].charAt(0) && ((-1 < args[i].indexOf('%') && -1 < args[i].indexOf(':')))) { key = args[i]; log.info("Searching " + key + "..."); if (null != key) { workWord = d.getWordBySenseKey(key); } if (null == workWord) { //parse sensekey lemma = key.substring(0, key.indexOf('%')).replace('_', ' '); String posId = key.substring(key.indexOf('%') + 1, key.indexOf(':')); if ("1".equals(posId) || "2".equals(posId) || "3".equals(posId) || "4".equals(posId) || "5".equals(posId)) { pos = POS.getPOSForId(Integer.parseInt(posId)); String lexFileString = key.substring(key.indexOf(':') + 1); if (-1 < lexFileString.indexOf(':')) { lexFileNum = Integer .parseInt(lexFileString.substring(0, lexFileString.indexOf(':'))); if (lexFileString.indexOf(':') + 1 < lexFileString.length()) { String lexIdString = lexFileString .substring(lexFileString.indexOf(':') + 1); if (-1 < lexIdString.indexOf(':')) { lexId = Integer .parseInt(lexIdString.substring(0, lexIdString.indexOf(':'))); // if (lexIdString.indexOf(':') + 1 < lexIdString.length()) { // headLemma = lexIdString.substring(lexIdString.indexOf(':') + 1); // if (-1 < headLemma.indexOf(':')) { // headLemma = headLemma.substring(0, headLemma.indexOf(':')); // if (null != headLemma && !"".equals(headLemma) && lexIdString.lastIndexOf(':') + 1 < lexIdString.length()) { // headLexId = Integer.parseInt(lexIdString.substring(lexIdString.lastIndexOf(':') + 1)); // } // } else { // log.error("Malformed sensekey " + key); // System.exit(1); // } // } } else { log.error("Malformed sensekey " + key); System.exit(1); } } else { log.error("Malformed sensekey " + key); System.exit(1); } } else { log.error("Malformed sensekey " + key); System.exit(1); } } else { log.error("Malformed sensekey " + key); System.exit(1); } } } else if (-1 < args[i].indexOf('#')) { if (2 < args[i].length()) { derivation = args[i].substring(2); if (null == derivation) { log.error("Missing derivation"); System.exit(1); } else { pos = POS.getPOSForKey(args[i].substring(0, 1)); if (null == pos) { log.error("POS " + args[i] + " is not recognized for derivation " + derivation); System.exit(1); } } } } if ("-add".equals(args[i])) { if (null == key) { log.error("Missing sensekey"); System.exit(1); } if (null != workWord) { log.error("Duplicate sensekey " + workWord.getSenseKey()); System.exit(1); } log.info("Creating " + pos.getLabel() + " synset..."); Synset tempSynset = d.createSynset(pos); log.info("Creating word " + lemma + "..."); workWord = new Word(d, tempSynset, 1, lemma); workWord.setLexId(lexId); tempSynset.getWords().add(workWord); tempSynset.setLexFileNum(lexFileNum); key = null; } if ("-remove".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { d.removeSynset(workWord.getSynset()); workWord = null; key = null; } } if ("-addword".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { Word tempWord = new Word(d, workWord.getSynset(), workWord.getSynset().getWords().size() + 1, args[i]); workWord.getSynset().getWords().add(tempWord); key = null; } else { log.error( "Missing word for addword command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-removeword".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { workWord.getSynset().getWords().remove(workWord); key = null; } } if ("-setgloss".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.getSynset().setGloss(args[i]); key = null; } else { log.error("Missing gloss for setgloss command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setadjclus".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.getSynset().setIsAdjectiveCluster(Boolean.parseBoolean(args[i])); key = null; } else { log.error("Missing flag for setadjclus command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setverbframe".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { if (workWord instanceof Verb) { Verb verb = (Verb) workWord; if ('-' == args[i].charAt(0)) { verb.getVerbFrameFlags().clear(Integer.parseInt(args[i].substring(1))); } else { verb.getVerbFrameFlags().set(Integer.parseInt(args[i])); } } else { log.error("Word at " + workWord.getSenseKey() + " should be verb"); System.exit(1); } key = null; } else { log.error("Missing index for setverbframe command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setverbframeall".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { if (workWord.getSynset() instanceof VerbSynset) { if ('-' == args[i].charAt(0)) { workWord.getSynset().getVerbFrameFlags() .clear(Integer.parseInt(args[i].substring(1))); } else { workWord.getSynset().getVerbFrameFlags().set(Integer.parseInt(args[i])); } } else { log.error("Synset at " + workWord.getSenseKey() + " should be verb"); System.exit(1); } key = null; } else { log.error("Missing index for setverbframeall command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setlexfile".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { if (-1 < args[i].indexOf('.')) { workWord.getSynset() .setLexFileNum(LexFileNameLexFileIdMap.getMap().get(args[i])); } else { workWord.getSynset().setLexFileNum(Integer.parseInt(args[i])); } } else { log.error("Missing file number or name for setlexfile command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-addptr".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { Word targetWord = d.getWordBySenseKey(args[i]); if (null != targetWord) { i++; if (i < args.length) { PointerType pt = PointerType.getPointerTypeForKey(args[i]); if (null != pt) { Pointer p; if (pt.isLexical()) { p = new Pointer(pt, workWord, targetWord); } else { p = new Pointer(pt, workWord.getSynset(), targetWord.getSynset()); } if (!workWord.getSynset().getPointers().contains(p)) { workWord.getSynset().getPointers().add(p); } else { log.error("Duplicate pointer of type " + pt + " to " + targetWord.getSenseKey() + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Invalid pointer type at " + args[i] + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing pointer type at " + args[i] + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing target at " + args[i] + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } key = null; } else { log.error("Missing sensekey for addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-removeptr".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { Word targetWord = d.getWordBySenseKey(args[i]); if (null != targetWord) { i++; if (i < args.length) { PointerType pt = PointerType.getPointerTypeForKey(args[i]); if (null != pt) { Pointer p; if (pt.isLexical()) { p = new Pointer(pt, workWord, targetWord); } else { p = new Pointer(pt, workWord.getSynset(), targetWord.getSynset()); } if (workWord.getSynset().getPointers().contains(p)) { workWord.getSynset().getPointers().remove(p); } else { log.error("Missing pointer of type " + pt + " to " + targetWord.getSenseKey() + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Invalid pointer type at " + args[i] + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing pointer type at " + args[i] + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing target at " + args[i] + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } key = null; } else { log.error("Missing sensekey for removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setlexid".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.setLexId(Integer.parseInt(args[i])); key = null; } else { log.error("Missing lexid for setlexid command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setusecount".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.setUseCount(Integer.parseInt(args[i])); key = null; } else { log.error("Missing count for setusecount command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-addexc".equals(args[i])) { i++; if (i < args.length && '-' != args[i].charAt(0)) { String baseform = args[i]; Exc e = d.getException(pos, derivation); if (null != e) { if (null != e.getExceptions()) { if (!e.getExceptions().contains(baseform)) { e.getExceptions().add(baseform); } } } else { ArrayList<String> list = new ArrayList<String>(1); list.add(baseform); d.createException(pos, derivation, list); } derivation = null; } else { log.error("Missing baseform for addexc command for derivation " + derivation); System.exit(1); } } if ("-removeexc".equals(args[i])) { Exc e = d.getException(pos, derivation); if (null != e) { i++; if (i < args.length && '-' != args[i].charAt(0)) { String baseform = args[i]; if (null != e.getExceptions()) { if (e.getExceptions().contains(baseform)) { e.getExceptions().remove(baseform); } if (0 == e.getExceptions().size()) { d.removeException(e); } } } else { d.removeException(e); } } else { log.error("Missing derivation " + derivation); System.exit(1); } derivation = null; } } d.save(); } else { //browse String key = args[0]; if (1 == args.length) { for (POS pos : POS.getAllPOS()) { IndexWord iw = d.getIndexWord(pos, key); if (null == iw) { System.out.println("\nNo information available for " + pos.getLabel() + " " + key); } else { System.out.println( "\nInformation available for " + iw.getPOS().getLabel() + " " + iw.getLemma()); printAvailableInfo(iw); } if (null != d.getMorphologicalProcessor()) { List<String> forms = d.getMorphologicalProcessor().lookupAllBaseForms(pos, key); if (null != forms) { for (String form : forms) { if (!key.equals(form)) { iw = d.getIndexWord(pos, form); if (null != iw) { System.out.println("\nInformation available for " + iw.getPOS().getLabel() + " " + iw.getLemma()); printAvailableInfo(iw); } } } } } } } else { boolean needHelp = false; boolean needGloss = false; boolean needLex = false; boolean needOffset = false; boolean needSenseNum = false; boolean needSenseKeys = false; int needSense = 0; for (String arg : args) { if ("-h".equals(arg)) { needHelp = true; } if ("-g".equals(arg)) { needGloss = true; } if ("-a".equals(arg)) { needLex = true; } if ("-o".equals(arg)) { needOffset = true; } if ("-s".equals(arg)) { needSenseNum = true; } if ("-k".equals(arg)) { needSenseKeys = true; } if (arg.startsWith("-n") && 2 < arg.length()) { needSense = Integer.parseInt(arg.substring(2)); } } for (String arg : args) { if (arg.startsWith("-ants") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display synsets containing direct antonyms of the search string.\n" + "\n" + "Direct antonyms are a pair of words between which there is an\n" + "associative bond built up by co-occurrences.\n" + "\n" + "Antonym synsets are preceded by \"=>\"."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nAntonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.ANTONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //ants if (arg.startsWith("-hype") && 6 == arg.length()) { if (needHelp) { System.out.println( "Recursively display hypernym (superordinate) tree for the search\n" + "string.\n" + "\n" + "Hypernym is the generic term used to designate a whole class of\n" + "specific instances. Y is a hypernym of X if X is a (kind of) Y.\n" + "\n" + "Hypernym synsets are preceded by \"=>\", and are indented from\n" + "the left according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHypernyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.HYPERNYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hype if (arg.startsWith("-hypo") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display immediate hyponyms (subordinates) for the search string.\n" + "\n" + "Hyponym is the generic term used to designate a member of a class.\n" + "X is a hyponym of Y if X is a (kind of) Y.\n" + "\n" + "Hyponym synsets are preceded by \"=>\"."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHyponyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.HYPONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hypo if (arg.startsWith("-tree") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display hyponym (subordinate) tree for the search string. This is\n" + "a recursive search that finds the hyponyms of each hyponym. \n" + "\n" + "Hyponym is the generic term used to designate a member of a class.\n" + "X is a hyponym of Y if X is a (kind of) Y. \n" + "\n" + "Hyponym synsets are preceded by \"=>\", and are indented from the left\n" + "according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHyponyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.HYPONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //tree if (arg.startsWith("-enta") && 6 == arg.length()) { if (needHelp) { System.out.println( "Recursively display entailment relations of the search string.\n" + "\n" + "The action represented by the verb X entails Y if X cannot be done\n" + "unless Y is, or has been, done.\n" + "\n" + "Entailment synsets are preceded by \"=>\", and are indented from the left\n" + "according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nEntailment of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.ENTAILMENT, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //enta if (arg.startsWith("-syns") && 6 == arg.length()) { POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nSynonyms of " + p.getLabel() + " " + iw.getLemma()); if (POS.ADJECTIVE == p) { if (needHelp) { System.out.println( "Display synonyms and synsets related to synsets containing\n" + "the search string. If the search string is in a head synset\n" + "the 'cluster's' satellite synsets are displayed. If the search\n" + "string is in a satellite synset, its head synset is displayed.\n" + "If the search string is a pertainym the word or synset that it\n" + "pertains to is displayed.\n" + "\n" + "A cluster is a group of adjective synsets that are organized around\n" + "antonymous pairs or triplets. An adjective cluster contains two or more\n" + "head synsets that contan antonyms. Each head synset has one or more\n" + "satellite synsets.\n" + "\n" + "A head synset contains at least one word that has a direct antonym\n" + "in another head synset of the same cluster.\n" + "\n" + "A satellite synset represents a concept that is similar in meaning to\n" + "the concept represented by its head synset.\n" + "\n" + "Direct antonyms are a pair of words between which there is an\n" + "associative bond built up by co-occurrences.\n" + "\n" + "Direct antonyms are printed in parentheses following the adjective.\n" + "The position of an adjective in relation to the noun may be restricted\n" + "to the prenominal, postnominal or predicative position. Where present\n" + "these restrictions are noted in parentheses.\n" + "\n" + "A pertainym is a relational adjective, usually defined by such phrases\n" + "as \"of or pertaining to\" and that does not have an antonym. It pertains\n" + "to a noun or another pertainym.\n" + "\n" + "Senses contained in head synsets are displayed above the satellites,\n" + "which are indented and preceded by \"=>\". Senses contained in\n" + "satellite synsets are displayed with the head synset below. The head\n" + "synset is preceded by \"=>\".\n" + "\n" + "Pertainym senses display the word or synsets that the search string\n" + "pertains to."); } tracePointers(iw, PointerType.SIMILAR_TO, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PARTICIPLE_OF, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } if (POS.ADVERB == p) { if (needHelp) { System.out.println( "Display synonyms and synsets related to synsets containing\n" + "the search string. If the search string is a pertainym the word\n" + "or synset that it pertains to is displayed.\n" + "\n" + "A pertainym is a relational adverb that is derived from an adjective.\n" + "\n" + "Pertainym senses display the word that the search string is derived from\n" + "and the adjective synset that contains the word. If the adjective synset\n" + "is a satellite synset, its head synset is also displayed."); } tracePointers(iw, PointerType.PERTAINYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } if (POS.NOUN == p || POS.VERB == p) { if (needHelp) { System.out.println( "Recursively display hypernym (superordinate) tree for the search\n" + "string.\n" + "\n" + "Hypernym is the generic term used to designate a whole class of\n" + "specific instances. Y is a hypernym of X if X is a (kind of) Y.\n" + "\n" + "Hypernym synsets are preceded by \"=>\", and are indented from\n" + "the left according to their level in the hierarchy."); } tracePointers(iw, PointerType.HYPERNYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } } //syns if (arg.startsWith("-smem") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMember Holonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //smem if (arg.startsWith("-ssub") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nSubstance Holonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.SUBSTANCE_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //ssub if (arg.startsWith("-sprt") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nPart Holonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.PART_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //sprt if (arg.startsWith("-memb") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMember Meronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //memb if (arg.startsWith("-subs") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nSubstance Meronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //subs if (arg.startsWith("-part") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nPart Meronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.PART_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //part if (arg.startsWith("-mero") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMeronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //mero if (arg.startsWith("-holo") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHolonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //holo if (arg.startsWith("-caus") && 6 == arg.length()) { if (needHelp) { System.out.println("Recursively display CAUSE TO relations of the search string.\n" + "\n" + "The action represented by the verb X causes the action represented by\n" + "the verb Y.\n" + "\n" + "CAUSE TO synsets are preceded by \"=>\", and are indented from the left\n" + "according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\n'Cause to' of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.CAUSE, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //caus if (arg.startsWith("-pert") && 6 == arg.length()) { POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nPertainyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.PERTAINYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //pert if (arg.startsWith("-attr") && 6 == arg.length()) { POS p = POS.getPOSForKey(arg.substring(5)); if (needHelp) { if (POS.NOUN == p) { System.out .println("Display adjectives for which search string is an attribute."); } if (POS.ADJECTIVE == p) { System.out.println("Display nouns that are attributes of search string."); } } IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nAttributes of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.ATTRIBUTE, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //attr if (arg.startsWith("-deri") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display derived forms - nouns and verbs that are related morphologically.\n" + "Each related synset is preceeded by its part of speech. Each word in the\n" + "synset is followed by its sense number."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nDerived forms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.NOMINALIZATION, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //deri if (arg.startsWith("-domn") && 6 == arg.length()) { if (needHelp) { System.out.println("Display domain to which this synset belongs."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nDomain of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.CATEGORY, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.USAGE, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.REGION, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //domn if (arg.startsWith("-domt") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all synsets belonging to the domain."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nDomain of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.CATEGORY_MEMBER, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.USAGE_MEMBER, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.REGION_MEMBER, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //domt if (arg.startsWith("-faml") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display familiarity and polysemy information for the search string.\n" + "The polysemy count is the number of senses in WordNet."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { String[] freqs = { "extremely rare", "very rare", "rare", "uncommon", "common", "familiar", "very familiar", "extremely familiar" }; String[] pos = { "a noun", "a verb", "an adjective", "an adverb" }; int cnt = iw.getSenses().size(); int familiar = 0; if (cnt == 0) { familiar = 0; } if (cnt == 1) { familiar = 1; } if (cnt == 2) { familiar = 2; } if (cnt >= 3 && cnt <= 4) { familiar = 3; } if (cnt >= 5 && cnt <= 8) { familiar = 4; } if (cnt >= 9 && cnt <= 16) { familiar = 5; } if (cnt >= 17 && cnt <= 32) { familiar = 6; } if (cnt > 32) { familiar = 7; } System.out.println("\n" + iw.getLemma() + " used as " + pos[p.getId() - 1] + " is " + freqs[familiar] + " (polysemy count = " + cnt + ")"); } } //faml if (arg.startsWith("-fram") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display applicable verb sentence frames for the search string.\n" + "\n" + "A frame is a sentence template illustrating the usage of a verb.\n" + "\n" + "Verb sentence frames are preceded with the string \"*>\" if a sentence\n" + "frame is acceptable for all of the words in the synset, and with \"=>\"\n" + "if a sentence frame is acceptable for the search string only."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nVerb frames of " + p.getLabel() + " " + iw.getLemma()); for (int i = 0; i < iw.getSenses().size(); i++) { Synset synset = iw.getSenses().get(i); for (String vf : synset.getVerbFrames()) { System.out.println("\t*> " + vf); } for (Word word : synset.getWords()) { if (iw.getLemma().equalsIgnoreCase(word.getLemma())) { if (word instanceof Verb) { Verb verb = (Verb) word; for (String vf : verb.getVerbFrames()) { System.out.println("\t=> " + vf); } } } } } } } //fram if (arg.startsWith("-hmer") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display meronyms for search string tree. This is a recursive search\n" + "the prints all the meronyms of the search string and all of its\n" + "hypernyms. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMeronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_MERONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hmer if (arg.startsWith("-hhol") && 6 == arg.length()) { if (needHelp) { System.out.println( "\"Display holonyms for search string tree. This is a recursive search\n" + "that prints all the holonyms of the search string and all of the\n" + "holonym's holonyms.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHolonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_HOLONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_HOLONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_HOLONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hhol if (arg.startsWith("-mero") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMeronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //mero if (arg.startsWith("-grep") && 6 == arg.length()) { if (needHelp) { System.out .println("Print all strings in the database containing the search string\n" + "as an individual word, or as the first or last string in a word or\n" + "collocation."); } POS p = POS.getPOSForKey(arg.substring(5)); System.out.println("\nGrep of " + p.getLabel() + " " + key); Iterator<IndexWord> ii = d.getIndexWordIterator(p, key); while (ii.hasNext()) { System.out.println(ii.next().getLemma()); } } //grep if ("-over".equals(arg)) { for (POS pos : POS.getAllPOS()) { if (null != d.getMorphologicalProcessor()) { IndexWord iw = d.getIndexWord(pos, key); //for plurals like species, glasses if (null != iw && key.equals(iw.getLemma())) { printOverview(pos, iw, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } List<String> forms = d.getMorphologicalProcessor().lookupAllBaseForms(pos, key); if (null != forms) { for (String form : forms) { if (!form.equals(key)) { iw = d.getIndexWord(pos, form); if (null != iw) { printOverview(pos, iw, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } } } } } } //over } } } } }
From source file:com.admc.jcreole.JCreole.java
/** * Run this method with no parameters to see syntax requirements and the * available parameters./*from w w w . jav a 2 s . c o m*/ * * N.b. do not call this method from a persistent program, because it * may call System.exit! * <p> * Any long-running program should use the lower-level methods in this * class instead (or directly use CreoleParser and CreoleScanner * instances). * </p> <p> * This method executes with all JCreole privileges. * </p> <p> * This method sets up the following htmlExpander mappings (therefore you * can reference these in both Creole and boilerplate text).<p> * <ul> * <li>sys|*: mappings for Java system properties * <li>isoDateTime * <li>isoDate * <li>pageTitle: Value derived from the specified Creole file name. * </ul> * * @throws IOException for any I/O problem that makes it impossible to * satisfy the request. * @throws CreoleParseException * if can not generate output, or if the run generates 0 output. * If the problem is due to input formatting, in most cases you * will get a CreoleParseException, which is a RuntimException, and * CreoleParseException has getters for locations in the source * data (though they will be offset for \r's in the provided * Creole source, if any). */ public static void main(String[] sa) throws IOException { String bpResPath = null; String bpFsPath = null; String outPath = null; String inPath = null; boolean debugs = false; boolean troubleshoot = false; boolean noBp = false; int param = -1; try { while (++param < sa.length) { if (sa[param].equals("-d")) { debugs = true; continue; } if (sa[param].equals("-t")) { troubleshoot = true; continue; } if (sa[param].equals("-r") && param + 1 < sa.length) { if (bpFsPath != null || bpResPath != null || noBp) throw new IllegalArgumentException(); bpResPath = sa[++param]; continue; } if (sa[param].equals("-f") && param + 1 < sa.length) { if (bpResPath != null || bpFsPath != null || noBp) throw new IllegalArgumentException(); bpFsPath = sa[++param]; continue; } if (sa[param].equals("-")) { if (noBp || bpFsPath != null || bpResPath != null) throw new IllegalArgumentException(); noBp = true; continue; } if (sa[param].equals("-o") && param + 1 < sa.length) { if (outPath != null) throw new IllegalArgumentException(); outPath = sa[++param]; continue; } if (inPath != null) throw new IllegalArgumentException(); inPath = sa[param]; } if (inPath == null) throw new IllegalArgumentException(); } catch (IllegalArgumentException iae) { System.err.println(SYNTAX_MSG); System.exit(1); } if (!noBp && bpResPath == null && bpFsPath == null) bpResPath = DEFAULT_BP_RES_PATH; String rawBoilerPlate = null; if (bpResPath != null) { if (bpResPath.length() > 0 && bpResPath.charAt(0) == '/') // Classloader lookups are ALWAYS relative to CLASSPATH roots, // and will abort if you specify a beginning "/". bpResPath = bpResPath.substring(1); InputStream iStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(bpResPath); if (iStream == null) throw new IOException("Boilerplate inaccessible: " + bpResPath); rawBoilerPlate = IOUtil.toString(iStream); } else if (bpFsPath != null) { rawBoilerPlate = IOUtil.toString(new File(bpFsPath)); } String creoleResPath = (inPath.length() > 0 && inPath.charAt(0) == '/') ? inPath.substring(1) : inPath; // Classloader lookups are ALWAYS relative to CLASSPATH roots, // and will abort if you specify a beginning "/". InputStream creoleStream = Thread.currentThread().getContextClassLoader() .getResourceAsStream(creoleResPath); File inFile = (creoleStream == null) ? new File(inPath) : null; JCreole jCreole = (rawBoilerPlate == null) ? (new JCreole()) : (new JCreole(rawBoilerPlate)); if (debugs) { jCreole.setInterWikiMapper(new InterWikiMapper() { // This InterWikiMapper is just for prototyping. // Use wiki name of "nil" to force lookup failure for path. // Use wiki page of "nil" to force lookup failure for label. public String toPath(String wikiName, String wikiPage) { if (wikiName != null && wikiName.equals("nil")) return null; return "{WIKI-LINK to: " + wikiName + '|' + wikiPage + '}'; } public String toLabel(String wikiName, String wikiPage) { if (wikiPage == null) throw new RuntimeException("Null page name sent to InterWikiMapper"); if (wikiPage.equals("nil")) return null; return "{LABEL for: " + wikiName + '|' + wikiPage + '}'; } }); Expander creoleExpander = new Expander(Expander.PairedDelims.RECTANGULAR); creoleExpander.put("testMacro", "\n\n<<prettyPrint>>\n{{{\n" + "!/bin/bash -p\n\ncp /etc/inittab /tmp\n}}}\n"); jCreole.setCreoleExpander(creoleExpander); } jCreole.setPrivileges(EnumSet.allOf(JCreolePrivilege.class)); Expander exp = jCreole.getHtmlExpander(); Date now = new Date(); exp.putAll("sys", System.getProperties(), false); exp.put("isoDateTime", isoDateTimeFormatter.format(now), false); exp.put("isoDate", isoDateFormatter.format(now), false); exp.put("pageTitle", (inFile == null) ? creoleResPath.replaceFirst("[.][^.]*$", "").replaceFirst(".*[/\\\\.]", "") : inFile.getName().replaceFirst("[.][^.]*$", "")); if (troubleshoot) { // We don't write any HMTL output here. // Goal is just to output diagnostics. StringBuilder builder = (creoleStream == null) ? IOUtil.toStringBuilder(inFile) : IOUtil.toStringBuilder(creoleStream); int newlineCount = 0; int lastOffset = -1; int offset = builder.indexOf("\n"); for (; offset >= 0; offset = builder.indexOf("\n", offset + 1)) { lastOffset = offset; newlineCount++; } // Accommodate input files with no terminating newline: if (builder.length() > lastOffset + 1) newlineCount++; System.out.println("Input lines: " + newlineCount); Exception lastException = null; while (true) { try { jCreole.parseCreole(builder); break; } catch (Exception e) { lastException = e; } if (builder.charAt(builder.length() - 1) == '\n') builder.setLength(builder.length() - 1); offset = builder.lastIndexOf("\n"); if (offset < 1) break; newlineCount--; builder.setLength(builder.lastIndexOf("\n")); } System.out.println((lastException == null) ? "Input validates" : String.format("Error around input line %d: %s", newlineCount, lastException.getMessage())); return; } String generatedHtml = (creoleStream == null) ? jCreole.parseCreole(inFile) : jCreole.parseCreole(IOUtil.toStringBuilder(creoleStream)); String html = jCreole.postProcess(generatedHtml, SystemUtils.LINE_SEPARATOR); if (outPath == null) { System.out.print(html); } else { FileUtils.writeStringToFile(new File(outPath), html, "UTF-8"); } }