List of usage examples for org.apache.commons.cli Option getValue
public String getValue()
null if there is no value. From source file:net.kahowell.xsd.fuzzer.XmlGenerator.java
/** * Drives the application, parsing command-line arguments to determine * options.//from ww w .ja v a 2 s .com * * @param args command line args */ public static void main(String[] args) { try { setupLog4j(); CommandLine commandLine = parser.parse(ConsoleOptions.OPTIONS, args); if (commandLine.hasOption("d")) { Logger.getLogger("net.kahowell.xsd.fuzzer").setLevel(Level.DEBUG); } for (Option option : commandLine.getOptions()) { if (option.getValue() != null) { log.debug("Using " + option.getDescription() + ": " + option.getValue()); } else { log.debug("Using " + option.getDescription()); } } Injector injector = Guice.createInjector( Modules.override(Modules.combine(new DefaultGeneratorsModule(), new DefaultOptionsModule())) .with(new CommandLineArgumentsModule(commandLine))); log.debug(injector.getBindings()); XsdParser xsdParser = injector.getInstance(XsdParser.class); XmlOptions xmlOptions = injector .getInstance(Key.get(XmlOptions.class, Names.named("xml save options"))); XmlGenerator xmlGenerator = injector.getInstance(XmlGenerator.class); XmlGenerationOptions xmlGenerationOptions = injector.getInstance(XmlGenerationOptions.class); doPostModuleConfig(commandLine, xmlGenerationOptions, injector); ByteArrayOutputStream stream = new ByteArrayOutputStream(); XmlObject generatedXml = xsdParser.generateXml(commandLine.getOptionValue("root")); generatedXml.save(stream, xmlOptions); if (commandLine.hasOption("v")) { if (xsdParser.validate(stream)) { log.info("Valid XML file produced."); } else { log.info("Invalid XML file produced."); System.exit(4); } } xmlGenerator.showOrSave(stream); } catch (MissingOptionException e) { if (e.getMissingOptions().size() != 0) { System.err.println("Missing argument(s): " + Arrays.toString(e.getMissingOptions().toArray())); } helpFormatter.printHelp(XmlGenerator.class.getSimpleName(), ConsoleOptions.OPTIONS); System.exit(1); } catch (ParseException e) { helpFormatter.printHelp(XmlGenerator.class.getSimpleName(), ConsoleOptions.OPTIONS); System.exit(2); } catch (Exception e) { e.printStackTrace(); System.exit(3); } }
From source file:com.medsavant.mailer.Mail.java
public static void main(String[] args) throws IOException { CommandLineParser parser = new GnuParser(); Options ops = getOptions();/*from w ww .j a v a2s . com*/ try { // parse the command line arguments CommandLine line = parser.parse(ops, args); // print help if (line.hasOption('h') || line.getOptions().length == 0) { printHelp(); return; } // parse args String email = null; String emailPass = null; String mailingList = null; String subject = null; String htmlFile = null; for (Option o : line.getOptions()) { switch (o.getOpt().charAt(0)) { case 's': subject = o.getValue(); break; case 'e': htmlFile = o.getValue(); break; case 'u': email = o.getValue(); break; case 'p': emailPass = o.getValue(); break; case 'l': mailingList = o.getValue(); break; } } setMailCredentials(email, emailPass, host, port); String text = readFileIntoString(new File(htmlFile)); sendEmail(mailingList, subject, text); } catch (org.apache.commons.cli.ParseException exp) { printHelp(); // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); } }
From source file:glacierpipe.GlacierPipeMain.java
public static void main(String[] args) throws IOException, ParseException { CommandLineParser parser = new GnuParser(); CommandLine cmd = parser.parse(OPTIONS, args); if (cmd.hasOption("help")) { try (PrintWriter writer = new PrintWriter(System.err)) { printHelp(writer);/*from w ww . java 2s.c o m*/ } System.exit(0); } else if (cmd.hasOption("upload")) { // Turn the CommandLine into Properties Properties cliProperties = new Properties(); for (Iterator<?> i = cmd.iterator(); i.hasNext();) { Option o = (Option) i.next(); String opt = o.getLongOpt(); opt = opt != null ? opt : o.getOpt(); String value = o.getValue(); value = value != null ? value : ""; cliProperties.setProperty(opt, value); } // Build up a configuration ConfigBuilder configBuilder = new ConfigBuilder(); // Archive name List<?> archiveList = cmd.getArgList(); if (archiveList.size() > 1) { throw new ParseException("Too many arguments"); } else if (archiveList.isEmpty()) { throw new ParseException("No archive name provided"); } configBuilder.setArchive(archiveList.get(0).toString()); // All other arguments on the command line configBuilder.setFromProperties(cliProperties); // Load any config from the properties file Properties fileProperties = new Properties(); try (InputStream in = new FileInputStream(configBuilder.propertiesFile)) { fileProperties.load(in); } catch (IOException e) { System.err.printf("Warning: unable to read properties file %s; %s%n", configBuilder.propertiesFile, e); } configBuilder.setFromProperties(fileProperties); // ... Config config = new Config(configBuilder); IOBuffer buffer = new MemoryIOBuffer(config.partSize); AmazonGlacierClient client = new AmazonGlacierClient( new BasicAWSCredentials(config.accessKey, config.secretKey)); client.setEndpoint(config.endpoint); // Actual upload try (InputStream in = new BufferedInputStream(System.in, 4096); PrintWriter writer = new PrintWriter(System.err); ObservableProperties configMonitor = config.reloadProperties ? new ObservableProperties(config.propertiesFile) : null; ProxyingThrottlingStrategy throttlingStrategy = new ProxyingThrottlingStrategy(config);) { TerminalGlacierPipeObserver observer = new TerminalGlacierPipeObserver(writer); if (configMonitor != null) { configMonitor.registerObserver(throttlingStrategy); } GlacierPipe pipe = new GlacierPipe(buffer, observer, config.maxRetries, throttlingStrategy); pipe.pipe(client, config.vault, config.archive, in); } catch (Exception e) { e.printStackTrace(System.err); } System.exit(0); } else { try (PrintWriter writer = new PrintWriter(System.err)) { writer.println("No action specified."); printHelp(writer); } System.exit(-1); } }
From source file:asl.seedscan.DQAWeb.java
public static void main(String args[]) { db = new MetricDatabase("", "", ""); findConsoleHandler();/*from www . j a v a2 s . c o m*/ consoleHandler.setLevel(Level.ALL); Logger.getLogger("").setLevel(Level.CONFIG); // Default locations of config and schema files File configFile = new File("dqaweb-config.xml"); File schemaFile = new File("schemas/DQAWebConfig.xsd"); boolean parseConfig = true; boolean testMode = false; ArrayList<File> schemaFiles = new ArrayList<File>(); schemaFiles.add(schemaFile); // ==== Command Line Parsing ==== Options options = new Options(); Option opConfigFile = new Option("c", "config-file", true, "The config file to use for seedscan. XML format according to SeedScanConfig.xsd."); Option opSchemaFile = new Option("s", "schema-file", true, "The schame file which should be used to verify the config file format. "); Option opTest = new Option("t", "test", false, "Run in test console mode rather than as a servlet."); OptionGroup ogConfig = new OptionGroup(); ogConfig.addOption(opConfigFile); OptionGroup ogSchema = new OptionGroup(); ogConfig.addOption(opSchemaFile); OptionGroup ogTest = new OptionGroup(); ogTest.addOption(opTest); options.addOptionGroup(ogConfig); options.addOptionGroup(ogSchema); options.addOptionGroup(ogTest); PosixParser optParser = new PosixParser(); CommandLine cmdLine = null; try { cmdLine = optParser.parse(options, args, true); } catch (org.apache.commons.cli.ParseException e) { logger.severe("Error while parsing command-line arguments."); System.exit(1); } Option opt; Iterator iter = cmdLine.iterator(); while (iter.hasNext()) { opt = (Option) iter.next(); if (opt.getOpt().equals("c")) { configFile = new File(opt.getValue()); } else if (opt.getOpt().equals("s")) { schemaFile = new File(opt.getValue()); } else if (opt.getOpt().equals("t")) { testMode = true; } } String query = ""; System.out.println("Entering Test Mode"); System.out.println("Enter a query string to view results or type \"help\" for example query strings"); InputStreamReader input = new InputStreamReader(System.in); BufferedReader reader = new BufferedReader(input); String result = ""; while (testMode == true) { try { System.out.printf("Query: "); query = reader.readLine(); if (query.equals("exit")) { testMode = false; } else if (query.equals("help")) { System.out.println("Need to add some help for people"); //TODO } else { result = processCommand(query); } System.out.println(result); } catch (IOException err) { System.err.println("Error reading line, in DQAWeb.java"); } } System.err.printf("DONE.\n"); }
From source file:dk.alexandra.fresco.demo.PrivateSetDemo.java
/** * The main method sets up application specific command line parameters, * parses command line arguments. Based on the command line arguments it * configures the SCE, instantiates the PrivateSetDemo and runs the PrivateSetDemo on the * SCE./* w w w . j a v a2 s. c o m*/ * */ public static void main(String[] args) { CmdLineUtil util = new CmdLineUtil(); SCEConfiguration sceConf = null; boolean[] key = null; int[] inputs = null; try { util.addOption(Option.builder("key") .desc("The key to use for encryption. " + "A " + INPUT_LENGTH + " char hex string. Required for player 1 and 2. " + "For both players this is interpreted as the AES key. ") .longOpt("key").hasArg().build()); util.addOption(Option.builder("in") .desc("The list of integers to use as input for the set intersection problem. " + "A comma separated list of integers. Required for player 1 and 2. " + "The lists must be of equal length for each player. ") .longOpt("input").hasArg().build()); CommandLine cmd = util.parse(args); sceConf = util.getSCEConfiguration(); // Get and validate the AES specific input. if (sceConf.getMyId() == 1 || sceConf.getMyId() == 2) { if (!cmd.hasOption("in") && !cmd.hasOption("key")) { throw new ParseException("Player 1 and 2 must submit inputs and keys"); } else { if (cmd.getOptionValue("key").length() != INPUT_LENGTH) { throw new IllegalArgumentException( "bad key hex string: must be hex string of length " + INPUT_LENGTH); } key = ByteArithmetic.toBoolean(cmd.getOptionValue("key")); for (Option o : cmd.getOptions()) { System.out.println("option: " + o.getValue()); } inputs = arrayFromString(cmd.getOptionValue("in")); } } else { if (cmd.hasOption("in")) throw new ParseException("Only player 1 and 2 should submit input"); } } catch (ParseException | IllegalArgumentException e) { System.out.println("Error: " + e); System.out.println(); util.displayHelp(); System.exit(-1); } // Do the secure computation using config from property files. PrivateSetDemo privateSetDemo = new PrivateSetDemo(sceConf.getMyId(), key, inputs); SCE sce = SCEFactory.getSCEFromConfiguration(sceConf); try { sce.runApplication(privateSetDemo); } catch (MPCException e) { System.out.println("Error while doing MPC: " + e.getMessage()); System.exit(-1); } // Print result. System.out.println("The resulting ciphertexts are:"); boolean[][] res = new boolean[privateSetDemo.result.length][BLOCK_SIZE]; for (int j = 0; j < privateSetDemo.result.length; j++) { for (int i = 0; i < BLOCK_SIZE; i++) { res[j][i] = privateSetDemo.result[j][i].getValue(); } System.out.println("result(" + j + "): " + ByteArithmetic.toHex(res[j])); } }
From source file:com.sludev.mssqlapplylog.MSSQLApplyLogMain.java
public static void main(String[] args) { CommandLineParser parser = new DefaultParser(); Options options = new Options(); // Most of the following defaults should be changed in // the --conf or "conf.properties" file String sqlURL = null;/* w ww .j a v a 2s . co m*/ String sqlUser = null; String sqlPass = null; String sqlDb = null; String sqlHost = "127.0.0.1"; String backupDirStr = null; String laterThanStr = ""; String fullBackupPathStr = null; String fullBackupPatternStr = "(?:[\\w_-]+?)(\\d+)\\.bak"; String fullBackupDatePatternStr = "yyyyMMddHHmm"; String sqlProcessUser = null; String logBackupPatternStr = "(.*)\\.trn"; String logBackupDatePatternStr = "yyyyMMddHHmmss"; boolean doFullRestore = false; Boolean useLogFileLastMode = null; Boolean monitorLogBackupDir = null; options.addOption(Option.builder().longOpt("conf").desc("Configuration file.").hasArg().build()); options.addOption(Option.builder().longOpt("laterthan").desc("'Later Than' file filter.").hasArg().build()); options.addOption(Option.builder().longOpt("restore-full") .desc("Restore the full backup before continuing.").build()); options.addOption(Option.builder().longOpt("use-lastmod") .desc("Sort/filter the log backups using their File-System 'Last Modified' date.").build()); options.addOption(Option.builder().longOpt("monitor-backup-dir") .desc("Monitor the backup directory for new log backups, and apply them.").build()); CommandLine line = null; try { try { line = parser.parse(options, args); } catch (ParseException ex) { throw new MSSQLApplyLogException(String.format("Error parsing command line.'%s'", ex.getMessage()), ex); } String confFile = null; // Process the command line arguments Iterator cmdI = line.iterator(); while (cmdI.hasNext()) { Option currOpt = (Option) cmdI.next(); String currOptName = currOpt.getLongOpt(); switch (currOptName) { case "conf": // Parse the configuration file confFile = currOpt.getValue(); break; case "laterthan": // "Later Than" file date filter laterThanStr = currOpt.getValue(); break; case "restore-full": // Do a full backup restore before restoring logs doFullRestore = true; break; case "monitor-backup-dir": // Monitor the backup directory for new logs monitorLogBackupDir = true; break; case "use-lastmod": // Use the last-modified date on Log Backup files for sorting/filtering useLogFileLastMode = true; break; } } Properties confProperties = null; if (StringUtils.isBlank(confFile) || Files.isReadable(Paths.get(confFile)) == false) { throw new MSSQLApplyLogException( "Missing or unreadable configuration file. Please specify --conf"); } else { // Process the conf.properties file confProperties = new Properties(); try { confProperties.load(Files.newBufferedReader(Paths.get(confFile))); } catch (IOException ex) { throw new MSSQLApplyLogException("Error loading properties file", ex); } sqlURL = confProperties.getProperty("sqlURL", ""); sqlUser = confProperties.getProperty("sqlUser", ""); sqlPass = confProperties.getProperty("sqlPass", ""); sqlDb = confProperties.getProperty("sqlDb", ""); sqlHost = confProperties.getProperty("sqlHost", ""); backupDirStr = confProperties.getProperty("backupDir", ""); if (StringUtils.isBlank(laterThanStr)) { laterThanStr = confProperties.getProperty("laterThan", ""); } fullBackupPathStr = confProperties.getProperty("fullBackupPath", fullBackupPathStr); fullBackupPatternStr = confProperties.getProperty("fullBackupPattern", fullBackupPatternStr); fullBackupDatePatternStr = confProperties.getProperty("fullBackupDatePattern", fullBackupDatePatternStr); sqlProcessUser = confProperties.getProperty("sqlProcessUser", ""); logBackupPatternStr = confProperties.getProperty("logBackupPattern", logBackupPatternStr); logBackupDatePatternStr = confProperties.getProperty("logBackupDatePattern", logBackupDatePatternStr); if (useLogFileLastMode == null) { String useLogFileLastModeStr = confProperties.getProperty("useLogFileLastMode", "false"); useLogFileLastMode = Boolean .valueOf(StringUtils.lowerCase(StringUtils.trim(useLogFileLastModeStr))); } if (monitorLogBackupDir == null) { String monitorBackupDirStr = confProperties.getProperty("monitorBackupDir", "false"); monitorLogBackupDir = Boolean .valueOf(StringUtils.lowerCase(StringUtils.trim(monitorBackupDirStr))); } } } catch (MSSQLApplyLogException ex) { try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { pw.append(String.format("Error : '%s'\n\n", ex.getMessage())); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(pw, 80, "\njava -jar mssqlapplylog.jar ", "\nThe MSSQLApplyLog application can be used in a variety of options and modes.\n", options, 0, 2, " All Rights Reserved.", true); System.out.println(sw.toString()); } catch (IOException iex) { LOGGER.debug("Error processing usage", iex); } System.exit(1); } MSSQLApplyLogConfig config = MSSQLApplyLogConfig.from(backupDirStr, fullBackupPathStr, fullBackupDatePatternStr, laterThanStr, fullBackupPatternStr, logBackupPatternStr, logBackupDatePatternStr, sqlHost, sqlDb, sqlUser, sqlPass, sqlURL, sqlProcessUser, useLogFileLastMode, doFullRestore, monitorLogBackupDir); MSSQLApplyLog logProc = MSSQLApplyLog.from(config); BasicThreadFactory thFactory = new BasicThreadFactory.Builder().namingPattern("restoreThread-%d").build(); ExecutorService mainThreadExe = Executors.newSingleThreadExecutor(thFactory); Future<Integer> currRunTask = mainThreadExe.submit(logProc); mainThreadExe.shutdown(); Integer resp = 0; try { resp = currRunTask.get(); } catch (InterruptedException ex) { LOGGER.error("Application 'main' thread was interrupted", ex); } catch (ExecutionException ex) { LOGGER.error("Application 'main' thread execution error", ex); } finally { // If main leaves for any reason, shutdown all threads mainThreadExe.shutdownNow(); } System.exit(resp); }
From source file:is.landsbokasafn.deduplicator.indexer.IndexingLauncher.java
public static void main(String[] args) throws Exception { loadConfiguration();//from w w w .j a v a 2s . co m // Set default values for all settings boolean verbose = readBooleanConfig(VERBOSE_CONF_KEY, true); boolean etag = readBooleanConfig(ETAG_CONF_KEY, false); boolean canonical = readBooleanConfig(CANONICAL_CONF_KEY, true); boolean indexURL = readBooleanConfig(INDEX_URL_KEY, true); boolean addToIndex = readBooleanConfig(ADD_TO_INDEX_CONF_KEY, false); String mimefilter = readStringConfig(MIME_CONF_KEY, "^text/.*"); boolean whitelist = readBooleanConfig(WHITELIST_CONF_KEY, false); String iteratorClassName = readStringConfig(ITERATOR_CONF_KEY, WarcIterator.class.getName()); // Parse command line options CommandLineParser clp = new CommandLineParser(args, new PrintWriter(System.out)); Option[] opts = clp.getCommandLineOptions(); for (int i = 0; i < opts.length; i++) { Option opt = opts[i]; switch (opt.getId()) { case 'w': whitelist = true; break; case 'a': addToIndex = true; break; case 'e': etag = true; break; case 'h': clp.usage(0); break; case 'i': iteratorClassName = opt.getValue(); break; case 'm': mimefilter = opt.getValue(); break; case 'u': indexURL = false; break; case 's': canonical = false; break; case 'v': verbose = true; break; } } if (!indexURL && canonical) { canonical = false; } List<String> cargs = clp.getCommandLineArguments(); if (cargs.size() != 2) { // Should be exactly two arguments. Source and target! clp.usage(0); } String source = cargs.get(0); String target = cargs.get(1); // Load the CrawlDataIterator CrawlDataIterator iterator = (CrawlDataIterator) Class.forName(iteratorClassName).newInstance(); // Print initial stuff System.out.println("Indexing: " + source); System.out.println(" - Index URL: " + indexURL); System.out.println(" - Mime filter: " + mimefilter + " (" + (whitelist ? "whitelist" : "blacklist") + ")"); System.out.println(" - Includes" + (canonical ? " <canonical URL>" : "") + (etag ? " <etag>" : "")); System.out.println(" - Iterator: " + iteratorClassName); System.out.println(" - " + iterator.getSourceType()); System.out.println("Target: " + target); if (addToIndex) { System.out.println(" - Add to existing index (if any)"); } else { System.out.println(" - New index (erases any existing index at " + "that location)"); } iterator.initialize(source); // Create the index long start = System.currentTimeMillis(); IndexBuilder di = new IndexBuilder(target, indexURL, canonical, etag, addToIndex); di.writeToIndex(iterator, mimefilter, !whitelist, verbose); // Clean-up di.close(); System.out.println("Total run time: " + DateUtils.formatMillisecondsToConventional(System.currentTimeMillis() - start)); }
From source file:com.stumbleupon.hbaseadmin.HBaseCompact.java
/** * Main entry point//from w w w .ja v a 2s. c o m * @param args command line arguments * @throws Exception */ public static void main(String[] args) throws Exception { CommandLineParser parser = new PosixParser(); CommandLine cmd = null; String hbaseSite = null; String jmxRemotePasswordFile = null; String jmxPort = null; Date startDate = null; Date endDate = null; int throttleFactor = 1; int numCycles = 1; int pauseInterval = DEFAULT_PAUSE_INTERVAL; int waitInterval = DEFAULT_WAIT_INTERVAL; int filesKeep = DEFAULT_FILES_KEEP; long regionCompactWaitTime = DEFAULT_REGION_COMPACT_WAIT_TIME; long maxStoreFileAge = 0; boolean excludeTables = false; String tableNamesString = ""; List<String> tableNames = new ArrayList<String>(); SimpleDateFormat sdf = new SimpleDateFormat("HH:mm"); // Parse command line options try { cmd = parser.parse(getOptions(), args); } catch (org.apache.commons.cli.ParseException e) { System.out.println(e.getMessage()); printOptions(); System.exit(-1); } for (Option option : cmd.getOptions()) { switch (option.getId()) { case 'c': hbaseSite = option.getValue(); break; case 'j': jmxRemotePasswordFile = option.getValue(); break; case 't': throttleFactor = Integer.parseInt(option.getValue()); break; case 'n': numCycles = Integer.parseInt(option.getValue()); break; case 'p': pauseInterval = Integer.parseInt(option.getValue()); break; case 'w': waitInterval = Integer.parseInt(option.getValue()); break; case 's': startDate = sdf.parse(option.getValue()); break; case 'e': endDate = sdf.parse(option.getValue()); break; case 'b': tableNamesString = option.getValue(); tableNames = Arrays.asList(option.getValue().split(",")); break; case 'f': filesKeep = Integer.parseInt(option.getValue()); break; case 'r': jmxPort = option.getValue(); break; case 'x': excludeTables = true; break; case 'm': regionCompactWaitTime = Long.parseLong(option.getValue()); break; case 'a': maxStoreFileAge = Long.parseLong(option.getValue()); break; default: throw new IllegalArgumentException("unexpected option " + option); } } LOG.info("Starting compactor"); LOG.info("--------------------------------------------------"); LOG.info("HBase site : {}", hbaseSite); LOG.info("RegionServer Jmx port : {}", jmxPort); LOG.info("Jmx password file : {}", jmxRemotePasswordFile); LOG.info("Compact interval : {}", pauseInterval); LOG.info("Check interval : {}", waitInterval); LOG.info("Throttle factor : {}", throttleFactor); LOG.info("Number of cycles : {}", numCycles); LOG.info("Off-peak start time : {}", Utils.dateString(startDate, "HH:mm")); LOG.info("Off-peak end time : {}", Utils.dateString(endDate, "HH:mm")); LOG.info("Minimum store files : {}", filesKeep); LOG.info("Table names : {}", tableNamesString); LOG.info("Exclude tables : {}", excludeTables); LOG.info("Region compact wait time: {}", regionCompactWaitTime); LOG.info("Max store file age : {}", maxStoreFileAge); LOG.info("--------------------------------------------------"); // Get command line options final Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path(hbaseSite)); HBaseCompact compact = new HBaseCompact(); ClusterUtils clusterUtils = new ClusterUtils(compact, regionCompactWaitTime); compact.setClusterUtils(clusterUtils); compact.setAdmin(new HBaseAdmin(conf)); compact.setSleepBetweenCompacts(pauseInterval); compact.setSleepBetweenChecks(waitInterval); compact.setThrottleFactor(throttleFactor); compact.setNumCycles(numCycles); compact.setStartDate(startDate); compact.setEndDate(endDate); compact.setNumStoreFiles(filesKeep); compact.setTableNames(tableNames); compact.setExcludeTables(excludeTables); compact.setMaxStoreFileAge(maxStoreFileAge); clusterUtils.setJmxPort(jmxPort); clusterUtils.setJmxPasswordFile(jmxRemotePasswordFile); compact.runCompactions(); }
From source file:asl.seedscan.SeedScan.java
public static void main(String args[]) { // Default locations of config and schema files File configFile = new File("config.xml"); File schemaFile = new File("schemas/SeedScanConfig.xsd"); boolean parseConfig = true; ArrayList<File> schemaFiles = new ArrayList<File>(); schemaFiles.add(schemaFile);/*from w w w.ja va2 s . c o m*/ // ==== Command Line Parsing ==== Options options = new Options(); Option opConfigFile = new Option("c", "config-file", true, "The config file to use for seedscan. XML format according to SeedScanConfig.xsd."); Option opSchemaFile = new Option("s", "schema-file", true, "The xsd schema file which should be used to verify the config file format. "); OptionGroup ogConfig = new OptionGroup(); ogConfig.addOption(opConfigFile); OptionGroup ogSchema = new OptionGroup(); ogConfig.addOption(opSchemaFile); options.addOptionGroup(ogConfig); options.addOptionGroup(ogSchema); PosixParser optParser = new PosixParser(); CommandLine cmdLine = null; try { cmdLine = optParser.parse(options, args, true); } catch (org.apache.commons.cli.ParseException e) { logger.error("Error while parsing command-line arguments."); System.exit(1); } Option opt; Iterator<?> iter = cmdLine.iterator(); while (iter.hasNext()) { opt = (Option) iter.next(); if (opt.getOpt().equals("c")) { configFile = new File(opt.getValue()); } else if (opt.getOpt().equals("s")) { schemaFile = new File(opt.getValue()); } } // ==== Configuration Read and Parse Actions ==== ConfigParser parser = new ConfigParser(schemaFiles); ConfigT config = parser.parseConfig(configFile); // Print out configuration file contents Formatter formatter = new Formatter(new StringBuilder(), Locale.US); // ===== CONFIG: LOCK FILE ===== File lockFile = new File(config.getLockfile()); logger.info("SeedScan lock file is '" + lockFile + "'"); LockFile lock = new LockFile(lockFile); if (!lock.acquire()) { logger.error("Could not acquire lock."); System.exit(1); } // ===== CONFIG: LOGGING ===== // MTH: This is now done in log4j.properties file // ===== CONFIG: DATABASE ===== MetricDatabase readDB = new MetricDatabase(config.getDatabase()); MetricDatabase writeDB = new MetricDatabase(config.getDatabase()); MetricReader reader = new MetricReader(readDB); MetricInjector injector = new MetricInjector(writeDB); // ===== CONFIG: SCANS ===== Hashtable<String, Scan> scans = new Hashtable<String, Scan>(); if (config.getScans().getScan() == null) { logger.error("No scans in configuration."); System.exit(1); } else { for (ScanT scanCfg : config.getScans().getScan()) { String name = scanCfg.getName(); if (scans.containsKey(name)) { logger.error("Duplicate scan name '" + name + "' encountered."); System.exit(1); } // This should really be handled by jaxb by setting it up in schemas/SeedScanConfig.xsd if (scanCfg.getStartDay() == null && scanCfg.getStartDate() == null) { logger.error( "== SeedScan Error: Must set EITHER cfg:start_day -OR- cfg:start_date in config.xml to start Scan!"); System.exit(1); } // Configure this Scan Scan scan = new Scan(scanCfg.getName()); scan.setPathPattern(scanCfg.getPath()); scan.setDatalessDir(scanCfg.getDatalessDir()); scan.setEventsDir(scanCfg.getEventsDir()); scan.setPlotsDir(scanCfg.getPlotsDir()); scan.setDaysToScan(scanCfg.getDaysToScan().intValue()); if (scanCfg.getStartDay() != null) { scan.setStartDay(scanCfg.getStartDay().intValue()); } if (scanCfg.getStartDate() != null) { scan.setStartDate(scanCfg.getStartDate().intValue()); } if (scanCfg.getNetworkSubset() != null) { logger.debug("Filter on Network Subset=[{}]", scanCfg.getNetworkSubset()); Filter filter = new Filter(false); for (String network : scanCfg.getNetworkSubset().split(",")) { logger.debug("Network =[{}]", network); filter.addFilter(network); } scan.setNetworks(filter); } if (scanCfg.getStationSubset() != null) { logger.debug("Filter on Station Subset=[{}]", scanCfg.getStationSubset()); Filter filter = new Filter(false); for (String station : scanCfg.getStationSubset().split(",")) { logger.debug("Station =[{}]", station); filter.addFilter(station); } scan.setStations(filter); } if (scanCfg.getLocationSubset() != null) { logger.debug("Filter on Location Subset=[{}]", scanCfg.getLocationSubset()); Filter filter = new Filter(false); for (String location : scanCfg.getLocationSubset().split(",")) { logger.debug("Location =[{}]", location); filter.addFilter(location); } scan.setLocations(filter); } if (scanCfg.getChannelSubset() != null) { logger.debug("Filter on Channel Subset=[{}]", scanCfg.getChannelSubset()); Filter filter = new Filter(false); for (String channel : scanCfg.getChannelSubset().split(",")) { logger.debug("Channel =[{}]", channel); filter.addFilter(channel); } scan.setChannels(filter); } for (MetricT met : scanCfg.getMetrics().getMetric()) { try { Class<?> metricClass = Class.forName(met.getClassName()); MetricWrapper wrapper = new MetricWrapper(metricClass); for (ArgumentT arg : met.getArgument()) { wrapper.add(arg.getName(), arg.getValue()); } scan.addMetric(wrapper); } catch (ClassNotFoundException ex) { logger.error("No such metric class '" + met.getClassName() + "'"); System.exit(1); } catch (InstantiationException ex) { logger.error("Could not dynamically instantiate class '" + met.getClassName() + "'"); System.exit(1); } catch (IllegalAccessException ex) { logger.error("Illegal access while loading class '" + met.getClassName() + "'"); System.exit(1); } catch (NoSuchFieldException ex) { logger.error("Invalid dynamic argument to Metric subclass '" + met.getClassName() + "'"); System.exit(1); } } scans.put(name, scan); } } // ==== Establish Database Connection ==== // TODO: State Tracking in the Database // - Record scan started in database. // - Track our progress as we go so a new process can pick up where // we left off if our process dies. // - Mark when each date-station-channel-operation is complete //LogDatabaseHandler logDB = new LogDatabaseHandler(configuration.get // For each day ((yesterday - scanDepth) to yesterday) // scan for these channel files, only process them if // they have not yet been scanned, or if changes have // occurred to the file since its last scan. Do this for // each scan type. Do not re-scan data for each type, // launch processes for each scan and use the same data set // for each. If we can pipe the data as it is read, do so. // If we need to push all of it at once, do these in sequence // in order to preserve overall system memory resources. Scan scan = null; // ==== Perform Scans ==== scan = scans.get("daily"); //MTH: This part could/should be moved up higher except that we need to know datalessDir, which, // at this point, is configured on a per scan basis ... so we need to know what scan we're doing MetaServer metaServer = null; if (config.getMetaserver() != null) { if (config.getMetaserver().getUseRemote().equals("yes") || config.getMetaserver().getUseRemote().equals("true")) { String remoteServer = config.getMetaserver().getRemoteUri(); try { metaServer = new MetaServer(new URI(remoteServer)); } catch (Exception e) { logger.error("caught URI exception:" + e.getMessage()); } } else { metaServer = new MetaServer(scan.getDatalessDir()); } } else { // Use local MetaServer metaServer = new MetaServer(scan.getDatalessDir()); } List<Station> stations = null; if (config.getStationList() == null) { // get StationList from MetaServer logger.info("Get StationList from MetaServer"); stations = metaServer.getStationList(); } else { // read StationList from config.xml logger.info("Read StationList from config.xml"); List<String> stationList = config.getStationList().getStation(); if (stationList.size() > 0) { stations = new ArrayList<Station>(); for (String station : stationList) { String[] words = station.split("_"); if (words.length != 2) { logger.warn(String.format("stationList: station=[%s] is NOT a valid station --> Skip", station)); } else { stations.add(new Station(words[0], words[1])); logger.info("config.xml: Read station:" + station); } } } else { logger.error("Error: No valid stations read from config.xml"); } } if (stations == null) { logger.error("Found NO stations to scan --> EXITTING SeedScan"); System.exit(1); } Thread readerThread = new Thread(reader); readerThread.start(); logger.info("Reader thread started."); Thread injectorThread = new Thread(injector); injectorThread.start(); logger.info("Injector thread started."); // Loop over scans and hand each one to a ScanManager logger.info("Hand scan to ScanManager"); for (String key : scans.keySet()) { scan = scans.get(key); logger.info(String.format("Scan=[%s] startDay=%d startDate=%d daysToScan=%d\n", key, scan.getStartDay(), scan.getStartDate(), scan.getDaysToScan())); ScanManager scanManager = new ScanManager(reader, injector, stations, scan, metaServer); } logger.info("ScanManager is [ FINISHED ] --> stop the injector and reader threads"); try { injector.halt(); logger.info("All stations processed. Waiting for injector thread to finish..."); synchronized (injectorThread) { //injectorThread.wait(); injectorThread.interrupt(); } logger.info("Injector thread halted."); } catch (InterruptedException ex) { logger.warn("The injector thread was interrupted while attempting to complete requests."); } try { reader.halt(); logger.info("All stations processed. Waiting for reader thread to finish..."); synchronized (readerThread) { //readerThread.wait(); readerThread.interrupt(); } logger.info("Reader thread halted."); } catch (InterruptedException ex) { logger.warn("The reader thread was interrupted while attempting to complete requests."); } try { lock.release(); } catch (IOException e) { ; } finally { logger.info("Release seedscan lock and quit metaServer"); lock = null; metaServer.quit(); } }
From source file:is.hi.bok.deduplicator.DigestIndexer.java
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) throws Exception {
CommandLineParser clp = new CommandLineParser(args, new PrintWriter(System.out));
long start = System.currentTimeMillis();
// Set default values for all settings.
boolean etag = false;
boolean equivalent = false;
boolean timestamp = false;
String indexMode = MODE_BOTH;
boolean addToIndex = false;
String mimefilter = "^text/.*";
boolean blacklist = true;
String iteratorClassName = CrawlLogIterator.class.getName();
String origin = null;/*w w w . j a v a 2 s. c o m*/
boolean skipDuplicates = false;
// Process the options
Option[] opts = clp.getCommandLineOptions();
for (int i = 0; i < opts.length; i++) {
Option opt = opts[i];
switch (opt.getId()) {
case 'w':
blacklist = false;
break;
case 'a':
addToIndex = true;
break;
case 'e':
etag = true;
break;
case 'h':
clp.usage(0);
break;
case 'i':
iteratorClassName = opt.getValue();
break;
case 'm':
mimefilter = opt.getValue();
break;
case 'o':
indexMode = opt.getValue();
break;
case 's':
equivalent = true;
break;
case 't':
timestamp = true;
break;
case 'r':
origin = opt.getValue();
break;
case 'd':
skipDuplicates = true;
break;
default:
System.err.println("Unhandled option id: " + opt.getId());
}
}
List cargs = clp.getCommandLineArguments();
if (cargs.size() != 2) {
// Should be exactly two arguments. Source and target!
clp.usage(0);
}
// Get the CrawlDataIterator
// Get the iterator classname or load default.
Class cl = Class.forName(iteratorClassName);
Constructor co = cl.getConstructor(new Class[] { String.class });
CrawlDataIterator iterator = (CrawlDataIterator) co.newInstance(new Object[] { (String) cargs.get(0) });
// Print initial stuff
System.out.println("Indexing: " + cargs.get(0));
System.out.println(" - Mode: " + indexMode);
System.out.println(" - Mime filter: " + mimefilter + " (" + (blacklist ? "blacklist" : "whitelist") + ")");
System.out.println(" - Includes" + (equivalent ? " <equivalent URL>" : "")
+ (timestamp ? " <timestamp>" : "") + (etag ? " <etag>" : ""));
System.out.println(" - Skip duplicates: " + (skipDuplicates ? "yes" : "no"));
System.out.println(" - Iterator: " + iteratorClassName);
System.out.println(" - " + iterator.getSourceType());
System.out.println("Target: " + cargs.get(1));
if (addToIndex) {
System.out.println(" - Add to existing index (if any)");
} else {
System.out.println(" - New index (erases any existing index at " + "that location)");
}
DigestIndexer di = new DigestIndexer((String) cargs.get(1), indexMode, equivalent, timestamp, etag,
addToIndex);
// Create the index
di.writeToIndex(iterator, mimefilter, blacklist, origin, true, skipDuplicates);
// Clean-up
di.close();
System.out.println("Total run time: "
+ ArchiveUtils.formatMillisecondsToConventional(System.currentTimeMillis() - start));
}