List of usage examples for org.apache.commons.cli OptionBuilder withArgName
public static OptionBuilder withArgName(String name)
From source file:de.prozesskraft.ptest.Fingerprint.java
public static void main(String[] args) throws org.apache.commons.cli.ParseException, IOException { // try/*w w w.ja v a 2s . co m*/ // { // if (args.length != 3) // { // System.out.println("Please specify processdefinition file (xml) and an outputfilename"); // } // // } // catch (ArrayIndexOutOfBoundsException e) // { // System.out.println("***ArrayIndexOutOfBoundsException: Please specify processdefinition.xml, openoffice_template.od*, newfile_for_processdefinitions.odt\n" + e.toString()); // } /*---------------------------- get options from ini-file ----------------------------*/ File inifile = new java.io.File( WhereAmI.getInstallDirectoryAbsolutePath(Fingerprint.class) + "/" + "../etc/ptest-fingerprint.ini"); if (inifile.exists()) { try { ini = new Ini(inifile); } catch (InvalidFileFormatException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } else { System.err.println("ini file does not exist: " + inifile.getAbsolutePath()); System.exit(1); } /*---------------------------- create boolean options ----------------------------*/ Option ohelp = new Option("help", "print this message"); Option ov = new Option("v", "prints version and build-date"); /*---------------------------- create argument options ----------------------------*/ Option opath = OptionBuilder.withArgName("PATH").hasArg() .withDescription( "[mandatory; default: .] the root path for the tree you want to make a fingerprint from.") // .isRequired() .create("path"); Option osizetol = OptionBuilder.withArgName("FLOAT").hasArg().withDescription( "[optional; default: 0.02] the sizeTolerance (as factor in percent) of all file entries will be set to this value. [0.0 < sizetol < 1.0]") // .isRequired() .create("sizetol"); Option omd5 = OptionBuilder.withArgName("no|yes").hasArg() .withDescription("[optional; default: yes] should be the md5sum of files determined? no|yes") // .isRequired() .create("md5"); Option oignore = OptionBuilder.withArgName("STRING").hasArgs() .withDescription("[optional] path-pattern that should be ignored when creating the fingerprint") // .isRequired() .create("ignore"); Option oignorefile = OptionBuilder.withArgName("FILE").hasArg().withDescription( "[optional] file with path-patterns (one per line) that should be ignored when creating the fingerprint") // .isRequired() .create("ignorefile"); Option ooutput = OptionBuilder.withArgName("FILE").hasArg() .withDescription("[mandatory; default: <path>/fingerprint.xml] fingerprint file") // .isRequired() .create("output"); Option of = new Option("f", "[optional] force overwrite fingerprint file if it already exists"); /*---------------------------- create options object ----------------------------*/ Options options = new Options(); options.addOption(ohelp); options.addOption(ov); options.addOption(opath); options.addOption(osizetol); options.addOption(omd5); options.addOption(oignore); options.addOption(oignorefile); options.addOption(ooutput); options.addOption(of); /*---------------------------- create the parser ----------------------------*/ CommandLineParser parser = new GnuParser(); try { // parse the command line arguments commandline = parser.parse(options, args); } catch (Exception exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); exiter(); } /*---------------------------- usage/help ----------------------------*/ if (commandline.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("fingerprint", options); System.exit(0); } else if (commandline.hasOption("v")) { System.out.println("web: " + web); System.out.println("author: " + author); System.out.println("version:" + version); System.out.println("date: " + date); System.exit(0); } /*---------------------------- ueberpruefen ob eine schlechte kombination von parametern angegeben wurde ----------------------------*/ String path = ""; String sizetol = ""; boolean md5 = false; Float sizetolFloat = null; String output = ""; java.io.File ignorefile = null; ArrayList<String> ignore = new ArrayList<String>(); if (!(commandline.hasOption("path"))) { System.err.println("setting default for -path=."); path = "."; } else { path = commandline.getOptionValue("path"); } if (!(commandline.hasOption("sizetol"))) { System.err.println("setting default for -sizetol=0.02"); sizetol = "0.02"; sizetolFloat = 0.02F; } else { sizetol = commandline.getOptionValue("sizetol"); sizetolFloat = Float.parseFloat(sizetol); if ((sizetolFloat > 1) || (sizetolFloat < 0)) { System.err.println("use only values >=0.0 and <1.0 for -sizetol"); System.exit(1); } } if (!(commandline.hasOption("md5"))) { System.err.println("setting default for -md5=yes"); md5 = true; } else if (commandline.getOptionValue("md5").equals("no")) { md5 = false; } else if (commandline.getOptionValue("md5").equals("yes")) { md5 = true; } else { System.err.println("use only values no|yes for -md5"); System.exit(1); } if (commandline.hasOption("ignore")) { ignore.addAll(Arrays.asList(commandline.getOptionValues("ignore"))); } if (commandline.hasOption("ignorefile")) { ignorefile = new java.io.File(commandline.getOptionValue("ignorefile")); if (!ignorefile.exists()) { System.err.println("warn: ignore file does not exist: " + ignorefile.getCanonicalPath()); } } if (!(commandline.hasOption("output"))) { System.err.println("setting default for -output=" + path + "/fingerprint.xml"); output = path + "/fingerprint.xml"; } else { output = commandline.getOptionValue("output"); } // wenn output bereits existiert -> abbruch java.io.File outputFile = new File(output); if (outputFile.exists()) { if (commandline.hasOption("f")) { outputFile.delete(); } else { System.err .println("error: output file (" + output + ") already exists. use -f to force overwrite."); System.exit(1); } } // if ( !( commandline.hasOption("output")) ) // { // System.err.println("option -output is mandatory."); // exiter(); // } /*---------------------------- die lizenz ueberpruefen und ggf abbrechen ----------------------------*/ // check for valid license ArrayList<String> allPortAtHost = new ArrayList<String>(); allPortAtHost.add(ini.get("license-server", "license-server-1")); allPortAtHost.add(ini.get("license-server", "license-server-2")); allPortAtHost.add(ini.get("license-server", "license-server-3")); MyLicense lic = new MyLicense(allPortAtHost, "1", "user-edition", "0.1"); // lizenz-logging ausgeben for (String actLine : (ArrayList<String>) lic.getLog()) { System.err.println(actLine); } // abbruch, wenn lizenz nicht valide if (!lic.isValid()) { System.exit(1); } /*---------------------------- die eigentliche business logic ----------------------------*/ Dir dir = new Dir(); dir.setBasepath(path); dir.setOutfilexml(output); // ignore file in ein Array lesen if ((ignorefile != null) && (ignorefile.exists())) { Scanner sc = new Scanner(ignorefile); while (sc.hasNextLine()) { ignore.add(sc.nextLine()); } sc.close(); } // // autoignore hinzufuegen // String autoIgnoreString = ini.get("autoignore", "autoignore"); // ignoreLines.addAll(Arrays.asList(autoIgnoreString.split(","))); // // debug // System.out.println("ignorefile content:"); // for(String actLine : ignore) // { // System.out.println("line: "+actLine); // } try { dir.genFingerprint(sizetolFloat, md5, ignore); } catch (NullPointerException e) { System.err.println("file/dir does not exist " + path); e.printStackTrace(); exiter(); } catch (IOException e) { e.printStackTrace(); exiter(); } System.out.println("writing to file: " + dir.getOutfilexml()); dir.writeXml(); }
From source file:com.cloudera.ByteCount.java
public static void main(String[] args) throws Exception { JobConf conf = new JobConf(new Configuration()); // Trim off the hadoop-specific args String[] remArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); // Pull in properties Options options = new Options(); Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator() .withDescription("use value for given property").create("D"); options.addOption(property);//from w w w .j ava 2 s. co m Option skipChecksums = new Option("skipChecksums", "skip checksums"); options.addOption(skipChecksums); Option profile = new Option("profile", "profile tasks"); options.addOption(profile); CommandLineParser parser = new BasicParser(); CommandLine line = parser.parse(options, remArgs); Properties properties = line.getOptionProperties("D"); for (Entry<Object, Object> prop : properties.entrySet()) { conf.set(prop.getKey().toString(), prop.getValue().toString()); System.out.println("Set config key " + prop.getKey() + " to " + prop.getValue()); } if (line.hasOption("skipChecksums")) { conf.setBoolean("bytecount.skipChecksums", true); System.out.println("Skipping checksums"); } if (line.hasOption("profile")) { conf.setBoolean("mapred.task.profile", true); conf.set("mapred.task.profile.params", "-agentlib:hprof=cpu=samples,depth=100,interval=1ms,lineno=y,thread=y,file=%s"); conf.set(MRJobConfig.NUM_MAP_PROFILES, "0"); conf.set("mapred.task.profile.maps", "1"); System.out.println("Profiling map tasks"); } // Get the positional arguments out remArgs = line.getArgs(); if (remArgs.length != 2) { System.err.println("Usage: ByteCount <inputBase> <outputBase>"); System.exit(1); } String inputBase = remArgs[0]; String outputBase = remArgs[1]; Job job = Job.getInstance(conf); job.setInputFormatClass(ByteBufferInputFormat.class); job.setMapOutputKeyClass(ByteWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setMapperClass(ByteCountMapper.class); job.setReducerClass(ByteCountReducer.class); job.setCombinerClass(ByteCountReducer.class); job.setOutputKeyClass(ByteWritable.class); job.setOutputValueClass(LongWritable.class); FileInputFormat.addInputPath(job, new Path(inputBase)); FileOutputFormat.setOutputPath(job, new Path(outputBase)); job.setJarByClass(ByteCount.class); boolean success = job.waitForCompletion(true); Counters counters = job.getCounters(); System.out.println("\tRead counters"); printCounter(counters, READ_COUNTER.BYTES_READ); printCounter(counters, READ_COUNTER.LOCAL_BYTES_READ); printCounter(counters, READ_COUNTER.SCR_BYTES_READ); printCounter(counters, READ_COUNTER.ZCR_BYTES_READ); System.exit(success ? 0 : 1); }
From source file:es.eucm.ead.exporter.ExporterMain.java
@SuppressWarnings("all") public static void main(String args[]) { Options options = new Options(); Option help = new Option("h", "help", false, "print this message"); Option quiet = new Option("q", "quiet", false, "be extra quiet"); Option verbose = new Option("v", "verbose", false, "be extra verbose"); Option legacy = OptionBuilder.withArgName("s> <t").hasArgs(3) .withDescription(//from ww w . ja v a 2 s .co m "source is a version 1.x game; must specify\n" + "<simplify> if 'true', simplifies result\n" + "<translate> if 'true', enables translation") .withLongOpt("import").create("i"); Option war = OptionBuilder.withArgName("web-base").hasArg() .withDescription("WAR packaging (web app); " + "must specify\n<web-base> the base WAR directory") .withLongOpt("war").create("w"); Option jar = OptionBuilder.withDescription("JAR packaging (desktop)").withLongOpt("jar").create("j"); Option apk = OptionBuilder.withArgName("props> <adk> <d").hasArgs(3) .withDescription("APK packaging (android); must specify \n" + "<props> (a properties file) \n" + "<adk> (location of the ADK to use) \n" + "<deploy> ('true' to install & deploy)") .withLongOpt("apk").create("a"); // EAD option Option ead = OptionBuilder.withDescription("EAD packaging (eAdventure)").withLongOpt("ead").create("e"); options.addOption(legacy); options.addOption(help); options.addOption(quiet); options.addOption(verbose); options.addOption(jar); options.addOption(war); options.addOption(apk); options.addOption(ead); CommandLineParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(options, args); } catch (ParseException pe) { System.err.println("Error parsing command-line: " + pe.getMessage()); showHelp(options); return; } // general options String[] extras = cmd.getArgs(); if (cmd.hasOption(help.getOpt()) || extras.length < 2) { showHelp(options); return; } if (cmd.hasOption(verbose.getOpt())) { verbosity = Verbose; } if (cmd.hasOption(quiet.getOpt())) { verbosity = Quiet; } // import String source = extras[0]; // optional import step if (cmd.hasOption(legacy.getOpt())) { String[] values = cmd.getOptionValues(legacy.getOpt()); AdventureConverter converter = new AdventureConverter(); if (values.length > 0 && values[0].equalsIgnoreCase("true")) { converter.setEnableSimplifications(true); } if (values.length > 1 && values[1].equalsIgnoreCase("true")) { converter.setEnableTranslations(true); } // set source for next steps to import-target source = converter.convert(source, null); } if (cmd.hasOption(jar.getOpt())) { if (checkFilesExist(cmd, options, source)) { JarExporter e = new JarExporter(); e.export(source, extras[1], verbosity.equals(Quiet) ? new QuietStream() : System.err); } } else if (cmd.hasOption(apk.getOpt())) { String[] values = cmd.getOptionValues(apk.getOpt()); if (checkFilesExist(cmd, options, values[0], values[1], source)) { AndroidExporter e = new AndroidExporter(); Properties props = new Properties(); File propsFile = new File(values[0]); try { props.load(new FileReader(propsFile)); props.setProperty(AndroidExporter.SDK_HOME, props.getProperty(AndroidExporter.SDK_HOME, values[1])); } catch (IOException ioe) { System.err.println("Could not load properties from " + propsFile.getAbsolutePath()); return; } e.export(source, extras[1], props, values.length > 2 && values[2].equalsIgnoreCase("true")); } } else if (cmd.hasOption(war.getOpt())) { if (checkFilesExist(cmd, options, extras[0])) { WarExporter e = new WarExporter(); e.setWarPath(cmd.getOptionValue(war.getOpt())); e.export(source, extras[1]); } } else if (cmd.hasOption(ead.getOpt())) { String destiny = extras[1]; if (!destiny.endsWith(".ead")) { destiny += ".ead"; } FileUtils.zip(new File(destiny), new File(source)); } else { showHelp(options); } }
From source file:de.burlov.amazon.s3.dirsync.CLI.java
/** * @param args//from ww w . ja v a 2 s.c o m */ @SuppressWarnings("static-access") public static void main(String[] args) { Logger.getLogger("").setLevel(Level.OFF); Logger deLogger = Logger.getLogger("de"); deLogger.setLevel(Level.INFO); Handler handler = new ConsoleHandler(); handler.setFormatter(new VerySimpleFormatter()); deLogger.addHandler(handler); deLogger.setUseParentHandlers(false); // if (true) // { // LogFactory.getLog(CLI.class).error("test msg", new Exception("test extception")); // return; // } Options opts = new Options(); OptionGroup gr = new OptionGroup(); /* * Befehlsgruppe initialisieren */ gr = new OptionGroup(); gr.setRequired(true); gr.addOption(OptionBuilder.withArgName("up|down").hasArg() .withDescription("Upload/Download changed or new files").create(CMD_UPDATE)); gr.addOption(OptionBuilder.withArgName("up|down").hasArg() .withDescription("Upload/Download directory snapshot").create(CMD_SNAPSHOT)); gr.addOption(OptionBuilder.withDescription("Delete remote folder").create(CMD_DELETE_DIR)); gr.addOption(OptionBuilder.withDescription("Delete a bucket").create(CMD_DELETE_BUCKET)); gr.addOption(OptionBuilder.create(CMD_HELP)); gr.addOption(OptionBuilder.create(CMD_VERSION)); gr.addOption(OptionBuilder.withDescription("Prints summary for stored data").create(CMD_SUMMARY)); gr.addOption(OptionBuilder.withDescription("Clean up orphaned objekts").create(CMD_CLEANUP)); gr.addOption(OptionBuilder.withDescription("Changes encryption password").withArgName("new password") .hasArg().create(CMD_CHANGE_PASSWORD)); gr.addOption(OptionBuilder.withDescription("Lists all buckets").create(CMD_LIST_BUCKETS)); gr.addOption(OptionBuilder.withDescription("Lists raw objects in a bucket").create(CMD_LIST_BUCKET)); gr.addOption(OptionBuilder.withDescription("Lists files in remote folder").create(CMD_LIST_DIR)); opts.addOptionGroup(gr); /* * Parametergruppe initialisieren */ opts.addOption(OptionBuilder.withArgName("key").isRequired(false).hasArg().withDescription("S3 access key") .create(OPT_S3S_KEY)); opts.addOption(OptionBuilder.withArgName("secret").isRequired(false).hasArg() .withDescription("Secret key for S3 account").create(OPT_S3S_SECRET)); opts.addOption(OptionBuilder.withArgName("bucket").isRequired(false).hasArg().withDescription( "Optional bucket name for storage. If not specified then an unique bucket name will be generated") .create(OPT_BUCKET)); // opts.addOption(OptionBuilder.withArgName("US|EU").hasArg(). // withDescription( // "Where the new bucket should be created. Default US").create( // OPT_LOCATION)); opts.addOption(OptionBuilder.withArgName("path").isRequired(false).hasArg() .withDescription("Local directory path").create(OPT_LOCAL_DIR)); opts.addOption(OptionBuilder.withArgName("name").isRequired(false).hasArg() .withDescription("Remote directory name").create(OPT_REMOTE_DIR)); opts.addOption(OptionBuilder.withArgName("password").isRequired(false).hasArg() .withDescription("Encryption password").create(OPT_ENC_PASSWORD)); opts.addOption(OptionBuilder.withArgName("patterns").hasArgs() .withDescription("Comma separated exclude file patterns like '*.tmp,*/dir/*.tmp'") .create(OPT_EXCLUDE_PATTERNS)); opts.addOption(OptionBuilder.withArgName("patterns").hasArgs().withDescription( "Comma separated include patterns like '*.java'. If not specified, then all files in specified local directory will be included") .create(OPT_INCLUDE_PATTERNS)); if (args.length == 0) { printUsage(opts); return; } CommandLine cmd = null; try { cmd = new GnuParser().parse(opts, args); if (cmd.hasOption(CMD_HELP)) { printUsage(opts); return; } if (cmd.hasOption(CMD_VERSION)) { System.out.println("s3dirsync version " + Version.CURRENT_VERSION); return; } String awsKey = cmd.getOptionValue(OPT_S3S_KEY); String awsSecret = cmd.getOptionValue(OPT_S3S_SECRET); String bucket = cmd.getOptionValue(OPT_BUCKET); String bucketLocation = cmd.getOptionValue(OPT_LOCATION); String localDir = cmd.getOptionValue(OPT_LOCAL_DIR); String remoteDir = cmd.getOptionValue(OPT_REMOTE_DIR); String password = cmd.getOptionValue(OPT_ENC_PASSWORD); String exclude = cmd.getOptionValue(OPT_EXCLUDE_PATTERNS); String include = cmd.getOptionValue(OPT_INCLUDE_PATTERNS); if (StringUtils.isBlank(awsKey) || StringUtils.isBlank(awsSecret)) { System.out.println("S3 account data required"); return; } if (StringUtils.isBlank(bucket)) { bucket = awsKey + ".dirsync"; } if (cmd.hasOption(CMD_DELETE_BUCKET)) { if (StringUtils.isBlank(bucket)) { System.out.println("Bucket name required"); return; } int deleted = S3Utils.deleteBucket(awsKey, awsSecret, bucket); System.out.println("Deleted objects: " + deleted); return; } if (cmd.hasOption(CMD_LIST_BUCKETS)) { for (String str : S3Utils.listBuckets(awsKey, awsSecret)) { System.out.println(str); } return; } if (cmd.hasOption(CMD_LIST_BUCKET)) { if (StringUtils.isBlank(bucket)) { System.out.println("Bucket name required"); return; } for (String str : S3Utils.listObjects(awsKey, awsSecret, bucket)) { System.out.println(str); } return; } if (StringUtils.isBlank(password)) { System.out.println("Encryption password required"); return; } char[] psw = password.toCharArray(); DirSync ds = new DirSync(awsKey, awsSecret, bucket, bucketLocation, psw); ds.setExcludePatterns(parseSubargumenths(exclude)); ds.setIncludePatterns(parseSubargumenths(include)); if (cmd.hasOption(CMD_SUMMARY)) { ds.printStorageSummary(); return; } if (StringUtils.isBlank(remoteDir)) { System.out.println("Remote directory name required"); return; } if (cmd.hasOption(CMD_DELETE_DIR)) { ds.deleteFolder(remoteDir); return; } if (cmd.hasOption(CMD_LIST_DIR)) { Folder folder = ds.getFolder(remoteDir); if (folder == null) { System.out.println("No such folder found: " + remoteDir); return; } for (Map.Entry<String, FileInfo> entry : folder.getIndexData().entrySet()) { System.out.println(entry.getKey() + " (" + FileUtils.byteCountToDisplaySize(entry.getValue().getLength()) + ")"); } return; } if (cmd.hasOption(CMD_CLEANUP)) { ds.cleanUp(); return; } if (cmd.hasOption(CMD_CHANGE_PASSWORD)) { String newPassword = cmd.getOptionValue(CMD_CHANGE_PASSWORD); if (StringUtils.isBlank(newPassword)) { System.out.println("new password required"); return; } char[] chars = newPassword.toCharArray(); ds.changePassword(chars); newPassword = null; Arrays.fill(chars, ' '); return; } if (StringUtils.isBlank(localDir)) { System.out.println(OPT_LOCAL_DIR + " argument required"); return; } String direction = ""; boolean up = false; boolean snapshot = false; if (StringUtils.isNotBlank(cmd.getOptionValue(CMD_UPDATE))) { direction = cmd.getOptionValue(CMD_UPDATE); } else if (StringUtils.isNotBlank(cmd.getOptionValue(CMD_SNAPSHOT))) { direction = cmd.getOptionValue(CMD_SNAPSHOT); snapshot = true; } if (StringUtils.isBlank(direction)) { System.out.println("Operation direction required"); return; } up = StringUtils.equalsIgnoreCase(OPT_UP, direction); File baseDir = new File(localDir); if (!baseDir.exists() && !baseDir.mkdirs()) { System.out.println("Invalid local directory: " + baseDir.getAbsolutePath()); return; } ds.syncFolder(baseDir, remoteDir, up, snapshot); } catch (DirSyncException e) { System.out.println(e.getMessage()); e.printStackTrace(); } catch (ParseException e) { System.out.println(e.getMessage()); printUsage(opts); } catch (Exception e) { e.printStackTrace(System.err); } }
From source file:cc.twittertools.index.IndexStatuses.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("dir").hasArg().withDescription("source collection directory") .create(COLLECTION_OPTION)); options.addOption(//from w w w.j a va2 s . co m OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(IndexStatuses.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); String indexPath = cmdline.getOptionValue(INDEX_OPTION); final FieldType textOptions = new FieldType(); textOptions.setIndexed(true); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) { textOptions.setStoreTermVectors(true); } LOG.info("collection: " + collectionPath); LOG.info("index: " + indexPath); LongOpenHashSet deletes = null; if (cmdline.hasOption(DELETES_OPTION)) { deletes = new LongOpenHashSet(); File deletesFile = new File(cmdline.getOptionValue(DELETES_OPTION)); if (!deletesFile.exists()) { System.err.println("Error: " + deletesFile + " does not exist!"); System.exit(-1); } LOG.info("Reading deletes from " + deletesFile); FileInputStream fin = new FileInputStream(deletesFile); byte[] ignoreBytes = new byte[2]; fin.read(ignoreBytes); // "B", "Z" bytes from commandline tools BufferedReader br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fin))); String s; while ((s = br.readLine()) != null) { if (s.contains("\t")) { deletes.add(Long.parseLong(s.split("\t")[0])); } else { deletes.add(Long.parseLong(s)); } } br.close(); fin.close(); LOG.info("Read " + deletes.size() + " tweetids from deletes file."); } long maxId = Long.MAX_VALUE; if (cmdline.hasOption(MAX_ID_OPTION)) { maxId = Long.parseLong(cmdline.getOptionValue(MAX_ID_OPTION)); LOG.info("index: " + maxId); } long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } StatusStream stream = new JsonStatusCorpusReader(file); Directory dir = FSDirectory.open(new File(indexPath)); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, IndexStatuses.ANALYZER); config.setOpenMode(OpenMode.CREATE); IndexWriter writer = new IndexWriter(dir, config); int cnt = 0; Status status; try { while ((status = stream.next()) != null) { if (status.getText() == null) { continue; } // Skip deletes tweetids. if (deletes != null && deletes.contains(status.getId())) { continue; } if (status.getId() > maxId) { continue; } cnt++; Document doc = new Document(); doc.add(new LongField(StatusField.ID.name, status.getId(), Field.Store.YES)); doc.add(new LongField(StatusField.EPOCH.name, status.getEpoch(), Field.Store.YES)); doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES)); doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions)); doc.add(new IntField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount(), Store.YES)); doc.add(new IntField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount(), Store.YES)); doc.add(new IntField(StatusField.STATUSES_COUNT.name, status.getStatusesCount(), Store.YES)); long inReplyToStatusId = status.getInReplyToStatusId(); if (inReplyToStatusId > 0) { doc.add(new LongField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId(), Field.Store.YES)); } String lang = status.getLang(); if (!lang.equals("unknown")) { doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES)); } long retweetStatusId = status.getRetweetedStatusId(); if (retweetStatusId > 0) { doc.add(new LongField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId(), Field.Store.YES)); doc.add(new IntField(StatusField.RETWEET_COUNT.name, status.getRetweetCount(), Store.YES)); if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) { LOG.warn("Error parsing retweet fields of " + status.getId()); } } writer.addDocument(doc); if (cnt % 100000 == 0) { LOG.info(cnt + " statuses indexed"); } } LOG.info(String.format("Total of %s statuses added", cnt)); if (cmdline.hasOption(OPTIMIZE_OPTION)) { LOG.info("Merging segments..."); writer.forceMerge(1); LOG.info("Done!"); } LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { writer.close(); dir.close(); stream.close(); } }
From source file:de.prozesskraft.ptest.Launch.java
public static void main(String[] args) throws org.apache.commons.cli.ParseException, IOException { // try/*from w ww . ja va2 s .c om*/ // { // if (args.length != 3) // { // System.out.println("Please specify processdefinition file (xml) and an outputfilename"); // } // // } // catch (ArrayIndexOutOfBoundsException e) // { // System.out.println("***ArrayIndexOutOfBoundsException: Please specify processdefinition.xml, openoffice_template.od*, newfile_for_processdefinitions.odt\n" + e.toString()); // } /*---------------------------- get options from ini-file ----------------------------*/ File inifile = new java.io.File( WhereAmI.getInstallDirectoryAbsolutePath(Launch.class) + "/" + "../etc/ptest-launch.ini"); if (inifile.exists()) { try { ini = new Ini(inifile); } catch (InvalidFileFormatException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } else { System.err.println("ini file does not exist: " + inifile.getAbsolutePath()); System.exit(1); } /*---------------------------- create boolean options ----------------------------*/ Option ohelp = new Option("help", "print this message"); Option ov = new Option("v", "prints version and build-date"); /*---------------------------- create argument options ----------------------------*/ Option ospl = OptionBuilder.withArgName("DIR").hasArg() .withDescription("[mandatory] directory with sample input data") // .isRequired() .create("spl"); Option oinstancedir = OptionBuilder.withArgName("DIR").hasArg() .withDescription("[mandatory, default: .] directory where the test will be performed") // .isRequired() .create("instancedir"); Option ocall = OptionBuilder.withArgName("FILE").hasArg() .withDescription("[mandatory, default: random call in spl-directory] file with call-string") // .isRequired() .create("call"); Option oaltapp = OptionBuilder.withArgName("STRING").hasArg() .withDescription( "[optional] alternative app. this String replaces the first line of the .call-file.") // .isRequired() .create("altapp"); Option oaddopt = OptionBuilder.withArgName("STRING").hasArg() .withDescription("[optional] add an option to the call.") // .isRequired() .create("addopt"); Option onolaunch = new Option("nolaunch", "only create instance directory, copy all spl files, but do NOT launch the process"); /*---------------------------- create options object ----------------------------*/ Options options = new Options(); options.addOption(ohelp); options.addOption(ov); options.addOption(ospl); options.addOption(oinstancedir); options.addOption(ocall); options.addOption(oaltapp); options.addOption(oaddopt); options.addOption(onolaunch); /*---------------------------- create the parser ----------------------------*/ CommandLineParser parser = new GnuParser(); try { // parse the command line arguments commandline = parser.parse(options, args); } catch (Exception exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); exiter(); } /*---------------------------- usage/help ----------------------------*/ if (commandline.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("launch", options); System.exit(0); } else if (commandline.hasOption("v")) { System.out.println("web: " + web); System.out.println("author: " + author); System.out.println("version:" + version); System.out.println("date: " + date); System.exit(0); } /*---------------------------- ueberpruefen ob eine schlechte kombination von parametern angegeben wurde ----------------------------*/ boolean error = false; String spl = null; String instancedir = null; String call = null; String altapp = null; ArrayList<String> addopt = new ArrayList<String>(); // spl initialisieren if (commandline.hasOption("spl")) { spl = commandline.getOptionValue("spl"); } else { System.err.println("option -spl is mandatory"); error = true; } // instancedir initialisieren if (commandline.hasOption("instancedir")) { instancedir = commandline.getOptionValue("instancedir"); } else { instancedir = System.getProperty("user.dir"); } // call initialisieren if (commandline.hasOption("call")) { call = commandline.getOptionValue("call"); } // altapp initialisieren if (commandline.hasOption("altapp")) { altapp = commandline.getOptionValue("altapp"); } // addopt initialisieren if (commandline.hasOption("addopt")) { for (String actString : commandline.getOptionValues("addopt")) { addopt.add(actString); } } // wenn fehler, dann exit if (error) { exiter(); } /*---------------------------- die lizenz ueberpruefen und ggf abbrechen ----------------------------*/ // check for valid license ArrayList<String> allPortAtHost = new ArrayList<String>(); allPortAtHost.add(ini.get("license-server", "license-server-1")); allPortAtHost.add(ini.get("license-server", "license-server-2")); allPortAtHost.add(ini.get("license-server", "license-server-3")); MyLicense lic = new MyLicense(allPortAtHost, "1", "user-edition", "0.1"); // lizenz-logging ausgeben for (String actLine : (ArrayList<String>) lic.getLog()) { System.err.println(actLine); } // abbruch, wenn lizenz nicht valide if (!lic.isValid()) { System.exit(1); } /*---------------------------- die eigentliche business logic ----------------------------*/ // das erste spl-objekt geben lassen Spl actSpl = new Splset(spl).getSpl().get(0); // den call, result und altapp ueberschreiben actSpl.setName("default"); if (call != null) { actSpl.setCall(new java.io.File(call)); } if (actSpl.getCall() == null) { System.err.println("error: no call information found"); System.exit(1); } if (altapp != null) { actSpl.setAltapp(altapp); } if (addopt.size() > 0) { actSpl.setAddopt(addopt); } actSpl.setResult(null); // das instancedir erstellen java.io.File actSplInstanceDir = new java.io.File(instancedir); System.err.println("info: creating directory " + actSplInstanceDir.getCanonicalPath()); actSplInstanceDir.mkdirs(); // Inputdaten in das InstanceDir exportieren actSpl.exportInput(actSplInstanceDir); // exit, wenn --nolaunch if (commandline.hasOption("nolaunch")) { System.err.println("info: exiting, because of -nolaunch"); System.exit(0); } // das logfile des Syscalls (zum debuggen des programms "process syscall" gedacht) String AbsLogSyscallWrapper = actSplInstanceDir.getCanonicalPath() + "/.log"; String AbsStdout = actSplInstanceDir.getCanonicalPath() + "/.stdout.txt"; String AbsStderr = actSplInstanceDir.getCanonicalPath() + "/.stderr.txt"; String AbsPid = actSplInstanceDir.getCanonicalPath() + "/.pid"; // beim starten von syscall werden parameter mit whitespaces an diesen auseinandergeschnitten und der nachfolgende aufruf schlaeft fehl // deshalb sollen whitespaces durch eine 'zeichensequenz' ersetzt werden // syscall ersetzt die zeichensequenz wieder zurueck in ein " " ArrayList<String> callFuerSyscall = actSpl.getCallAsArrayList(); ArrayList<String> callFuerSyscallMitTrennzeichen = new ArrayList<String>(); for (String actString : callFuerSyscall) { callFuerSyscallMitTrennzeichen.add(actString.replaceAll("\\s+", "%WHITESPACE%")); } try { // den Aufrufstring fuer die externe App (process syscall --version 0.6.0)) splitten // beim aufruf muss das erste argument im path zu finden sein, sonst gibt die fehlermeldung 'no such file or directory' ArrayList<String> processSyscallWithArgs = new ArrayList<String>( Arrays.asList(ini.get("apps", "pkraft-syscall").split(" "))); // die sonstigen argumente hinzufuegen processSyscallWithArgs.add("-call"); processSyscallWithArgs.add(String.join(" ", callFuerSyscallMitTrennzeichen)); // processSyscallWithArgs.add("\""+call+"\""); processSyscallWithArgs.add("-stdout"); processSyscallWithArgs.add(AbsStdout); processSyscallWithArgs.add("-stderr"); processSyscallWithArgs.add(AbsStderr); processSyscallWithArgs.add("-pid"); processSyscallWithArgs.add(AbsPid); processSyscallWithArgs.add("-mylog"); processSyscallWithArgs.add(AbsLogSyscallWrapper); processSyscallWithArgs.add("-maxrun"); processSyscallWithArgs.add("" + 3000); // erstellen prozessbuilder ProcessBuilder pb = new ProcessBuilder(processSyscallWithArgs); // erweitern des PATHs um den prozesseigenen path // Map<String,String> env = pb.environment(); // String path = env.get("PATH"); // log("debug", "$PATH="+path); // path = this.parent.getAbsPath()+":"+path; // env.put("PATH", path); // log("info", "path: "+path); // setzen der aktuellen directory (in der syscall ausgefuehrt werden soll) java.io.File directory = new java.io.File(instancedir); System.err.println("info: setting execution directory to: " + directory.getCanonicalPath()); pb.directory(directory); // zum debuggen ein paar ausgaben // java.lang.Process p1 = Runtime.getRuntime().exec("date >> ~/tmp.debug.work.txt"); // p1.waitFor(); // java.lang.Process p2 = Runtime.getRuntime().exec("ls -la "+this.getParent().getAbsdir()+" >> ~/tmp.debug.work.txt"); // p2.waitFor(); // java.lang.Process pro = Runtime.getRuntime().exec("nautilus"); // java.lang.Process superpro = Runtime.getRuntime().exec(processSyscallWithArgs.toArray(new String[processSyscallWithArgs.size()])); // p3.waitFor(); System.err.println("info: calling: " + pb.command()); // starten des prozesses java.lang.Process sysproc = pb.start(); // einfangen der stdout- und stderr des subprozesses InputStream is_stdout = sysproc.getInputStream(); InputStream is_stderr = sysproc.getErrorStream(); // Send your InputStream to an InputStreamReader: InputStreamReader isr_stdout = new InputStreamReader(is_stdout); InputStreamReader isr_stderr = new InputStreamReader(is_stderr); // That needs to go to a BufferedReader: BufferedReader br_stdout = new BufferedReader(isr_stdout); BufferedReader br_stderr = new BufferedReader(isr_stderr); // // oeffnen der OutputStreams zu den Ausgabedateien // FileWriter fw_stdout = new FileWriter(sStdout); // FileWriter fw_stderr = new FileWriter(sStderr); // zeilenweise in die files schreiben String line_out = new String(); String line_err = new String(); while (br_stdout.readLine() != null) { } // while (((line_out = br_stdout.readLine()) != null) || ((line_err = br_stderr.readLine()) != null)) // { // if (!(line_out == null)) // { // System.out.println(line_out); // System.out.flush(); // } // if (!(line_err == null)) // { // System.err.println(line_err); // System.err.flush(); // } // } int exitValue = sysproc.waitFor(); // fw_stdout.close(); // fw_stderr.close(); System.err.println("exitvalue: " + exitValue); sysproc.destroy(); System.exit(exitValue); // alternativer aufruf // java.lang.Process sysproc = Runtime.getRuntime().exec(StringUtils.join(args_for_syscall, " ")); // log("info", "call executed. pid="+sysproc.hashCode()); // wait 2 seconds for becoming the pid-file visible // Thread.sleep(2000); // int exitValue = sysproc.waitFor(); // // der prozess soll bis laengstens // if(exitValue != 0) // { // System.err.println("error: call returned a value indicating an error: "+exitValue); // } // else // { // System.err.println("info: call returned value: "+exitValue); // } // System.err.println("info: "+new Date().toString()); // System.err.println("info: bye"); // // sysproc.destroy(); // // System.exit(sysproc.exitValue()); } catch (Exception e2) { System.err.println("error: " + e2.getMessage()); System.exit(1); } }
From source file:com.ligadata.EncryptUtils.GenerateKeys.java
public static void main(String[] args) { CommandLine commandLine;/*w w w. j a v a 2 s .c om*/ if (args.length == 0) { PrintUsage(); System.exit(-1); } Option o_help = new Option("help", "The help option"); Option o_generateSampleKeys = new Option("generateSampleKeys", "a flag to generate sample keys"); Option o_algorithm = OptionBuilder.withArgName("algorithm").hasArg() .withDescription("The encryption algorithm").create("algorithm"); Option o_password = OptionBuilder.withArgName("password").hasArg().withDescription("ascii password") .create("password"); Option o_publicKeyFile = OptionBuilder.withArgName("publicKeyFile").hasArg() .withDescription("File containing public key").create("publicKeyFile"); Option o_privateKeyFile = OptionBuilder.withArgName("privateKeyFile").hasArg() .withDescription("File containing private key").create("privateKeyFile"); Options options = new Options(); CommandLineParser parser = new GnuParser(); options.addOption(o_help); options.addOption(o_generateSampleKeys); options.addOption(o_algorithm); options.addOption(o_password); options.addOption(o_publicKeyFile); options.addOption(o_privateKeyFile); try { commandLine = parser.parse(options, args); if (commandLine.hasOption("algorithm")) { logger.info("Option algorithm is present. The value is: "); algorithm = commandLine.getOptionValue("algorithm"); logger.info(algorithm); } if (commandLine.hasOption("password")) { logger.info("Option password is present. The value is: "); password = commandLine.getOptionValue("password"); logger.info(password); } if (commandLine.hasOption("publicKeyFile")) { logger.info("Option publicKeyFile is present. The value is: "); publicKeyFile = commandLine.getOptionValue("publicKeyFile"); logger.info(publicKeyFile); } if (commandLine.hasOption("privateKeyFile")) { logger.info("Option privateKeyFile is present. The value is: "); privateKeyFile = commandLine.getOptionValue("privateKeyFile"); logger.info(privateKeyFile); } if (commandLine.hasOption("help")) { logger.info("Option help is present. This is a flag option."); help = true; } if (commandLine.hasOption("generateSampleKeys")) { logger.info("Option generateSampleKeys is present. This is a flag option."); generateSampleKeys = true; } String[] remainder = commandLine.getArgs(); for (String argument : remainder) { logger.info(argument); logger.info(" "); } int rc = generateKeys(); System.exit(rc); } catch (ParseException exception) { System.out.print("Parse error: "); System.out.println(exception.getMessage()); System.exit(-2); } }
From source file:gobblin.test.TestWorker.java
@SuppressWarnings("all") public static void main(String[] args) throws Exception { // Build command-line options Option configOption = OptionBuilder.withArgName("framework config file") .withDescription("Configuration properties file for the framework").hasArgs().withLongOpt("config") .create('c'); Option jobConfigsOption = OptionBuilder.withArgName("job config files") .withDescription("Comma-separated list of job configuration files").hasArgs() .withLongOpt("jobconfigs").create('j'); Option modeOption = OptionBuilder.withArgName("run mode") .withDescription("Test mode (schedule|run); 'schedule' means scheduling the jobs, " + "whereas 'run' means running the jobs immediately") .hasArg().withLongOpt("mode").create('m'); Option helpOption = OptionBuilder.withArgName("help").withDescription("Display usage information") .withLongOpt("help").create('h'); Options options = new Options(); options.addOption(configOption);/*from w w w.j a va 2s.c o m*/ options.addOption(jobConfigsOption); options.addOption(modeOption); options.addOption(helpOption); // Parse command-line options CommandLineParser parser = new BasicParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption('h')) { printUsage(options); System.exit(0); } // Start the test worker with the given configuration properties Configuration config = new PropertiesConfiguration(cmd.getOptionValue('c')); Properties properties = ConfigurationConverter.getProperties(config); TestWorker testWorker = new TestWorker(properties); testWorker.start(); // Job running mode Mode mode = Mode.valueOf(cmd.getOptionValue('m').toUpperCase()); // Get the list of job configuration files List<String> jobConfigFiles = Lists .newArrayList(Splitter.on(',').omitEmptyStrings().trimResults().split(cmd.getOptionValue('j'))); CountDownLatch latch = new CountDownLatch(jobConfigFiles.size()); for (String jobConfigFile : jobConfigFiles) { // For each job, load the job configuration, then run or schedule the job. Properties jobProps = new Properties(); jobProps.load(new FileReader(jobConfigFile)); jobProps.putAll(properties); testWorker.runJob(jobProps, mode, new TestJobListener(latch)); } // Wait for all jobs to finish latch.await(); testWorker.stop(); }
From source file:io.anserini.index.IndexTweetsUpdatePlace.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("collection").hasArg() .withDescription("source collection directory").create(COLLECTION_OPTION)); options.addOption(// w w w .ja v a 2s. c o m OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(IndexTweetsUpdatePlace.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); String indexPath = cmdline.getOptionValue(INDEX_OPTION); System.out.println(collectionPath + " " + indexPath); LOG.info("collection: " + collectionPath); LOG.info("index: " + indexPath); long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) { textOptions.setStoreTermVectors(true); } final StatusStream stream = new JsonStatusCorpusReader(file); final Directory dir = new SimpleFSDirectory(Paths.get(cmdline.getOptionValue(INDEX_OPTION))); final IndexWriterConfig config = new IndexWriterConfig(ANALYZER); config.setOpenMode(IndexWriterConfig.OpenMode.APPEND); final IndexWriter writer = new IndexWriter(dir, config); System.out.print("Original # of docs " + writer.numDocs()); int updateCount = 0; Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { stream.close(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } try { writer.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { dir.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("Shutting down"); } }); int cnt = 0; Status status; try { while ((status = stream.next()) != null) { if (status.getPlace() != null) { // Query q = NumericRangeQuery.newLongRange(TweetStreamReader.StatusField.ID.name, status.getId(), // status.getId(), true, true); // System.out.print("Deleting docCount="+writer.numDocs()); // writer.deleteDocuments(q); // writer.commit(); // System.out.print(" Deleted docCount="+writer.numDocs()); Document doc = new Document(); doc.add(new LongField(StatusField.ID.name, status.getId(), Field.Store.YES)); doc.add(new LongField(StatusField.EPOCH.name, status.getEpoch(), Field.Store.YES)); doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES)); doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions)); doc.add(new IntField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount(), Store.YES)); doc.add(new IntField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount(), Store.YES)); doc.add(new IntField(StatusField.STATUSES_COUNT.name, status.getStatusesCount(), Store.YES)); doc.add(new DoubleField(StatusField.LONGITUDE.name, status.getLongitude(), Store.YES)); doc.add(new DoubleField(StatusField.LATITUDE.name, status.getlatitude(), Store.YES)); doc.add(new StringField(StatusField.PLACE.name, status.getPlace(), Store.YES)); long inReplyToStatusId = status.getInReplyToStatusId(); if (inReplyToStatusId > 0) { doc.add(new LongField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId(), Field.Store.YES)); } String lang = status.getLang(); if (!lang.equals("unknown")) { doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES)); } long retweetStatusId = status.getRetweetedStatusId(); if (retweetStatusId > 0) { doc.add(new LongField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId(), Field.Store.YES)); doc.add(new IntField(StatusField.RETWEET_COUNT.name, status.getRetweetCount(), Store.YES)); if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) { LOG.warn("Error parsing retweet fields of " + status.getId()); } } long id = status.getId(); BytesRefBuilder brb = new BytesRefBuilder(); NumericUtils.longToPrefixCodedBytes(id, 0, brb); Term term = new Term(StatusField.ID.name, brb.get()); writer.updateDocument(term, doc); // writer.addDocument(doc); updateCount += 1; if (updateCount % 10000 == 0) { LOG.info(updateCount + " statuses updated"); writer.commit(); System.out.println("Updated docCount=" + writer.numDocs()); } } } LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { writer.close(); dir.close(); stream.close(); } }
From source file:de.prozesskraft.pkraft.Manager.java
public static void main(String[] args) throws org.apache.commons.cli.ParseException, CloneNotSupportedException { // try/*from w ww . ja va 2 s. c o m*/ // { // if (args.length != 1) // { // System.out.println("Please specify Inputfile and Outputfile (prozessinstanz.lri)"); // } // // } // catch (ArrayIndexOutOfBoundsException e) // { // System.out.println("***ArrayIndexOutOfBoundsException: Please specify procesdefinition.lrd and processinstance.lri\n" + e.toString()); // } /*---------------------------- get options from ini-file ----------------------------*/ File inifile = new java.io.File( WhereAmI.getInstallDirectoryAbsolutePath(Manager.class) + "/" + "../etc/pkraft-manager.ini"); if (inifile.exists()) { try { ini = new Ini(inifile); } catch (InvalidFileFormatException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } else { System.err.println("ini file does not exist: " + inifile.getAbsolutePath()); System.exit(1); exit = true; } /*---------------------------- create boolean options ----------------------------*/ Option help = new Option("help", "print this message"); Option v = new Option("v", "prints version and build-date"); /*---------------------------- create argument options ----------------------------*/ Option instance = OptionBuilder.withArgName("instance").hasArg() .withDescription("[mandatory] process instance file") // .isRequired() .create("instance"); Option stop = OptionBuilder.withArgName("stop") // .hasArg() .withDescription("[optional] stops a running manager for given instance") // .isRequired() .create("stop"); Option kill = OptionBuilder.withArgName("kill") // .hasArg() .withDescription("[optional] kills all applications that have been started by steps") // .isRequired() .create("kill"); /*---------------------------- create options object ----------------------------*/ Options options = new Options(); options.addOption(help); options.addOption(v); options.addOption(instance); options.addOption(stop); options.addOption(kill); /*---------------------------- create the parser ----------------------------*/ CommandLineParser parser = new GnuParser(); try { // parse the command line arguments line = parser.parse(options, args); } // catch ( ParseException exp ) catch (Exception exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); exiter(); } /*---------------------------- usage/help ----------------------------*/ if (line.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("manager", options); exit = true; System.exit(0); } else if (line.hasOption("v")) { System.out.println("author: info@prozesskraft.de"); System.out.println("version: [% version %]"); System.out.println("date: [% date %]"); exit = true; System.exit(0); } else if (!(line.hasOption("instance"))) { exiter(); } /*---------------------------- die lizenz ueberpruefen und ggf abbrechen ----------------------------*/ // check for valid license ArrayList<String> allPortAtHost = new ArrayList<String>(); allPortAtHost.add(ini.get("license-server", "license-server-1")); allPortAtHost.add(ini.get("license-server", "license-server-2")); allPortAtHost.add(ini.get("license-server", "license-server-3")); MyLicense lic = new MyLicense(allPortAtHost, "1", "user-edition", "0.1"); // lizenz-logging ausgeben for (String actLine : (ArrayList<String>) lic.getLog()) { System.err.println(actLine); } // abbruch, wenn lizenz nicht valide if (!lic.isValid()) { exit = true; System.exit(1); } /*---------------------------- business logic ----------------------------*/ Process actualProcess = null; try { Process p1 = new Process(); // die dauer des loops festlegen. Dies soll kein standardwert sein, da sonst bei vielen subprozessen die Prozessorlast stark oszilliert // zwischen 12 und 17 sekunden // Random rand = new Random(System.currentTimeMillis()); // int loop_period_seconds = rand.nextInt((17 - 12) + 1) + 12; // System.err.println("loop period is randomly set to: "+loop_period_seconds); fileBinary = new java.io.File(line.getOptionValue("instance")); String pathBinary = ""; if (fileBinary.exists()) { pathBinary = fileBinary.getAbsolutePath(); System.err.println("file does exist: " + pathBinary); } else { System.err.println("file does not exist: " + fileBinary.getAbsolutePath()); exiter(); } if (line.hasOption("stop") || line.hasOption("kill")) { p1.setInfilebinary(pathBinary); Process p2 = p1.readBinary(); p2.log("debug", "setting new manager-Id (0) to signal actual manager (" + p2.getManagerid() + ") that he is no longer in charge "); System.err.println("info: stopping instance"); System.err.println("debug: setting new manager-Id (0) to signal actual manager (" + p2.getManagerid() + ") that he is no longer in charge "); p2.setManagerid(0); p2.run = false; p2.setOutfilebinary(pathBinary); p2.writeBinary(); if (line.hasOption("kill")) { System.err.println("info: killing all steps of instance"); String returnStringOfKills = p2.kill(); System.err.println("info: killing returns: " + returnStringOfKills); } boolean pradar = (!(p2.isWrapper())); // pradar checkout if (pradar) { pradarAttend(p2.getRootdir() + "/process.pmb"); // pradarCheckout(p2.getId(), p2.getName(), "0"); } exit = true; System.exit(0); } startZyklischerThread(0); // prozessinstanz einlesen p1.setInfilebinary(pathBinary); managerid = p1.genManagerid(); Process p2; p2 = p1.readBinary(); // beim aufruf des programms wird erstmal die instanz occupiert p2.setManagerid(managerid); System.err.println("debug: manager " + managerid + ": occupying instance."); p2.log("info", "manager " + managerid + ": occupying instance."); p2.log("debug", "manager " + managerid + ": setting new manager-id to signal other running managers that they are not longer needed."); p2.log("debug", "manager " + managerid + ": setting binary file for input to: " + pathBinary); // System.out.println("setting binary file for input to: "+line.getOptionValue("instance")); p2.log("debug", "manager " + managerid + ": reading binary file: " + pathBinary); p2.setInfilebinary(pathBinary); p2.setOutfilebinary(pathBinary); p2.log("debug", "manager " + managerid + ": setting binary file for output: " + pathBinary); // instanz auf platte schreiben (um anderen managern zu signalisieren, dass sie nicht mehr gebraucht werden // System.out.println("setting manager-id to: "+managerid); p2.log("debug", "manager " + managerid + ": writing process to binary file to occupy instance."); // wenn es kein wrapper-prozess ist, dann soll die komunikation mit pradar vom manager uebernommen werden boolean pradar = (!(p2.isWrapper())); System.err.println("debug: setting instance to run"); p2.run = true; // pradar checkin if (pradar && p2.run && p2.touchInMillis == 0) { pradarAttend(p2.getRootdir() + "/process.pmb"); // p2.log("debug", "pradar-checkin id="+p2.getId()+", process="+p2.getName()+", processversion="+p2.getVersion()+", id2="+p2.getId2()+", parentid="+p2.getParentid()+", resource="+p2.getRootdir()+"/process.pmb"); // pradarCheckin(p2.getId(), p2.getName(), p2.getVersion(), p2.getId2(), p2.getParentid(), getPid(), p2.getRootdir()+"/process.pmb"); } System.err.println("debug: writing binary"); p2.writeBinary(); // process weiter schubsen pushProcessAsFarAsPossible(pathBinary, false); // try // { // // der thread soll so lange schlafen, wie die periode lang ist. die schlafdauer wird mit der anzahl multipliziert, wie oft das loadAverage zu hoch war (max 5) // int faktorForPeriod = Math.min(10, p2.counterLoadAverageTooHigh + 1); // // int secondsToSleep = loop_period_seconds * faktorForPeriod; // System.err.println("debug: sleeping for " + secondsToSleep + " seconds"); // // int millisecondsToSleep = secondsToSleep*1000; // System.err.println("debug: sleeping for " + millisecondsToSleep + " milliseconds"); // // Thread.sleep(millisecondsToSleep); // } // catch (InterruptedException e) // { // // TODO Auto-generated catch block // e.printStackTrace(); // // // ausgabe in das debugLogFile // exiterException(actualProcess.getOutfilebinary(), e); // } } catch (Exception e) { if (actualProcess != null) { actualProcess.log("fatal", e.getMessage() + "\n" + Arrays.toString(e.getStackTrace())); updateFile(actualProcess); e.printStackTrace(); // ausgabe in das debugLogFile exiterException(actualProcess.getOutfilebinary(), e); } exit = true; System.exit(10); } }