List of usage examples for org.apache.commons.cli OptionBuilder withArgName
public static OptionBuilder withArgName(String name)
From source file:edu.umd.cloud9.collection.wikipedia.BuildWikipediaDocnoMapping.java
@SuppressWarnings("static-access") @Override//from w w w . jav a2s . c om public int run(String[] args) throws Exception { Options options = new Options(); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("XML dump file").create(INPUT_OPTION)); options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("tmp output directory") .create(OUTPUT_PATH_OPTION)); options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output file") .create(OUTPUT_FILE_OPTION)); options.addOption(KEEP_ALL_OPTION, false, "keep all pages"); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(OUTPUT_PATH_OPTION) || !cmdline.hasOption(OUTPUT_FILE_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String inputPath = cmdline.getOptionValue(INPUT_OPTION); String outputPath = cmdline.getOptionValue(OUTPUT_PATH_OPTION); String outputFile = cmdline.getOptionValue(OUTPUT_FILE_OPTION); boolean keepAll = cmdline.hasOption(KEEP_ALL_OPTION); LOG.info("Tool name: " + this.getClass().getName()); LOG.info(" - input: " + inputPath); LOG.info(" - output path: " + outputPath); LOG.info(" - output file: " + outputFile); LOG.info(" - keep all pages: " + keepAll); JobConf conf = new JobConf(getConf(), BuildWikipediaDocnoMapping.class); conf.setJobName(String.format("BuildWikipediaDocnoMapping[%s: %s, %s: %s]", INPUT_OPTION, inputPath, OUTPUT_FILE_OPTION, outputFile)); conf.setBoolean(KEEP_ALL_OPTION, keepAll); conf.setNumReduceTasks(1); FileInputFormat.setInputPaths(conf, new Path(inputPath)); FileOutputFormat.setOutputPath(conf, new Path(outputPath)); FileOutputFormat.setCompressOutput(conf, false); conf.setInputFormat(WikipediaPageInputFormat.class); conf.setOutputKeyClass(IntWritable.class); conf.setOutputValueClass(IntWritable.class); conf.setOutputFormat(TextOutputFormat.class); conf.setMapperClass(MyMapper.class); conf.setReducerClass(MyReducer.class); // Delete the output directory if it exists already. FileSystem.get(conf).delete(new Path(outputPath), true); RunningJob job = JobClient.runJob(conf); Counters c = job.getCounters(); long cnt = keepAll ? c.getCounter(PageTypes.TOTAL) : c.getCounter(PageTypes.ARTICLE); WikipediaDocnoMapping.writeDocnoMappingData(outputPath + "/part-00000", (int) cnt, outputFile); return 0; }
From source file:com.aravind.flazr.android.rtmp.client.ClientOptions.java
protected static Options getCliOptions() { final Options options = new Options(); options.addOption(new Option("help", "print this message")); options.addOption(OptionBuilder.withArgName("host").hasArg().withDescription("host name").create("host")); options.addOption(OptionBuilder.withArgName("port").hasArg().withDescription("port number").create("port")); options.addOption(OptionBuilder.withArgName("app").hasArg().withDescription("app name").create("app")); options.addOption(OptionBuilder.withArgName("start").hasArg() .withDescription("start position (milliseconds)").create("start")); options.addOption(OptionBuilder.withArgName("length").hasArg().withDescription("length (milliseconds)") .create("length")); options.addOption(OptionBuilder.withArgName("buffer").hasArg() .withDescription("buffer duration (milliseconds)").create("buffer")); options.addOption(new Option("rtmpe", "use RTMPE (encryption)")); options.addOption(new Option("live", "publish local file to server in 'live' mode")); options.addOption(new Option("record", "publish local file to server in 'record' mode")); options.addOption(new Option("append", "publish local file to server in 'append' mode")); options.addOption(OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator() .withDescription("add / override connection param").create("D")); options.addOption(OptionBuilder.withArgName("swf").hasArg() .withDescription("path to (decompressed) SWF for verification").create("swf")); options.addOption(OptionBuilder.withArgName("version").hasArg() .withDescription("client version to use in RTMP handshake (hex)").create("version")); options.addOption(OptionBuilder.withArgName("load").hasArg() .withDescription("no. of client connections (load testing)").create("load")); options.addOption(OptionBuilder.withArgName("loop").hasArg().withDescription("for publish mode, loop count") .create("loop")); options.addOption(OptionBuilder.withArgName("threads").hasArg() .withDescription("for load testing (load) mode, thread pool size").create("threads")); options.addOption(new Option("file", "spawn connections listed in file (load testing)")); return options; }
From source file:de.fischer.thotti.core.runner.NDRunner.java
private static Options createCommandLineOption() { Options options = new Options(); Option version = new Option("slave", "Run in slave mode"); Option executionid = OptionBuilder.withArgName("id").hasArg() .withDescription("Run the test with this execution id.").create("executionid"); options.addOption(version);// w ww .ja va2 s. c o m options.addOption(executionid); return options; }
From source file:com.virtualparadigm.packman.cli.Main.java
private static Options buildCommandLineOptions() { Options cliOptions = new Options(); OptionBuilder.withArgName(CMD_OPTION_LONG_PACKAGE_NAME); OptionBuilder.withLongOpt(CMD_OPTION_LONG_PACKAGE_NAME); OptionBuilder.withDescription("package name"); OptionBuilder.hasArg(true);/* w w w . ja va 2 s . c o m*/ OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_PACKAGE_NAME)); OptionBuilder.withArgName(CMD_OPTION_LONG_PACKAGE_VERSION); OptionBuilder.withLongOpt(CMD_OPTION_LONG_PACKAGE_VERSION); OptionBuilder.withDescription("package version"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_PACKAGE_VERSION)); OptionBuilder.withArgName(CMD_OPTION_LONG_PACKAGE_FILE); OptionBuilder.withLongOpt(CMD_OPTION_LONG_PACKAGE_FILE); OptionBuilder.withDescription("package file"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_PACKAGE_FILE)); OptionBuilder.withArgName(CMD_OPTION_LONG_LICENSE_FILE); OptionBuilder.withLongOpt(CMD_OPTION_LONG_LICENSE_FILE); OptionBuilder.withDescription("license file"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_LICENSE_FILE)); OptionBuilder.withArgName(CMD_OPTION_LONG_AUTORUN_INSTALL_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_AUTORUN_INSTALL_DIR); OptionBuilder.withDescription("install directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_AUTORUN_INSTALL_DIR)); OptionBuilder.withArgName(CMD_OPTION_LONG_AUTORUN_UNINSTALL_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_AUTORUN_UNINSTALL_DIR); OptionBuilder.withDescription("uninstall directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_AUTORUN_UNINSTALL_DIR)); OptionBuilder.withArgName(CMD_OPTION_LONG_NEW_STATE_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_NEW_STATE_DIR); OptionBuilder.withDescription("new state directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_NEW_STATE_DIR)); OptionBuilder.withArgName(CMD_OPTION_LONG_OLD_STATE_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_OLD_STATE_DIR); OptionBuilder.withDescription("old state directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_OLD_STATE_DIR)); OptionBuilder.withArgName(CMD_OPTION_LONG_DEV_MODE); OptionBuilder.withLongOpt(CMD_OPTION_LONG_DEV_MODE); OptionBuilder.withDescription("development mode"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_DEV_MODE)); OptionBuilder.withArgName(CMD_OPTION_LONG_TARGET_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_TARGET_DIR); OptionBuilder.withDescription("target directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_TARGET_DIR)); OptionBuilder.withArgName(CMD_OPTION_LONG_DATA_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_DATA_DIR); OptionBuilder.withDescription("jpackage manager data directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_DATA_DIR)); OptionBuilder.withArgName(CMD_OPTION_LONG_LOCAL_CONFIG_FILE); OptionBuilder.withLongOpt(CMD_OPTION_LONG_LOCAL_CONFIG_FILE); OptionBuilder.withDescription("local install configuration values"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_LOCAL_CONFIG_FILE)); OptionBuilder.withArgName(CMD_OPTION_LONG_TEMP_DIR); OptionBuilder.withLongOpt(CMD_OPTION_LONG_TEMP_DIR); OptionBuilder.withDescription("temp directory"); OptionBuilder.hasArg(true); OptionBuilder.isRequired(false); cliOptions.addOption(OptionBuilder.create(CMD_OPTION_LONG_TEMP_DIR)); return cliOptions; }
From source file:com.boundary.sdk.event.EventCLI.java
@SuppressWarnings("static-access") private void addCreatedAtOption() { optionCreatedAt = OptionBuilder.withArgName("yyyy-mm-dd HH-MM-SS").hasArg() .withDescription("Date and time of event creation (UTC)").withLongOpt("created-at") .create(OPTION_CREATED_AT);//from w w w . j av a 2 s . c om options.addOption(optionCreatedAt); }
From source file:edu.umd.cloud9.example.hbase.HBaseWordCount.java
/** * Runs this tool.//from w w w. ja va 2 s. c o m */ @SuppressWarnings({ "static-access" }) public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT)); options.addOption( OptionBuilder.withArgName("table").hasArg().withDescription("HBase table name").create(OUTPUT)); options.addOption(OptionBuilder.withArgName("num").hasArg().withDescription("number of reducers") .create(NUM_REDUCERS)); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } if (!cmdline.hasOption(INPUT) || !cmdline.hasOption(OUTPUT)) { System.out.println("args: " + Arrays.toString(args)); HelpFormatter formatter = new HelpFormatter(); formatter.setWidth(120); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String inputPath = cmdline.getOptionValue(INPUT); String outputTable = cmdline.getOptionValue(OUTPUT); int reduceTasks = cmdline.hasOption(NUM_REDUCERS) ? Integer.parseInt(cmdline.getOptionValue(NUM_REDUCERS)) : 1; // If the table doesn't already exist, create it. Configuration conf = getConf(); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); Configuration hbaseConfig = HBaseConfiguration.create(conf); HBaseAdmin admin = new HBaseAdmin(hbaseConfig); if (admin.tableExists(outputTable)) { LOG.info(String.format("Table '%s' exists: dropping table and recreating.", outputTable)); LOG.info(String.format("Disabling table '%s'", outputTable)); admin.disableTable(outputTable); LOG.info(String.format("Droppping table '%s'", outputTable)); admin.deleteTable(outputTable); } HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(outputTable)); for (int i = 0; i < FAMILIES.length; i++) { HColumnDescriptor hColumnDesc = new HColumnDescriptor(FAMILIES[i]); tableDesc.addFamily(hColumnDesc); } admin.createTable(tableDesc); LOG.info(String.format("Successfully created table '%s'", outputTable)); admin.close(); // Now we're ready to start running MapReduce. LOG.info("Tool: " + HBaseWordCount.class.getSimpleName()); LOG.info(" - input path: " + inputPath); LOG.info(" - output table: " + outputTable); LOG.info(" - number of reducers: " + reduceTasks); Job job = Job.getInstance(conf); job.setJobName(HBaseWordCount.class.getSimpleName()); job.setJarByClass(HBaseWordCount.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setMapperClass(MyMapper.class); job.setCombinerClass(MyReducer.class); job.setNumReduceTasks(reduceTasks); FileInputFormat.setInputPaths(job, new Path(inputPath)); TableMapReduceUtil.initTableReducerJob(outputTable, MyTableReducer.class, job); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); return 0; }
From source file:com.google.code.linkedinapi.client.examples.InvitationApiExample.java
/** * Build command line options object.//from ww w .j a v a 2s . c om */ private static Options buildOptions() { Options opts = new Options(); String helpMsg = "Print this message."; Option help = new Option(HELP_OPTION, helpMsg); opts.addOption(help); String consumerKeyMsg = "You API Consumer Key."; OptionBuilder.withArgName("consumerKey"); OptionBuilder.hasArg(); OptionBuilder.withDescription(consumerKeyMsg); Option consumerKey = OptionBuilder.create(CONSUMER_KEY_OPTION); opts.addOption(consumerKey); String consumerSecretMsg = "You API Consumer Secret."; OptionBuilder.withArgName("consumerSecret"); OptionBuilder.hasArg(); OptionBuilder.withDescription(consumerSecretMsg); Option consumerSecret = OptionBuilder.create(CONSUMER_SECRET_OPTION); opts.addOption(consumerSecret); String accessTokenMsg = "You OAuth Access Token."; OptionBuilder.withArgName("accessToken"); OptionBuilder.hasArg(); OptionBuilder.withDescription(accessTokenMsg); Option accessToken = OptionBuilder.create(ACCESS_TOKEN_OPTION); opts.addOption(accessToken); String tokenSecretMsg = "You OAuth Access Token Secret."; OptionBuilder.withArgName("accessTokenSecret"); OptionBuilder.hasArg(); OptionBuilder.withDescription(tokenSecretMsg); Option accessTokenSecret = OptionBuilder.create(ACCESS_TOKEN_SECRET_OPTION); opts.addOption(accessTokenSecret); String subjectMsg = "Subject of the invitation."; OptionBuilder.withArgName("subject"); OptionBuilder.hasArg(); OptionBuilder.withDescription(subjectMsg); Option subject = OptionBuilder.create(SUBJECT_OPTION); opts.addOption(subject); String messageMsg = "Content of the invitation."; OptionBuilder.withArgName("message"); OptionBuilder.hasArg(); OptionBuilder.withDescription(messageMsg); Option message = OptionBuilder.create(MESSAGE_OPTION); opts.addOption(message); String idMsg = "ID of the user to whom invitation has to be sent."; OptionBuilder.withArgName("id"); OptionBuilder.hasArg(); OptionBuilder.withDescription(idMsg); Option id = OptionBuilder.create(ID_OPTION); opts.addOption(id); String emailMsg = "Email of the user to whom invitation has to be sent."; OptionBuilder.withArgName("email"); OptionBuilder.hasArg(); OptionBuilder.withDescription(emailMsg); Option email = OptionBuilder.create(EMAIL_OPTION); opts.addOption(email); String firstNameMsg = "First name of the user to whom invitation has to be sent."; OptionBuilder.withArgName("firstName"); OptionBuilder.hasArg(); OptionBuilder.withDescription(firstNameMsg); Option firstName = OptionBuilder.create(FIRST_NAME_OPTION); opts.addOption(firstName); String lastNameMsg = "Last name of the user to whom invitation has to be sent."; OptionBuilder.withArgName("lastName"); OptionBuilder.hasArg(); OptionBuilder.withDescription(lastNameMsg); Option lastName = OptionBuilder.create(LAST_NAME_OPTION); opts.addOption(lastName); String authHeaderMsg = "Auth Header of the user to whom invitation has to be sent."; OptionBuilder.withArgName("authHeader"); OptionBuilder.hasArg(); OptionBuilder.withDescription(authHeaderMsg); Option authHeader = OptionBuilder.create(AUTH_HEADER_OPTION); opts.addOption(authHeader); return opts; }
From source file:com.ibm.bi.dml.debug.DMLDebuggerInterface.java
/** * Set DML debugger CLI functionality menu *///www. j av a 2 s . c o m @SuppressWarnings("static-access") public void setOptions() { //add help option options.addOption("h", "help", false, "list debugger functions"); //add run option options.addOption("r", "run", false, "start your DML script"); //add quit option options.addOption("q", "quit", false, "exit debug mode"); //add resume option options.addOption("c", "continue", false, "continue running your DML script"); //add step over //options.addOption("n", "next", false, "next line, stepping over function calls"); //add single-stepping options.addOption("s", "step", false, "next line, stepping into function calls"); //add single-stepping options.addOption("si", "stepi", false, "next runtime instruction rather than DML source lines (for advanced users)"); // No step return for now //add step return // Option stepReturn = OptionBuilder.withArgName( "function-name" ) // .hasOptionalArg() // .withDescription( "execute instructions associated with current function as single step") // .create( "step_return" ); // options.addOption(stepReturn); //add set breakpoint option Option setBreakpoint = OptionBuilder.withLongOpt("break").withArgName("line-number").hasArg() .withDescription("set breakpoint at given line number").create("b"); options.addOption(setBreakpoint); // The key assumption here is that user doesnot keep toggling breakpoints too often //add delete breakpoint option Option disableBreakpoint = OptionBuilder.withLongOpt("delete").withArgName("line-number").hasArg() .withDescription("delete breakpoint at given line number").create("d"); options.addOption(disableBreakpoint); //add list breakpoints option Option infoOption = OptionBuilder.withLongOpt("info").withArgName("[break | frame]").hasOptionalArgs(1) .withDescription("show all breakpoints or frames (info <break | frame>)").create("i"); options.addOption(infoOption); //add display DML script option Option displayScript = OptionBuilder.withLongOpt("list") .withArgName("[next numlines] | [prev numlines] | [all]").hasOptionalArgs(2).withValueSeparator(' ') .withDescription("display DML script source lines. Default: numlines = 10").create("l"); options.addOption(displayScript); //add display DML script interspersed with runtime instructions option Option displayInst = OptionBuilder.withLongOpt("listi") .withArgName("[next numlines] | [prev numlines] | [all]").hasOptionalArgs(2).withValueSeparator(' ') .withDescription( "display corresponding instructions for DML script source lines. Default: numlines = 10 (for advanced users)") .create("li"); options.addOption(displayInst); //add set value of DML scalar variable option Option setVar = OptionBuilder.withArgName("varName value").hasArgs(2).withValueSeparator(' ') .withDescription( "set value of a scalar or specified cell of a matrix variable. (Eg: \'set alpha 0.1\' or \'set A[1,2] 20\')") .create("set"); options.addOption(setVar); //add display DML matrix (or vector) variable option Option displayMatrix = OptionBuilder.withLongOpt("print").withArgName("varName").hasArg().withDescription( "display contents of a scalar or matrix variable or rows/columns/cell of matrix. (Eg: \'p alpha\' or \'p A\' or \'p A[1,]\')") .create("p"); options.addOption(displayMatrix); Option displayTypeMatrix = OptionBuilder //.withLongOpt( "whatis" ) .withArgName("varName").hasArg() .withDescription( "display the type (and metadata) of a variable. (Eg: \'whatis alpha\' or \'whatis A\')") .create("whatis"); options.addOption(displayTypeMatrix); }
From source file:edu.umd.gorden2.BuildInvertedIndexHBase.java
/** * Runs this tool./* w ww. j a va 2 s.c o m*/ */ @SuppressWarnings({ "static-access" }) public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("HBase table name").create(OUTPUT)); options.addOption(OptionBuilder.withArgName("num").hasArg().withDescription("number of reducers") .create(NUM_REDUCERS)); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } if (!cmdline.hasOption(INPUT) || !cmdline.hasOption(OUTPUT)) { System.out.println("args: " + Arrays.toString(args)); HelpFormatter formatter = new HelpFormatter(); formatter.setWidth(120); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String inputPath = cmdline.getOptionValue(INPUT); String outputTable = cmdline.getOptionValue(OUTPUT); int reduceTasks = cmdline.hasOption(NUM_REDUCERS) ? Integer.parseInt(cmdline.getOptionValue(NUM_REDUCERS)) : 1; Configuration conf = getConf(); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); Configuration hbaseConfig = HBaseConfiguration.create(conf); HBaseAdmin admin = new HBaseAdmin(hbaseConfig); if (admin.tableExists(outputTable)) { LOG.info(String.format("Table '%s' exists: dropping table and recreating.", outputTable)); LOG.info(String.format("Disabling table '%s'", outputTable)); admin.disableTable(outputTable); LOG.info(String.format("Droppping table '%s'", outputTable)); admin.deleteTable(outputTable); } HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(outputTable)); for (int i = 0; i < FAMILIES.length; i++) { HColumnDescriptor hColumnDesc = new HColumnDescriptor(FAMILIES[i]); tableDesc.addFamily(hColumnDesc); } admin.createTable(tableDesc); LOG.info(String.format("Successfully created table '%s'", outputTable)); admin.close(); LOG.info("Tool name: " + BuildInvertedIndexHBase.class.getSimpleName()); LOG.info(" - input path: " + inputPath); LOG.info(" - output path: " + outputTable); LOG.info(" - num reducers: " + reduceTasks); Job job = Job.getInstance(conf); job.setJobName(BuildInvertedIndexHBase.class.getSimpleName()); job.setJarByClass(BuildInvertedIndexHBase.class); job.setNumReduceTasks(reduceTasks); FileInputFormat.setInputPaths(job, new Path(inputPath)); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(PairOfInts.class); job.setMapperClass(MyMapper.class); //job.setReducerClass(MyReducer.class); // Delete the output directory if it exists already. // Path outputDir = new Path(outputPath); // FileSystem.get(getConf()).delete(outputDir, true); TableMapReduceUtil.initTableReducerJob(outputTable, MyTableReducer.class, job); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); System.out.println("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); return 0; }
From source file:edu.umd.cloud9.collection.clue.CountClueWarcRecords.java
/** * Runs this tool./*from ww w .ja v a 2s.c o m*/ */ @SuppressWarnings("static-access") public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(ORIGINAL_OPTION, "use original ClueWeb09 distribution")); options.addOption(new Option(REPACKED_OPTION, "use repacked SequenceFiles")); options.addOption(OptionBuilder.withArgName("path").hasArg() .withDescription("path: base path for 'original', actual path for 'repacked'").create(PATH_OPTION)); options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("DocnoMapping data path") .create(MAPPING_OPTION)); options.addOption(OptionBuilder.withArgName("num").hasArg() .withDescription("segment number (required if 'original')").create(SEGMENT_OPTION)); options.addOption(OptionBuilder.withArgName("path").hasArg() .withDescription("output file to write the number of records").create(COUNT_OPTION)); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } boolean repacked; if (cmdline.hasOption(REPACKED_OPTION)) { repacked = true; } else if (cmdline.hasOption(ORIGINAL_OPTION)) { repacked = false; } else { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); System.err.println("Expecting either -original or -repacked"); return -1; } if (!cmdline.hasOption(PATH_OPTION) || !cmdline.hasOption(MAPPING_OPTION) || (!repacked && !cmdline.hasOption(SEGMENT_OPTION))) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String path = cmdline.getOptionValue(PATH_OPTION); String mappingFile = cmdline.getOptionValue(MAPPING_OPTION); int segment = 1; if (!repacked) { segment = Integer.parseInt(cmdline.getOptionValue(SEGMENT_OPTION)); } LOG.info("Tool name: " + CountClueWarcRecords.class.getSimpleName()); LOG.info(" - repacked: " + repacked); LOG.info(" - path: " + path); LOG.info(" - mapping file: " + mappingFile); if (!repacked) { LOG.info(" - segment number: " + segment); } FileSystem fs = FileSystem.get(getConf()); int mapTasks = 10; JobConf conf = new JobConf(getConf(), CountClueWarcRecords.class); conf.setJobName( CountClueWarcRecords.class.getSimpleName() + (repacked ? ":" + path : ":segment" + segment)); conf.setNumMapTasks(mapTasks); conf.setNumReduceTasks(0); if (repacked) { // Note, we have to add the files one by one, otherwise, SequenceFileInputFormat // thinks its a MapFile. for (FileStatus status : fs.listStatus(new Path(path))) { FileInputFormat.addInputPath(conf, status.getPath()); } } else { ClueCollectionPathConstants.addEnglishCollectionPart(conf, path, segment); } DistributedCache.addCacheFile(new URI(mappingFile), conf); if (repacked) { conf.setInputFormat(SequenceFileInputFormat.class); } else { conf.setInputFormat(ClueWarcInputFormat.class); } conf.setOutputFormat(NullOutputFormat.class); conf.setMapperClass(MyMapper.class); RunningJob job = JobClient.runJob(conf); Counters counters = job.getCounters(); int numDocs = (int) counters.findCounter(Records.PAGES).getCounter(); LOG.info("Read " + numDocs + " docs."); if (cmdline.hasOption(COUNT_OPTION)) { String f = cmdline.getOptionValue(COUNT_OPTION); FSDataOutputStream out = fs.create(new Path(f)); out.write(new Integer(numDocs).toString().getBytes()); out.close(); } return 0; }