List of usage examples for org.apache.hadoop.util GenericOptionsParser GenericOptionsParser
public GenericOptionsParser(Configuration conf, Options options, String[] args) throws IOException
GenericOptionsParser to parse given options as well as generic Hadoop options. From source file:com.gsinnovations.howdah.CommandLineUtil.java
License:Apache License
/** * Print the options supported by <code>GenericOptionsParser</code>. * In addition to the options supported by the job, passed in as the * group parameter./* w w w. j a v a 2s. c om*/ * * @param group job-specific command-line options. */ public static void printHelpWithGenericOptions(Group group) { org.apache.commons.cli.Options ops = new org.apache.commons.cli.Options(); new GenericOptionsParser(new Configuration(), ops, new String[0]); org.apache.commons.cli.HelpFormatter fmt = new org.apache.commons.cli.HelpFormatter(); fmt.printHelp("<command> [Generic Options] [Job-Specific Options]", "Generic Options:", ops, ""); PrintWriter pw = new PrintWriter(System.out); HelpFormatter formatter = new HelpFormatter(); formatter.setGroup(group); formatter.setPrintWriter(pw); formatter.printHelp(); pw.flush(); }
From source file:com.iflytek.spider.parse.ParseText.java
License:Apache License
public static void main(String argv[]) throws Exception { String usage = "ParseText (-local | -dfs <namenode:port>) recno segment"; if (argv.length < 3) { System.out.println("usage:" + usage); return;/*w w w.j a va 2 s . c o m*/ } Options opts = new Options(); Configuration conf = SpiderConfiguration.create(); GenericOptionsParser parser = new GenericOptionsParser(conf, opts, argv); String[] remainingArgs = parser.getRemainingArgs(); FileSystem fs = FileSystem.get(conf); try { int recno = Integer.parseInt(remainingArgs[0]); String segment = remainingArgs[1]; String filename = new Path(segment, ParseText.DIR_NAME).toString(); ParseText parseText = new ParseText(); ArrayFile.Reader parseTexts = new ArrayFile.Reader(fs, filename, conf); parseTexts.get(recno, parseText); System.out.println("Retrieved " + recno + " from file " + filename); System.out.println(parseText); parseTexts.close(); } finally { fs.close(); } }
From source file:eu.larkc.iris.Main.java
License:Apache License
@Override public int run(String[] args) throws Exception { GenericOptionsParser gop = new GenericOptionsParser(getConf(), new org.apache.commons.cli.Options(), args); processUserArguments(gop.getRemainingArgs()); rules = createRules();/*from w w w .ja va2 s .com*/ Configuration hadoopConf = gop.getConfiguration(); defaultConfiguration.hadoopConfiguration = hadoopConf; defaultConfiguration.jobConf = setupJob(hadoopConf); defaultConfiguration.project = project; logger.info("predicate indexing is " + (defaultConfiguration.doPredicateIndexing ? "ON" : "OFF")); if (rdfImporter) { return doRdfImport(defaultConfiguration); } else if (ntripleImporter) { return doNTripleImport(defaultConfiguration); } else if (tester) { return doTester(defaultConfiguration); } else if (processer) { return doProcess(); } else if (rdfExporter) { return doRdfExport(defaultConfiguration); } else if (ntripleExporter) { return doNTripleExport(defaultConfiguration); } else if (viewConfig) { return doViewConfig(defaultConfiguration); } return -1; }
From source file:it.crs4.seal.common.SealToolParser.java
License:Open Source License
/** * Parses command line./*from w w w . j av a2s. co m*/ * * Override this method to implement additional command line options, * but do make sure you call this method to parse the default options. */ protected CommandLine parseOptions(Configuration conf, String[] args) throws ParseException, IOException { myconf = conf; setDefaultProperties(conf); // load settings from configuration file // first, parse the command line (in getRcFile) looking for an option overriding the default seal configuration file File configFile = getRcFile(args); if (configFile != null) loadConfig(conf, configFile); // now parse the entire command line using the default hadoop parser. Now // the user can override properties specified in the config file with properties // specified on the command line. CommandLine line = new GenericOptionsParser(conf, options, args).getCommandLine(); if (line == null) throw new ParseException("Error parsing command line"); // getCommandLine returns an null if there was a parsing error ////////////////////// input/output formats ////////////////////// // set the configuration property. Then, we'll check the property // to ensure it has a valid value, regardless of whether we just set it, // so that the check will also be valid if the property is set directly. if (line.hasOption(opt_inputFormat.getOpt())) myconf.set(INPUT_FORMAT_CONF, line.getOptionValue(opt_inputFormat.getOpt())); validateIOFormat(INPUT_FORMAT_CONF, acceptedInputFormats); if (line.hasOption(opt_outputFormat.getOpt())) myconf.set(OUTPUT_FORMAT_CONF, line.getOptionValue(opt_outputFormat.getOpt())); validateIOFormat(OUTPUT_FORMAT_CONF, acceptedOutputFormats); if (conf.get(INPUT_FORMAT_ENCODING) != null) { String value = conf.get(INPUT_FORMAT_ENCODING); if (value.equals("sanger") || value.equals("illumina")) conf.set(fi.tkk.ics.hadoop.bam.FormatConstants.CONF_INPUT_BASE_QUALITY_ENCODING, value); else throw new ParseException("Invalid " + INPUT_FORMAT_ENCODING + ". Expected 'sanger' or 'illumina'"); } /////////////////////// output compression ///////////////////// if (line.hasOption(opt_compressOutput.getOpt())) { myconf.setBoolean("mapred.output.compress", true); String codec = line.getOptionValue(opt_compressOutput.getOpt()); if (codec != null) { String codecClass = "org.apache.hadoop.io.compress.GzipCodec"; // default if ("auto".equalsIgnoreCase(codec) || "gzip".equalsIgnoreCase(codec)) { // pass. Already set } else if ("bzip2".equalsIgnoreCase(codec)) codecClass = "org.apache.hadoop.io.compress.BZip2Codec"; else if ("snappy".equalsIgnoreCase(codec)) codecClass = "org.apache.hadoop.io.compress.SnappyCodec"; else { throw new ParseException("Unknown codec " + codec + ". Valid values are gzip, bzip2, snappy and auto.\n" + "If you want to use an unsupported codec pass 'auto' and set the property mapred.output.compression.codec directly"); } myconf.set("mapred.output.compression.codec", codecClass); } } ////////////////////// number of reducers ////////////////////// if (line.hasOption(opt_nReduceTasks.getOpt())) { String rString = line.getOptionValue(opt_nReduceTasks.getOpt()); try { int r = Integer.parseInt(rString); if (r >= minReduceTasks) nReduceTasks = r; else throw new ParseException("Number of reducers must be greater than or equal to " + minReduceTasks + " (got " + rString + ")"); } catch (NumberFormatException e) { throw new ParseException("Invalid number of reduce tasks '" + rString + "'"); } } ////////////////////// positional arguments ////////////////////// String[] otherArgs = line.getArgs(); if (otherArgs.length < 2) // require at least two: one input and one output throw new ParseException("You must provide input and output paths"); else { // FileSystem fs; for (int i = 0; i < otherArgs.length - 1; ++i) { Path p = new Path(otherArgs[i]); fs = p.getFileSystem(conf); p = p.makeQualified(fs); FileStatus[] files = fs.globStatus(p); if (files != null && files.length > 0) { for (FileStatus status : files) inputs.add(status.getPath()); } else throw new ParseException("Input path " + p.toString() + " doesn't exist"); } // now the last one, should be the output path outputDir = new Path(otherArgs[otherArgs.length - 1]); fs = outputDir.getFileSystem(conf); outputDir = outputDir.makeQualified(fs); if (fs.exists(outputDir)) throw new ParseException( "Output path " + outputDir.toString() + " already exists. Won't overwrite"); } return line; }
From source file:org.apache.ambari.servicemonitor.utils.ToolRunnerPlus.java
License:Apache License
/** * Get ready to run a tool but don't actually run it. This is for test purposes. * @param conf Configuration to start with * @param tool tool to set up/*from w ww . j a v a 2 s .c o m*/ * @param args command line arguments * @return either an array of unparsed elements, or null, meaning "preparation failed, do not execute" * @throws IOException on problems */ public static String[] prepareToExecute(Configuration conf, ToolPlus tool, String[] args) throws IOException { boolean canRun = true; if (conf == null) { conf = new Configuration(); } Options options = tool.createToolSpecificOptions(); if (options == null) { options = new Options(); } options.addOption("p", "dump", false, "dump the current configuration"); options.addOption("u", "usage", false, "Print the Usage"); GenericOptionsParser parser = new GenericOptionsParser(conf, options, args); //process our local values //set the configuration back, so that Tool can configure itself Configuration configuration = parser.getConfiguration(); CommandLine commandLine = parser.getCommandLine(); if (commandLine == null) { dumpArguments(args); canRun = false; } else { if (commandLine.hasOption("u")) { usage(args, tool, options); canRun = false; } else { tool.setConf(configuration); tool.setCommandLine(commandLine); if (commandLine.hasOption("p")) { //dump the commands dumpArguments(args); //dump the configuration dumpConf(conf); dumpSystemState(); String toolDump = tool.dump(); if (toolDump != null && !toolDump.isEmpty()) { println(toolDump); } } } } String[] toolArgs; if (canRun) { toolArgs = parser.getRemainingArgs(); } else { toolArgs = null; } return toolArgs; }
From source file:org.apache.mahout.common.CommandLineUtil.java
License:Apache License
/** * Print the options supported by {@code GenericOptionsParser}. * In addition to the options supported by the job, passed in as the * group parameter./*from w ww .jav a 2s . co m*/ * * @param group job-specific command-line options. */ public static void printHelpWithGenericOptions(Group group) throws IOException { new GenericOptionsParser(new Configuration(), new org.apache.commons.cli.Options(), new String[0]); PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.out, Charsets.UTF_8), true); HelpFormatter formatter = new HelpFormatter(); formatter.setGroup(group); formatter.setPrintWriter(pw); formatter.setFooter( "Specify HDFS directories while running on hadoop; else specify local file system directories"); formatter.print(); }
From source file:org.apache.mahout.common.CommandLineUtil.java
License:Apache License
public static void printHelpWithGenericOptions(Group group, OptionException oe) throws IOException { new GenericOptionsParser(new Configuration(), new org.apache.commons.cli.Options(), new String[0]); PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.out, Charsets.UTF_8), true); HelpFormatter formatter = new HelpFormatter(); formatter.setGroup(group);//from w w w . j a v a 2 s . co m formatter.setPrintWriter(pw); formatter.setException(oe); formatter.print(); }