List of usage examples for org.apache.hadoop.util ProgramDriver driver
public void driver(String[] argv) throws Throwable
From source file:boostingPL.driver.BoostingPLDriver.java
License:Open Source License
public static void main(String[] args) throws Throwable { ProgramDriver programDriver = new ProgramDriver(); addClass(programDriver, "boostingPL.driver.AdaBoostPLDriver", "AdaBoostPL:Parallel boosting for binary classifier problem"); addClass(programDriver, "boostingPL.driver.SAMMEPLDriver", "SAMMEPL:Parallel boosting for multiclass classifier problem"); programDriver.driver(args); }
From source file:ca.dealsaccess.scout.driver.ScoutDriver.java
License:Apache License
public static void main(String[] args) throws Throwable { ProgramDriver programDriver = new ProgramDriver(); Properties mainClasses = loadProperties("driver.classes.props"); if (mainClasses == null) { mainClasses = loadProperties("driver.classes.default.props"); }/* www.j a v a2 s . co m*/ if (mainClasses == null) { throw new IOException("Can't load any properties file?"); } boolean foundShortName = false; for (Object key : mainClasses.keySet()) { String keyString = (String) key; if (args.length > 0 && shortName(mainClasses.getProperty(keyString)).equals(args[0])) { foundShortName = true; } if (args.length > 0 && keyString.equalsIgnoreCase(args[0]) && isDeprecated(mainClasses, keyString)) { log.error(desc(mainClasses.getProperty(keyString))); return; } if (isDeprecated(mainClasses, keyString)) { continue; } addClass(programDriver, keyString, mainClasses.getProperty(keyString)); } if (args.length < 1 || args[0] == null || "-h".equals(args[0]) || "--help".equals(args[0])) { programDriver.driver(args); } String progName = args[0]; if (!foundShortName) { addClass(programDriver, progName, progName); } shift(args); Properties mainProps = loadProperties(progName + ".props"); if (mainProps == null) { log.warn("No {}.props found on classpath, will use command-line arguments only", progName); mainProps = new Properties(); } Map<String, String[]> argMap = Maps.newHashMap(); int i = 0; while (i < args.length && args[i] != null) { List<String> argValues = Lists.newArrayList(); String arg = args[i]; i++; if (arg.startsWith("-D")) { // '-Dkey=value' or '-Dkey=value1,value2,etc' case String[] argSplit = arg.split("="); arg = argSplit[0]; if (argSplit.length == 2) { argValues.add(argSplit[1]); } } else { // '-key [values]' or '--key [values]' case. while (i < args.length && args[i] != null) { if (args[i].startsWith("-")) { break; } argValues.add(args[i]); i++; } } argMap.put(arg, argValues.toArray(new String[argValues.size()])); } // Add properties from the .props file that are not overridden on the command line for (String key : mainProps.stringPropertyNames()) { String[] argNamePair = key.split("\\|"); String shortArg = '-' + argNamePair[0].trim(); String longArg = argNamePair.length < 2 ? null : "--" + argNamePair[1].trim(); if (!argMap.containsKey(shortArg) && (longArg == null || !argMap.containsKey(longArg))) { argMap.put(longArg, new String[] { mainProps.getProperty(key) }); } } // Now add command-line args List<String> argsList = Lists.newArrayList(); argsList.add(progName); for (Map.Entry<String, String[]> entry : argMap.entrySet()) { String arg = entry.getKey(); if (arg.startsWith("-D")) { // arg is -Dkey - if value for this !isEmpty(), then arg -> -Dkey + "=" + value String[] argValues = entry.getValue(); if (argValues.length > 0 && !argValues[0].trim().isEmpty()) { arg += '=' + argValues[0].trim(); } argsList.add(1, arg); } else { argsList.add(arg); for (String argValue : Arrays.asList(argMap.get(arg))) { if (!argValue.isEmpty()) { argsList.add(argValue); } } } } long start = System.currentTimeMillis(); programDriver.driver(argsList.toArray(new String[argsList.size()])); if (log.isInfoEnabled()) { log.info("Program took {} ms (Minutes: {})", System.currentTimeMillis() - start, (System.currentTimeMillis() - start) / 60000.0); } }
From source file:ca.etsmtl.lasi.hbasewikipedialoader.Driver.java
License:Apache License
/** * @param args//from www . j a v a 2s . co m * @throws Throwable */ public static void main(String[] args) throws Throwable { ProgramDriver pgd = new ProgramDriver(); pgd.addClass(HBaseWikipediaLoader.NAME, HBaseWikipediaLoader.class, "Load the Wikipedia articles dump in a HBase table"); pgd.driver(args); }
From source file:ca.uwaterloo.iss4e.hadoop.Driver.java
License:Open Source License
public static void main(String argv[]) { ProgramDriver pgd = new ProgramDriver(); try {/*from w ww. j ava 2 s . c o m*/ pgd.addClass("ThreelMain", ThreelMain.class, "Threeline program"); pgd.driver(argv); } catch (Throwable e) { e.printStackTrace(); } }
From source file:cn.clickwise.bigdata.Main.java
License:Apache License
public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try {//w ww. j av a2s .com pgd.addClass("preprocess", Preprocess.class, "A map/reduce program that preprocess all the input data sources."); pgd.driver(argv); // Success exitCode = 0; } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); }
From source file:com.benchmark.mapred.ExampleDriver.java
License:Apache License
public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try {/* ww w . j a v a 2s .c o m*/ pgd.addClass("wordcount", WordCount.class, "A map/reduce program that counts the words in the input files."); pgd.addClass("aggregatewordcount", AggregateWordCount.class, "An Aggregate based map/reduce program that counts the words in the input files."); pgd.addClass("aggregatewordhist", AggregateWordHistogram.class, "An Aggregate based map/reduce program that computes the histogram of the words in the input files."); pgd.addClass("grep", Grep.class, "A map/reduce program that counts the matches of a regex in the input."); pgd.addClass("randomwriter", RandomWriter.class, "A map/reduce program that writes 10GB of random data per node."); pgd.addClass("randomtextwriter", RandomTextWriter.class, "A map/reduce program that writes 10GB of random textual data per node."); pgd.addClass("sort", Sort.class, "A map/reduce program that sorts the data written by the random writer."); pgd.addClass("pi", PiEstimator.class, "A map/reduce program that estimates Pi using monte-carlo method."); pgd.addClass("pentomino", DistributedPentomino.class, "A map/reduce tile laying program to find solutions to pentomino problems."); pgd.addClass("secondarysort", SecondarySort.class, "An example defining a secondary sort to the reduce."); pgd.addClass("sudoku", Sudoku.class, "A sudoku solver."); pgd.addClass("sleep", SleepJob.class, "A job that sleeps at each map and reduce task."); pgd.addClass("join", Join.class, "A job that effects a join over sorted, equally partitioned datasets"); pgd.addClass("multifilewc", MultiFileWordCount.class, "A job that counts words from several files."); pgd.addClass("dbcount", DBCountPageView.class, "An example job that count the pageview counts from a database."); pgd.addClass("teragen", TeraGen.class, "Generate data for the terasort"); pgd.addClass("terasort", TeraSort.class, "Run the terasort"); pgd.addClass("teravalidate", TeraValidate.class, "Checking results of terasort"); pgd.addClass("kmeans", Kmeans.class, "Kmeans on movies data"); pgd.addClass("classification", Classification.class, "Classify movies into clusters"); pgd.addClass("histogram_movies", HistogramMovies.class, "A map/reduce program that gives a histogram of movies based on ratings."); pgd.addClass("histogram_ratings", HistogramRatings.class, "A map/reduce program that gives a histogram of users ratings on movies."); pgd.addClass("selfjoin", SelfJoin.class, "A map/reduce program that creates k+1 associations given set of k-field associations"); pgd.addClass("invertedindex", InvertedIndex.class, "A map/reduce program that creates an inverted index of documents."); pgd.addClass("adjlist", AdjList.class, "A map/reduce program that finds adjacency list of graph nodes."); pgd.addClass("termvectorperhost", TermVectorPerHost.class, "A map/reduce program that creates the term-vectors (frequency of words) per document."); pgd.addClass("sequencecount", SequenceCount.class, "A map/reduce program that counts the occurrence of consecutive words in the input files."); pgd.addClass("rankedinvertedindex", RankedInvertedIndex.class, "A map/reduce program that creates the top k document lists per word"); pgd.driver(argv); // Success exitCode = 0; } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); }
From source file:com.chinamobile.bcbsp.examples.driver.ExampleDriver.java
License:Apache License
/** * main method.//from www . ja v a 2 s. c o m * @param argv command parameter * @throws Exception */ public static void main(String[] argv) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try { pgd.addClass("pagerank", PageRankDriver.class, "An example program that computes the pagerank value of web pages."); pgd.addClass("sssp", SSPDriver.class, "An example program that computes the single " + "source shortest path problem."); pgd.addClass("kmeans", KMeansDriver.class, "An example program that computes" + " the k-means algorithm."); pgd.addClass("hits", HitsDriver.class, "An example program that computes " + "the authority and hub value of nodes "); pgd.addClass("simrank", SRDriver.class, "An example program that computes " + "the similarity "); pgd.addClass("copra", Driver.class, "An example program that computes" + " the commuty"); pgd.addClass("cc", CCBSPDriverNew.class, "An example program that computes " + "the connectedcomponent"); pgd.addClass("lpcluster", LPClusterDriver.class, "An example program that computes " + "the graph cluster"); pgd.addClass("smc", SemiClusterDriver.class, "An example program that computes" + " culster"); pgd.addClass("subG", SubGraphDriver.class, "An example program that computes" + " the subgraphmining"); pgd.driver(argv); exitCode = 0; } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); }
From source file:com.cloudera.hbase.Driver.java
License:Open Source License
public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try {//from w w w. j a v a2 s . c o m pgd.addClass("wordcount", WordCount.class, "MapReduce program to count word frequencies."); pgd.addClass("oldwordcount", OldWordCount.class, "MapReduce program to count word frequencies (old API)."); pgd.driver(argv); // Success exitCode = 0; } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); }
From source file:com.example.bigtable.sample.WordCountDriver.java
License:Open Source License
public static void main(String[] args) { ProgramDriver programDriver = new ProgramDriver(); int exitCode = -1; try {//from ww w . j a v a2 s . com programDriver.addClass("wordcount-hbase", WordCountHBase.class, "A map/reduce program that counts the words in the input files."); programDriver.addClass("export-table", Export.class, "A map/reduce program that exports a table to a file."); //programDriver.addClass("cellcounter", CellCounter.class, "Count them cells!"); programDriver.driver(args); exitCode = programDriver.run(args); } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); }
From source file:com.github.gaoyangthu.demo.mapred.ExampleDriver.java
License:Apache License
public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try {/* w w w. j a v a 2 s . c o m*/ pgd.addClass("wordcount", WordCount.class, "A map/reduce program that counts the words in the input files."); pgd.addClass("aggregatewordcount", AggregateWordCount.class, "An Aggregate based map/reduce program that counts the words in the input files."); pgd.addClass("aggregatewordhist", AggregateWordHistogram.class, "An Aggregate based map/reduce program that computes the histogram of the words in the input files."); pgd.addClass("grep", Grep.class, "A map/reduce program that counts the matches of a regex in the input."); pgd.addClass("randomwriter", RandomWriter.class, "A map/reduce program that writes 10GB of random data per node."); pgd.addClass("randomtextwriter", RandomTextWriter.class, "A map/reduce program that writes 10GB of random textual data per node."); pgd.addClass("sort", Sort.class, "A map/reduce program that sorts the data written by the random writer."); pgd.addClass("pi", PiEstimator.class, "A map/reduce program that estimates Pi using monte-carlo method."); pgd.addClass("pentomino", DistributedPentomino.class, "A map/reduce tile laying program to find solutions to pentomino problems."); pgd.addClass("secondarysort", SecondarySort.class, "An example defining a secondary sort to the reduce."); pgd.addClass("sudoku", Sudoku.class, "A sudoku solver."); pgd.addClass("sleep", SleepJob.class, "A job that sleeps at each map and reduce task."); pgd.addClass("join", Join.class, "A job that effects a join over sorted, equally partitioned datasets"); pgd.addClass("multifilewc", MultiFileWordCount.class, "A job that counts words from several files."); pgd.addClass("dbcount", DBCountPageView.class, "An example job that count the pageview counts from a database."); pgd.addClass("teragen", TeraGen.class, "Generate data for the terasort"); pgd.addClass("terasort", TeraSort.class, "Run the terasort"); pgd.addClass("teravalidate", TeraValidate.class, "Checking results of terasort"); pgd.driver(argv); // Success exitCode = 0; } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); }