Example usage for org.apache.hadoop.util ToolRunner printGenericCommandUsage

Introduction

In this page you can find the example usage for org.apache.hadoop.util ToolRunner printGenericCommandUsage.

Prototype

public static void printGenericCommandUsage(PrintStream out)

Source Link

Document

Prints generic command-line argurments and usage information.

Usage

From source file:org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample.java

License:Apache License

private static void printUsage() {
    System.err.println("broadcastAndOneToOneExample " + skipLocalityCheck);
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.BroadcastLoadGen.java

License:Apache License

@Override
protected void printUsage() {
    System.err.println(//from   w  ww.  j a  v  a 2 s .co m
            "Usage: " + "BroadcastLoadGen <num_source_tasks>  <total_source_data> <num_destination_tasks>");
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.FilterLinesByWord.java

License:Apache License

private static void printUsage() {
    System.err//from   w  w w.  jav a 2  s  .  c  o  m
            .println("Usage filtelinesrbyword <in> <out> <filter_word> [-generateSplitsInClient true/<false>]");
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.FilterLinesByWordOneToOne.java

License:Apache License

private static void printUsage() {
    System.err.println("Usage filterLinesByWordOneToOne <in> <out> <filter_word>"
            + " [-generateSplitsInClient true/<false>]");
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.GroupByOrderByMRRTest.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
        ToolRunner.printGenericCommandUsage(System.err);
        return 2;
    }/*from  w w  w .ja  v a  2  s .c o  m*/

    String inputPath = otherArgs[0];
    String outputPath = otherArgs[1];

    UserGroupInformation.setConfiguration(conf);

    TezConfiguration tezConf = new TezConfiguration(conf);
    FileSystem fs = FileSystem.get(conf);

    if (fs.exists(new Path(outputPath))) {
        throw new FileAlreadyExistsException("Output directory " + outputPath + " already exists");
    }

    Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();

    String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
            TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR
            + Long.toString(System.currentTimeMillis());
    Path stagingDir = new Path(stagingDirStr);
    FileSystem pathFs = stagingDir.getFileSystem(tezConf);
    pathFs.mkdirs(new Path(stagingDirStr));

    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
    stagingDir = pathFs.makeQualified(new Path(stagingDirStr));

    TezClient tezClient = TezClient.create("groupbyorderbymrrtest", tezConf);
    tezClient.start();

    LOG.info("Submitting groupbyorderbymrrtest DAG as a new Tez Application");

    try {
        DAG dag = createDAG(conf, localResources, stagingDir, inputPath, outputPath, true);

        tezClient.waitTillReady();

        DAGClient dagClient = tezClient.submitDAG(dag);

        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
        if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
            LOG.error("groupbyorderbymrrtest failed, state=" + dagStatus.getState() + ", diagnostics="
                    + dagStatus.getDiagnostics());
            return -1;
        }
        LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
        return 0;
    } finally {
        tezClient.stop();
    }
}

From source file:org.apache.tez.mapreduce.examples.IntersectDataGen.java

License:Apache License

private static void printUsage() {
    System.err.println("Usage: "
            + "intersectdatagen <outPath1> <path1Size> <outPath2> <path2Size> <expectedResultPath> <numTasks>");
    ;/*www.  ja  v a  2 s. c  o  m*/
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.IntersectExample.java

License:Apache License

private static void printUsage() {
    System.err.println("Usage: " + "intersect <file1> <file2> <numPartitions> <outPath>");
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.IntersectValidate.java

License:Apache License

private static void printUsage() {
    System.err.println("Usage: " + "intersectvalidate <path1> <path2>");
    ToolRunner.printGenericCommandUsage(System.err);
}

From source file:org.apache.tez.mapreduce.examples.MRRSleepJob.java

License:Apache License

public int run(String[] args) throws Exception {

    if (args.length < 1) {
        System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" + " [-ir numIntermediateReducer]"
                + " [-irs numIntermediateReducerStages]"
                + " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]"
                + " [-irt intermediateReduceSleepTime]" + " [-recordt recordSleepTime (msec)]"
                + " [-generateSplitsInAM (false)/true]" + " [-writeSplitsToDfs (false)/true]");
        ToolRunner.printGenericCommandUsage(System.err);
        return 2;
    }/*from   w w  w .  ja va2  s .  c o  m*/

    int numMapper = 1, numReducer = 1, numIReducer = 1;
    long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100, iReduceSleepTime = 1;
    int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
    int iReduceStagesCount = 1;
    boolean writeSplitsToDfs = false;
    boolean generateSplitsInAM = false;
    boolean splitsOptionFound = false;

    for (int i = 0; i < args.length; i++) {
        if (args[i].equals("-m")) {
            numMapper = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-r")) {
            numReducer = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-ir")) {
            numIReducer = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-mt")) {
            mapSleepTime = Long.parseLong(args[++i]);
        } else if (args[i].equals("-rt")) {
            reduceSleepTime = Long.parseLong(args[++i]);
        } else if (args[i].equals("-irt")) {
            iReduceSleepTime = Long.parseLong(args[++i]);
        } else if (args[i].equals("-irs")) {
            iReduceStagesCount = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-recordt")) {
            recSleepTime = Long.parseLong(args[++i]);
        } else if (args[i].equals("-generateSplitsInAM")) {
            if (splitsOptionFound) {
                throw new RuntimeException(
                        "Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
            }
            splitsOptionFound = true;
            generateSplitsInAM = Boolean.parseBoolean(args[++i]);

        } else if (args[i].equals("-writeSplitsToDfs")) {
            if (splitsOptionFound) {
                throw new RuntimeException(
                        "Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
            }
            splitsOptionFound = true;
            writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
        }
    }

    if (numIReducer > 0 && numReducer <= 0) {
        throw new RuntimeException("Cannot have intermediate reduces without" + " a final reduce");
    }

    // sleep for *SleepTime duration in Task by recSleepTime per record
    mapSleepCount = (int) Math.ceil(mapSleepTime / ((double) recSleepTime));
    reduceSleepCount = (int) Math.ceil(reduceSleepTime / ((double) recSleepTime));
    iReduceSleepCount = (int) Math.ceil(iReduceSleepTime / ((double) recSleepTime));

    TezConfiguration conf = new TezConfiguration(getConf());
    FileSystem remoteFs = FileSystem.get(conf);

    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
            conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));

    Path remoteStagingDir = remoteFs.makeQualified(
            new Path(conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
                    Long.toString(System.currentTimeMillis())));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

    DAG dag = createDAG(remoteFs, conf, remoteStagingDir, numMapper, numReducer, iReduceStagesCount,
            numIReducer, mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount, iReduceSleepTime,
            iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);

    TezClient tezSession = TezClient.create("MRRSleep", conf, false, null, credentials);
    tezSession.start();
    DAGClient dagClient = tezSession.submitDAG(dag);
    dagClient.waitForCompletion();
    tezSession.stop();

    return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
}

From source file:org.apache.tez.mapreduce.examples.RPCLoadGen.java

License:Apache License

@Override
protected void printUsage() {
    System.err.println("Usage: " + "RPCLoadGen <numTasks> <max_sleep_time_millis> <get_task_payload_size> ["
            + "<" + VIA_RPC + ">|" + VIA_HDFS_DIST_CACHE + "|" + VIA_HDFS_DIRECT_READ + "]");
    ToolRunner.printGenericCommandUsage(System.err);
}