Example usage for org.apache.hadoop.util ClassUtil findContainingJar

Introduction

In this page you can find the example usage for org.apache.hadoop.util ClassUtil findContainingJar.

Prototype

public static String findContainingJar(Class<?> clazz) 

Source Link

Document

Find a jar that contains a class of the same name, if any.

Usage

From source file:gobblin.runtime.embedded.EmbeddedGobblin.java

License:Apache License

/**
 * Specify that the input jar should be added to workers' classpath on distributed mode. Jars with lower priority value
 * will appear first in the classpath. Default priority is 0.
 *//*w w w .  ja  v  a  2  s .  c om*/
public EmbeddedGobblin distributeJarByClassWithPriority(Class<?> klazz, int priority) {
    return distributeJarWithPriority(ClassUtil.findContainingJar(klazz), priority);
}

---

From source file:gobblin.runtime.embedded.EmbeddedGobblinDistcp.java

License:Apache License

public EmbeddedGobblinDistcp(Path from, Path to) throws JobTemplate.TemplateException, IOException {
    super("Distcp");
    try {// w  w w.j  a  v a 2 s .  co m
        setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/distcp.template"));
    } catch (URISyntaxException | SpecNotFoundException exc) {
        throw new RuntimeException("Could not instantiate an " + EmbeddedGobblinDistcp.class.getName(), exc);
    }
    this.setConfiguration("from", from.toString());
    this.setConfiguration("to", to.toString());
    // Infer source and target fs uris from the input paths
    this.setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_FS_URI,
            from.getFileSystem(new Configuration()).getUri().toString());
    this.setConfiguration(ConfigurationKeys.WRITER_FILE_SYSTEM_URI,
            to.getFileSystem(new Configuration()).getUri().toString());

    // add gobblin-data-management jar to distributed jars
    this.distributeJar(ClassUtil.findContainingJar(CopySource.class));
}

---

From source file:kr.co.bitnine.octopus.util.VersionInfo.java

License:Apache License

public static void main(String[] args) {
    System.out.println("Octopus " + getVersion());
    System.out.println("SCM: " + getUrl() + ", revision: " + getRevision());
    System.out.println("Compiled by " + getUser() + " on " + getDate());
    System.out.println("From source with checksum " + getSrcChecksum());
    System.out.println("This command was run using " + ClassUtil.findContainingJar(VersionInfo.class));
}

---

From source file:org.apache.flink.tez.client.RemoteTezEnvironment.java

License:Apache License

public void registerMainClass(Class mainClass) {
    jarPath = new Path(ClassUtil.findContainingJar(mainClass));
    LOG.info("Registering main class " + mainClass.getName() + " contained in " + jarPath.toString());
}

---

From source file:org.apache.gobblin.runtime.embedded.EmbeddedGobblin.java

License:Apache License

/**
 * Specify that the input jar should be added to workers' classpath on distributed mode. Jars with lower priority value
 * will appear first in the classpath. Default priority is 0.
 *//*from  w  w w. java 2 s  .c o m*/
public EmbeddedGobblin distributeJarByClassWithPriority(Class<?> klazz, int priority) {
    String jar = ClassUtil.findContainingJar(klazz);
    if (jar == null) {
        log.warn(String.format("Could not find jar for class %s. This is normal in test runs.", klazz));
        return this;
    }
    return distributeJarWithPriority(jar, priority);
}

---

From source file:org.apache.kylin.engine.mr.common.AbstractHadoopJob.java

License:Apache License

protected void setJobClasspath(Job job, KylinConfig kylinConf) {
    String jarPath = kylinConf.getKylinJobJarPath();
    File jarFile = new File(jarPath);
    if (jarFile.exists()) {
        job.setJar(jarPath);/*from  ww  w.j av  a2  s . co  m*/
        logger.info("append job jar: " + jarPath);
    } else {
        job.setJarByClass(this.getClass());
    }

    String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
    String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
    String kylinKafkaDependency = System.getProperty("kylin.kafka.dependency");
    logger.info("append kylin.hbase.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);

    Configuration jobConf = job.getConfiguration();
    String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
    if (classpath == null || classpath.length() == 0) {
        logger.info("Didn't find " + MAP_REDUCE_CLASSPATH
                + " in job configuration, will run 'mapred classpath' to get the default value.");
        classpath = getDefaultMapRedClasspath();
        logger.info("The default mapred classpath is: " + classpath);
    }

    if (kylinHBaseDependency != null) {
        // yarn classpath is comma separated
        kylinHBaseDependency = kylinHBaseDependency.replace(":", ",");
        classpath = classpath + "," + kylinHBaseDependency;
    }

    jobConf.set(MAP_REDUCE_CLASSPATH, classpath);
    logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));

    /*
     *  set extra dependencies as tmpjars & tmpfiles if configured
     */
    StringBuilder kylinDependency = new StringBuilder();

    // for hive dependencies
    if (kylinHiveDependency != null) {
        // yarn classpath is comma separated
        kylinHiveDependency = kylinHiveDependency.replace(":", ",");

        logger.info("Hive Dependencies Before Filtered: " + kylinHiveDependency);
        String filteredHive = filterKylinHiveDependency(kylinHiveDependency, kylinConf);
        logger.info("Hive Dependencies After Filtered: " + filteredHive);

        StringUtil.appendWithSeparator(kylinDependency, filteredHive);
    } else {

        logger.info("No hive dependency jars set in the environment, will find them from classpath:");

        try {
            String hiveExecJarPath = ClassUtil
                    .findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));

            StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath);
            logger.info("hive-exec jar file: " + hiveExecJarPath);

            String hiveHCatJarPath = ClassUtil
                    .findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
            StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath);
            logger.info("hive-catalog jar file: " + hiveHCatJarPath);

            String hiveMetaStoreJarPath = ClassUtil
                    .findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
            StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath);
            logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
        } catch (ClassNotFoundException e) {
            logger.error("Cannot found hive dependency jars: " + e);
        }
    }

    // for kafka dependencies
    if (kylinKafkaDependency != null) {
        kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
        logger.info("Kafka Dependencies: " + kylinKafkaDependency);
        StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency);
    } else {
        logger.info("No Kafka dependency jar set in the environment, will find them from classpath:");
        try {
            String kafkaClientJarPath = ClassUtil
                    .findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
            StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath);
            logger.info("kafka jar file: " + kafkaClientJarPath);

        } catch (ClassNotFoundException e) {
            logger.debug("Not found kafka client jar from classpath, it is optional for normal build: " + e);
        }
    }

    // for KylinJobMRLibDir
    String mrLibDir = kylinConf.getKylinJobMRLibDir();
    StringUtil.appendWithSeparator(kylinDependency, mrLibDir);

    setJobTmpJarsAndFiles(job, kylinDependency.toString());

    overrideJobConfig(job.getConfiguration(), kylinConf.getMRConfigOverride());
}

---

From source file:org.apache.tajo.util.VersionInfo.java

License:Apache License

public static void main(String[] args) {
    LOG.debug("version: " + getVersion());
    System.out.println("Tajo " + getVersion());
    System.out.println("Git " + getUrl() + " -r " + getRevision());
    System.out.println("Compiled by " + getUser() + " on " + getDate());
    System.out.println("Compiled with protoc " + getProtocVersion());
    System.out.println("From source with checksum " + getSrcChecksum());
    System.out.println("This command was run using " + ClassUtil.findContainingJar(VersionInfo.class));
}

---

From source file:org.apache.tez.common.VersionInfo.java

License:Apache License

public static void main(String[] args) {
    if (args.length != 1) {
        System.err.println("Invalid no. of args. Usage: VersionInfo <component-name>");
        System.exit(-1);/*  w w  w.  j a va 2s . c  om*/
    }

    VersionInfo versionInfo = new VersionInfo(args[0]);
    System.out.println("VersionInfo: " + versionInfo.toString());
    System.out.println("This command was run using " + ClassUtil.findContainingJar(VersionInfo.class));
}

---

From source file:org.apache.tez.mapreduce.examples.FilterLinesByWord.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    Credentials credentials = new Credentials();

    boolean generateSplitsInClient = false;

    SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
    try {/*  w  ww.  j av a2s . com*/
        generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
        otherArgs = splitCmdLineParser.getRemainingArgs();
    } catch (ParseException e1) {
        System.err.println("Invalid options");
        printUsage();
        return 2;
    }

    if (otherArgs.length != 3) {
        printUsage();
        return 2;
    }

    String inputPath = otherArgs[0];
    String outputPath = otherArgs[1];
    String filterWord = otherArgs[2];

    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(new Path(outputPath))) {
        System.err.println("Output directory : " + outputPath + " already exists");
        return 2;
    }

    TezConfiguration tezConf = new TezConfiguration(conf);

    fs.getWorkingDirectory();
    Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);

    String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
    if (jarPath == null) {
        throw new TezUncheckedException(
                "Could not find any jar containing" + FilterLinesByWord.class.getName() + " in the classpath");
    }

    Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
    fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
    FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
    TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, conf);

    Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
    LocalResource dagJarLocalRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
            remoteJarStatus.getModificationTime());
    commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);

    TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources,
            credentials);
    tezSession.start(); // Why do I need to start the TezSession.

    Configuration stage1Conf = new JobConf(conf);
    stage1Conf.set(FILTER_PARAM_NAME, filterWord);

    Configuration stage2Conf = new JobConf(conf);
    stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
    stage2Conf.setBoolean("mapred.mapper.new-api", false);

    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    // Setup stage1 Vertex
    Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor
            .create(FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
            .addTaskLocalFiles(commonLocalResources);

    DataSourceDescriptor dsd;
    if (generateSplitsInClient) {
        // TODO TEZ-1406. Dont' use MRInputLegacy
        stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
        stage1Conf.setBoolean("mapred.mapper.new-api", false);
        dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
    } else {
        dsd = MRInputLegacy.createConfigBuilder(stage1Conf, TextInputFormat.class, inputPath).groupSplits(false)
                .build();
    }
    stage1Vertex.addDataSource("MRInput", dsd);

    // Setup stage2 Vertex
    Vertex stage2Vertex = Vertex.create("stage2",
            ProcessorDescriptor.create(FilterByWordOutputProcessor.class.getName())
                    .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
            1);
    stage2Vertex.addTaskLocalFiles(commonLocalResources);

    // Configure the Output for stage2
    OutputDescriptor od = OutputDescriptor.create(MROutput.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf));
    OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(MROutputCommitter.class.getName());
    stage2Vertex.addDataSink("MROutput", DataSinkDescriptor.create(od, ocd, null));

    UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
            .newBuilder(Text.class.getName(), TextLongPair.class.getName()).setFromConfiguration(tezConf)
            .build();

    DAG dag = DAG.create("FilterLinesByWord");
    Edge edge = Edge.create(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty());
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);

    LOG.info("Submitting DAG to Tez Session");
    DAGClient dagClient = tezSession.submitDAG(dag);
    LOG.info("Submitted DAG to Tez Session");

    DAGStatus dagStatus = null;
    String[] vNames = { "stage1", "stage2" };
    try {
        while (true) {
            dagStatus = dagClient.getDAGStatus(null);
            if (dagStatus.getState() == DAGStatus.State.RUNNING
                    || dagStatus.getState() == DAGStatus.State.SUCCEEDED
                    || dagStatus.getState() == DAGStatus.State.FAILED
                    || dagStatus.getState() == DAGStatus.State.KILLED
                    || dagStatus.getState() == DAGStatus.State.ERROR) {
                break;
            }
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
                // continue;
            }
        }

        while (dagStatus.getState() == DAGStatus.State.RUNNING) {
            try {
                ExampleDriver.printDAGStatus(dagClient, vNames);
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    // continue;
                }
                dagStatus = dagClient.getDAGStatus(null);
            } catch (TezException e) {
                LOG.fatal("Failed to get application progress. Exiting");
                return -1;
            }
        }

        dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));

    } finally {
        fs.delete(stagingDir, true);
        tezSession.stop();
    }

    ExampleDriver.printDAGStatus(dagClient, vNames, true, true);
    LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
    return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}

---

From source file:org.apache.tez.mapreduce.examples.FilterLinesByWordOneToOne.java

License:Apache License

@Override
public int run(String[] otherArgs) throws Exception {
    boolean generateSplitsInClient = false;
    SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
    try {//  w  w  w  . jav a 2  s . co  m
        generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
        otherArgs = splitCmdLineParser.getRemainingArgs();
    } catch (ParseException e1) {
        System.err.println("Invalid options");
        printUsage();
        return 2;
    }

    if (otherArgs.length != 3) {
        printUsage();
        return 2;
    }

    String inputPath = otherArgs[0];
    String outputPath = otherArgs[1];
    String filterWord = otherArgs[2];

    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(new Path(outputPath))) {
        System.err.println("Output directory : " + outputPath + " already exists");
        return 2;
    }

    TezConfiguration tezConf = new TezConfiguration(conf);

    fs.getWorkingDirectory();
    Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);

    String jarPath = ClassUtil.findContainingJar(FilterLinesByWordOneToOne.class);
    if (jarPath == null) {
        throw new TezUncheckedException("Could not find any jar containing"
                + FilterLinesByWordOneToOne.class.getName() + " in the classpath");
    }

    Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
    fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
    FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);

    Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
    LocalResource dagJarLocalRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
            remoteJarStatus.getModificationTime());
    commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);

    TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, null);
    tezSession.start(); // Why do I need to start the TezSession.

    Configuration stage1Conf = new JobConf(conf);
    stage1Conf.set(FILTER_PARAM_NAME, filterWord);

    Configuration stage2Conf = new JobConf(conf);

    stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
    stage2Conf.setBoolean("mapred.mapper.new-api", false);

    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    // Setup stage1 Vertex
    Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor
            .create(FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
            .addTaskLocalFiles(commonLocalResources);

    DataSourceDescriptor dsd;
    if (generateSplitsInClient) {
        // TODO TEZ-1406. Dont' use MRInputLegacy
        stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
        stage1Conf.setBoolean("mapred.mapper.new-api", false);
        dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
    } else {
        dsd = MRInputLegacy.createConfigBuilder(stage1Conf, TextInputFormat.class, inputPath).groupSplits(false)
                .build();
    }
    stage1Vertex.addDataSource("MRInput", dsd);

    // Setup stage2 Vertex
    Vertex stage2Vertex = Vertex.create("stage2",
            ProcessorDescriptor.create(FilterByWordOutputProcessor.class.getName())
                    .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
            dsd.getNumberOfShards());
    stage2Vertex.addTaskLocalFiles(commonLocalResources);

    // Configure the Output for stage2
    stage2Vertex.addDataSink("MROutput",
            DataSinkDescriptor.create(
                    OutputDescriptor.create(MROutput.class.getName())
                            .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
                    OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));

    UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
            .newBuilder(Text.class.getName(), TextLongPair.class.getName()).setFromConfiguration(tezConf)
            .build();

    DAG dag = DAG.create("FilterLinesByWord");
    Edge edge = Edge.create(stage1Vertex, stage2Vertex, edgeConf.createDefaultOneToOneEdgeProperty());
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);

    LOG.info("Submitting DAG to Tez Session");
    DAGClient dagClient = tezSession.submitDAG(dag);
    LOG.info("Submitted DAG to Tez Session");

    DAGStatus dagStatus = null;
    String[] vNames = { "stage1", "stage2" };
    try {
        while (true) {
            dagStatus = dagClient.getDAGStatus(null);
            if (dagStatus.getState() == DAGStatus.State.RUNNING
                    || dagStatus.getState() == DAGStatus.State.SUCCEEDED
                    || dagStatus.getState() == DAGStatus.State.FAILED
                    || dagStatus.getState() == DAGStatus.State.KILLED
                    || dagStatus.getState() == DAGStatus.State.ERROR) {
                break;
            }
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
                // continue;
            }
        }

        while (dagStatus.getState() == DAGStatus.State.RUNNING) {
            try {
                ExampleDriver.printDAGStatus(dagClient, vNames);
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    // continue;
                }
                dagStatus = dagClient.getDAGStatus(null);
            } catch (TezException e) {
                LOG.fatal("Failed to get application progress. Exiting");
                return -1;
            }
        }
    } finally {
        fs.delete(stagingDir, true);
        tezSession.stop();
    }

    ExampleDriver.printDAGStatus(dagClient, vNames);
    LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
    return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}

---

• «
• 1
• 2
• »