Example usage for org.apache.hadoop.util JarFinder getJar

Introduction

In this page you can find the example usage for org.apache.hadoop.util JarFinder getJar.

Prototype

public static String getJar(Class klass) 

Source Link

Document

Returns the full path to the Jar containing the class.

Usage

From source file:com.datatorrent.stram.StramClient.java

License:Apache License

public static LinkedHashSet<String> findJars(LogicalPlan dag, Class<?>[] defaultClasses) {
    List<Class<?>> jarClasses = new ArrayList<Class<?>>();

    for (String className : dag.getClassNames()) {
        try {/* www. j  a v  a  2  s  .c  om*/
            Class<?> clazz = Thread.currentThread().getContextClassLoader().loadClass(className);
            jarClasses.add(clazz);
        } catch (ClassNotFoundException e) {
            throw new IllegalArgumentException("Failed to load class " + className, e);
        }
    }

    for (Class<?> clazz : Lists.newArrayList(jarClasses)) {
        // process class and super classes (super does not require deploy annotation)
        for (Class<?> c = clazz; c != Object.class && c != null; c = c.getSuperclass()) {
            //LOG.debug("checking " + c);
            jarClasses.add(c);
            jarClasses.addAll(Arrays.asList(c.getInterfaces()));
        }
    }

    jarClasses.addAll(Arrays.asList(defaultClasses));

    if (dag.isDebug()) {
        LOG.debug("Deploy dependencies: {}", jarClasses);
    }

    LinkedHashSet<String> localJarFiles = new LinkedHashSet<String>(); // avoid duplicates
    HashMap<String, String> sourceToJar = new HashMap<String, String>();

    for (Class<?> jarClass : jarClasses) {
        if (jarClass.getProtectionDomain().getCodeSource() == null) {
            // system class
            continue;
        }
        String sourceLocation = jarClass.getProtectionDomain().getCodeSource().getLocation().toString();
        String jar = sourceToJar.get(sourceLocation);
        if (jar == null) {
            // don't create jar file from folders multiple times
            jar = JarFinder.getJar(jarClass);
            sourceToJar.put(sourceLocation, jar);
            LOG.debug("added sourceLocation {} as {}", sourceLocation, jar);
        }
        if (jar == null) {
            throw new AssertionError("Cannot resolve jar file for " + jarClass);
        }
        localJarFiles.add(jar);
    }

    String libJarsPath = dag.getValue(LogicalPlan.LIBRARY_JARS);
    if (!StringUtils.isEmpty(libJarsPath)) {
        String[] libJars = StringUtils.splitByWholeSeparator(libJarsPath, LIB_JARS_SEP);
        localJarFiles.addAll(Arrays.asList(libJars));
    }

    LOG.info("Local jar file dependencies: " + localJarFiles);

    return localJarFiles;
}

---

From source file:com.datatorrent.stram.StramMiniClusterTest.java

License:Apache License

@Test
public void testSetupShutdown() throws Exception {
    GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
    ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
    GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
    List<NodeReport> nodeReports = response.getNodeReports();
    LOG.info("{}", nodeReports);

    for (NodeReport nr : nodeReports) {
        LOG.info("Node: {}", nr.getNodeId());
        LOG.info("Total memory: {}", nr.getCapability());
        LOG.info("Used memory: {}", nr.getUsed());
        LOG.info("Number containers: {}", nr.getNumContainers());
    }/*from   www  .j  ava 2 s.c o  m*/

    String appMasterJar = JarFinder.getJar(StreamingAppMaster.class);
    LOG.info("appmaster jar: " + appMasterJar);
    String testJar = JarFinder.getJar(StramMiniClusterTest.class);
    LOG.info("testJar: " + testJar);

    // create test application
    Properties dagProps = new Properties();

    // input module (ensure shutdown works while windows are generated)
    dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.classname",
            TestGeneratorInputOperator.class.getName());
    dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.maxTuples", "1");

    // fake output adapter - to be ignored when determine shutdown
    //props.put(DAGContext.DT_PREFIX + "stream.output.classname", HDFSOutputStream.class.getName());
    //props.put(DAGContext.DT_PREFIX + "stream.output.inputNode", "module2");
    //props.put(DAGContext.DT_PREFIX + "stream.output.filepath", "miniclustertest-testSetupShutdown.out");

    dagProps.put(StreamingApplication.DT_PREFIX + "operator.module1.classname",
            GenericTestOperator.class.getName());

    dagProps.put(StreamingApplication.DT_PREFIX + "operator.module2.classname",
            GenericTestOperator.class.getName());

    dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.source", "numGen.outport");
    dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.sinks", "module1.inport1");

    dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.source", "module1.outport1");
    dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.sinks", "module2.inport1");

    dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(),
            "-Dlog4j.properties=custom_log4j.properties");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(),
            "64");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.VCORES.getName(),
            "1");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + "operator.*.port.*."
            + Context.PortContext.BUFFER_MEMORY_MB.getName(), "32");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.DEBUG.getName(), "true");
    //dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINERS_MAX_COUNT.getName(), "2");
    LOG.info("dag properties: {}", dagProps);

    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf);
    tb.addFromProperties(dagProps, null);
    LogicalPlan dag = createDAG(tb);
    Configuration yarnConf = new Configuration(yarnCluster.getConfig());
    StramClient client = new StramClient(yarnConf, dag);
    try {
        client.start();
        if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
            client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
        }
        LOG.info("Running client");
        client.startApplication();
        boolean result = client.monitorApplication();

        LOG.info("Client run completed. Result=" + result);
        Assert.assertTrue(result);
    } finally {
        client.stop();
    }
}

---

From source file:org.apache.metron.integration.components.YarnComponent.java

License:Apache License

public YarnComponent withApplicationMasterClass(Class clazz) {
    appmasterJar = JarFinder.getJar(clazz);
    return this;
}

---

From source file:org.apache.solr.hadoop.hack.MiniMRClientClusterFactory.java

License:Apache License

public static MiniMRClientCluster create(Class<?> caller, String identifier, int noOfNMs, Configuration conf,
        File testWorkDir) throws IOException {

    if (conf == null) {
        conf = new Configuration();
    }//from w w w  .ja  v  a 2s.c o m

    FileSystem fs = FileSystem.get(conf);

    Path testRootDir = new Path(testWorkDir.getPath(), identifier + "-tmpDir").makeQualified(fs);
    Path appJar = new Path(testRootDir, "MRAppJar.jar");

    // Copy MRAppJar and make it private.
    Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR);

    fs.copyFromLocalFile(appMasterJar, appJar);
    fs.setPermission(appJar, new FsPermission("744"));

    Job job = Job.getInstance(conf);

    job.addFileToClassPath(appJar);

    Path callerJar = new Path(JarFinder.getJar(caller));
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));
    job.addFileToClassPath(remoteCallerJar);

    MiniMRYarnCluster miniMRYarnCluster;
    try {
        miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs, testWorkDir);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    job.getConfiguration().set("minimrclientcluster.caller.name", identifier);
    job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", noOfNMs);
    miniMRYarnCluster.init(job.getConfiguration());
    miniMRYarnCluster.start();

    return new MiniMRYarnClusterAdapter(miniMRYarnCluster, testWorkDir);
}

---