Example usage for org.apache.hadoop.util RunJar unJar

List of usage examples for org.apache.hadoop.util RunJar unJar

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.util.*
  5. RunJar
  6. unJar

Introduction

In this page you can find the example usage for org.apache.hadoop.util RunJar unJar.

Prototype

public void unJar(File jarFile, File toDir) throws IOException

Source Link

Document

Unpack a jar file into a directory.

Usage

From source file:com.chinamobile.bcbsp.bspstaff.StaffRunner.java

License:Apache License

/**
 * Start to run a BSP staff.//from   www.  ja va2  s.co  m
 */
@Override
public void run() {
    try {
        String sep = System.getProperty("path.separator");
        File workDir = new File(new File(staff.getJobFile()).getParent(), "work");
        boolean isCreated = workDir.mkdirs();
        if (!isCreated) {
            LOG.debug("StaffRunner.workDir : " + workDir);
        }
        StringBuffer classPath = new StringBuffer();
        classPath.append(System.getProperty("java.class.path"));
        classPath.append(sep);
        if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(this.conf.getJobType())) {
            String exe = conf.getJobExe();
            if (exe != null) {
                classPath.append(sep);
                classPath.append(exe);
                classPath.append(sep);
                classPath.append(workDir);
            }
        } else {
            String jar = conf.getJar();
            // if jar exists, it into workDir
            if (jar != null) {
                RunJar.unJar(new File(jar), workDir);
                File[] libs = new File(workDir, "lib").listFiles();
                if (libs != null) {
                    for (int i = 0; i < libs.length; i++) {
                        // add libs from jar to classpath
                        classPath.append(sep);
                        classPath.append(libs[i]);
                    }
                }
                classPath.append(sep);
                classPath.append(new File(workDir, "classes"));
                classPath.append(sep);
                classPath.append(workDir);
            }
        }
        // Build exec child jmv args.
        Vector<String> vargs = new Vector<String>();
        File jvm = new File(new File(System.getProperty("java.home"), "bin"), "java");
        vargs.add(jvm.toString());

        // bsp.child.java.opts
        String javaOpts = conf.getConf().get("bsp.child.java.opts", "-Xmx200m");
        javaOpts = javaOpts.replace("@taskid@", staff.getStaffID().toString());
        String[] javaOptsSplit = javaOpts.split(" ");
        for (int i = 0; i < javaOptsSplit.length; i++) {
            vargs.add(javaOptsSplit[i]);
        }

        // Add classpath.
        vargs.add("-classpath");
        vargs.add(classPath.toString());

        // Setup the log4j prop
        long logSize = StaffLog.getStaffLogLength(((BSPConfiguration) conf.getConf()));
        vargs.add("-Dbcbsp.log.dir=" + new File(System.getProperty("bcbsp.log.dir")).getAbsolutePath());
        vargs.add("-Dbcbsp.root.logger=INFO,TLA");

        LOG.info("debug: staff ID is " + staff.getStaffID());

        vargs.add("-Dbcbsp.tasklog.taskid=" + staff.getStaffID());
        vargs.add("-Dbcbsp.tasklog.totalLogFileSize=" + logSize);

        // Add main class and its arguments
        vargs.add(WorkerManager.Child.class.getName());
        InetSocketAddress addr = workerManager.getStaffTrackerReportAddress();

        vargs.add(addr.getHostName());
        vargs.add(Integer.toString(addr.getPort()));
        vargs.add(staff.getStaffID().toString());
        vargs.add(Integer.toString(getFaultSSStep()));
        vargs.add(workerManager.getHostName());
        vargs.add(this.conf.getJobType());

        // Run java
        runChild(vargs.toArray(new String[0]), workDir);
    } catch (Exception e) {
        LOG.error("[run]", e);
    }
}

From source file:com.chinamobile.bcbsp.workermanager.WorkerManager.java

License:Apache License

/**
 * Localization of job./*from  w  ww  .j a  v  a2  s  .c  o m*/
 * @param sip StaffInProgress
 * @param directive {@link Directive}
 */
private void localizeJob(StaffInProgress sip, Directive directive) throws IOException {
    Staff staff = sip.getStaff();
    conf.addResource(staff.getJobFile());
    BSPJob defaultJobConf = new BSPJob((BSPConfiguration) conf);
    Path localJobFile = defaultJobConf.getLocalPath(
            Constants.BC_BSP_LOCAL_SUBDIR_WORKERMANAGER + "/" + staff.getStaffID() + "/" + "job.xml");
    Path localJarFile = null;
    // systemFS.copyToLocalFile(new Path(staff.getJobFile()), localJobFile);
    bspsystemFS.copyToLocalFile(new BSPHdfsImpl().newPath(staff.getJobFile()), localJobFile);
    BSPConfiguration confBsp = new BSPConfiguration();
    confBsp.addResource(localJobFile);
    LOG.info("debug: conf.get(Constants.USER_BC_BSP_JOB_TYPE) " + confBsp.get(Constants.USER_BC_BSP_JOB_TYPE));
    BSPJob jobConf = new BSPJob(confBsp, staff.getJobID().toString());
    LOG.info("debug: conf.get(Constants.USER_BC_BSP_JOB_TYPE) " + confBsp.get(Constants.USER_BC_BSP_JOB_TYPE));
    LOG.info("debug: job type is " + jobConf.getJobType());
    if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(jobConf.getJobType())) {
        LOG.info("debug: in LocalizeJob job.exe");
        localJarFile = defaultJobConf.getLocalPath(
                Constants.BC_BSP_LOCAL_SUBDIR_WORKERMANAGER + "/" + staff.getStaffID() + "/" + "jobC");
    } else {
        LOG.info("debug: in in LocalizeJob  job.jar");
        localJarFile = defaultJobConf.getLocalPath(
                Constants.BC_BSP_LOCAL_SUBDIR_WORKERMANAGER + "/" + staff.getStaffID() + "/" + "job.jar");
    }
    Path jarFile = null;
    LOG.info("debug: job type is" + jobConf.getJobType());
    if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(jobConf.getJobType())) {
        LOG.info("debug: in LocalizeJob bofore jobConf.getJobExe =" + jobConf.getJobExe());
        if (jobConf.getJobExe() != null) {
            jarFile = new Path(jobConf.getJobExe());
        }
        LOG.info("jarFile is" + jarFile);
        jobConf.setJobExe(localJarFile.toString());
    } else {
        if (jobConf.getJar() != null) {
            jarFile = new Path(jobConf.getJar());
        }
        jobConf.setJar(localJarFile.toString());
    }
    if (jarFile != null) {
        LOG.info("jarFile != null");
        bspsystemFS.copyToLocalFile(jarFile, localJarFile);
        File workDir = new File(new File(localJobFile.toString()).getParent(), "work");
        if (!workDir.mkdirs()) {
            if (!workDir.isDirectory()) {
                throw new IOException("Mkdirs failed to create " + workDir.toString());
            }
        }
        if (!Constants.USER_BC_BSP_JOB_TYPE_C.equals(jobConf.getJobType())) {
            RunJar.unJar(new File(localJarFile.toString()), workDir);
            /** Add the user program jar to the system's classpath. */
            ClassLoaderUtil.addClassPath(localJarFile.toString());
        }
    }
    RunningJob rjob = addStaffToJob(staff.getJobID(), localJobFile, sip, directive, jobConf);
    LOG.info("debug:after addStaffToJob(staff.getJobID(), " + "localJobFile, sip, directive, jobConf); ");
    rjob.localized = true;
    sip.setFaultSSStep(directive.getFaultSSStep());
    LOG.info("debug:before launchStaffForJob(sip, jobConf);");
    launchStaffForJob(sip, jobConf);
}

From source file:org.apache.hama.bsp.GroomServer.java

License:Apache License

private void localizeJob(TaskInProgress tip) throws IOException {
    Task task = tip.getTask();//w  w  w .  j  av a 2 s . co  m
    conf.addResource(task.getJobFile());
    BSPJob defaultJobConf = new BSPJob((HamaConfiguration) conf);
    Path localJobFile = defaultJobConf.getLocalPath(SUBDIR + "/" + task.getTaskID() + "/" + "job.xml");

    RunningJob rjob = addTaskToJob(task.getJobID(), localJobFile, tip);
    BSPJob jobConf = null;

    synchronized (rjob) {
        if (!rjob.localized) {
            FileSystem dfs = FileSystem.get(conf);
            FileSystem localFs = FileSystem.getLocal(conf);
            Path jobDir = localJobFile.getParent();
            if (localFs.exists(jobDir)) {
                localFs.delete(jobDir, true);
                boolean b = localFs.mkdirs(jobDir);
                if (!b)
                    throw new IOException("Not able to create job directory " + jobDir.toString());
            }

            Path localJarFile = defaultJobConf.getLocalPath(SUBDIR + "/" + task.getTaskID() + "/" + "job.jar");

            Path jobFilePath = new Path(task.getJobFile());

            //wait a while for file to finish being written
            for (int i = 0; i < 300 & !dfs.exists(jobFilePath); i++) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    LOG.warn("Sleep failed", e);
                }
            }

            dfs.copyToLocalFile(jobFilePath, localJobFile);

            HamaConfiguration conf = new HamaConfiguration();
            conf.addResource(localJobFile);
            jobConf = new BSPJob(conf, task.getJobID().toString());

            Path jarFile = null;
            if (jobConf.getJar() != null) {
                jarFile = new Path(jobConf.getJar());
            } else {
                LOG.warn("No jar file for job " + task.getJobID() + " has been defined!");
            }
            jobConf.setJar(localJarFile.toString());

            if (jarFile != null) {
                dfs.copyToLocalFile(jarFile, localJarFile);

                // also unjar the job.jar files in workdir
                File workDir = new File(new File(localJobFile.toString()).getParent(), "work");
                if (!workDir.mkdirs()) {
                    if (!workDir.isDirectory()) {
                        throw new IOException("Mkdirs failed to create " + workDir.toString());
                    }
                }
                RunJar.unJar(new File(localJarFile.toString()), workDir);
            }
            rjob.localized = true;
        } else {
            HamaConfiguration conf = new HamaConfiguration();
            conf.addResource(rjob.getJobFile());
            jobConf = new BSPJob(conf, rjob.getJobId().toString());
        }
    }

    launchTaskForJob(tip, jobConf);
}

From source file:org.apache.hama.bsp.TaskRunner.java

License:Apache License

private static String assembleClasspath(BSPJob jobConf, File workDir) {
    StringBuffer classPath = new StringBuffer();
    // start with same classpath as parent process
    classPath.append(System.getProperty("java.class.path"));
    classPath.append(SYSTEM_PATH_SEPARATOR);

    String jar = jobConf.getJar();
    if (jar != null) { // if jar exists, it into workDir
        try {//from   w w w . j a v a  2s .c o m
            RunJar.unJar(new File(jar), workDir);
        } catch (IOException ioe) {
            LOG.error("Unable to uncompressing file to " + workDir.toString(), ioe);
        }
        File[] libs = new File(workDir, "lib").listFiles();
        if (libs != null) {
            for (File lib : libs) {
                // add libs from jar to classpath
                classPath.append(SYSTEM_PATH_SEPARATOR);
                classPath.append(lib);
            }
        }
        classPath.append(SYSTEM_PATH_SEPARATOR);
        classPath.append(new File(workDir, "classes"));
        classPath.append(SYSTEM_PATH_SEPARATOR);
        classPath.append(workDir);
    }
    return classPath.toString();
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2JobResourceManager.java

License:Apache License

/**
 * Process list of resources.//  w  w  w  .  j  a v  a2  s.c o  m
 *
 * @param jobLocDir Job working directory.
 * @param files Array of {@link URI} or {@link org.apache.hadoop.fs.Path} to process resources.
 * @param download {@code true}, if need to download. Process class path only else.
 * @param extract {@code true}, if need to extract archive.
 * @param clsPathUrls Collection to add resource as classpath resource.
 * @param rsrcNameProp Property for resource name array setting.
 * @throws IOException If failed.
 */
private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
        @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
    if (F.isEmptyOrNulls(files))
        return;

    Collection<String> res = new ArrayList<>();

    for (Object pathObj : files) {
        Path srcPath;

        if (pathObj instanceof URI) {
            URI uri = (URI) pathObj;

            srcPath = new Path(uri);
        } else
            srcPath = (Path) pathObj;

        String locName = srcPath.getName();

        File dstPath = new File(jobLocDir.getAbsolutePath(), locName);

        res.add(locName);

        rsrcSet.add(dstPath);

        if (clsPathUrls != null)
            clsPathUrls.add(dstPath.toURI().toURL());

        if (!download)
            continue;

        JobConf cfg = ctx.getJobConf();

        FileSystem dstFs = FileSystem.getLocal(cfg);

        FileSystem srcFs = job.fileSystem(srcPath.toUri(), cfg);

        if (extract) {
            File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");

            if (!archivesPath.exists() && !archivesPath.mkdir())
                throw new IOException(
                        "Failed to create directory " + "[path=" + archivesPath + ", jobId=" + jobId + ']');

            File archiveFile = new File(archivesPath, locName);

            FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);

            String archiveNameLC = archiveFile.getName().toLowerCase();

            if (archiveNameLC.endsWith(".jar"))
                RunJar.unJar(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".zip"))
                FileUtil.unZip(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".tar.gz") || archiveNameLC.endsWith(".tgz")
                    || archiveNameLC.endsWith(".tar"))
                FileUtil.unTar(archiveFile, dstPath);
            else
                throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
        } else
            FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
    }

    if (!res.isEmpty() && rsrcNameProp != null)
        ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
}

From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2JobResourceManager.java

License:Apache License

/**
 * Process list of resources./*  ww w  . j  a  v a  2 s  .c o  m*/
 *
 * @param jobLocDir Job working directory.
 * @param files Array of {@link java.net.URI} or {@link org.apache.hadoop.fs.Path} to process resources.
 * @param download {@code true}, if need to download. Process class path only else.
 * @param extract {@code true}, if need to extract archive.
 * @param clsPathUrls Collection to add resource as classpath resource.
 * @param rsrcNameProp Property for resource name array setting.
 * @throws IOException If failed.
 */
private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
        @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
    if (F.isEmptyOrNulls(files))
        return;

    Collection<String> res = new ArrayList<>();

    for (Object pathObj : files) {
        String locName = null;
        Path srcPath;

        if (pathObj instanceof URI) {
            URI uri = (URI) pathObj;

            locName = uri.getFragment();

            srcPath = new Path(uri);
        } else
            srcPath = (Path) pathObj;

        if (locName == null)
            locName = srcPath.getName();

        File dstPath = new File(jobLocDir.getAbsolutePath(), locName);

        res.add(locName);

        rsrcSet.add(dstPath);

        if (clsPathUrls != null)
            clsPathUrls.add(dstPath.toURI().toURL());

        if (!download)
            continue;

        JobConf cfg = ctx.getJobConf();

        FileSystem dstFs = FileSystem.getLocal(cfg);

        FileSystem srcFs = srcPath.getFileSystem(cfg);

        if (extract) {
            File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");

            if (!archivesPath.exists() && !archivesPath.mkdir())
                throw new IOException(
                        "Failed to create directory " + "[path=" + archivesPath + ", jobId=" + jobId + ']');

            File archiveFile = new File(archivesPath, locName);

            FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);

            String archiveNameLC = archiveFile.getName().toLowerCase();

            if (archiveNameLC.endsWith(".jar"))
                RunJar.unJar(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".zip"))
                FileUtil.unZip(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".tar.gz") || archiveNameLC.endsWith(".tgz")
                    || archiveNameLC.endsWith(".tar"))
                FileUtil.unTar(archiveFile, dstPath);
            else
                throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
        } else
            FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
    }

    if (!res.isEmpty() && rsrcNameProp != null)
        ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
}

From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2JobResourceManager.java

License:Open Source License

/**
 * Process list of resources.//from   ww  w .  ja va 2s.  com
 *
 * @param jobLocDir Job working directory.
 * @param files Array of {@link java.net.URI} or {@link org.apache.hadoop.fs.Path} to process resources.
 * @param download {@code true}, if need to download. Process class path only else.
 * @param extract {@code true}, if need to extract archive.
 * @param clsPathUrls Collection to add resource as classpath resource.
 * @param rsrcNameProp Property for resource name array setting.
 * @throws IOException If failed.
 */
private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
        @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
    if (F.isEmptyOrNulls(files))
        return;

    Collection<String> res = new ArrayList<>();

    for (Object pathObj : files) {
        String locName = null;
        Path srcPath;

        if (pathObj instanceof URI) {
            URI uri = (URI) pathObj;

            locName = uri.getFragment();

            srcPath = new Path(uri);
        } else
            srcPath = (Path) pathObj;

        if (locName == null)
            locName = srcPath.getName();

        File dstPath = new File(jobLocDir.getAbsolutePath(), locName);

        res.add(locName);

        rsrcList.add(dstPath);

        if (clsPathUrls != null)
            clsPathUrls.add(dstPath.toURI().toURL());

        if (!download)
            continue;

        JobConf cfg = ctx.getJobConf();

        FileSystem dstFs = FileSystem.getLocal(cfg);

        FileSystem srcFs = srcPath.getFileSystem(cfg);

        if (extract) {
            File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");

            if (!archivesPath.exists() && !archivesPath.mkdir())
                throw new IOException(
                        "Failed to create directory " + "[path=" + archivesPath + ", jobId=" + jobId + ']');

            File archiveFile = new File(archivesPath, locName);

            FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);

            String archiveNameLC = archiveFile.getName().toLowerCase();

            if (archiveNameLC.endsWith(".jar"))
                RunJar.unJar(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".zip"))
                FileUtil.unZip(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".tar.gz") || archiveNameLC.endsWith(".tgz")
                    || archiveNameLC.endsWith(".tar"))
                FileUtil.unTar(archiveFile, dstPath);
            else
                throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
        } else
            FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
    }

    if (!res.isEmpty() && rsrcNameProp != null)
        ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
}