Example usage for org.apache.hadoop.fs FileUtil unZip

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil unZip.

Prototype

public static void unZip(File inFile, File unzipDir) throws IOException 

Source Link

Document

Given a File input it will unzip it in the unzip directory.

Usage

From source file:io.apigee.lembos.utils.RunnerUtils.java

License:Apache License

/**
 * Takes a module path, which could be a local filesystem path or a url, and returns the local path to the module.
 *
 * <b>Note:</b> If the value is a URL, the URL will be downloaded locally to create the necessary filesystem
 * location for the Node.js module to allow for archiving and adding to DistributedCache.
 *
 * @param conf the Hadoop configuration/*ww  w .  j  a va2 s. com*/
 *
 * @return the local filesystem path to the module
 *
 * @throws IOException if anything goes wrong
 */
public static File createLocalCopyOfModule(final Configuration conf) throws IOException {
    final String moduleName = conf.get(LembosConstants.MR_MODULE_NAME);
    final String modulePath = conf.get(LembosConstants.MR_MODULE_PATH);
    File localTempModule = null;

    if (moduleName != null && !moduleName.trim().isEmpty() && modulePath != null
            && !modulePath.trim().isEmpty()) {
        URL moduleUrl;

        // Test if this is a URL or a file
        try {
            moduleUrl = new URL(modulePath);
        } catch (MalformedURLException e) {
            // This is to be expected if the configuration path is not a URL
            moduleUrl = null;
        }

        // Create a local temporary directory to contain the Node.js module
        final java.nio.file.Path tmpDir = Files.createTempDirectory("LembosMapReduceModule");
        FileSystem fs;

        // Delete the temp directory
        tmpDir.toFile().deleteOnExit();

        // Create the proper FileSystem
        if (moduleUrl == null) {
            fs = FileSystem.getLocal(conf);
        } else {
            try {
                fs = FileSystem.get(moduleUrl.toURI(), conf);
            } catch (URISyntaxException e) {
                throw new IOException(e);
            }
        }

        final org.apache.hadoop.fs.Path pathObj = new org.apache.hadoop.fs.Path(modulePath);

        if (fs.exists(pathObj)) {
            final org.apache.hadoop.fs.Path tmpPathObj = new org.apache.hadoop.fs.Path(
                    tmpDir.toAbsolutePath().toString());

            // Copy the local/remote file(s) to the temporary directory
            fs.copyToLocalFile(pathObj, tmpPathObj);

            final File moduleFile = new File(
                    new org.apache.hadoop.fs.Path(tmpPathObj, pathObj.getName()).toString());

            // Set the MapReduce module path accordingly
            if (moduleFile.isFile()) {
                final String fileName = moduleFile.getName();
                boolean wasArchive = false;

                if (fileName.endsWith(".tar") || fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
                    FileUtil.unTar(moduleFile, tmpDir.toFile());
                    wasArchive = true;
                } else if (fileName.endsWith(".zip")) {
                    FileUtil.unZip(moduleFile, tmpDir.toFile());
                    wasArchive = true;
                }

                if (wasArchive) {
                    for (final String extension : KNOWN_NODE_MODULE_EXTENSIONS) {
                        final File potentialModuleFile = new File(tmpDir.toFile(), moduleName + extension);

                        if (potentialModuleFile.exists()) {
                            localTempModule = potentialModuleFile;
                            break;
                        }
                    }
                } else {
                    localTempModule = moduleFile;
                }
            } else {
                localTempModule = new File(tmpDir.toFile(), moduleName);
            }
        } else {
            throw new RuntimeException("Unable to create/locate Node.js module locally: " + modulePath);
        }
    }

    if (localTempModule == null) {
        throw new RuntimeException("Unable to create local copy of Node.js module from path: "
                + conf.get(LembosConstants.MR_MODULE_PATH));
    }

    return localTempModule;
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2JobResourceManager.java

License:Apache License

/**
 * Process list of resources.//from w ww  . j  a v a 2s.c o m
 *
 * @param jobLocDir Job working directory.
 * @param files Array of {@link URI} or {@link org.apache.hadoop.fs.Path} to process resources.
 * @param download {@code true}, if need to download. Process class path only else.
 * @param extract {@code true}, if need to extract archive.
 * @param clsPathUrls Collection to add resource as classpath resource.
 * @param rsrcNameProp Property for resource name array setting.
 * @throws IOException If failed.
 */
private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
        @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
    if (F.isEmptyOrNulls(files))
        return;

    Collection<String> res = new ArrayList<>();

    for (Object pathObj : files) {
        Path srcPath;

        if (pathObj instanceof URI) {
            URI uri = (URI) pathObj;

            srcPath = new Path(uri);
        } else
            srcPath = (Path) pathObj;

        String locName = srcPath.getName();

        File dstPath = new File(jobLocDir.getAbsolutePath(), locName);

        res.add(locName);

        rsrcSet.add(dstPath);

        if (clsPathUrls != null)
            clsPathUrls.add(dstPath.toURI().toURL());

        if (!download)
            continue;

        JobConf cfg = ctx.getJobConf();

        FileSystem dstFs = FileSystem.getLocal(cfg);

        FileSystem srcFs = job.fileSystem(srcPath.toUri(), cfg);

        if (extract) {
            File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");

            if (!archivesPath.exists() && !archivesPath.mkdir())
                throw new IOException(
                        "Failed to create directory " + "[path=" + archivesPath + ", jobId=" + jobId + ']');

            File archiveFile = new File(archivesPath, locName);

            FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);

            String archiveNameLC = archiveFile.getName().toLowerCase();

            if (archiveNameLC.endsWith(".jar"))
                RunJar.unJar(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".zip"))
                FileUtil.unZip(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".tar.gz") || archiveNameLC.endsWith(".tgz")
                    || archiveNameLC.endsWith(".tar"))
                FileUtil.unTar(archiveFile, dstPath);
            else
                throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
        } else
            FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
    }

    if (!res.isEmpty() && rsrcNameProp != null)
        ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
}

---

From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2JobResourceManager.java

License:Apache License

/**
 * Process list of resources.//from   ww  w .  j  a v a  2 s . c o  m
 *
 * @param jobLocDir Job working directory.
 * @param files Array of {@link java.net.URI} or {@link org.apache.hadoop.fs.Path} to process resources.
 * @param download {@code true}, if need to download. Process class path only else.
 * @param extract {@code true}, if need to extract archive.
 * @param clsPathUrls Collection to add resource as classpath resource.
 * @param rsrcNameProp Property for resource name array setting.
 * @throws IOException If failed.
 */
private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
        @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
    if (F.isEmptyOrNulls(files))
        return;

    Collection<String> res = new ArrayList<>();

    for (Object pathObj : files) {
        String locName = null;
        Path srcPath;

        if (pathObj instanceof URI) {
            URI uri = (URI) pathObj;

            locName = uri.getFragment();

            srcPath = new Path(uri);
        } else
            srcPath = (Path) pathObj;

        if (locName == null)
            locName = srcPath.getName();

        File dstPath = new File(jobLocDir.getAbsolutePath(), locName);

        res.add(locName);

        rsrcSet.add(dstPath);

        if (clsPathUrls != null)
            clsPathUrls.add(dstPath.toURI().toURL());

        if (!download)
            continue;

        JobConf cfg = ctx.getJobConf();

        FileSystem dstFs = FileSystem.getLocal(cfg);

        FileSystem srcFs = srcPath.getFileSystem(cfg);

        if (extract) {
            File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");

            if (!archivesPath.exists() && !archivesPath.mkdir())
                throw new IOException(
                        "Failed to create directory " + "[path=" + archivesPath + ", jobId=" + jobId + ']');

            File archiveFile = new File(archivesPath, locName);

            FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);

            String archiveNameLC = archiveFile.getName().toLowerCase();

            if (archiveNameLC.endsWith(".jar"))
                RunJar.unJar(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".zip"))
                FileUtil.unZip(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".tar.gz") || archiveNameLC.endsWith(".tgz")
                    || archiveNameLC.endsWith(".tar"))
                FileUtil.unTar(archiveFile, dstPath);
            else
                throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
        } else
            FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
    }

    if (!res.isEmpty() && rsrcNameProp != null)
        ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2JobResourceManager.java

License:Open Source License

/**
 * Process list of resources.//from   w  ww  .  jav  a2 s. c  om
 *
 * @param jobLocDir Job working directory.
 * @param files Array of {@link java.net.URI} or {@link org.apache.hadoop.fs.Path} to process resources.
 * @param download {@code true}, if need to download. Process class path only else.
 * @param extract {@code true}, if need to extract archive.
 * @param clsPathUrls Collection to add resource as classpath resource.
 * @param rsrcNameProp Property for resource name array setting.
 * @throws IOException If failed.
 */
private void processFiles(File jobLocDir, @Nullable Object[] files, boolean download, boolean extract,
        @Nullable Collection<URL> clsPathUrls, @Nullable String rsrcNameProp) throws IOException {
    if (F.isEmptyOrNulls(files))
        return;

    Collection<String> res = new ArrayList<>();

    for (Object pathObj : files) {
        String locName = null;
        Path srcPath;

        if (pathObj instanceof URI) {
            URI uri = (URI) pathObj;

            locName = uri.getFragment();

            srcPath = new Path(uri);
        } else
            srcPath = (Path) pathObj;

        if (locName == null)
            locName = srcPath.getName();

        File dstPath = new File(jobLocDir.getAbsolutePath(), locName);

        res.add(locName);

        rsrcList.add(dstPath);

        if (clsPathUrls != null)
            clsPathUrls.add(dstPath.toURI().toURL());

        if (!download)
            continue;

        JobConf cfg = ctx.getJobConf();

        FileSystem dstFs = FileSystem.getLocal(cfg);

        FileSystem srcFs = srcPath.getFileSystem(cfg);

        if (extract) {
            File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives");

            if (!archivesPath.exists() && !archivesPath.mkdir())
                throw new IOException(
                        "Failed to create directory " + "[path=" + archivesPath + ", jobId=" + jobId + ']');

            File archiveFile = new File(archivesPath, locName);

            FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg);

            String archiveNameLC = archiveFile.getName().toLowerCase();

            if (archiveNameLC.endsWith(".jar"))
                RunJar.unJar(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".zip"))
                FileUtil.unZip(archiveFile, dstPath);
            else if (archiveNameLC.endsWith(".tar.gz") || archiveNameLC.endsWith(".tgz")
                    || archiveNameLC.endsWith(".tar"))
                FileUtil.unTar(archiveFile, dstPath);
            else
                throw new IOException("Cannot unpack archive [path=" + srcPath + ", jobId=" + jobId + ']');
        } else
            FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
    }

    if (!res.isEmpty() && rsrcNameProp != null)
        ctx.getJobConf().setStrings(rsrcNameProp, res.toArray(new String[res.size()]));
}

---