Example usage for org.apache.hadoop.fs FileUtil copy

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil copy.

Prototype

private static boolean copy(FileSystem srcFS, FileStatus srcStatus, File dst, boolean deleteSource,
        Configuration conf) throws IOException 

Source Link

Document

Copy FileSystem files to local files.

Usage

From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2JobResourceManager.java

License:Open Source License

/**
 * Prepare job resources. Resolve the classpath list and download it if needed.
 *
 * @param download {@code true} If need to download resources.
 * @param jobLocDir Work directory for the job.
 * @throws GridException If failed./*from w  w w  . j  ava  2 s  .  c  om*/
 */
public void prepareJobEnvironment(boolean download, File jobLocDir) throws GridException {
    try {
        if (jobLocDir.exists())
            throw new GridException("Local job directory already exists: " + jobLocDir.getAbsolutePath());

        JobConf cfg = ctx.getJobConf();

        String mrDir = cfg.get("mapreduce.job.dir");

        if (mrDir != null) {
            stagingDir = new Path(new URI(mrDir));

            if (download) {
                FileSystem fs = FileSystem.get(stagingDir.toUri(), cfg);

                if (!fs.exists(stagingDir))
                    throw new GridException(
                            "Failed to find map-reduce submission directory (does not exist): " + stagingDir);

                if (!FileUtil.copy(fs, stagingDir, jobLocDir, false, cfg))
                    throw new GridException(
                            "Failed to copy job submission directory contents to local file system " + "[path="
                                    + stagingDir + ", locDir=" + jobLocDir.getAbsolutePath() + ", jobId="
                                    + jobId + ']');
            }

            File jarJobFile = new File(jobLocDir, "job.jar");

            Collection<URL> clsPathUrls = new ArrayList<>();

            clsPathUrls.add(jarJobFile.toURI().toURL());

            rsrcList.add(jarJobFile);
            rsrcList.add(new File(jobLocDir, "job.xml"));

            processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
            processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null,
                    MRJobConfig.CACHE_LOCALARCHIVES);
            processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
            processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);

            if (!clsPathUrls.isEmpty()) {
                clsPath = new URL[clsPathUrls.size()];

                clsPathUrls.toArray(clsPath);
            }
        } else if (!jobLocDir.mkdirs())
            throw new GridException("Failed to create local job directory: " + jobLocDir.getAbsolutePath());

        setLocalFSWorkingDirectory(jobLocDir);
    } catch (URISyntaxException | IOException e) {
        throw new GridException(e);
    }
}

---

From source file:org.kitesdk.maven.plugins.DeployAppMojo.java

License:Apache License

public void execute() throws MojoExecutionException, MojoFailureException {
    try {//from  w  w w  . j a  v  a  2  s  . c o m
        Configuration conf = new Configuration();
        Path appPath = getAppPath();
        File appFile = localApplicationFile != null ? localApplicationFile
                : new File(mavenProject.getBuild().getDirectory(), applicationName);
        getLog().info("Deploying " + appFile + " to " + appPath);

        FileSystem destFileSystem = FileSystem.get(new URI(deployFileSystem), conf);
        if (destFileSystem.exists(appPath)) {
            if (!updateApplication) {
                throw new MojoExecutionException("Application already exists at " + appPath
                        + ". Use 'updateApplication' option to force deployment.");
            }
            boolean success = destFileSystem.delete(appPath, true);
            if (!success) {
                throw new MojoExecutionException("Error deleting existing application at " + appPath);
            }
        }
        boolean success = FileUtil.copy(appFile, destFileSystem, appPath, false, conf);
        if (!success) {
            throw new MojoExecutionException(
                    "Error creating parent directories " + "for deploying Oozie application");
        }
    } catch (URISyntaxException e) {
        throw new MojoExecutionException("Syntax error in 'deployFileSystem': " + deployFileSystem, e);
    } catch (IOException e) {
        throw new MojoExecutionException("Error deploying application", e);
    }
}

---

From source file:org.springframework.data.hadoop.fs.FsShell.java

License:Apache License

private void copyToLocal(final FileSystem srcFS, final Path src, final File dst, final boolean copyCrc)
        throws IOException {

    final String COPYTOLOCAL_PREFIX = "_copyToLocal_";

    /* Keep the structure similar to ChecksumFileSystem.copyToLocal(). 
    * Ideal these two should just invoke FileUtil.copy() and not repeat
    * recursion here. Of course, copy() should support two more options :
    * copyCrc and useTmpFile (may be useTmpFile need not be an option).
    */// ww w .j a  v a  2 s.  c om
    if (!srcFS.getFileStatus(src).isDir()) {
        if (dst.exists()) {
            // match the error message in FileUtil.checkDest():
            throw new IOException("Target " + dst + " already exists");
        }

        // use absolute name so that tmp file is always created under dest dir
        File tmp = FileUtil.createLocalTempFile(dst.getAbsoluteFile(), COPYTOLOCAL_PREFIX, true);
        if (!FileUtil.copy(srcFS, src, tmp, false, srcFS.getConf())) {
            throw new IOException("Failed to copy " + src + " to " + dst);
        }

        if (!tmp.renameTo(dst)) {
            throw new IOException(
                    "Failed to rename tmp file " + tmp + " to local destination \"" + dst + "\".");
        }

        if (copyCrc) {
            if (!(srcFS instanceof ChecksumFileSystem)) {
                throw new IOException("Source file system does not have crc files");
            }

            ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS;
            File dstcs = FileSystem.getLocal(srcFS.getConf())
                    .pathToFile(csfs.getChecksumFile(new Path(dst.getCanonicalPath())));
            copyToLocal(csfs.getRawFileSystem(), csfs.getChecksumFile(src), dstcs, false);
        }
    } else {
        // once FileUtil.copy() supports tmp file, we don't need to mkdirs().
        dst.mkdirs();
        for (FileStatus path : srcFS.listStatus(src)) {
            copyToLocal(srcFS, path.getPath(), new File(dst, path.getPath().getName()), copyCrc);
        }
    }
}

---

• «
• 1
• 2
• 3
• »