Example usage for org.apache.hadoop.fs FileUtil copy

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil copy.

Prototype

public static boolean copy(FileSystem srcFS, Path src, FileSystem dstFS, Path dst, boolean deleteSource,
        Configuration conf) throws IOException 

Source Link

Document

Copy files between FileSystems.

Usage

From source file:org.springframework.data.hadoop.fs.FsShell.java

License:Apache License

public void cp(String src, String src2, String... dst) {
    Object[] va = parseVarargs(src, src2, dst);
    @SuppressWarnings("unchecked")
    List<Path> srcs = (List<Path>) va[0];
    Path dstPath = (Path) va[1];

    try {//w  ww .ja  v  a  2s.  c  o  m

        FileSystem dstFs = dstPath.getFileSystem(configuration);
        boolean isDestDir = !dstFs.isFile(dstPath);

        if (StringUtils.hasText(src2) || (ObjectUtils.isEmpty(dst) && dst.length > 2)) {
            if (!isDestDir) {
                throw new IllegalArgumentException("When copying multiple files, destination " + dstPath.toUri()
                        + " should be a directory.");
            }
        }

        for (Path path : srcs) {
            FileSystem srcFs = path.getFileSystem(configuration);
            Path[] from = FileUtil.stat2Paths(srcFs.globStatus(path), path);
            if (!ObjectUtils.isEmpty(from) && from.length > 1 && !isDestDir) {
                throw new IllegalArgumentException(
                        "When copying multiple files, destination should be a directory.");
            }
            for (Path fromPath : from) {
                FileUtil.copy(srcFs, fromPath, dstFs, dstPath, false, configuration);
            }
        }
    } catch (IOException ex) {
        throw new HadoopException("Cannot copy resources " + ex.getMessage(), ex);
    }
}

---

From source file:org.wso2.carbon.hdfs.mgt.HDFSAdmin.java

License:Open Source License

/**
 * Copy  a given path to another path./*from ww w  .  j av a 2s.c  om*/
 * @param srcPath the src path to copy.
 * @param dstPath the destination to copy to.
 * @throws HDFSServerManagementException
 */
public void copy(String srcPath, String dstPath) throws HDFSServerManagementException {

    FileSystem hdfsFS = null;
    try {
        hdfsFS = hdfsAdminHelperInstance.getFSforUser();
    } catch (IOException e) {
        String msg = "Error occurred while mouting the file system";
        handleException(msg, e);
    }

    Path[] srcs = new Path[0];
    if (hdfsFS != null) {
        try {
            srcs = FileUtil.stat2Paths(hdfsFS.globStatus(new Path(srcPath)), new Path(srcPath));
        } catch (IOException e) {
            String msg = "Error occurred while trying to copy file.";
            handleException(msg, e);
        }
    }
    try {
        if (srcs.length > 1 && !hdfsFS.getFileStatus(new Path(dstPath)).isDir()) {
            throw new IOException("When copying multiple files, " + "destination should be a directory.");
        }
    } catch (IOException e) {
        String msg = "Error occurred while trying to copy file.";
        handleException(msg, e);
    }
    Configuration configuration = new Configuration();
    configuration.set("io.file.buffer.size", Integer.toString(4096));
    for (int i = 0; i < srcs.length; i++) {
        try {
            FileUtil.copy(hdfsFS, srcs[i], hdfsFS, new Path(dstPath), false, configuration);
        } catch (IOException e) {
            String msg = "Error occurred while trying to copy file.";
            handleException(msg, e);
        }
    }
}

---

From source file:simsql.code_generator.MyPhysicalDatabase.java

License:Apache License

public void backupTo(String toHere) {

    ConsoleReader cr = null;//from  w ww.  jav a 2s. c  o m
    try {
        cr = new ConsoleReader();
    } catch (Exception e) {
        throw new RuntimeException("Could not create a console for reading!", e);
    }

    try {

        // first, we see if the directory that we are writing to exists
        Configuration conf = new Configuration();
        FileSystem dfs = FileSystem.get(conf);

        Path pathTo = new Path(toHere);
        if (dfs.exists(pathTo)) {
            System.out.format("The specified path already exists. Would you like to overwrite it? [Y/N] ");

            char answer = (char) cr.readCharacter(new char[] { 'Y', 'N', 'y', 'n' });
            System.out.println(answer);

            if (answer == 'Y' || answer == 'y') {
                dfs.delete(pathTo, true);
            } else {

                // otherwise, we don't proceed.
                pathTo = null;
            }
        }

        // do we continue?
        if (pathTo != null) {

            // make it a directory
            // dfs.mkdirs(pathTo);

            // get our current directory.
            Path pathFrom = new Path(myDir);

            // and all the paths of the in our directory
            Path[] sourcePaths = FileUtil.stat2Paths(dfs.globStatus(pathFrom), pathFrom);
            for (Path sp : sourcePaths) {

                // copy all of it.
                FileUtil.copy(dfs, sp, dfs, pathTo, false, conf);
            }
        }

    } catch (Exception e) {
        throw new RuntimeException("Could not back up data into directory " + toHere, e);
    }
}

---

From source file:simsql.code_generator.MyPhysicalDatabase.java

License:Apache License

public void restoreFrom(String fromHere) {

    try {//w  w w  .jav  a2  s  .c  o m

        // first, we see if the directory that we are reading from
        // exists and is a directory.
        Configuration conf = new Configuration();
        FileSystem dfs = FileSystem.get(conf);

        Path pathFrom = new Path(fromHere);
        if (!dfs.exists(pathFrom) || !dfs.isDirectory(pathFrom)) {
            System.out.println("The specified restoration path does not exist or is not a directory!");
            return;
        }

        // now, get the destination path.
        Path pathTo = new Path(myDir);
        if (dfs.exists(pathTo)) {

            // destroy it, if it's there.
            dfs.delete(pathTo, true);
        }

        // make the directory
        // dfs.mkdirs(pathTo);

        // and all the paths we will be copying
        Path[] sourcePaths = FileUtil.stat2Paths(dfs.globStatus(pathFrom), pathFrom);
        for (Path sp : sourcePaths) {

            // restore all of it.
            FileUtil.copy(dfs, sp, dfs, pathTo, false, conf);
        }

    } catch (Exception e) {
        throw new RuntimeException("Could not restore data from directory " + fromHere, e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »