Example usage for org.apache.hadoop.fs FileUtil copy

List of usage examples for org.apache.hadoop.fs FileUtil copy

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileUtil
  6. copy

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil copy.

Prototype

public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst,
        boolean deleteSource, boolean overwrite, Configuration conf) throws IOException

Source Link

Document

Copy files between FileSystems.

Usage

From source file:org.oclc.firefly.hadoop.backup.Import.java

License:Apache License

/**
 * Import the given tables from the given backup directory
 * @param tables An array of table names
 * @throws TableNotFoundException If a table is not found in backup copy
 * @throws TableExistsException If a table already exists
 * @throws IOException If failed to read from file system
 */// w ww .  j  av a 2  s. c o m
public void importAll(String[] tables) throws TableNotFoundException, TableExistsException, IOException {
    numFailedImports = 0;
    numTablesImported = 0;

    doChecks(tables);

    // make a copy of backup
    if (retainOriginal) {
        LOG.info("Making copy of tables as requested. This may take a while...");

        Path tmpPath = new Path(getTmpBackupDirectory() + "/" + backupDirPath.getName());
        fs.delete(tmpPath, true);

        // Only copy the tables that are being imported
        for (String tableName : tables) {
            Path tmpTablePath = new Path(tmpPath + "/" + tableName);
            Path bakupDirTablePath = new Path(backupDirPath + "/" + tableName);

            LOG.info(". Copying " + bakupDirTablePath + " to " + tmpTablePath);
            FileUtil.copy(fs, bakupDirTablePath, fs, tmpTablePath, false, true, conf);
        }

        backupDirPath = tmpPath;
    }

    // Import one table at a time
    LOG.info("Importing tables");
    for (String tableName : tables) {
        LOG.info(". " + tableName);

        boolean imported = importTable(backupDirPath, tableName);
        if (!imported) {
            LOG.error("Table not imported");
            numFailedImports++;
        } else {
            numTablesImported++;
        }
    }
}

From source file:org.smartfrog.services.hadoop.operations.dfs.DfsCopyFilesImpl.java

License:Open Source License

/**
 * do the work//from   w w w.  j  ava2 s.c  o  m
 *
 * @throws Exception on any failure
 */
@Override
protected void performDfsOperation() throws Exception {
    ManagedConfiguration conf = createConfiguration();

    String sourceFSURL = sfResolve(ATTR_SOURCEFS, "", true);
    String destFSURL = sfResolve(ATTR_DESTFS, "", true);
    Path source = resolveDfsPath(ATTR_SOURCE);
    Path dest = resolveDfsPath(ATTR_DEST);
    boolean overwrite = sfResolve(ATTR_OVERWRITE, false, true);

    String matchPattern = sfResolve(ATTR_PATTERN, "", true);

    int minFileCount = sfResolve(DfsPathOperation.ATTR_MIN_FILE_COUNT, 0, true);
    int maxFileCount = sfResolve(DfsPathOperation.ATTR_MAX_FILE_COUNT, 0, true);

    //URL to the destination filesystem
    FileSystem sourceFS = null;
    FileSystem destFS = null;
    try {
        sourceFS = DfsUtils.createFileSystem(sourceFSURL, conf);
        destFS = DfsUtils.createFileSystem(destFSURL, conf);
        DfsUtils.assertNotDependent(sourceFS, source, destFS, dest);
        DfsUtils.mkdirs(destFS, dest);

        //build a list of sourcefiles
        List<Path> sourceFiles = listFiles(sourceFS, source, matchPattern);
        int count = sourceFiles.size();
        if (count < minFileCount) {
            throw new SmartFrogDeploymentException(
                    "File count " + count + " is below the minFileCount value of " + minFileCount, this);
        }
        if (maxFileCount > -1 && count > maxFileCount) {
            throw new SmartFrogDeploymentException(
                    "File count " + count + " is above the maxFileCount value of " + minFileCount, this);
        }
        Path[] sourcePaths = new Path[count];
        sourcePaths = sourceFiles.toArray(sourcePaths);

        boolean deleteSource = false;
        FileUtil.copy(sourceFS, sourcePaths, destFS, dest, deleteSource, overwrite, conf);
    } finally {
        DfsUtils.closeQuietly(sourceFS);
        DfsUtils.closeQuietly(destFS);
    }
}

From source file:org.terrier.utility.io.HadoopUtility.java

License:Mozilla Public License

protected static void saveApplicationSetupToJob(JobConf jobConf, boolean getFreshProperties) throws Exception {
    // Do we load a fresh properties File?
    //TODO fix, if necessary
    //if (getFreshProperties)
    //   loadApplicationSetup(new Path(ApplicationSetup.TERRIER_HOME));

    FileSystem remoteFS = FileSystem.get(jobConf);
    URI remoteFSURI = remoteFS.getUri();
    //make a copy of the current application setup properties, these may be amended
    //as some files are more globally accessible
    final Properties propertiesDuringJob = new Properties();
    Properties appProperties = ApplicationSetup.getProperties();
    for (Object _key : appProperties.keySet()) {
        String key = (String) _key;
        propertiesDuringJob.put(key, appProperties.get(key));
    }/*from   ww  w.  ja v a2  s.  c o  m*/

    //the share folder is needed during indexing, save this on DFS
    if (Files.getFileSystemName(ApplicationSetup.TERRIER_SHARE).equals("local")) {
        Path tempTRShare = makeTemporaryFile(jobConf, "terrier.share");
        propertiesDuringJob.setProperty("terrier.share", remoteFSURI.resolve(tempTRShare.toUri()).toString());
        if (Files.exists(ApplicationSetup.TERRIER_SHARE)) {
            jobConf.set("terrier.share.copied", remoteFSURI.resolve(tempTRShare.toUri()).toString());
            logger.info("Copying terrier share/ directory (" + ApplicationSetup.TERRIER_SHARE
                    + ") to shared storage area (" + remoteFSURI.resolve(tempTRShare.toUri()).toString() + ")");
            FileUtil.copy(FileSystem.getLocal(jobConf), new Path(ApplicationSetup.TERRIER_SHARE), remoteFS,
                    tempTRShare, false, false, jobConf);
        } else {
            logger.warn(
                    "No terrier.share folder found at " + ApplicationSetup.TERRIER_SHARE + ", copying skipped");
        }
    }

    //copy the terrier.properties content over
    Path tempTRProperties = makeTemporaryFile(jobConf, "terrier.properties");
    logger.debug("Writing terrier properties out to DFS " + tempTRProperties.toString());
    OutputStream out = remoteFS.create(tempTRProperties);
    remoteFS.deleteOnExit(tempTRProperties);
    propertiesDuringJob.store(out, "Automatically generated by HadoopUtility.saveApplicationSetupToJob()");
    out.close();
    out = null;
    DistributedCache.addCacheFile(tempTRProperties.toUri().resolve(new URI("#terrier.properties")), jobConf);
    DistributedCache.createSymlink(jobConf);

    //copy the non-JVM system properties over as well
    Path tempSysProperties = makeTemporaryFile(jobConf, "system.properties");
    DataOutputStream dos = FileSystem.get(jobConf).create(tempSysProperties);
    logger.debug("Writing system properties out to DFS " + tempSysProperties.toString());
    for (Object _propertyKey : System.getProperties().keySet()) {
        String propertyKey = (String) _propertyKey;
        if (!startsWithAny(propertyKey, checkSystemProperties)) {
            dos.writeUTF(propertyKey);
            dos.writeUTF(System.getProperty(propertyKey));
        }
    }
    dos.writeUTF("FIN");
    dos.close();
    dos = null;
    DistributedCache.addCacheFile(tempSysProperties.toUri().resolve(new URI("#system.properties")), jobConf);
}

From source file:ras.test.hadoop.fs.InMemoryFileSystem.java

License:Apache License

static void copy(InMemoryFileSystem srcFs, Path srcPath, InMemoryFileSystem dstFs, Path dstParentPath)
        throws IOException {

    FileUtil.copy(srcFs, srcPath, dstFs, dstParentPath, false, false, new Configuration());
}

From source file:ras.test.hadoop.fs.LocalInMemoryFileSystem.java

License:Apache License

/**
 * /* ww  w.  j a va 2s  . com*/
 * @param conf
 *            The configuration whose distributed cache files are to be
 *            localized.
 * @throws IOException
 * @throws IllegalStateException
 *             If the {@code conf} has not been properly initialized via a
 *             call to {@link InMemoryFileSystem#configure(Configuration)}.
 */
@SuppressWarnings("resource")
public static void localizeCacheFiles(Configuration conf) throws IOException {
    Validate.notNull(conf, "conf == null not allowed!");

    LocalInMemoryFileSystem dstFs = new LocalInMemoryFileSystem();
    dstFs.setConf(conf);
    Path dstPath = new Path(BASE_LOCAL_CACHE_FILE_DIR);
    dstFs.mkdirs(dstPath);

    List<String> localFiles = new ArrayList<String>();
    URI[] cacheFiles = DistributedCache.getCacheFiles(conf);

    if (cacheFiles == null || cacheFiles.length == 0) {
        throw new IllegalStateException("No files added to distributed cache!");
    }

    for (URI uri : cacheFiles) {
        FileSystem srcFs = FileSystem.get(uri, conf);
        Path srcPath = new Path(uri.getPath());
        localFiles.add(new Path(dstPath, srcPath.getName()).toUri().toString());
        FileUtil.copy(srcFs, srcPath, dstFs, dstPath, false, false, conf);
    }
    conf.set(LOCAL_FILES_KEY, StringUtils.join(localFiles, ","));
}

From source file:wwh.exec.Main.java

private static void prepareEnvironment() throws IOException {
    FileSystem lfs = FileSystem.get(new Configuration());

    for (int i = 0; i < CLUSTERS.length; i++) {
        YarnConfiguration configuration = createClusterConfiguration(CLUSTERS[i]);
        FileSystem fs = FileSystem.get(configuration);

        fs.delete(new Path(REMOTE_DATA_FOLDER), true);
        fs.delete(new Path("/user/bennyl/cloud-burst"), true);

        Path localData = new Path(LOCAL_DATA_FOLDER + "c" + i);

        FileUtil.copy(lfs, localData, fs, new Path(REMOTE_DATA_FOLDER), false, true, configuration);

    }/*  www.java2  s.c o  m*/

    final YarnConfiguration mainClusterConfiguration = createClusterConfiguration(MAIN_CLUSTER_RM);
    FileUtil.copy(new File(LOCAL_WORKSPACE), FileSystem.get(mainClusterConfiguration),
            new Path(REMOTE_DATA_FOLDER), false, mainClusterConfiguration);

}