Example usage for org.apache.hadoop.mapreduce.filecache DistributedCache getCacheArchives

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.filecache DistributedCache getCacheArchives.

Prototype

@Deprecated
public static URI[] getCacheArchives(Configuration conf) throws IOException 

Source Link

Document

Get cache archives set in the Configuration.

Usage

From source file:org.apache.tez.mapreduce.hadoop.mapreduce.JobContextImpl.java

License:Apache License

/**
 * Get cache archives set in the Configuration
 * @return A URI array of the caches set in the Configuration
 * @throws IOException/*w  w  w.  j av  a 2 s.  c  o  m*/
 */
public URI[] getCacheArchives() throws IOException {
    return DistributedCache.getCacheArchives(conf);
}

---

From source file:org.apache.tez.mapreduce.processor.MRTask.java

License:Apache License

/**
 * Set up the DistributedCache related configs to make
 * {@link DistributedCache#getLocalCacheFiles(Configuration)} and
 * {@link DistributedCache#getLocalCacheArchives(Configuration)} working.
 *
 * @param job//from  ww  w  .  j ava 2 s . c om
 * @throws IOException
 */
private static void setupDistributedCacheConfig(final JobConf job) throws IOException {

    String localWorkDir = (job.get(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR));
    // ^ ^ all symlinks are created in the current work-dir

    // Update the configuration object with localized archives.
    URI[] cacheArchives = DistributedCache.getCacheArchives(job);
    if (cacheArchives != null) {
        List<String> localArchives = new ArrayList<String>();
        for (int i = 0; i < cacheArchives.length; ++i) {
            URI u = cacheArchives[i];
            Path p = new Path(u);
            Path name = new Path((null == u.getFragment()) ? p.getName() : u.getFragment());
            String linkName = name.toUri().getPath();
            localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
        }
        if (!localArchives.isEmpty()) {
            job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils.join(localArchives, ','));
        }
    }

    // Update the configuration object with localized files.
    URI[] cacheFiles = DistributedCache.getCacheFiles(job);
    if (cacheFiles != null) {
        List<String> localFiles = new ArrayList<String>();
        for (int i = 0; i < cacheFiles.length; ++i) {
            URI u = cacheFiles[i];
            Path p = new Path(u);
            Path name = new Path((null == u.getFragment()) ? p.getName() : u.getFragment());
            String linkName = name.toUri().getPath();
            localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
        }
        if (!localFiles.isEmpty()) {
            job.set(MRJobConfig.CACHE_LOCALFILES, StringUtils.join(localFiles, ','));
        }
    }
}

---

From source file:org.apache.tez.mapreduce.task.MRRuntimeTask.java

License:Apache License

/**
 * Set up the DistributedCache related configs to make
 * {@link DistributedCache#getLocalCacheFiles(Configuration)} and
 * {@link DistributedCache#getLocalCacheArchives(Configuration)} working.
 * //from  w ww  .jav a 2s. com
 * @param job
 * @throws IOException
 */
private static void setupDistributedCacheConfig(final JobConf job) throws IOException {

    String localWorkDir = (job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
    // ^ ^ all symlinks are created in the current work-dir

    // Update the configuration object with localized archives.
    URI[] cacheArchives = DistributedCache.getCacheArchives(job);
    if (cacheArchives != null) {
        List<String> localArchives = new ArrayList<String>();
        for (int i = 0; i < cacheArchives.length; ++i) {
            URI u = cacheArchives[i];
            Path p = new Path(u);
            Path name = new Path((null == u.getFragment()) ? p.getName() : u.getFragment());
            String linkName = name.toUri().getPath();
            localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
        }
        if (!localArchives.isEmpty()) {
            job.set(MRJobConfig.CACHE_LOCALARCHIVES,
                    StringUtils.arrayToString(localArchives.toArray(new String[localArchives.size()])));
        }
    }

    // Update the configuration object with localized files.
    URI[] cacheFiles = DistributedCache.getCacheFiles(job);
    if (cacheFiles != null) {
        List<String> localFiles = new ArrayList<String>();
        for (int i = 0; i < cacheFiles.length; ++i) {
            URI u = cacheFiles[i];
            Path p = new Path(u);
            Path name = new Path((null == u.getFragment()) ? p.getName() : u.getFragment());
            String linkName = name.toUri().getPath();
            localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
        }
        if (!localFiles.isEmpty()) {
            job.set(MRJobConfig.CACHE_LOCALFILES,
                    StringUtils.arrayToString(localFiles.toArray(new String[localFiles.size()])));
        }
    }
}

---