Example usage for org.apache.hadoop.filecache DistributedCache DistributedCache

Introduction

In this page you can find the example usage for org.apache.hadoop.filecache DistributedCache DistributedCache.

Prototype

DistributedCache

Source Link

Usage

From source file:org.springframework.data.hadoop.fs.DistributedCacheFactoryBean.java

License:Apache License

@Override
public void afterPropertiesSet() throws Exception {
    Assert.notNull(conf, "A Hadoop configuration is required");
    Assert.notEmpty(entries, "No entries specified");

    // fall back to system discovery
    if (fs == null) {
        fs = FileSystem.get(conf);
    }//from  w  w  w.  j a v  a 2 s.  c  om

    ds = new DistributedCache();

    if (createSymlink) {
        DistributedCache.createSymlink(conf);
    }

    HdfsResourceLoader loader = new HdfsResourceLoader(fs);

    boolean warnCpEntry = !":".equals(System.getProperty("path.separator"));

    try {
        for (CacheEntry entry : entries) {
            Resource[] resources = loader.getResources(entry.value);
            if (!ObjectUtils.isEmpty(resources)) {
                for (Resource resource : resources) {
                    HdfsResource res = (HdfsResource) resource;

                    URI uri = res.getURI();
                    String path = getPathWithFragment(uri);

                    String defaultLink = resource.getFilename();
                    boolean isArchive = (defaultLink.endsWith(".tgz") || defaultLink.endsWith(".tar")
                            || defaultLink.endsWith(".tar.gz") || defaultLink.endsWith(".zip"));

                    switch (entry.type) {
                    case CP:
                        // Path does not handle fragments so use the URI instead
                        Path p = new Path(URI.create(path));

                        if (FILE_SEPARATOR_WARNING && warnCpEntry) {
                            LogFactory.getLog(DistributedCacheFactoryBean.class).warn(
                                    "System path separator is not ':' - this will likely cause invalid classpath entries within the DistributedCache. See the docs and HADOOP-9123 for more information.");
                            // show the warning once per CL
                            FILE_SEPARATOR_WARNING = false;
                        }

                        if (isArchive) {
                            DistributedCache.addArchiveToClassPath(p, conf, fs);
                        } else {
                            DistributedCache.addFileToClassPath(p, conf, fs);
                        }

                        break;

                    case LOCAL:

                        if (isArchive) {
                            if (VersionUtils.isHadoop2X()) {
                                // TODO - Need to figure out how to add local archive
                            } else {
                                Method addLocalArchives = ReflectionUtils.findMethod(DistributedCache.class,
                                        "addLocalArchives", Configuration.class, String.class);
                                addLocalArchives.invoke(null, conf, path);
                            }
                        } else {
                            if (VersionUtils.isHadoop2X()) {
                                // TODO - Need to figure out how to add local files
                            } else {
                                Method addLocalFiles = ReflectionUtils.findMethod(DistributedCache.class,
                                        "addLocalFiles", Configuration.class, String.class);
                                addLocalFiles.invoke(null, conf, path);
                            }
                        }

                        break;

                    case CACHE:

                        if (!path.contains("#")) {
                            // use the path to avoid adding the host:port into the uri
                            uri = URI.create(path + "#" + defaultLink);
                        }

                        if (isArchive) {
                            DistributedCache.addCacheArchive(uri, conf);
                        } else {
                            DistributedCache.addCacheFile(uri, conf);
                        }

                        break;
                    }
                }
            }
        }
    } finally {
        loader.close();
    }
}

---