Example usage for org.apache.hadoop.yarn.api.records LocalResource setPattern

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource setPattern.

Prototype

@Public
@Stable
public abstract void setPattern(String pattern);

Source Link

Document

Set the pattern that should be used to extract entries from the archive (only used when type is PATTERN).

Usage

From source file:cascading.flow.tez.util.TezUtil.java

License:Open Source License

protected static void addResource(Map<String, LocalResource> localResources, Map<String, String> environment,
        String fileName, FileStatus stats, Path fullPath, LocalResourceType type) throws IOException {
    if (localResources.containsKey(fileName))
        throw new FlowException("duplicate filename added to classpath resources: " + fileName);

    URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(fullPath);
    long len = stats.getLen();
    long modificationTime = stats.getModificationTime();

    LocalResource resource = LocalResource.newInstance(yarnUrlFromPath, type,
            LocalResourceVisibility.APPLICATION, len, modificationTime);

    if (type == LocalResourceType.PATTERN) {
        // todo: parametrize this for dynamic inclusion below
        String pattern = "(?:classes/|lib/).*";

        resource.setPattern(pattern);

        if (environment != null) {
            String current = "";

            current += PWD.$$() + File.separator + fileName + File.separator + "*" + CLASS_PATH_SEPARATOR;
            current += PWD.$$() + File.separator + fileName + File.separator + "lib" + File.separator + "*"
                    + CLASS_PATH_SEPARATOR;
            current += PWD.$$() + File.separator + fileName + File.separator + "classes" + File.separator + "*"
                    + CLASS_PATH_SEPARATOR;

            String classPath = environment.get(CLASSPATH.name());

            if (classPath == null)
                classPath = "";
            else if (!classPath.startsWith(CLASS_PATH_SEPARATOR))
                classPath += CLASS_PATH_SEPARATOR;

            classPath += current;/*from w w  w  . j ava2  s  . c om*/

            LOG.info("adding to cluster side classpath: {} ", classPath);

            environment.put(CLASSPATH.name(), classPath);
        }
    }

    localResources.put(fileName, resource);
}

---

From source file:com.continuuity.weave.yarn.utils.YarnUtils.java

License:Open Source License

private static LocalResource setLocalResourceType(LocalResource localResource, LocalFile localFile) {
    if (localFile.isArchive()) {
        if (localFile.getPattern() == null) {
            localResource.setType(LocalResourceType.ARCHIVE);
        } else {//from   ww w  .  j  ava2s.c o m
            localResource.setType(LocalResourceType.PATTERN);
            localResource.setPattern(localFile.getPattern());
        }
    } else {
        localResource.setType(LocalResourceType.FILE);
    }
    return localResource;
}

---

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
        List<PlanLocalResource> localResourcesList) {
    Map<String, LocalResource> map = new HashMap<String, LocalResource>();
    for (PlanLocalResource res : localResourcesList) {
        LocalResource r = new LocalResourcePBImpl();

        //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
        //else we will receive a default value back, eg ""
        if (res.hasPattern()) {
            r.setPattern(res.getPattern());
        }/*  w ww  . j a v  a2  s. co m*/
        r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri()))); // see above notes on HDFS URL handling
        r.setSize(res.getSize());
        r.setTimestamp(res.getTimeStamp());
        r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
        r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
        map.put(res.getName(), r);
    }
    return map;
}

---