Example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE

List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType FILE

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResourceType
  6. FILE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Prototype

LocalResourceType FILE

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Click Source Link

Document

Regular file i.e.

Usage

From source file:org.apache.metron.maas.service.callback.LaunchContainer.java

License:Apache License

private Map.Entry<String, LocalResource> localizeResource(FileStatus status) {
    URL url = ConverterUtils.getYarnUrlFromURI(status.getPath().toUri());
    LocalResource resource = LocalResource.newInstance(url, LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime());
    String name = status.getPath().getName();
    return new AbstractMap.SimpleEntry<>(name, resource);
}

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {/* w  w w. j  a  va  2s. co m*/
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return dst;
}

From source file:org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager.java

License:Apache License

public Map<String, LocalResource> getTezResources(Set<String> resourceNames) throws Exception {
    Map<String, LocalResource> tezResources = new HashMap<String, LocalResource>();
    for (String resourceName : resourceNames) {
        // The resource name will be symlinked to the resource path in the
        // container's working directory.
        Path resourcePath = resources.get(resourceName);
        FileStatus fstat = remoteFs.getFileStatus(resourcePath);

        LocalResource tezResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromPath(fstat.getPath()), LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION, fstat.getLen(), fstat.getModificationTime());

        tezResources.put(resourceName, tezResource);
    }//from   w  w w  .j  a  v  a2 s.  c  o m
    return tezResources;
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceConfig.java

License:Apache License

@Test
public void testResourceConfigIncluded() {
    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");

    Config conf = new MapConfig(configMap);

    LocalizerResourceConfig manager = new LocalizerResourceConfig(conf);
    assertEquals(1, manager.getResourceNames().size());
    assertEquals("myResource1", manager.getResourceNames().get(0));
    assertEquals("readme", manager.getResourceLocalName("myResource1"));
    assertEquals(LocalResourceType.FILE, manager.getResourceLocalType("myResource1"));
    assertEquals(LocalResourceVisibility.PUBLIC, manager.getResourceLocalVisibility("myResource1"));
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapSuccess() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");

    configMap.put("yarn.resources.myResource2.path", "https://host2.com/package");
    configMap.put("yarn.resources.myResource2.local.name", "__package");
    configMap.put("yarn.resources.myResource2.local.type", "archive");
    configMap.put("yarn.resources.myResource2.local.visibility", "private");

    configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr");
    configMap.put("yarn.resources.myResource3.local.name", "csr");
    configMap.put("yarn.resources.myResource3.local.type", "file");
    configMap.put("yarn.resources.myResource3.local.visibility", "application");

    configMap.put("otherconfig", "https://host4.com/not_included");
    configMap.put("yarn.resources.myResource4.local.name", "notExisting");
    configMap.put("yarn.resources.myResource4.local.type", "file");
    configMap.put("yarn.resources.myResource4.local.visibility", "application");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);/*from w w  w  . j  a v a  2s . co  m*/
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertEquals("resourceMap has 3 resources", 3, resourceMap.size());

    // resource1
    assertEquals("host1.com", resourceMap.get("readme").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType());
    assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility());

    // resource 2
    assertEquals("host2.com", resourceMap.get("__package").getResource().getHost());
    assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType());
    assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility());

    // resource 3
    assertEquals("host3.com", resourceMap.get("csr").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility());

    // resource 4 should not exist
    assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4"));
    assertNull("Resource does not exist with the defined config name notExisting for myResource4 either",
            resourceMap.get("notExisting"));
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithDefaultValues() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);// ww  w.  j  a  v a2 s . co  m
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
    assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
    assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}

From source file:org.apache.slider.providers.agent.AgentProviderService.java

License:Apache License

@Override
public void buildContainerLaunchContext(ContainerLauncher launcher, AggregateConf instanceDefinition,
        Container container, String role, SliderFileSystem fileSystem, Path generatedConfPath,
        MapOperations resourceComponent, MapOperations appComponent, Path containerTmpDirPath)
        throws IOException, SliderException {

    String appDef = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getMandatoryOption(AgentKeys.APP_DEF);

    initializeApplicationConfiguration(instanceDefinition, fileSystem);

    log.info("Build launch context for Agent");
    log.debug(instanceDefinition.toString());

    // Set the environment
    launcher.putEnv(SliderUtils.buildEnvMap(appComponent));

    String workDir = ApplicationConstants.Environment.PWD.$();
    launcher.setEnv("AGENT_WORK_ROOT", workDir);
    log.info("AGENT_WORK_ROOT set to {}", workDir);
    String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
    launcher.setEnv("AGENT_LOG_ROOT", logDir);
    log.info("AGENT_LOG_ROOT set to {}", logDir);
    if (System.getenv(HADOOP_USER_NAME) != null) {
        launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
    }//from   w  w w.ja va  2  s .c  o  m
    // for 2-Way SSL
    launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase());

    //local resources

    // TODO: Should agent need to support App Home
    String scriptPath = new File(AgentKeys.AGENT_MAIN_SCRIPT_ROOT, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
    String appHome = instanceDefinition.getAppConfOperations().getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
    if (SliderUtils.isSet(appHome)) {
        scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
    }

    // set PYTHONPATH
    List<String> pythonPaths = new ArrayList<String>();
    pythonPaths.add(AgentKeys.AGENT_MAIN_SCRIPT_ROOT);
    String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
    launcher.setEnv(PYTHONPATH, pythonPath);
    log.info("PYTHONPATH set to {}", pythonPath);

    Path agentImagePath = null;
    String agentImage = instanceDefinition.getInternalOperations()
            .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
    if (SliderUtils.isUnset(agentImage)) {
        agentImagePath = new Path(new Path(
                new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
                        container.getId().getApplicationAttemptId().getApplicationId().toString()),
                AgentKeys.PROVIDER_AGENT), SliderKeys.AGENT_TAR);
    } else {
        agentImagePath = new Path(agentImage);
    }

    // TODO: throw exception when agent tarball is not available

    if (fileSystem.getFileSystem().exists(agentImagePath)) {
        LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
        launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
    } else {
        log.error("Required agent image slider-agent.tar.gz is unavailable.");
    }

    log.info("Using {} for agent.", scriptPath);
    LocalResource appDefRes = fileSystem.createAmResource(
            fileSystem.getFileSystem().resolvePath(new Path(appDef)), LocalResourceType.ARCHIVE);
    launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes);

    String agentConf = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getOption(AgentKeys.AGENT_CONF, "");
    if (SliderUtils.isSet(agentConf)) {
        LocalResource agentConfRes = fileSystem.createAmResource(
                fileSystem.getFileSystem().resolvePath(new Path(agentConf)), LocalResourceType.FILE);
        launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
    }

    String agentVer = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getOption(AgentKeys.AGENT_VERSION, null);
    if (agentVer != null) {
        LocalResource agentVerRes = fileSystem.createAmResource(
                fileSystem.getFileSystem().resolvePath(new Path(agentVer)), LocalResourceType.FILE);
        launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
    }

    if (SliderUtils.isHadoopClusterSecure(getConfig())) {
        localizeServiceKeytabs(launcher, instanceDefinition, fileSystem);
    }

    MapOperations amComponent = instanceDefinition.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
    boolean twoWayEnabled = amComponent != null
            ? Boolean.valueOf(amComponent.getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false))
            : false;
    if (twoWayEnabled) {
        localizeContainerSSLResources(launcher, container, fileSystem);
    }

    //add the configuration resources
    launcher.addLocalResources(
            fileSystem.submitDirectory(generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME));

    String label = getContainerLabel(container, role);
    CommandLineBuilder operation = new CommandLineBuilder();

    String pythonExec = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH, AgentKeys.PYTHON_EXE);

    operation.add(pythonExec);

    operation.add(scriptPath);
    operation.add(ARG_LABEL, label);
    operation.add(ARG_ZOOKEEPER_QUORUM);
    operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM));
    operation.add(ARG_ZOOKEEPER_REGISTRY_PATH);
    operation.add(getZkRegistryPath());

    String debugCmd = agentLaunchParameter.getNextLaunchParameter(role);
    if (SliderUtils.isSet(debugCmd)) {
        operation.add(ARG_DEBUG);
        operation.add(debugCmd);
    }

    operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1");

    launcher.addCommand(operation.build());

    // initialize the component instance state
    getComponentStatuses().put(label, new ComponentInstanceState(role, container.getId(),
            getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}

From source file:org.apache.slider.providers.agent.AgentProviderService.java

License:Apache License

private void localizeContainerSSLResources(ContainerLauncher launcher, Container container,
        SliderFileSystem fileSystem) throws SliderException {
    try {/*from  w w  w .  j a v a2 s  .c  o  m*/
        // localize server cert
        Path certsDir = new Path(fileSystem.buildClusterDirPath(getClusterName()), "certs");
        LocalResource certResource = fileSystem.createAmResource(new Path(certsDir, SliderKeys.CRT_FILE_NAME),
                LocalResourceType.FILE);
        launcher.addLocalResource(AgentKeys.CERT_FILE_LOCALIZATION_PATH, certResource);

        // generate and localize agent cert
        CertificateManager certMgr = new CertificateManager();
        String hostname = container.getNodeId().getHost();
        String containerId = container.getId().toString();
        certMgr.generateAgentCertificate(hostname, containerId);
        LocalResource agentCertResource = fileSystem.createAmResource(
                uploadSecurityResource(CertificateManager.getAgentCertficateFilePath(containerId), fileSystem),
                LocalResourceType.FILE);
        // still using hostname as file name on the agent side, but the files
        // do end up under the specific container's file space
        launcher.addLocalResource("certs/" + hostname + ".crt", agentCertResource);
        LocalResource agentKeyResource = fileSystem.createAmResource(
                uploadSecurityResource(CertificateManager.getAgentKeyFilePath(containerId), fileSystem),
                LocalResourceType.FILE);
        launcher.addLocalResource("certs/" + hostname + ".key", agentKeyResource);

    } catch (Exception e) {
        throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
                "Unable to localize certificates.  Two-way SSL cannot be enabled");
    }
}

From source file:org.apache.slider.providers.agent.AgentProviderService.java

License:Apache License

private void localizeServiceKeytabs(ContainerLauncher launcher, AggregateConf instanceDefinition,
        SliderFileSystem fileSystem) throws IOException {
    String keytabPathOnHost = instanceDefinition.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM)
            .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
    if (SliderUtils.isUnset(keytabPathOnHost)) {
        String amKeytabName = instanceDefinition.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM)
                .get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
        String keytabDir = instanceDefinition.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM)
                .get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
        // we need to localize the keytab files in the directory
        Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, getClusterName());
        FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath);
        LocalResource keytabRes;// ww w . j a va  2s. c om
        boolean serviceKeytabsDeployed = false;
        for (FileStatus keytab : keytabs) {
            if (!amKeytabName.equals(keytab.getPath().getName())
                    && keytab.getPath().getName().endsWith(".keytab")) {
                serviceKeytabsDeployed = true;
                log.info("Localizing keytab {}", keytab.getPath().getName());
                keytabRes = fileSystem.createAmResource(keytab.getPath(), LocalResourceType.FILE);
                launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" + keytab.getPath().getName(), keytabRes);
            }
        }
        if (!serviceKeytabsDeployed) {
            log.warn(
                    "No service keytabs for the application have been localized.  "
                            + "If the application requires keytabs for secure operation, "
                            + "please ensure that the required keytabs have been uploaded "
                            + "to the folder designated by the property {}: {}",
                    SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR, keytabDirPath);
        }
    }
}

From source file:org.apache.slider.providers.slideram.SliderAMClientProvider.java

License:Apache License

/**
 * If the cluster is secure, and an HDFS installed keytab is available for AM
 * authentication, add this keytab as a local resource for the AM launch.
 *
 * @param fileSystem/*from   w ww .  j  ava 2  s  . c  o m*/
 * @param launcher
 * @param instanceDescription
 * @param providerResources
 * @throws IOException
 */
protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem, AbstractLauncher launcher,
        AggregateConf instanceDescription, Map<String, LocalResource> providerResources) throws IOException {
    if (UserGroupInformation.isSecurityEnabled()) {
        String keytabPathOnHost = instanceDescription.getAppConfOperations()
                .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
        if (SliderUtils.isUnset(keytabPathOnHost)) {
            String amKeytabName = instanceDescription.getAppConfOperations()
                    .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
            String keytabDir = instanceDescription.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM)
                    .get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
            Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName,
                    instanceDescription.getName());
            LocalResource keytabRes = fileSystem.createAmResource(keytabPath, LocalResourceType.FILE);

            providerResources.put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes);
        }
    }
    launcher.addLocalResources(providerResources);
}