List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType ARCHIVE
LocalResourceType ARCHIVE
To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType ARCHIVE.
Click Source Link
NodeManager
. From source file:org.apache.reef.runtime.yarn.client.uploader.JobFolder.java
License:Apache License
/** * Creates a LocalResource instance for the JAR file referenced by the given Path. *///from w ww .java 2 s .co m public LocalResource getLocalResourceForPath(final Path jarPath) throws IOException { final LocalResource localResource = Records.newRecord(LocalResource.class); final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath); localResource.setType(LocalResourceType.ARCHIVE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath())); localResource.setTimestamp(status.getModificationTime()); localResource.setSize(status.getLen()); return localResource; }
From source file:org.apache.reef.runtime.yarn.client.YarnJobSubmissionHandler.java
License:Apache License
/** * Creates a LocalResource instance for the JAR file referenced by the given Path *//* www .j a v a 2 s . com*/ private LocalResource makeLocalResourceForJarFile(final Path path) throws IOException { final LocalResource localResource = Records.newRecord(LocalResource.class); final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(path); localResource.setType(LocalResourceType.ARCHIVE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath())); localResource.setTimestamp(status.getModificationTime()); localResource.setSize(status.getLen()); return localResource; }
From source file:org.apache.reef.runtime.yarn.driver.UploaderToJobFolder.java
License:Apache License
/** * Creates a LocalResource instance for the JAR file referenced by the given Path * * @param path//from www . java 2s. c o m * @return * @throws IOException */ LocalResource makeLocalResourceForJarFile(final Path path) throws IOException { final LocalResource localResource = Records.newRecord(LocalResource.class); final FileStatus status = FileContext.getFileContext(this.fileSystem.getUri()).getFileStatus(path); localResource.setType(LocalResourceType.ARCHIVE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath())); localResource.setTimestamp(status.getModificationTime()); localResource.setSize(status.getLen()); return localResource; }
From source file:org.apache.samza.job.yarn.ContainerUtil.java
License:Apache License
protected void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus;/*from www. j a v a2 s . c o m*/ try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaException("IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { ioe.printStackTrace(); throw new SamzaException("IO Exception when writing credentials to output buffer"); } ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(Collections.singletonMap("__package", packageResource)); log.debug("setting package to {}", packageResource); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaException("Received YarnException when starting container: " + container.getId()); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaException("Received IOException when starting container: " + container.getId()); } }
From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath.//from w ww. j av a 2 s .c o m */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws SamzaContainerLaunchException { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaContainerLaunchException( "IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { log.error("IOException when writing credentials.", ioe); throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer"); } ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(Collections.singletonMap("__package", packageResource)); log.debug("setting package to {}", packageResource); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaContainerLaunchException( "Received YarnException when starting container: " + container.getId(), ye); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaContainerLaunchException( "Received IOException when starting container: " + container.getId(), ioe); } }
From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java
License:Apache License
@Test public void testResourceMapSuccess() { Map<String, String> configMap = new HashMap<>(); configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme"); configMap.put("yarn.resources.myResource1.local.name", "readme"); configMap.put("yarn.resources.myResource1.local.type", "file"); configMap.put("yarn.resources.myResource1.local.visibility", "public"); configMap.put("yarn.resources.myResource2.path", "https://host2.com/package"); configMap.put("yarn.resources.myResource2.local.name", "__package"); configMap.put("yarn.resources.myResource2.local.type", "archive"); configMap.put("yarn.resources.myResource2.local.visibility", "private"); configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr"); configMap.put("yarn.resources.myResource3.local.name", "csr"); configMap.put("yarn.resources.myResource3.local.type", "file"); configMap.put("yarn.resources.myResource3.local.visibility", "application"); configMap.put("otherconfig", "https://host4.com/not_included"); configMap.put("yarn.resources.myResource4.local.name", "notExisting"); configMap.put("yarn.resources.myResource4.local.type", "file"); configMap.put("yarn.resources.myResource4.local.visibility", "application"); Config conf = new MapConfig(configMap); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName()); yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName()); LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);//from www. ja va 2 s . co m Map<String, LocalResource> resourceMap = mapper.getResourceMap(); assertEquals("resourceMap has 3 resources", 3, resourceMap.size()); // resource1 assertEquals("host1.com", resourceMap.get("readme").getResource().getHost()); assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType()); assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility()); // resource 2 assertEquals("host2.com", resourceMap.get("__package").getResource().getHost()); assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType()); assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility()); // resource 3 assertEquals("host3.com", resourceMap.get("csr").getResource().getHost()); assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType()); assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility()); // resource 4 should not exist assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4")); assertNull("Resource does not exist with the defined config name notExisting for myResource4 either", resourceMap.get("notExisting")); }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath.// www . java2 s . c om */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws IOException { LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); packageResource.setResource(packageUrl); log.debug("Set package resource in YarnContainerRunner for {}", packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens to start the container Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((org.apache.hadoop.security.token.Token) iter.next()).decodeIdentifier(); if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); Map<String, LocalResource> localResourceMap = new HashMap<>(); localResourceMap.put("__package", packageResource); // include the resources from the universal resource configurations LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration); localResourceMap.putAll(resourceMapper.getResourceMap()); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(localResourceMap); if (UserGroupInformation.isSecurityEnabled()) { Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls(); if (!acls.isEmpty()) { context.setApplicationACLs(acls); } } log.debug("Setting localResourceMap to {}", localResourceMap); log.debug("Setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); log.info( "Making an async start request for Container ID: {} on host: {} with local resource map: {} and context: {}", container.getId(), container.getNodeHttpAddress(), localResourceMap.toString(), context); nmClientAsync.startContainerAsync(container, context); }
From source file:org.apache.samza.job.yarn.YarnContainerRunner.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath.//from ww w .j a v a 2s . c o m */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws SamzaContainerLaunchException { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // TODO: SAMZA-1144 remove the customized approach for package resource and use the common one. // But keep it now for backward compatibility. // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaContainerLaunchException( "IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); log.info("set package Resource in YarnContainerRunner for {}", packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { log.error("IOException when writing credentials.", ioe); throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer"); } Map<String, LocalResource> localResourceMap = new HashMap<>(); localResourceMap.put("__package", packageResource); // include the resources from the universal resource configurations LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration); localResourceMap.putAll(resourceMapper.getResourceMap()); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(localResourceMap); log.debug("setting localResourceMap to {}", localResourceMap); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaContainerLaunchException( "Received YarnException when starting container: " + container.getId(), ye); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaContainerLaunchException( "Received IOException when starting container: " + container.getId(), ioe); } }
From source file:org.apache.slider.common.tools.CoreFileSystem.java
License:Apache License
public boolean maybeAddImagePath(Map<String, LocalResource> localResources, Path imagePath) throws IOException { if (imagePath != null) { LocalResource resource = createAmResource(imagePath, LocalResourceType.ARCHIVE); localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource); return true; } else {/* w w w . ja v a 2 s. co m*/ return false; } }
From source file:org.apache.slider.providers.agent.AgentProviderService.java
License:Apache License
@Override public void buildContainerLaunchContext(ContainerLauncher launcher, AggregateConf instanceDefinition, Container container, String role, SliderFileSystem fileSystem, Path generatedConfPath, MapOperations resourceComponent, MapOperations appComponent, Path containerTmpDirPath) throws IOException, SliderException { String appDef = instanceDefinition.getAppConfOperations().getGlobalOptions() .getMandatoryOption(AgentKeys.APP_DEF); initializeApplicationConfiguration(instanceDefinition, fileSystem); log.info("Build launch context for Agent"); log.debug(instanceDefinition.toString()); // Set the environment launcher.putEnv(SliderUtils.buildEnvMap(appComponent)); String workDir = ApplicationConstants.Environment.PWD.$(); launcher.setEnv("AGENT_WORK_ROOT", workDir); log.info("AGENT_WORK_ROOT set to {}", workDir); String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR; launcher.setEnv("AGENT_LOG_ROOT", logDir); log.info("AGENT_LOG_ROOT set to {}", logDir); if (System.getenv(HADOOP_USER_NAME) != null) { launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); }/*from www.j a va2 s . c o m*/ // for 2-Way SSL launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase()); //local resources // TODO: Should agent need to support App Home String scriptPath = new File(AgentKeys.AGENT_MAIN_SCRIPT_ROOT, AgentKeys.AGENT_MAIN_SCRIPT).getPath(); String appHome = instanceDefinition.getAppConfOperations().getGlobalOptions().get(AgentKeys.PACKAGE_PATH); if (SliderUtils.isSet(appHome)) { scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath(); } // set PYTHONPATH List<String> pythonPaths = new ArrayList<String>(); pythonPaths.add(AgentKeys.AGENT_MAIN_SCRIPT_ROOT); String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths); launcher.setEnv(PYTHONPATH, pythonPath); log.info("PYTHONPATH set to {}", pythonPath); Path agentImagePath = null; String agentImage = instanceDefinition.getInternalOperations() .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); if (SliderUtils.isUnset(agentImage)) { agentImagePath = new Path(new Path( new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR), container.getId().getApplicationAttemptId().getApplicationId().toString()), AgentKeys.PROVIDER_AGENT), SliderKeys.AGENT_TAR); } else { agentImagePath = new Path(agentImage); } // TODO: throw exception when agent tarball is not available if (fileSystem.getFileSystem().exists(agentImagePath)) { LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes); } else { log.error("Required agent image slider-agent.tar.gz is unavailable."); } log.info("Using {} for agent.", scriptPath); LocalResource appDefRes = fileSystem.createAmResource( fileSystem.getFileSystem().resolvePath(new Path(appDef)), LocalResourceType.ARCHIVE); launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes); String agentConf = instanceDefinition.getAppConfOperations().getGlobalOptions() .getOption(AgentKeys.AGENT_CONF, ""); if (SliderUtils.isSet(agentConf)) { LocalResource agentConfRes = fileSystem.createAmResource( fileSystem.getFileSystem().resolvePath(new Path(agentConf)), LocalResourceType.FILE); launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes); } String agentVer = instanceDefinition.getAppConfOperations().getGlobalOptions() .getOption(AgentKeys.AGENT_VERSION, null); if (agentVer != null) { LocalResource agentVerRes = fileSystem.createAmResource( fileSystem.getFileSystem().resolvePath(new Path(agentVer)), LocalResourceType.FILE); launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes); } if (SliderUtils.isHadoopClusterSecure(getConfig())) { localizeServiceKeytabs(launcher, instanceDefinition, fileSystem); } MapOperations amComponent = instanceDefinition.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM); boolean twoWayEnabled = amComponent != null ? Boolean.valueOf(amComponent.getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) : false; if (twoWayEnabled) { localizeContainerSSLResources(launcher, container, fileSystem); } //add the configuration resources launcher.addLocalResources( fileSystem.submitDirectory(generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME)); String label = getContainerLabel(container, role); CommandLineBuilder operation = new CommandLineBuilder(); String pythonExec = instanceDefinition.getAppConfOperations().getGlobalOptions() .getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH, AgentKeys.PYTHON_EXE); operation.add(pythonExec); operation.add(scriptPath); operation.add(ARG_LABEL, label); operation.add(ARG_ZOOKEEPER_QUORUM); operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM)); operation.add(ARG_ZOOKEEPER_REGISTRY_PATH); operation.add(getZkRegistryPath()); String debugCmd = agentLaunchParameter.getNextLaunchParameter(role); if (SliderUtils.isSet(debugCmd)) { operation.add(ARG_DEBUG); operation.add(debugCmd); } operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1"); launcher.addCommand(operation.build()); // initialize the component instance state getComponentStatuses().put(label, new ComponentInstanceState(role, container.getId(), getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME))); }