List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType FILE
LocalResourceType FILE
To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType FILE.
Click Source Link
From source file:org.springframework.yarn.config.LocalresourcesNamespaceTest.java
License:Apache License
/** * When only single path is defined configuration should assume * it is accessed from local hdfs access point and other settings * set to defaults.//from ww w . j ava 2s .c o m */ @Test public void testDefaultsWithMinimalSetup() throws Exception { assertNotNull(defaultResources); assertNotNull(defaultLocalResourcesFactoryBean); Configuration configuration = TestUtils.readField("configuration", defaultLocalResourcesFactoryBean); assertNotNull(configuration); //should be something like fs.defaultFS=hdfs://x.x.x.x:9000 String defaultFs = configuration.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); assertThat(defaultFs, startsWith("hdfs")); assertThat(defaultFs, endsWith("8020")); Collection<TransferEntry> hdfsEntries = TestUtils.readField("hdfsEntries", defaultLocalResourcesFactoryBean); assertNotNull(hdfsEntries); assertThat(hdfsEntries.size(), is(1)); TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0]; String path = TestUtils.readField("path", entry); assertThat(path, is("/tmp/foo.jar")); LocalResourceType type = TestUtils.readField("type", entry); assertThat(type, is(LocalResourceType.FILE)); LocalResourceVisibility visibility = TestUtils.readField("visibility", entry); assertThat(visibility, is(LocalResourceVisibility.APPLICATION)); String local = TestUtils.readField("local", entry); assertThat(local, notNullValue()); assertThat(local, startsWith("hdfs")); assertThat(local, endsWith("8020")); String remote = TestUtils.readField("remote", entry); assertThat(remote, notNullValue()); assertThat(remote, startsWith("hdfs")); assertThat(remote, endsWith("8020")); }
From source file:org.springframework.yarn.config.LocalresourcesNamespaceTest.java
License:Apache License
@Test public void testGlobalLocalMixed() throws Exception { assertNotNull(resourcesWithMixed);// w w w .ja va 2 s . co m assertNotNull(localResourcesFactoryBeanWithMixed); Configuration configuration = TestUtils.readField("configuration", localResourcesFactoryBeanWithMixed); assertNotNull(configuration); Collection<TransferEntry> hdfsEntries = TestUtils.readField("hdfsEntries", localResourcesFactoryBeanWithMixed); assertNotNull(hdfsEntries); assertThat(hdfsEntries.size(), is(3)); TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0]; String path = TestUtils.readField("path", entry); assertThat(path, is("/tmp/foo.jar")); LocalResourceType type = TestUtils.readField("type", entry); assertThat(type, is(LocalResourceType.FILE)); LocalResourceVisibility visibility = TestUtils.readField("visibility", entry); assertThat(visibility, is(LocalResourceVisibility.PUBLIC)); String local = TestUtils.readField("local", entry); assertThat(local, is("hdfs://0.0.0.0:9001")); String remote = TestUtils.readField("remote", entry); assertThat(remote, is("hdfs://192.168.223.139:9000")); entry = (TransferEntry) hdfsEntries.toArray()[1]; path = TestUtils.readField("path", entry); assertThat(path, is("/tmp/jee.jar")); type = TestUtils.readField("type", entry); assertThat(type, is(LocalResourceType.FILE)); visibility = TestUtils.readField("visibility", entry); assertThat(visibility, is(LocalResourceVisibility.PUBLIC)); local = TestUtils.readField("local", entry); assertThat(local, is("hdfs://0.0.0.0:9000")); remote = TestUtils.readField("remote", entry); assertThat(remote, is("hdfs://192.168.223.139:9001")); entry = (TransferEntry) hdfsEntries.toArray()[2]; path = TestUtils.readField("path", entry); assertThat(path, is("/tmp/bar.jar")); type = TestUtils.readField("type", entry); assertThat(type, is(LocalResourceType.ARCHIVE)); visibility = TestUtils.readField("visibility", entry); assertThat(visibility, is(LocalResourceVisibility.PRIVATE)); local = TestUtils.readField("local", entry); assertThat(local, is("hdfs://0.0.0.0:9000")); remote = TestUtils.readField("remote", entry); assertThat(remote, is("hdfs://192.168.223.139:9000")); }
From source file:org.springframework.yarn.fs.LocalResourcesFactoryBean.java
License:Apache License
@Override public void afterPropertiesSet() throws Exception { String defaultFs = configuration.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); // defaults if defined for (TransferEntry entry : hdfsEntries) { if (entry.type == null) { entry.type = (defaultType != null ? defaultType : LocalResourceType.FILE); }/*from w w w.j a v a 2s. c o m*/ if (entry.visibility == null) { entry.visibility = (defaultVisibility != null ? defaultVisibility : LocalResourceVisibility.APPLICATION); } if (entry.local == null) { entry.local = (defaultLocal != null ? defaultLocal : defaultFs); } if (entry.remote == null) { entry.remote = (defaultRemote != null ? defaultRemote : defaultFs); } Assert.isTrue(entry.local != null && entry.remote != null, "Entry local/remote hdfs address can't be null"); } resources = new DefaultResourceLocalizer(configuration, hdfsEntries, copyEntries); }
From source file:oz.hadoop.yarn.api.core.ApplicationContainerLauncherImpl.java
License:Apache License
/** * *//* www .j a va2 s . c om*/ private Map<String, LocalResource> buildLocalResources() { try { Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String suffix = this.applicationSpecification.getString(YayaConstants.APPLICATION_NAME) + "_master/" + this.applicationSpecification.getInt(YayaConstants.APP_ID) + "/"; FileSystem fs = FileSystem.get(this.yarnConfig); Path dst = new Path(fs.getHomeDirectory(), suffix); FileStatus[] deployedResources = fs.listStatus(dst); for (FileStatus fileStatus : deployedResources) { if (logger.isDebugEnabled()) { logger.debug("Creating local resource for: " + fileStatus.getPath()); } LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(fileStatus.getPath().toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime()); localResources.put(fileStatus.getPath().getName(), scRsrc); } return localResources; } catch (Exception e) { StringWriter sw = new StringWriter(); PrintWriter writer = new PrintWriter(sw); e.printStackTrace(writer); throw new IllegalStateException("Failed to build LocalResources\n " + sw.toString(), e); } }
From source file:oz.hadoop.yarn.api.core.ApplicationMasterLauncherImpl.java
License:Apache License
/** * *///from ww w. ja v a2 s .com private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, int appId, Map<String, LocalResource> localResources) { String suffix = this.applicationName + "_master/" + appId + "/" + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); try { Path sourcePath = new Path(fileSrcPath); if (logger.isDebugEnabled()) { logger.debug("Copying '" + sourcePath + "' to " + dst); } fs.copyFromLocalFile(sourcePath, dst); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); } catch (Exception e) { throw new IllegalStateException("Failed to communicate with FileSystem: " + fs, e); } }
From source file:oz.hadoop.yarn.api.core.YayaUtilsTests.java
License:Apache License
@Test public void validateClassPathFromLocalSource() throws Exception { LocalResource l1 = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(new URI("file://foo.jar")), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1234L, 678L); LocalResource l2 = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(new URI("file://bar.jar")), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1234L, 678L); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put("foo.jar", l1); localResources.put("bar.jar", l2); assertEquals("./bar.jar:./foo.jar:", YayaUtils.calculateClassPath(localResources)); }
From source file:proxyyarn.ProxyYarn.java
License:Apache License
public boolean run() throws Exception { Configuration conf = new YarnConfiguration(new Configuration()); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);/* ww w . jav a2 s. co m*/ yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); log.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); log.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { log.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo("default"); log.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { log.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } FileSystem fs = FileSystem.get(conf); if (!fs.getClass().equals(DistributedFileSystem.class)) { log.error("Expected DistributedFileSystem, but was {}", fs.getClass().getSimpleName()); System.exit(1); } // ApplicationClientProtocol applicationsManager; // InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); // log.info("Connecting to ResourceManager at {}", rmAddress); // Configuration appManagerServerConf = new Configuration(conf); // YarnRPC rpc = YarnRPC.create(appManagerServerConf); // ApplicationClientProtocol applicationManager = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, appManagerServerConf); String appName = "AccumuloProxyYarn"; YarnClientApplication app = yarnClient.createApplication(); // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class); // GetNewApplicationResponse response = applicationManager.getNewApplication(request); // log.info("Got new ApplicationId=" + response.getApplicationId()); // ApplicationId appId = response.getApplicationId(); // Create a new ApplicationSubmissionContext // ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the ApplicationId // appContext.setApplicationId(appId); // set the application name // appContext.setApplicationName(appName); // Create a new container launch context for the AM's container ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // Define the local resources required Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // Lets assume the jar we need for our ApplicationMaster is available in // HDFS at a certain known path to us and we want to make it available to // the ApplicationMaster in the launched container Path localJarPath = new Path( "file:///Users/jelser/projects/accumulo-proxy-yarn/target/accumulo-proxy-yarn-0.0.1-SNAPSHOT.jar"); Path jarPath = new Path("hdfs:///accumulo-proxy-yarn-0.0.1-SNAPSHOT.jar"); fs.copyFromLocalFile(false, true, localJarPath, jarPath); FileStatus jarStatus = fs.getFileStatus(jarPath); LocalResource amJarRsrc = Records.newRecord(LocalResource.class); // Set the type of resource - file or archive // archives are untarred at the destination by the framework amJarRsrc.setType(LocalResourceType.FILE); // Set visibility of the resource // Setting to most private option i.e. this file will only // be visible to this instance of the running application amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION); // Set the location of resource to be copied over into the // working directory amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application amJarRsrc.setTimestamp(jarStatus.getModificationTime()); amJarRsrc.setSize(jarStatus.getLen()); // The framework will create a symlink called AppMaster.jar in the // working directory that will be linked back to the actual file. // The ApplicationMaster, if needs to reference the jar file, would // need to use the symlink filename. localResources.put("AppMaster.jar", amJarRsrc); // Set the local resources into the launch context amContainer.setLocalResources(localResources); // Set up the environment needed for the launch context Map<String, String> env = new HashMap<String, String>(); // For example, we could setup the classpath needed. // Assuming our classes or jars are available as local resources in the // working directory from which the command will be run, we need to append // "." to the path. // By default, all the hadoop specific classpaths will already be available // in $CLASSPATH, so we should be careful not to overwrite it. String classPathEnv = "$CLASSPATH:./*:/Users/jelser/projects/accumulo-proxy-yarn/target/lib/*"; env.put("CLASSPATH", classPathEnv); amContainer.setEnvironment(env); // Construct the command to be executed on the launched container String command = "${JAVA_HOME}" + "/bin/java" + " proxyyarn.ProxyYarnAppMaster 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; List<String> commands = new ArrayList<String>(); commands.add(command); // add additional commands if needed // Set the command array into the container spec amContainer.setCommands(commands); // Define the resource requirements for the container // For now, YARN only supports memory so we set the memory // requirements. // If the process takes more than its allocated memory, it will // be killed by the framework. // Memory being requested for should be less than max capability // of the cluster and all asks should be a multiple of the min capability. Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); appContext.setResource(capability); // Create the request to send to the ApplicationsManager // SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); // appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the ApplicationsManager // Ignore the response as either a valid response object is returned on // success or an exception thrown to denote the failure // applicationManager.submitApplication(appRequest); // Set the container launch content into the ApplicationSubmissionContext appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure log.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); return monitorApplication(yarnClient, appId); /* Thread.sleep(200); boolean running = false; while(true) { GetApplicationReportRequest reportRequest = Records.newRecord(GetApplicationReportRequest.class); reportRequest.setApplicationId(appId); GetApplicationReportResponse reportResponse = applicationManager.getApplicationReport(reportRequest); ApplicationReport report = reportResponse.getApplicationReport(); log.info(report.toString()); YarnApplicationState state = report.getYarnApplicationState(); switch (state) { case NEW: case NEW_SAVING: case SUBMITTED: case ACCEPTED: log.info("State: {}", state); break; case RUNNING: log.info("Running application"); running = true; break; case FINISHED: case FAILED: case KILLED: log.info("State: {}", state); return; default: log.info("Unknown state: {}", state); return; } if (!running) { Thread.sleep(1000); } }*/ }
From source file:shuffle.ShuffleCheck.java
License:Apache License
LocalResource createLocalResource(FileSystem fs, Path file) throws IOException { final LocalResourceType type = LocalResourceType.FILE; final LocalResourceVisibility visibility = LocalResourceVisibility.APPLICATION; FileStatus fstat = fs.getFileStatus(file); org.apache.hadoop.yarn.api.records.URL resourceURL = ConverterUtils.getYarnUrlFromPath(file); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(resourceURL);/* w ww.j av a2 s. c o m*/ lr.setType(type); lr.setSize(resourceSize); lr.setVisibility(visibility); lr.setTimestamp(resourceModificationTime); return lr; }
From source file:tachyon.yarn.Client.java
License:Apache License
private void setupAppMasterJar(LocalResource appMasterJar) throws IOException { Path jarHdfsPath = new Path(mAppMasterJarHdfs); // known path to jar file on HDFS FileStatus jarStat = FileSystem.get(mYarnConf).getFileStatus(jarHdfsPath); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarHdfsPath)); appMasterJar.setSize(jarStat.getLen()); appMasterJar.setTimestamp(jarStat.getModificationTime()); appMasterJar.setType(LocalResourceType.FILE); appMasterJar.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:tachyon.yarn.Utils.java
License:Apache License
/** * Creates a local resource for a file on HDFS. * * @param yarnConf YARN configuration/* w w w . ja v a 2s .c o m*/ * @param resourcePath known path of resource file on HDFS * @throws IOException if the file can not be found on HDFS */ public static LocalResource createLocalResourceOfFile(YarnConfiguration yarnConf, String resourcePath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); Path jarHdfsPath = new Path(resourcePath); FileStatus jarStat = FileSystem.get(yarnConf).getFileStatus(jarHdfsPath); localResource.setResource(ConverterUtils.getYarnUrlFromPath(jarHdfsPath)); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); return localResource; }