List of usage examples for org.apache.hadoop.yarn.api.records ApplicationId newInstance
@Public
@Unstable
public static ApplicationId newInstance(long clusterTimestamp, int id)
From source file:azkaban.jobtype.HadoopJobUtils.java
License:Apache License
/** * <pre>//from ww w.j av a 2s . c om * Uses YarnClient to kill the job on HDFS. * Using JobClient only works partially: * If yarn container has started but spark job haven't, it will kill * If spark job has started, the cancel will hang until the spark job is complete * If the spark job is complete, it will return immediately, with a job not found on job tracker * </pre> * * @param applicationId * @throws IOException * @throws YarnException */ public static void killJobOnCluster(String applicationId, Logger log) throws YarnException, IOException { YarnConfiguration yarnConf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); yarnClient.start(); String[] split = applicationId.split("_"); ApplicationId aid = ApplicationId.newInstance(Long.parseLong(split[1]), Integer.parseInt(split[2])); log.info("start klling application: " + aid); yarnClient.killApplication(aid); log.info("successfully killed application: " + aid); }
From source file:com.mellanox.hadoop.mapred.UdaPluginSH.java
License:Apache License
static IndexRecordBridge getPathIndex(String jobIDStr, String mapId, int reduce) { String user = userRsrc.get(jobIDStr); /////////////////////// JobID jobID = JobID.forName(jobIDStr); ApplicationId appID = ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()), jobID.getId()); final String base = ContainerLocalizer.USERCACHE + "/" + user + "/" + ContainerLocalizer.APPCACHE + "/" + ConverterUtils.toString(appID) + "/output" + "/" + mapId; if (LOG.isDebugEnabled()) { LOG.debug("DEBUG0 " + base); }//w ww. ja v a 2s. c o m // Index file IndexRecordBridge data = null; try { Path indexFileName = lDirAlloc.getLocalPathToRead(base + "/file.out.index", mjobConf); // Map-output file Path mapOutputFileName = lDirAlloc.getLocalPathToRead(base + "/file.out", mjobConf); if (LOG.isDebugEnabled()) { LOG.debug("DEBUG1 " + base + " : " + mapOutputFileName + " : " + indexFileName); } /////////////////////// // TODO: is this correct ?? - why user and not runAsUserName like in hadoop-1 ?? // on 2nd thought, this sounds correct, because probably we registered the runAsUser and not the "user" data = indexCache.getIndexInformationBridge(mapId, reduce, indexFileName, user); data.pathMOF = mapOutputFileName.toString(); } catch (IOException e) { LOG.error("got an exception while retrieving the Index Info"); } return data; }
From source file:io.amient.yarn1.YarnMaster.java
License:Open Source License
private void initialize() throws Exception { this.restartEnabled = Boolean.valueOf(appConfig.getProperty("yarn1.restart.enabled", "false")); this.restartFailedRetries = Integer.valueOf(appConfig.getProperty("yarn1.restart.failed.retries", "5")); URL url = getTrackingURL();//from w ww .j a va 2s .c om log.info("APPLICATION TRACKING URL: " + url); if (localMode) { executor = Executors.newCachedThreadPool(); } else { this.appId = ApplicationId.newInstance(Long.parseLong(appConfig.getProperty("am.timestamp")), Integer.parseInt(appConfig.getProperty("am.id"))); log.info("APPLICATION ID: " + appId.toString()); rmClient = AMRMClientAsync.createAMRMClientAsync(100, listener); rmClient.init(yarnConfig); nmClient = NMClient.createNMClient(); nmClient.init(yarnConfig); rmClient.start(); rmClient.registerApplicationMaster("", 0, url == null ? null : url.toString()); nmClient.start(); YarnClient.distributeResources(yarnConfig, appConfig, applicationName); } }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
private static ApplicationId getApplicationId(int id) { return ApplicationId.newInstance(123456, id); }
From source file:org.apache.ambari.view.slider.SliderAppsViewControllerImpl.java
License:Apache License
private ApplicationId getApplicationId(String appIdString) { if (appIdString != null) { int index = appIdString.indexOf('_'); if (index > -1 && index < appIdString.length() - 1) { ApplicationId appId = ApplicationId.newInstance(Long.parseLong(appIdString.substring(0, index)), Integer.parseInt(appIdString.substring(index + 1))); return appId; }/*from www . j a v a 2s. c om*/ } return null; }
From source file:org.apache.flink.yarn.AbstractYarnClusterTest.java
License:Apache License
/** * Tests that the cluster retrieval of a finished YARN application fails. */// w w w. ja v a2 s .c o m @Test(expected = ClusterRetrieveException.class) public void testClusterClientRetrievalOfFinishedYarnApplication() throws Exception { final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 42); final ApplicationReport applicationReport = createApplicationReport(applicationId, YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED); final YarnClient yarnClient = new TestingYarnClient( Collections.singletonMap(applicationId, applicationReport)); final YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnClient.init(yarnConfiguration); yarnClient.start(); final TestingAbstractYarnClusterDescriptor clusterDescriptor = new TestingAbstractYarnClusterDescriptor( new Configuration(), yarnConfiguration, temporaryFolder.newFolder().getAbsolutePath(), yarnClient, false); try { clusterDescriptor.retrieve(applicationId); } finally { clusterDescriptor.close(); } }
From source file:org.apache.flink.yarn.CliFrontendYarnAddressConfigurationTest.java
License:Apache License
@Test(expected = IllegalConfigurationException.class) public void testResumeFromInvalidYarnID() throws Exception { File directoryPath = writeYarnPropertiesFile(validPropertiesFile); // start CLI Frontend TestCLI frontend = new CustomYarnTestCLI(directoryPath.getAbsolutePath(), FinalApplicationStatus.SUCCEEDED); RunOptions options = CliFrontendParser .parseRunCommand(new String[] { "-yid", ApplicationId.newInstance(0, 666).toString() }); frontend.retrieveClient(options);//from ww w . ja va2s. c o m checkJobManagerAddress(frontend.getConfiguration(), TEST_YARN_JOB_MANAGER_ADDRESS, TEST_YARN_JOB_MANAGER_PORT); }
From source file:org.apache.flink.yarn.YarnFileStageTest.java
License:Apache License
/** * Verifies that nested directories are properly copied with the given filesystem and paths. * * @param targetFileSystem//from w ww . ja v a 2s . c o m * file system of the target path * @param targetDir * target path (URI like <tt>hdfs://...</tt>) * @param tempFolder * JUnit temporary folder rule to create the source directory with * @param addSchemeToLocalPath * whether add the <tt>file://</tt> scheme to the local path to copy from */ static void testCopyFromLocalRecursive(FileSystem targetFileSystem, Path targetDir, TemporaryFolder tempFolder, boolean addSchemeToLocalPath) throws Exception { // directory must not yet exist assertFalse(targetFileSystem.exists(targetDir)); final File srcDir = tempFolder.newFolder(); final Path srcPath; if (addSchemeToLocalPath) { srcPath = new Path("file://" + srcDir.getAbsolutePath()); } else { srcPath = new Path(srcDir.getAbsolutePath()); } HashMap<String /* (relative) path */, /* contents */ String> srcFiles = new HashMap<>(4); // create and fill source files srcFiles.put("1", "Hello 1"); srcFiles.put("2", "Hello 2"); srcFiles.put("nested/3", "Hello nested/3"); srcFiles.put("nested/4/5", "Hello nested/4/5"); for (Map.Entry<String, String> src : srcFiles.entrySet()) { File file = new File(srcDir, src.getKey()); //noinspection ResultOfMethodCallIgnored file.getParentFile().mkdirs(); try (DataOutputStream out = new DataOutputStream(new FileOutputStream(file))) { out.writeUTF(src.getValue()); } } // copy the created directory recursively: try { List<Path> remotePaths = new ArrayList<>(); HashMap<String, LocalResource> localResources = new HashMap<>(); AbstractYarnClusterDescriptor.uploadAndRegisterFiles( Collections.singletonList(new File(srcPath.toUri().getPath())), targetFileSystem, targetDir, ApplicationId.newInstance(0, 0), remotePaths, localResources, new StringBuilder()); assertEquals(srcFiles.size(), localResources.size()); Path workDir = ConverterUtils .getPathFromYarnURL(localResources.get(srcPath.getName() + "/1").getResource()).getParent(); RemoteIterator<LocatedFileStatus> targetFilesIterator = targetFileSystem.listFiles(workDir, true); HashMap<String /* (relative) path */, /* contents */ String> targetFiles = new HashMap<>(4); final int workDirPrefixLength = workDir.toString().length() + 1; // one more for the concluding "/" while (targetFilesIterator.hasNext()) { LocatedFileStatus targetFile = targetFilesIterator.next(); int retries = 5; do { try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { String absolutePathString = targetFile.getPath().toString(); String relativePath = absolutePathString.substring(workDirPrefixLength); targetFiles.put(relativePath, in.readUTF()); assertEquals("extraneous data in file " + relativePath, -1, in.read()); break; } catch (FileNotFoundException e) { // For S3, read-after-write may be eventually consistent, i.e. when trying // to access the object before writing it; see // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel // -> try again a bit later Thread.sleep(50); } } while ((retries--) > 0); } assertThat(targetFiles, equalTo(srcFiles)); } finally { // clean up targetFileSystem.delete(targetDir, true); } }
From source file:org.apache.flink.yarn.YarnFlinkResourceManagerTest.java
License:Apache License
@Test public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Exception { new JavaTestKit(system) { {// w w w. jav a 2 s . c o m final Deadline deadline = new FiniteDuration(3, TimeUnit.MINUTES).fromNow(); Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null); String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters(1L, 1L, 1L, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class); int yarnHeartbeatIntervalMillis = 1000; int maxFailedContainers = 10; int numInitialTaskManagers = 5; final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); NMClient nodeManagerClient = mock(NMClient.class); UUID leaderSessionID = UUID.randomUUID(); final List<Container> containerList = new ArrayList<>(); for (int i = 0; i < numInitialTaskManagers; i++) { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(ContainerId.newInstance(ApplicationAttemptId .newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); containerList.add(mockContainer); } doAnswer(new Answer() { int counter = 0; @Override public Object answer(InvocationOnMock invocation) throws Throwable { if (counter < containerList.size()) { callbackHandler .onContainersAllocated(Collections.singletonList(containerList.get(counter++))); } return null; } }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); final CompletableFuture<AkkaActorGateway> resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture<AkkaActorGateway> leaderGatewayFuture = new CompletableFuture<>(); doAnswer((InvocationOnMock invocation) -> { Container container = (Container) invocation.getArguments()[0]; resourceManagerFuture.thenCombine(leaderGatewayFuture, (resourceManagerGateway, leaderGateway) -> { resourceManagerGateway.tell( new NotifyResourceStarted( YarnFlinkResourceManager.extractResourceID(container)), leaderGateway); return null; }); return null; }).when(nodeManagerClient).startContainer(Matchers.any(Container.class), Matchers.any(ContainerLaunchContext.class)); ActorRef resourceManager = null; ActorRef leader1; try { leader1 = system.actorOf(Props.create(TestingUtils.ForwardingActor.class, getRef(), Option.apply(leaderSessionID))); resourceManager = system.actorOf(Props.create(TestingYarnFlinkResourceManager.class, flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient)); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); final AkkaActorGateway resourceManagerGateway = new AkkaActorGateway(resourceManager, leaderSessionID); leaderGatewayFuture.complete(leader1Gateway); resourceManagerFuture.complete(resourceManagerGateway); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> taskManagerRegisteredFuture = resourceManagerGateway .ask(new NotifyWhenResourcesRegistered(numInitialTaskManagers), deadline.timeLeft()); Await.ready(taskManagerRegisteredFuture, deadline.timeLeft()); leaderRetrievalService.notifyListener(null, null); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (Container container : containerList) { resourceManagerGateway.tell( new NotifyResourceStarted(YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); } for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> numberOfRegisteredResourcesFuture = resourceManagerGateway .ask(RequestNumberOfRegisteredResources.INSTANCE, deadline.timeLeft()); int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { resourceManager.tell(PoisonPill.getInstance(), ActorRef.noSender()); } } } }; }
From source file:org.apache.flink.yarn.YarnResourceManagerTest.java
License:Apache License
private static Container mockContainer(String host, int port, int containerId) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), containerId);//from w ww . java 2s . com when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }