List of usage examples for org.apache.hadoop.yarn.api.records ApplicationAttemptId toString
@Override
public String toString()
From source file:com.yahoo.storm.yarn.MasterServer.java
License:Open Source License
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { LOG.info("Starting the AM!!!!"); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used " + "unless for testing purposes"); CommandLine cl = new GnuParser().parse(opts, args); ApplicationAttemptId appAttemptID; Map<String, String> envs = System.getenv(); if (cl.hasOption("app_attempt_id")) { String appIdStr = cl.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else if (envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { ContainerId containerId = ConverterUtils .toContainerId(envs.get(ApplicationConstants.Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); LOG.info("appAttemptID from env:" + appAttemptID.toString()); } else {/*from ww w. j ava 2 s. c o m*/ LOG.error("appAttemptID is not specified for storm master"); throw new Exception("appAttemptID is not specified for storm master"); } @SuppressWarnings("rawtypes") Map storm_conf = Config.readStormConfig(null); Util.rmNulls(storm_conf); YarnConfiguration hadoopConf = new YarnConfiguration(); final String host = InetAddress.getLocalHost().getHostName(); storm_conf.put("nimbus.host", host); StormAMRMClient rmClient = new StormAMRMClient(appAttemptID, storm_conf, hadoopConf); rmClient.init(hadoopConf); rmClient.start(); BlockingQueue<Container> launcherQueue = new LinkedBlockingQueue<Container>(); MasterServer server = new MasterServer(storm_conf, rmClient); try { final int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT)); final String target = host + ":" + port; InetSocketAddress addr = NetUtils.createSocketAddr(target); RegisterApplicationMasterResponse resp = rmClient.registerApplicationMaster(addr.getHostName(), port, null); LOG.info("Got a registration response " + resp); LOG.info("Max Capability " + resp.getMaximumResourceCapability()); rmClient.setMaxResource(resp.getMaximumResourceCapability()); LOG.info("Starting HB thread"); server.initAndStartHeartbeat(rmClient, launcherQueue, (Integer) storm_conf.get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS)); LOG.info("Starting launcher"); initAndStartLauncher(rmClient, launcherQueue); rmClient.startAllSupervisors(); LOG.info("Starting Master Thrift Server"); server.serve(); LOG.info("StormAMRMClient::unregisterApplicationMaster"); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "AllDone", null); } finally { if (server.isServing()) { LOG.info("Stop Master Thrift Server"); server.stop(); } LOG.info("Stop RM client"); rmClient.stop(); } System.exit(0); }
From source file:io.hops.ha.common.SchedulerApplicationInfo.java
License:Apache License
public FiCaSchedulerAppInfo getFiCaSchedulerAppInfo(ApplicationAttemptId appAttemptId) { if (fiCaSchedulerAppInfo.get(appAttemptId.toString()) == null) { fiCaSchedulerAppInfo.put(appAttemptId.toString(), new FiCaSchedulerAppInfo(appAttemptId)); }/*from ww w . j a va 2 s.c o m*/ return fiCaSchedulerAppInfo.get(appAttemptId.toString()); }
From source file:io.hops.ha.common.TransactionStateImpl.java
License:Apache License
private void persistAllocateResponsesToAdd() throws IOException { if (!allocateResponsesToAdd.isEmpty()) { AllocateResponseDataAccess da = (AllocateResponseDataAccess) RMStorageFactory .getDataAccess(AllocateResponseDataAccess.class); List<AllocateResponse> toAdd = new ArrayList<AllocateResponse>(); for (ApplicationAttemptId id : allocateResponsesToAdd.keySet()) { if (!allocateResponsesToRemove.remove(id)) { AllocateResponseLock lock = allocateResponsesToAdd.get(id); synchronized (lock) { AllocateResponsePBImpl lastResponse = (AllocateResponsePBImpl) lock.getAllocateResponse(); if (lastResponse != null) { toAdd.add(new AllocateResponse(id.toString(), lastResponse.getProto().toByteArray())); }/*from ww w. j a v a2 s . c o m*/ } } } da.addAll(toAdd); } }
From source file:io.hops.ha.common.TransactionStateImpl.java
License:Apache License
private void persistAllocateResponsesToRemove() throws IOException { if (!allocateResponsesToRemove.isEmpty()) { AllocateResponseDataAccess da = (AllocateResponseDataAccess) RMStorageFactory .getDataAccess(AllocateResponseDataAccess.class); List<AllocateResponse> toRemove = new ArrayList<AllocateResponse>(); for (ApplicationAttemptId id : allocateResponsesToRemove) { toRemove.add(new AllocateResponse(id.toString())); }//w w w. ja v a2s. c om da.removeAll(toRemove); } }
From source file:io.hops.metadata.util.RMUtilities.java
License:Apache License
/** * Retrieve applications (RMApp, RMAppAttempt) from NDB. MUST be used only * by ResourceTrackerService as some fields of the objects are not set. * * @param rmContext/*from w ww . j a v a 2 s . c o m*/ * @param conf * @param applicationId * @return * @throws java.io.IOException */ public static RMApp getRMApp(RMContext rmContext, Configuration conf, String applicationId) throws IOException { //Retrieve all applicationIds from NDB ApplicationState hopAppState = RMUtilities.getApplicationState(applicationId); if (hopAppState != null) { //Create ApplicationState for every application ApplicationId appId = ConverterUtils.toApplicationId(hopAppState.getApplicationid()); ApplicationStateDataPBImpl appStateData = new ApplicationStateDataPBImpl( ApplicationStateDataProto.parseFrom(hopAppState.getAppstate())); RMStateStore.ApplicationState appState = new RMStateStore.ApplicationState(appStateData.getSubmitTime(), appStateData.getStartTime(), appStateData.getApplicationSubmissionContext(), appStateData.getUser(), appStateData.getState(), appStateData.getDiagnostics(), appStateData.getFinishTime(), appStateData.getStateBeforeKilling(), appStateData.getUpdatedNodesId()); LOG.debug("loadRMAppState for app " + appState.getAppId() + " state " + appState.getState()); //Create RMApp //Null fields are not required by ResourceTrackerService RMAppImpl application = new RMAppImpl(appId, rmContext, conf, appState.getApplicationSubmissionContext().getApplicationName(), appState.getUser(), appState.getApplicationSubmissionContext().getQueue(), appState.getApplicationSubmissionContext(), null, null, appState.getSubmitTime(), appState.getApplicationSubmissionContext().getApplicationType(), appState.getApplicationSubmissionContext().getApplicationTags(), null); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, appState.getAttemptCount() + 1); ApplicationAttemptState hopAppAttemptState = RMUtilities.getApplicationAttemptState(applicationId, appAttemptId.toString()); if (hopAppAttemptState != null) { ApplicationAttemptStateDataPBImpl attemptStateData = new ApplicationAttemptStateDataPBImpl( YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto .parseFrom(hopAppAttemptState.getApplicationattemptstate())); RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, rmContext, null, null, appState.getApplicationSubmissionContext(), conf, application.getMaxAppAttempts() == application.getAppAttempts().size()); ((RMAppAttemptImpl) attempt).setMasterContainer(attemptStateData.getMasterContainer()); application.addRMAppAttempt(appAttemptId, attempt); return application; } } return null; }
From source file:org.apache.samza.validation.YarnJobValidationTool.java
License:Apache License
public ApplicationAttemptId validateRunningAttemptId(ApplicationId appId) throws Exception { ApplicationAttemptId attemptId = this.client.getApplicationReport(appId).getCurrentApplicationAttemptId(); ApplicationAttemptReport attemptReport = this.client.getApplicationAttemptReport(attemptId); if (attemptReport.getYarnApplicationAttemptState() == YarnApplicationAttemptState.RUNNING) { log.info("Job is running. AttempId " + attemptId.toString()); return attemptId; } else {/*from w w w. j a va 2 s . c o m*/ throw new SamzaException("Job not running " + this.jobName); } }
From source file:org.apache.samza.webapp.TestApplicationMasterRestClient.java
License:Apache License
@Test public void testGetAmStateSuccess() throws IOException { SamzaApplicationState samzaAppState = createSamzaApplicationState(); ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(APP_ATTEMPT_ID); ContainerId containerId = ConverterUtils.toContainerId(YARN_CONTAINER_ID_1); YarnAppState yarnAppState = createYarnAppState(containerId); String response = ApplicationMasterRestServlet.getAmState(jsonMapper, samzaAppState, yarnAppState); setupMockClientResponse(HttpStatus.SC_OK, "Success", response); ApplicationMasterRestClient client = new ApplicationMasterRestClient(mockClient, AM_HOST_NAME, AM_RPC_PORT); Map<String, Object> amStateResult = client.getAmState(); assertEquals(4, amStateResult.size()); assertEquals(String.format("%s:%s", yarnAppState.nodeHost, yarnAppState.rpcUrl.getPort()), amStateResult.get("host")); assertEquals(containerId.toString(), amStateResult.get("container-id")); // Can only validate the keys because up-time changes everytime it's requested assertEquals(buildExpectedContainerResponse(yarnAppState.runningProcessors, samzaAppState).keySet(), ((Map<String, Object>) amStateResult.get("containers")).keySet()); assertEquals(attemptId.toString(), amStateResult.get("app-attempt-id")); }
From source file:org.apache.tez.dag.history.logging.proto.HistoryEventProtoConverter.java
License:Apache License
private HistoryEventProto.Builder makeBuilderForEvent(HistoryEvent event, long time, TezDAGID dagId, ApplicationId appId, ApplicationAttemptId appAttemptId, TezVertexID vertexId, TezTaskID taskId, TezTaskAttemptID taskAttemptId, String user) { HistoryEventProto.Builder builder = HistoryEventProto.newBuilder(); builder.setEventType(event.getEventType().name()); builder.setEventTime(time);/*from www . j ava2 s . co m*/ if (taskAttemptId != null) { builder.setTaskAttemptId(taskAttemptId.toString()); taskId = taskAttemptId.getTaskID(); } if (taskId != null) { builder.setTaskId(taskId.toString()); vertexId = taskId.getVertexID(); } if (vertexId != null) { builder.setVertexId(vertexId.toString()); dagId = vertexId.getDAGId(); } if (dagId != null) { builder.setDagId(dagId.toString()); if (appId == null) { appId = dagId.getApplicationId(); } } if (appAttemptId != null) { builder.setAppAttemptId(appAttemptId.toString()); if (appId == null) { appId = appAttemptId.getApplicationId(); } } if (appId != null) { builder.setAppId(appId.toString()); } if (user != null) { builder.setUser(user); } return builder; }