List of usage examples for org.apache.hadoop.yarn.api.records Priority newInstance
@Public
@Stable
public static Priority newInstance(int p)
From source file:x10.x10rt.yarn.ApplicationMaster.java
License:Open Source License
private void setup() throws IOException, YarnException { LOG.info("Starting ApplicationMaster"); // Remove the AM->RM token so that containers cannot access it. Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token);/*from w w w . j ava 2 s . c o m*/ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); UserGroupInformation appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, new RMCallbackHandler()); resourceManager.init(conf); resourceManager.start(); nodeManager = new NMClientAsyncImpl(new NMCallbackHandler(this)); nodeManager.init(conf); nodeManager.start(); // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); { int slash = appMasterHostname.indexOf('/'); if (slash != -1) appMasterHostname = appMasterHostname.substring(0, slash); } // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. // TODO: should we reject instead of modifying to fit? if (memoryPerPlaceInMb > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + memoryPerPlaceInMb + ", max=" + maxMem); memoryPerPlaceInMb = maxMem; } if (coresPerPlace > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + coresPerPlace + ", max=" + maxVCores); coresPerPlace = maxVCores; } else if (coresPerPlace == 0) { LOG.info("Container virtual cores specified as auto (X10_NTHREADS=0)." + " Using max value." + ", specified=" + coresPerPlace + ", max=" + maxVCores); coresPerPlace = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = initialNumPlaces - previousAMRunningContainers.size(); // open a local port for X10rt management, and register it with the selector launcherChannel = ServerSocketChannel.open(); //launcherChannel.bind(new InetSocketAddress(appMasterHostname, 0)); // bind to the visible network hostname and random port launcherChannel.bind(null); launcherChannel.configureBlocking(false); appMasterPort = launcherChannel.socket().getLocalPort(); launcherChannel.register(selector, SelectionKey.OP_ACCEPT); numRequestedContainers.set(initialNumPlaces); // Send request for containers to RM for (int i = 0; i < numTotalContainersToRequest; ++i) { Resource capability = Resource.newInstance(memoryPerPlaceInMb, coresPerPlace); ContainerRequest request = new ContainerRequest(capability, null, null, Priority.newInstance(0)); LOG.info("Requested container ask: " + request.toString()); resourceManager.addContainerRequest(request); pendingRequests.add(request); } }
From source file:x10.x10rt.yarn.ApplicationMaster.java
License:Open Source License
private void processMessage(ByteBuffer msg, SocketChannel sc) { assert (msg.capacity() >= headerLength); msg.rewind(); // reset the buffer for reading from the beginning CTRL_MSG_TYPE type = CTRL_MSG_TYPE.values()[msg.getInt()]; int destination = msg.getInt(); final int source = msg.getInt(); int datalen = msg.getInt(); assert (datalen == msg.remaining()); LOG.info("Processing message of type " + type + ", size " + (headerLength + datalen) + " from place " + source);//www. j a va2 s .c o m /* System.err.print("Message contents:"); for (int i=0; i<msg.capacity(); i++) System.err.print(" "+Integer.toHexString(msg.get(i)).toUpperCase()); System.err.println(); */ switch (type) { case HELLO: { // read the port information, and update the record for this place assert (datalen == 4 && source < numRequestedContainers.get() && source >= 0); final CommunicationLink linkInfo; LOG.info("Getting link for place " + source); synchronized (links) { linkInfo = links.get(source); } linkInfo.port = ((msg.get() & 0xFF) << 8) | (msg.get() & 0xFF); // unsigned short in network order linkInfo.sc = sc; // check if there are pending port requests for this place if (linkInfo.pendingPortRequests != null) { // prepare response message String linkString = linkInfo.node.getHost() + ":" + linkInfo.port; byte[] linkBytes = linkString.getBytes(); // matches existing code. TODO: switch to UTF8 ByteBuffer response = ByteBuffer.allocateDirect(headerLength + linkBytes.length) .order(ByteOrder.nativeOrder()); response.putInt(CTRL_MSG_TYPE.PORT_RESPONSE.ordinal()); response.putInt(-1); response.putInt(source); response.putInt(linkBytes.length); response.put(linkBytes); // send response to each waiting place for (int place : linkInfo.pendingPortRequests) { response.putInt(4, place); // set the destination to the requesting place response.rewind(); // TODO: this code may get stuck here if the reciever isn't reading properly try { while (response.hasRemaining()) links.get(place).sc.write(response); } catch (IOException e) { LOG.warn("Unable to send out port response for place " + place + " to place " + source, e); } } linkInfo.pendingPortRequests = null; } LOG.info("HELLO from place " + source + " at port " + linkInfo.port); if (pendingKills != null && pendingKills.containsKey(source)) { int delay = pendingKills.remove(source); LOG.info("Scheduling a takedown of place " + source + " in " + delay + " seconds"); placeKiller.schedule(new Runnable() { @Override public void run() { LOG.info("KILLING CONTAINER FOR PLACE " + source); nodeManager.stopContainerAsync(linkInfo.container, linkInfo.node); } }, delay, TimeUnit.SECONDS); } } break; case GOODBYE: { try { CommunicationLink link = links.get(source); assert (link.pendingPortRequests == null); sc.close(); link.port = PORT_DEAD; } catch (IOException e) { LOG.warn("Error closing socket channel", e); } LOG.info("GOODBYE to place " + source); } break; case PORT_REQUEST: { LOG.info("Got PORT_REQUEST from place " + source + " for place " + destination); // check to see if we know the requested information CommunicationLink linkInfo = links.get(destination); if (linkInfo.port != PORT_UNKNOWN) { String linkString; if (linkInfo.port == PORT_DEAD) linkString = DEAD; else linkString = linkInfo.node.getHost() + ":" + linkInfo.port; LOG.info("Telling place " + source + " that place " + destination + " is at " + linkString); byte[] linkBytes = linkString.getBytes(); // matches existing code. TODO: switch to UTF8 ByteBuffer response = ByteBuffer.allocateDirect(headerLength + linkBytes.length) .order(ByteOrder.nativeOrder()); response.putInt(CTRL_MSG_TYPE.PORT_RESPONSE.ordinal()); response.putInt(source); response.putInt(destination); response.putInt(linkBytes.length); response.put(linkBytes); response.rewind(); // TODO: this code may get stuck here if the reciever isn't reading properly try { while (response.hasRemaining()) sc.write(response); } catch (IOException e) { LOG.warn("Unable to send out port response for place " + destination + " to place " + source, e); } } else { // port is not known. remember we have a place asking for it when it becomes available if (linkInfo.pendingPortRequests == null) linkInfo.pendingPortRequests = new ArrayList<Integer>(2); linkInfo.pendingPortRequests.add(source); LOG.info("Stashing PORT_REQUEST from place " + source + " for place " + destination + " until the answer is known"); } } break; case LAUNCH_REQUEST: { assert (datalen == 8); int numPlacesRequested = (int) msg.getLong(); int oldvalue = numRequestedContainers.getAndAdd((int) numPlacesRequested); // Send request for containers to RM for (int i = 0; i < numPlacesRequested; i++) { Resource capability = Resource.newInstance(memoryPerPlaceInMb, coresPerPlace); ContainerRequest request = new ContainerRequest(capability, null, null, Priority.newInstance(0)); LOG.info("Adding a new container request " + request.toString()); resourceManager.addContainerRequest(request); pendingRequests.add(request); } LOG.info("Requested an increase of " + numPlacesRequested + " places on top of the previous " + oldvalue + " places"); msg.rewind(); msg.putInt(CTRL_MSG_TYPE.LAUNCH_RESPONSE.ordinal()); msg.rewind(); try { while (msg.hasRemaining()) sc.write(msg); } catch (IOException e) { LOG.warn("Unable to send out launch response to place " + source, e); } } break; default: LOG.warn("unknown message type " + type); } LOG.info("Finished processing message of size " + (headerLength + datalen) + " from place " + source); }