List of usage examples for org.apache.hadoop.yarn.exceptions YarnException YarnException
public YarnException(Throwable cause)
From source file:com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer.java
License:Open Source License
/** * Analyzes properties of Yarn cluster and Hadoop configurations. *//*ww w . j av a 2 s. c om*/ public static void analyzeYarnCluster(YarnClient yarnClient, YarnConfiguration conf, boolean verbose) { try { List<NodeReport> nodesReport = yarnClient.getNodeReports(); if (verbose) System.out.println("There are " + nodesReport.size() + " nodes in the cluster"); if (nodesReport.isEmpty()) throw new YarnException("There are zero available nodes in the yarn cluster"); nodesMaxPhySorted = new ArrayList<Long>(nodesReport.size()); clusterTotalMem = 0; clusterTotalCores = 0; clusterTotalNodes = 0; minimumMRContainerPhyMB = -1; for (NodeReport node : nodesReport) { Resource resource = node.getCapability(); Resource used = node.getUsed(); if (used == null) used = Resource.newInstance(0, 0); int mb = resource.getMemory(); int cores = resource.getVirtualCores(); if (mb <= 0) throw new YarnException("A node has non-positive memory " + mb); int myMinMRPhyMB = mb / cores / CPU_HYPER_FACTOR; if (minimumMRContainerPhyMB < myMinMRPhyMB) minimumMRContainerPhyMB = myMinMRPhyMB; // minimumMRContainerPhyMB needs to be the largest among the mins clusterTotalMem += (long) mb * 1024 * 1024; nodesMaxPhySorted.add((long) mb * 1024 * 1024); clusterTotalCores += cores; clusterTotalNodes++; if (verbose) System.out.println("\t" + node.getNodeId() + " has " + mb + " MB (" + used.getMemory() + " MB used) memory and " + resource.getVirtualCores() + " (" + used.getVirtualCores() + " used) cores"); } Collections.sort(nodesMaxPhySorted, Collections.reverseOrder()); nodesMaxBudgetSorted = new ArrayList<Double>(nodesMaxPhySorted.size()); for (int i = 0; i < nodesMaxPhySorted.size(); i++) nodesMaxBudgetSorted.add(ResourceOptimizer.phyToBudget(nodesMaxPhySorted.get(i))); _remotePar = nodesReport.size(); if (_remotePar == 0) throw new YarnException("There are no available nodes in the yarn cluster"); // Now get the default cluster settings _remoteMRSortMem = (1024 * 1024) * conf.getLong("io.sort.mb", 100); //100MB //handle jvm max mem (map mem budget is relevant for map-side distcache and parfor) //(for robustness we probe both: child and map configuration parameters) String javaOpts1 = conf.get("mapred.child.java.opts"); //internally mapred/mapreduce synonym String javaOpts2 = conf.get("mapreduce.map.java.opts", null); //internally mapred/mapreduce synonym String javaOpts3 = conf.get("mapreduce.reduce.java.opts", null); //internally mapred/mapreduce synonym if (javaOpts2 != null) //specific value overrides generic _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts2); else _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts1); if (javaOpts3 != null) //specific value overrides generic _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts3); else _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts1); //HDFS blocksize String blocksize = conf.get(MRConfigurationNames.DFS_BLOCK_SIZE, "134217728"); _blocksize = Long.parseLong(blocksize); minimalPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); maximumPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); mrAMPhy = (long) conf.getInt("yarn.app.mapreduce.am.resource.mb", 1536) * 1024 * 1024; } catch (Exception e) { throw new RuntimeException("Unable to analyze yarn cluster ", e); } /* * This is for AppMaster to query available resource in the cluster during heartbeat * AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); AllocateResponse response = rmClient.allocate(0); int nodeCount = response.getNumClusterNodes(); Resource resource = response.getAvailableResources(); List<NodeReport> nodeUpdate = response.getUpdatedNodes(); LOG.info("This is a " + nodeCount + " node cluster with totally " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); LOG.info(nodereport.size() + " updatedNode reports received"); for (NodeReport node : nodeUpdate) { resource = node.getCapability(); LOG.info(node.getNodeId() + " updated with " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); }*/ }
From source file:com.inforefiner.hdata.ApplicationMaster.java
License:Apache License
@VisibleForTesting void startTimelineClient(final Configuration conf) throws YarnException, IOException, InterruptedException { try {//from w w w. j a va 2 s . c om appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { // Creating the Timeline Client timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); } else { timelineClient = null; LOG.warn("Timeline service is not enabled"); } return null; } }); } catch (UndeclaredThrowableException e) { throw new YarnException(e.getCause()); } }
From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * @param c/* w w w .j a v a2 s.c om*/ * The cluster exception to attempt to alocate with the RM * @throws YarnException */ private void requestResources(Cluster c) throws YarnException, UnknownHostException { //set memory if (c.getCcContainerMem() != null) { ccMem = Integer.parseInt(c.getCcContainerMem()); } else { ccMem = CC_MEMORY_MBS_DEFAULT; } if (c.getNcContainerMem() != null) { ncMem = Integer.parseInt(c.getNcContainerMem()); } else { ncMem = CC_MEMORY_MBS_DEFAULT; } //request CC int numNodes = 0; ContainerRequest ccAsk = hostToRequest(cC.getClusterIp(), true); resourceManager.addContainerRequest(ccAsk); LOG.info("Asked for CC: " + Arrays.toString(ccAsk.getNodes().toArray())); numNodes++; //now we wait to be given the CC before starting the NCs... //we will wait a minute. int deathClock = 60; while (ccUp.get() == false && deathClock > 0) { try { Thread.sleep(1000); } catch (InterruptedException ex) { LOG.debug(ex); } --deathClock; } if (deathClock == 0 && ccUp.get() == false) { throw new YarnException("Couldn't allocate container for CC. Abort!"); } LOG.info("Waiting for CC process to start"); //TODO: inspect for actual liveness instead of waiting. // is there a good way to do this? maybe try opening a socket to it... try { Thread.sleep(10000); } catch (InterruptedException ex) { LOG.debug(ex); } //request NCs for (Node n : c.getNode()) { resourceManager.addContainerRequest(hostToRequest(n.getClusterIp(), false)); LOG.info("Asked for NC: " + n.getClusterIp()); numNodes++; synchronized (pendingNCs) { pendingNCs.add(n); } } LOG.info("Requested all NCs and CCs. Wait for things to settle!"); numRequestedContainers.set(numNodes); numTotalContainers = numNodes; doneAllocating = true; }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private static void startAction(AsterixYARNClient client) throws YarnException { YarnClientApplication app;//from www .java 2 s .c om List<DFSResourceCoordinate> res; ApplicationId appId; try { app = client.makeApplicationContext(); res = client.deployConfig(); res.addAll(client.distributeBinaries()); appId = client.deployAM(app, res, client.mode); LOG.info("Asterix started up with Application ID: " + appId.toString()); if (Utils.waitForLiveness(appId, "Waiting for AsterixDB instance to resume ", client.yarnClient, client.instanceName, client.conf, client.ccRestPort)) { System.out.println("Asterix successfully deployed and is now running."); } else { LOG.fatal("AsterixDB appears to have failed to install and start"); throw new YarnException("AsterixDB appears to have failed to install and start"); } } catch (IOException e) { throw new YarnException(e); } }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private static void installAction(AsterixYARNClient client) throws YarnException { YarnClientApplication app;/*from w w w .java 2 s . c o m*/ List<DFSResourceCoordinate> res; ApplicationId appId; try { app = client.makeApplicationContext(); client.installConfig(); client.writeAsterixConfig(Utils.parseYarnClusterConfig(client.asterixConf)); client.installAsterixConfig(false); res = client.deployConfig(); res.addAll(client.distributeBinaries()); appId = client.deployAM(app, res, client.mode); LOG.info("Asterix started up with Application ID: " + appId.toString()); if (Utils.waitForLiveness(appId, "Waiting for new AsterixDB Instance to start ", client.yarnClient, client.instanceName, client.conf, client.ccRestPort)) { System.out.println("Asterix successfully deployed and is now running."); } else { LOG.fatal("AsterixDB appears to have failed to install and start"); throw new YarnException("AsterixDB appears to have failed to install and start"); } } catch (IOException e) { LOG.fatal("Asterix failed to deploy on to cluster"); throw new YarnException(e); } }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Upload the Asterix cluster description on to the DFS. This will persist the state of the instance. * /*from w ww. j a v a 2 s. c o m*/ * @return * @throws YarnException * @throws IOException */ private List<DFSResourceCoordinate> deployConfig() throws YarnException, IOException { FileSystem fs = FileSystem.get(conf); List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); FileStatus destStatus; try { destStatus = fs.getFileStatus(dstConf); } catch (IOException e) { throw new YarnException("Asterix instance by that name does not appear to exist in DFS"); } LocalResource asterixConfLoc = Records.newRecord(LocalResource.class); asterixConfLoc.setType(LocalResourceType.FILE); asterixConfLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixConfLoc.setResource(ConverterUtils.getYarnUrlFromPath(dstConf)); asterixConfLoc.setTimestamp(destStatus.getModificationTime()); DFSResourceCoordinate conf = new DFSResourceCoordinate(); conf.envs.put(dstConf.toUri().toString(), AConstants.CONFLOCATION); conf.envs.put(Long.toString(asterixConfLoc.getSize()), AConstants.CONFLEN); conf.envs.put(Long.toString(asterixConfLoc.getTimestamp()), AConstants.CONFTIMESTAMP); conf.name = CONFIG_DEFAULT_NAME; conf.res = asterixConfLoc; resources.add(conf); return resources; }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Asks YARN to kill a given application by appId * @param appId The application to kill. * @param yarnClient The YARN client object that is connected to the RM. * @throws YarnException//w w w. ja va 2s . c o m * @throws IOException */ public static void killApplication(ApplicationId appId, YarnClient yarnClient) throws YarnException, IOException { if (appId == null) { throw new YarnException("No Application given to kill"); } if (yarnClient.isInState(STATE.INITED)) { yarnClient.start(); } YarnApplicationState st; ApplicationReport rep = yarnClient.getApplicationReport(appId); st = rep.getYarnApplicationState(); if (st == YarnApplicationState.FINISHED || st == YarnApplicationState.KILLED || st == YarnApplicationState.FAILED) { LOG.info("Application " + appId + " already exited."); return; } LOG.info("Killing applicaiton with ID: " + appId); yarnClient.killApplication(appId); }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Tries to stop a running AsterixDB instance gracefully. * @throws IOException//from w w w . j a v a 2s . c o m * @throws YarnException */ private void stopInstanceIfRunning() throws IOException, YarnException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); //if the instance is up, fix that if (isRunning()) { try { this.stopInstance(); } catch (IOException e) { throw new YarnException(e); } } else if (!fs.exists(dstConf)) { throw new YarnException("No instance configured with that name exists"); } }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private ApplicationId getLockFile() throws IOException, YarnException { if (instanceFolder == "") { throw new IllegalStateException("Instance name not given."); }//from w w w. j av a 2 s .com FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); if (!fs.exists(lockPath)) { throw new YarnException("Instance appears to not be running. If you know it is, try using kill"); } BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(lockPath))); String lockAppId = br.readLine(); br.close(); return ConverterUtils.toApplicationId(lockAppId); }
From source file:edu.uci.ics.asterix.aoya.Utils.java
License:Apache License
/** * Simply parses out the YARN cluster config and instantiates it into a nice object. * //from w w w . j av a2s . c om * @return The object representing the configuration * @throws FileNotFoundException * @throws JAXBException */ public static Cluster parseYarnClusterConfig(String path) throws YarnException { try { File f = new File(path); JAXBContext configCtx = JAXBContext.newInstance(Cluster.class); Unmarshaller unmarshaller = configCtx.createUnmarshaller(); Cluster cl = (Cluster) unmarshaller.unmarshal(f); return cl; } catch (JAXBException e) { throw new YarnException(e); } }