List of usage examples for org.apache.hadoop.yarn.exceptions YarnException YarnException
public YarnException(Throwable cause)
From source file:edu.uci.ics.asterix.aoya.Utils.java
License:Apache License
public static void writeYarnClusterConfig(String path, Cluster cl) throws YarnException { try {// ww w .j av a2s.c o m File f = new File(path); JAXBContext configCtx = JAXBContext.newInstance(Cluster.class); Marshaller marhsaller = configCtx.createMarshaller(); marhsaller.marshal(cl, f); } catch (JAXBException e) { throw new YarnException(e); } }
From source file:edu.uci.ics.asterix.aoya.Utils.java
License:Apache License
public static boolean waitForLiveness(ApplicationId appId, boolean probe, boolean print, String message, YarnClient yarnClient, String instanceName, Configuration conf, int port) throws YarnException { ApplicationReport report;/* w w w .j a v a2 s . c o m*/ try { report = yarnClient.getApplicationReport(appId); } catch (IOException e) { throw new YarnException(e); } YarnApplicationState st = report.getYarnApplicationState(); for (int i = 0; i < 120; i++) { if (st != YarnApplicationState.RUNNING) { try { report = yarnClient.getApplicationReport(appId); st = report.getYarnApplicationState(); if (print) { System.out.print(message + Utils.makeDots(i) + "\r"); } Thread.sleep(1000); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); } catch (IOException e1) { throw new YarnException(e1); } if (st == YarnApplicationState.FAILED || st == YarnApplicationState.FINISHED || st == YarnApplicationState.KILLED) { return false; } } if (probe) { String host; host = getCCHostname(instanceName, conf); try { for (int j = 0; j < 60; j++) { if (!Utils.probeLiveness(host, port)) { try { if (print) { System.out.print(message + Utils.makeDots(i) + "\r"); } Thread.sleep(1000); } catch (InterruptedException e2) { Thread.currentThread().interrupt(); } } else { if (print) { System.out.println(""); } return true; } } } catch (IOException e1) { throw new YarnException(e1); } } else { if (print) { System.out.println(""); } return true; } } if (print) { System.out.println(""); } return false; }
From source file:edu.uci.ics.asterix.aoya.Utils.java
License:Apache License
public static String getCCHostname(String instanceName, Configuration conf) throws YarnException { try {//from w ww . ja v a 2s . com FileSystem fs = FileSystem.get(conf); String instanceFolder = instanceName + "/"; String pathSuffix = CONF_DIR_REL + instanceFolder + "cluster-config.xml"; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); File tmp = File.createTempFile("cluster-config", "xml"); tmp.deleteOnExit(); fs.copyToLocalFile(dstConf, new Path(tmp.getPath())); JAXBContext clusterConf = JAXBContext.newInstance(Cluster.class); Unmarshaller unm = clusterConf.createUnmarshaller(); Cluster cl = (Cluster) unm.unmarshal(tmp); String ccIp = cl.getMasterNode().getClientIp(); return ccIp; } catch (IOException | JAXBException e) { throw new YarnException(e); } }
From source file:io.hops.tensorflow.TimelineHandler.java
License:Apache License
public void startClient(final Configuration conf) throws YarnException, IOException, InterruptedException { try {//from ww w . j a v a 2s . c o m ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { // Creating the Timeline Client timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); } else { timelineClient = null; LOG.warn("Timeline service is not enabled"); } return null; } }); } catch (UndeclaredThrowableException e) { throw new YarnException(e.getCause()); } }
From source file:org.apache.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * @param c/*w ww. jav a 2s.c om*/ * The cluster exception to attempt to alocate with the RM * @throws YarnException */ private void requestResources(Cluster c) throws YarnException, UnknownHostException { //set memory if (c.getCcContainerMem() != null) { ccMem = Integer.parseInt(c.getCcContainerMem()); } else { ccMem = CC_MEMORY_MBS_DEFAULT; } if (c.getNcContainerMem() != null) { ncMem = Integer.parseInt(c.getNcContainerMem()); } else { ncMem = CC_MEMORY_MBS_DEFAULT; } //request CC int numNodes = 0; ContainerRequest ccAsk = hostToRequest(cC.getClusterIp(), true); resourceManager.addContainerRequest(ccAsk); LOG.info("Asked for CC: " + Arrays.toString(ccAsk.getNodes().toArray())); numNodes++; //now we wait to be given the CC before starting the NCs... //we will wait a minute. int deathClock = 60; while (ccUp.get() == false && deathClock > 0) { try { Thread.sleep(1000); } catch (InterruptedException ex) { LOG.debug(ex); } --deathClock; } if (deathClock == 0 && ccUp.get() == false) { throw new YarnException("Couldn't allocate container for CC. Abort!"); } LOG.info("Waiting for CC process to start"); //TODO: inspect for actual liveness instead of waiting. // is there a good way to do this? maybe try opening a socket to it... try { Thread.sleep(10000); } catch (InterruptedException ex) { LOG.debug(ex); } //request NCs for (Node n : c.getNode()) { resourceManager.addContainerRequest(hostToRequest(n.getClusterIp(), false)); LOG.info("Asked for NC: " + n.getClusterIp()); numNodes++; synchronized (pendingNCs) { pendingNCs.add(n); } } LOG.info("Requested all NCs and CCs. Wait for things to settle!"); numRequestedContainers.set(numNodes); numTotalContainers = numNodes; doneAllocating = true; }
From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java
License:Apache License
/** * Analyzes properties of Yarn cluster and Hadoop configurations. * /*from w w w .j a v a2 s. co m*/ * @param yarnClient hadoop yarn client * @param conf hadoop yarn configuration * @param verbose output info to standard output */ public static void analyzeYarnCluster(YarnClient yarnClient, YarnConfiguration conf, boolean verbose) { try { List<NodeReport> nodesReport = yarnClient.getNodeReports(); if (verbose) System.out.println("There are " + nodesReport.size() + " nodes in the cluster"); if (nodesReport.isEmpty()) throw new YarnException("There are zero available nodes in the yarn cluster"); nodesMaxPhySorted = new ArrayList<>(nodesReport.size()); clusterTotalMem = 0; clusterTotalCores = 0; clusterTotalNodes = 0; minimumMRContainerPhyMB = -1; for (NodeReport node : nodesReport) { Resource resource = node.getCapability(); Resource used = node.getUsed(); if (used == null) used = Resource.newInstance(0, 0); int mb = resource.getMemory(); int cores = resource.getVirtualCores(); if (mb <= 0) throw new YarnException("A node has non-positive memory " + mb); int myMinMRPhyMB = mb / cores / CPU_HYPER_FACTOR; if (minimumMRContainerPhyMB < myMinMRPhyMB) minimumMRContainerPhyMB = myMinMRPhyMB; // minimumMRContainerPhyMB needs to be the largest among the mins clusterTotalMem += (long) mb * 1024 * 1024; nodesMaxPhySorted.add((long) mb * 1024 * 1024); clusterTotalCores += cores; clusterTotalNodes++; if (verbose) System.out.println("\t" + node.getNodeId() + " has " + mb + " MB (" + used.getMemory() + " MB used) memory and " + resource.getVirtualCores() + " (" + used.getVirtualCores() + " used) cores"); } Collections.sort(nodesMaxPhySorted, Collections.reverseOrder()); nodesMaxBudgetSorted = new ArrayList<>(nodesMaxPhySorted.size()); for (int i = 0; i < nodesMaxPhySorted.size(); i++) nodesMaxBudgetSorted.add(ResourceOptimizer.phyToBudget(nodesMaxPhySorted.get(i))); _remotePar = nodesReport.size(); if (_remotePar == 0) throw new YarnException("There are no available nodes in the yarn cluster"); // Now get the default cluster settings _remoteMRSortMem = (1024 * 1024) * conf.getLong(MRConfigurationNames.MR_TASK_IO_SORT_MB, 100); //100MB //handle jvm max mem (map mem budget is relevant for map-side distcache and parfor) //(for robustness we probe both: child and map configuration parameters) String javaOpts1 = conf.get(MRConfigurationNames.MR_CHILD_JAVA_OPTS); //internally mapred/mapreduce synonym String javaOpts2 = conf.get(MRConfigurationNames.MR_MAP_JAVA_OPTS, null); //internally mapred/mapreduce synonym String javaOpts3 = conf.get(MRConfigurationNames.MR_REDUCE_JAVA_OPTS, null); //internally mapred/mapreduce synonym if (javaOpts2 != null) //specific value overrides generic _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts2); else _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts1); if (javaOpts3 != null) //specific value overrides generic _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts3); else _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts1); //HDFS blocksize String blocksize = conf.get(MRConfigurationNames.DFS_BLOCKSIZE, "134217728"); _blocksize = Long.parseLong(blocksize); minimalPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); maximumPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); mrAMPhy = (long) conf.getInt(MRConfigurationNames.YARN_APP_MR_AM_RESOURCE_MB, 1536) * 1024 * 1024; } catch (Exception e) { throw new RuntimeException("Unable to analyze yarn cluster ", e); } /* * This is for AppMaster to query available resource in the cluster during heartbeat * AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); AllocateResponse response = rmClient.allocate(0); int nodeCount = response.getNumClusterNodes(); Resource resource = response.getAvailableResources(); List<NodeReport> nodeUpdate = response.getUpdatedNodes(); LOG.info("This is a " + nodeCount + " node cluster with totally " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); LOG.info(nodereport.size() + " updatedNode reports received"); for (NodeReport node : nodeUpdate) { resource = node.getCapability(); LOG.info(node.getNodeId() + " updated with " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); }*/ }