List of usage examples for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer
public DataOutputBuffer()
From source file:crunch.MaxTemperature.java
License:Apache License
public static DataOutputBuffer serialize(Writable writable) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); DataOutputStream dataOut = new DataOutputStream(out); writable.write(dataOut);/*from www . j a v a2 s .co m*/ dataOut.close(); return out; }
From source file:de.huberlin.wbi.hiway.am.HiWay.java
License:Apache License
/** * Main run function for the application master * //www. j a v a2s . c o m * @return True if there were no errors * @throws YarnException * YarnException * @throws IOException * IOException */ @SuppressWarnings("unchecked") public boolean run() throws YarnException, IOException { System.out.println("Starting ApplicationMaster"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); allocListener = new RMCallbackHandler(this); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = new NMCallbackHandler(this); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); Data workflowData = new Data(workflowPath); workflowData.stageIn(); // Register self with ResourceManager. This will start heartbeating to the RM. appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); switch (schedulerName) { case staticRoundRobin: case heft: scheduler = schedulerName.equals(HiWayConfiguration.HIWAY_SCHEDULER_OPTS.staticRoundRobin) ? new RoundRobin(getWorkflowName(), hdfs, conf) : new HEFT(getWorkflowName(), hdfs, conf); break; case greedyQueue: scheduler = new GreedyQueue(getWorkflowName(), conf, hdfs); break; default: C3PO c3po = new C3PO(getWorkflowName(), hdfs, conf); switch (schedulerName) { case conservative: c3po.setConservatismWeight(12d); c3po.setnClones(0); c3po.setPlacementAwarenessWeight(0.01d); c3po.setOutlookWeight(0.01d); break; case cloning: c3po.setConservatismWeight(0.01d); c3po.setnClones(1); c3po.setPlacementAwarenessWeight(0.01d); c3po.setOutlookWeight(0.01d); break; case placementAware: c3po.setConservatismWeight(0.01d); c3po.setnClones(0); c3po.setPlacementAwarenessWeight(12d); c3po.setOutlookWeight(0.01d); break; case outlooking: c3po.setConservatismWeight(0.01d); c3po.setnClones(0); c3po.setPlacementAwarenessWeight(0.01d); c3po.setOutlookWeight(12d); break; default: c3po.setConservatismWeight(3d); c3po.setnClones(2); c3po.setPlacementAwarenessWeight(1d); c3po.setOutlookWeight(2d); } scheduler = c3po; } scheduler.initialize(); writeEntryToLog(new JsonReportEntry(getRunId(), null, null, null, null, null, HiwayDBI.KEY_WF_NAME, getWorkflowName())); parseWorkflow(); scheduler.updateRuntimeEstimates(getRunId().toString()); federatedReport = new Data(appId + ".log"); // Dump out information about cluster capability as seen by the resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); int maxCores = response.getMaximumResourceCapability().getVirtualCores(); System.out.println("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { System.out.println("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerCores > maxCores) { System.out.println("Container vcores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerCores + ", max=" + maxCores); containerCores = maxCores; } while (!done) { try { while (scheduler.hasNextNodeRequest()) { ContainerRequest containerAsk = setupContainerAskForRM(scheduler.getNextNodeRequest()); amRMClient.addContainerRequest(containerAsk); } Thread.sleep(1000); System.out.println("Current application state: requested=" + numRequestedContainers + ", completed=" + numCompletedContainers + ", failed=" + numFailedContainers + ", killed=" + numKilledContainers + ", allocated=" + numAllocatedContainers); } catch (InterruptedException e) { e.printStackTrace(); System.exit(-1); } } finish(); } catch (Exception e) { e.printStackTrace(); System.exit(-1); } return success; }
From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java
License:Apache License
/** * Main run function for the application master. Does more initialization (sic!). * Calls the abstract {@link #parseWorkflow()}, then {@link #executeWorkflow()} and finally {@link #finish()}. * @return True if there were no errors/*w ww .j av a2 s. co m*/ */ protected boolean run() throws IOException { /* log */ Logger.writeToStdout("Starting ApplicationMaster"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); // remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Resource Manager communications setup RMCallbackHandler allocListener = new RMCallbackHandler(this); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); // Node Managers communications setup containerListener = new NMCallbackHandler(this); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // get workflow file if (hdfs.exists(workflowPath)) { Path localPath = new Path(workflowPath.getName()); hdfs.copyToLocalFile(false, workflowPath, localPath); workflowPath = localPath; workflowFile = new Data(workflowPath); workflowFile.stageOut(); } else { // TODO this doesn't work; the path is triggered when running the application e.g., as hiway workflows/test.dax // but stageIn then fails, because in the HDFS, there is only test.dax and not workflows/test.dax workflowFile = new Data(workflowPath); workflowFile.stageIn(); } // Register self with ResourceManager. This will start heartbeating to the RM. /* the hostname of the container running the Hi-WAY ApplicationMaster */ String appMasterHostname = NetUtils.getHostname(); /* the port on which the ApplicationMaster listens for status updates from clients */ int appMasterRpcPort = -1; /* the tracking URL to which the ApplicationMaster publishes info for clients to monitor */ String appMasterTrackingUrl = ""; RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // initialize scheduler switch (schedulerEnumValue) { case roundRobin: case heft: int workerMemory = conf.getInt(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB); scheduler = schedulerEnumValue.equals(HiWayConfiguration.HIWAY_SCHEDULERS.roundRobin) ? new RoundRobin(getWorkflowName()) : new HEFT(getWorkflowName(), workerMemory / containerMemory); break; case greedy: scheduler = new GreedyQueue(getWorkflowName()); break; case memoryAware: scheduler = new MemoryAware(getWorkflowName(), amRMClient); break; case perfectDaxGQ: scheduler = new PerfectDaxGreedyQueue(getWorkflowName()); break; default: C3PO c3po = new C3PO(getWorkflowName()); switch (schedulerEnumValue) { case dataAware: c3po.setConservatismWeight(0.01d); c3po.setnClones(0); c3po.setPlacementAwarenessWeight(12d); c3po.setOutlookWeight(0.01d); break; default: c3po.setConservatismWeight(3d); c3po.setnClones(2); c3po.setPlacementAwarenessWeight(1d); c3po.setOutlookWeight(2d); } scheduler = c3po; } scheduler.init(conf, hdfs, containerMemory, customMemoryMap, containerCores, requestPriority); scheduler.initializeProvenanceManager(); /* log */ logger.writeEntryToLog(new JsonReportEntry(getRunId(), null, null, null, null, null, HiwayDBI.KEY_WF_NAME, getWorkflowName())); logger.federatedReport = new Data(appId + ".log"); // parse workflow, obtain ready tasks Collection<TaskInstance> readyTasks = parseWorkflow(); // scheduler updates runtime estimates for all tasks comprising the workflow scheduler.updateRuntimeEstimates(getRunId().toString()); scheduler.addTasks(readyTasks); // Dump out information about cluster capability as seen by the resource manager maxMem = response.getMaximumResourceCapability().getMemory(); maxCores = response.getMaximumResourceCapability().getVirtualCores(); /* log */ Logger.writeToStdout("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { /* log */ Logger.writeToStdout("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerCores > maxCores) { /* log */ Logger.writeToStdout("Container vcores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerCores + ", max=" + maxCores); containerCores = maxCores; } // this is the actual work loop: // ask for resources until the workflow is done. executeWorkflow(); finish(); } catch (Exception e) { e.printStackTrace(System.out); System.exit(-1); } return success; }
From source file:de.huberlin.wbi.hiway.common.Client.java
License:Apache License
/** * Main run function for the client.// w w w . j a v a 2s .c om * * @return true if application completed successfully. */ private boolean run() throws IOException, YarnException { /* log */ System.out.println("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); /* log */ System.out.println( "Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); /* log */ System.out.println("Got Cluster node info from ASM"); /* log */ for (NodeReport node : clusterNodeReports) System.out.println("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); /* log */ System.out.println("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); /* log */ for (QueueUserACLInfo aclInfo : listAclInfo) for (QueueACL userAcl : aclInfo.getUserAcls()) System.out.println("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // Get min/max resource capabilities from RM and change memory ask if needed int maxVC = appResponse.getMaximumResourceCapability().getVirtualCores(); /* log */ System.out.println("Max vCores capabililty of resources in this cluster " + maxVC); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); /* log */ System.out.println("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amVCores > maxVC) { /* log */ System.out.println("AM vCores specified above max threshold of cluster. Using max value." + ", specified=" + amVCores + ", max=" + maxVC); amVCores = maxVC; } if (amMemory > maxMem) { /* log */ System.out.println("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationType(conf.get(HiWayConfiguration.HIWAY_AM_APPLICATION_TYPE, HiWayConfiguration.HIWAY_AM_APPLICATION_TYPE_DEFAULT)); appContext.setApplicationName("run " + workflowParam + " (type: " + workflowType.toString() + ")"); ApplicationId appId = appContext.getApplicationId(); String hdfsBaseDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE, HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE_DEFAULT); String hdfsSandboxDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE, HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE_DEFAULT); Path hdfsBaseDirectory = new Path(new Path(hdfs.getUri()), hdfsBaseDirectoryName); Data.setHdfsBaseDirectory(hdfsBaseDirectory); Path hdfsSandboxDirectory = new Path(hdfsBaseDirectory, hdfsSandboxDirectoryName); Path hdfsApplicationDirectory = new Path(hdfsSandboxDirectory, appId.toString()); Data.setHdfsApplicationDirectory(hdfsApplicationDirectory); Data.setHdfs(hdfs); Path wfSource, wfDest, wfTemp = null; try { wfSource = new Path(new URI(workflowParam).getPath()); } catch (URISyntaxException e) { wfSource = new Path(workflowParam); } wfDest = new Path(hdfsApplicationDirectory + "/" + wfSource.getName()); // (1) if workflow file in hdfs, then transfer to temp file in local fs if (hdfs.exists(wfSource)) { wfTemp = new Path("./." + wfSource.getName()); System.out.println("Workflow found in HDFS at location " + wfSource); hdfs.copyToLocalFile(false, wfSource, wfTemp); } // (2) if galaxy workflow, then copy and replace input ports if (workflowType.equals(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_OPTS.galaxy)) { wfTemp = preProcessGalaxyWorkflow(wfSource, wfTemp); } if (wfTemp != null) { hdfs.copyFromLocalFile(wfTemp, wfDest); new File(wfTemp.toString()).delete(); } else { hdfs.copyFromLocalFile(wfSource, wfDest); } if (summaryPath != null) summary = new Data(summaryPath); if (customMemPath != null) (new Data(customMemPath)).stageOut(); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); /* set the env variables to be setup in the env where the application master will be run */ System.out.println("Set the environment for the application master"); Map<String, String> env = new HashMap<>(); StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(':'); classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<>(30); // Set java executable command System.out.println("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); if (HiWayConfiguration.debug) vargs.add( "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); vargs.add("-Xss" + "16m"); // Set class name switch (workflowType) { case dax: vargs.add(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_DAX_AM_CLASS); break; case log: vargs.add(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_LOG_AM_CLASS); break; case galaxy: vargs.add(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_GALAXY_AM_CLASS); break; case cuneiformE: vargs.add(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_CUNEIFORME_AM_CLASS); break; default: vargs.add(HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_CUNEIFORMJ_AM_CLASS); } vargs.add("--scheduler " + schedulerName.toString()); if (memory != null) vargs.add("--memory " + memory); if (summary != null) vargs.add("--summary " + summary.getName()); if (customMemPath != null) vargs.add("--custom " + customMemPath); vargs.add("--appid " + appId.toString()); if (HiWayConfiguration.debug) vargs.add("--debug"); if (HiWayConfiguration.verbose) vargs.add("--verbose"); vargs.add(workflowParam); vargs.add("> >(tee AppMaster.stdout " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout)"); vargs.add("2> >(tee AppMaster.stderr " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr >&2)"); // Get final command StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } System.out.println("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements Resource capability = Records.newRecord(Resource.class); capability.setVirtualCores(amVCores); capability.setMemory(amMemory); appContext.setResource(capability); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = hdfs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { System.out.println("Got dt for " + hdfs.getUri() + "; " + token); } } try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager /* log */ System.out.println("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application boolean success = monitorApplication(appId); if (success && summary != null) { summary.stageIn(); } return success; }
From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException/*from w w w. j ava2 s.com*/ * @throws IOException */ @SuppressWarnings({ "unchecked" }) public boolean run() throws YarnException, IOException { yarnClient.start(); LOG.info("Starting ApplicationMaster"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amRMClient = AMRMClient.createAMRMClient(); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); //TODO: Figure out how to do this. List<NodeReport> reports = this.yarnClient.getNodeReports(); LOG.info("Cluster Status"); List<Resource> availableResources = new ArrayList<Resource>(); for (NodeReport nr : reports) { LOG.info(" NodeId: " + nr.getNodeId() + " Capabilities " + nr.getCapability() + " Used Resources " + nr.getUsed()); int availableMem = nr.getCapability().getMemory() - nr.getUsed().getMemory(); int availableVCores = nr.getCapability().getVirtualCores() - nr.getUsed().getVirtualCores(); Resource resource = Resource.newInstance(availableMem, availableVCores); availableResources.add(resource); } /*Based on available resources scheduler should decide the best allocation of recommenders to resources and return a list of resources that should be requested from the ResourceManager*/ DataSetDescription datasetDesc = new DataSetDescription(this.setup.dataMetadataFile); List<RecommenderPool> recommenderPools = RecommenderScheduler.splitRecommenderPool(availableResources, recommenders, datasetDesc, this.setup.nShards); for (RecommenderPool res : recommenderPools) { ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority); LOG.info("CONTAINER ASK: " + containerAsk); amRMClient.addContainerRequest(containerAsk); } float progress = 0; List<RecommenderPool> pendingPools = new ArrayList<RecommenderPool>(); for (RecommenderPool p : recommenderPools) pendingPools.add(p); this.numTotalContainers = recommenderPools.size(); this.numCompletedContainers.set(0); this.numAllocatedContainers.set(0); while (numCompletedContainers.get() != numTotalContainers) { try { Thread.sleep(200); AllocateResponse allocResp = amRMClient.allocate(progress); List<Container> newContainers = allocResp.getAllocatedContainers(); List<ContainerStatus> completedContainers = allocResp.getCompletedContainersStatuses(); if (this.numAllocatedContainers.get() >= this.numTotalContainers && pendingPools.size() != 0) { //Ask for new containers for pending pools LOG.warn("The number of allocated containers has exceeded number of total containers, but " + "the pending pool size is still not 0. Asking for new containers from RM"); for (RecommenderPool res : pendingPools) { ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority); LOG.info("NEW CONTAINER ASK: " + containerAsk); amRMClient.addContainerRequest(containerAsk); } } if (newContainers.size() > 0) { LOG.info("Allocated " + newContainers.size() + " new containers"); numAllocatedContainers.addAndGet(newContainers.size()); for (Container container : newContainers) { //Find matching recommender pool from pendingRecommender pools. RecommenderPool pool = null; for (RecommenderPool p : pendingPools) { if (p.getTotalMemory() == container.getResource().getMemory()) { pool = p; break; } } if (pool == null) { LOG.warn("No Takers for Container " + container + " Releasing container"); amRMClient.releaseAssignedContainer(container.getId()); } else { startContainer(container, pool); //This pool has now got a container. Remove it from pending pools pendingPools.remove(pool); } } } onContainersCompleted(completedContainers); } catch (InterruptedException ex) { } } finish(); return success; }
From source file:edu.indiana.d2i.htrc.io.mem.HadoopWritableTranscoder.java
License:Apache License
@Override public CachedData encode(T obj) { CachedData data = null;/*from w ww .ja v a2 s . c om*/ try { DataOutputBuffer encodeBuffer = new DataOutputBuffer(); obj.write(encodeBuffer); byte[] bytes = encodeBuffer.getData(); data = new CachedData(0, bytes, bytes.length); } catch (IOException e) { e.printStackTrace(); } return data; }
From source file:edu.indiana.d2i.htrc.kmeans.MemRandomSeedGenerator.java
License:Apache License
public static void buildRandom(Configuration conf, Path input, int k, DistanceMeasure measure) throws IOException { // build id list FileSystem fs = FileSystem.get(conf); DataInputStream fsinput = new DataInputStream(fs.open(input)); Iterator<Text> idIterator = new IDList(fsinput).iterator(); List<String> idlist = new ArrayList<String>(); while (idIterator.hasNext()) { Text id = idIterator.next(); idlist.add(id.toString());/* w ww. ja v a 2 s . c om*/ } ThreadedMemcachedClient client = ThreadedMemcachedClient.getThreadedMemcachedClient(conf); MemcachedClient cache = client.getCache(); Transcoder<VectorWritable> vectorTranscoder = new HadoopWritableTranscoder<VectorWritable>(conf, VectorWritable.class); // pick k random id List<Text> chosenTexts = Lists.newArrayListWithCapacity(k); List<Cluster> chosenClusters = Lists.newArrayListWithCapacity(k); int nextClusterId = 0; // Random random = RandomUtils.getRandom(); // for (String id : idlist) { // VectorWritable vectorWritable = cache.get(id, vectorTranscoder); // if (vectorWritable != null) { // Cluster newCluster = new Cluster(vectorWritable.get(), // nextClusterId++, measure); // newCluster.observe(vectorWritable.get(), 1); // Text newText = new Text(id); // int currentSize = chosenTexts.size(); //// if (currentSize < k) { //// chosenTexts.add(newText); //// chosenClusters.add(newCluster); //// } else if (random.nextInt(currentSize + 1) != 0) { //// int indexToRemove = random.nextInt(currentSize); //// chosenTexts.remove(indexToRemove); //// chosenClusters.remove(indexToRemove); //// chosenTexts.add(newText); //// chosenClusters.add(newCluster); //// } // } else { // logger.error("cannot find VectorWritable for " + id); //// throw new RuntimeException("cannot find VectorWritable for " + id); // } // } for (int i = 0; i < k; i++) { String id = idlist.get(i); VectorWritable vectorWritable = cache.get(id, vectorTranscoder); System.out.println("pick " + id); if (vectorWritable != null) { Cluster newCluster = new Cluster(vectorWritable.get(), nextClusterId++, measure); chosenClusters.add(newCluster); } else { logger.error("cannot find VectorWritable for " + id); throw new RuntimeException("cannot find VectorWritable for " + id); } } // write out the seeds to Memcached int maxExpir = conf.getInt(HTRCConstants.MEMCACHED_MAX_EXPIRE, -1); Transcoder<Cluster> clusterTranscoder = new HadoopWritableTranscoder<Cluster>(conf, Cluster.class); for (int i = 0; i < chosenClusters.size(); i++) { System.out.println("set cluster " + MemKMeansUtil.toClusterName(i)); cache.set(MemKMeansUtil.toClusterName(i), maxExpir, chosenClusters.get(i), clusterTranscoder); Cluster cluster = cache.get(MemKMeansUtil.toClusterName(i), clusterTranscoder); if (cluster != null) { DataOutputBuffer buf = new DataOutputBuffer(); cluster.write(buf); System.out.println("read from memcached " + cluster.getIdentifier() + " size " + buf.size()); } } client.close(); }
From source file:eu.stratosphere.yarn.ApplicationMaster.java
License:Apache License
private void run() throws Exception { //Utils.logFilesInCurrentDirectory(LOG); // Initialize clients to ResourceManager and NodeManagers Configuration conf = Utils.initializeYarnConfiguration(); FileSystem fs = FileSystem.get(conf); Map<String, String> envs = System.getenv(); final String currDir = envs.get(Environment.PWD.key()); final String logDirs = envs.get(Environment.LOG_DIRS.key()); final String ownHostname = envs.get(Environment.NM_HOST.key()); final String appId = envs.get(Client.ENV_APP_ID); final String clientHomeDir = envs.get(Client.ENV_CLIENT_HOME_DIR); final String applicationMasterHost = envs.get(Environment.NM_HOST.key()); final String remoteStratosphereJarPath = envs.get(Client.STRATOSPHERE_JAR_PATH); final String shipListString = envs.get(Client.ENV_CLIENT_SHIP_FILES); final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME); final int taskManagerCount = Integer.valueOf(envs.get(Client.ENV_TM_COUNT)); final int memoryPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_MEMORY)); final int coresPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_CORES)); int heapLimit = Utils.calculateHeapSize(memoryPerTaskManager); if (currDir == null) { throw new RuntimeException("Current directory unknown"); }/* w w w .j a v a 2 s .c om*/ if (ownHostname == null) { throw new RuntimeException("Own hostname (" + Environment.NM_HOST + ") not set."); } LOG.info("Working directory " + currDir); // load Stratosphere configuration. Utils.getStratosphereConfiguration(currDir); final String localWebInterfaceDir = currDir + "/resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME; // Update yaml conf -> set jobManager address to this machine's address. FileInputStream fis = new FileInputStream(currDir + "/stratosphere-conf.yaml"); BufferedReader br = new BufferedReader(new InputStreamReader(fis)); Writer output = new BufferedWriter(new FileWriter(currDir + "/stratosphere-conf-modified.yaml")); String line; while ((line = br.readLine()) != null) { if (line.contains(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)) { output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n"); } else if (line.contains(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY)) { output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + "\n"); } else { output.append(line + "\n"); } } // just to make sure. output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n"); output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + localWebInterfaceDir + "\n"); output.append(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY + ": " + logDirs + "\n"); output.close(); br.close(); File newConf = new File(currDir + "/stratosphere-conf-modified.yaml"); if (!newConf.exists()) { LOG.warn("modified yaml does not exist!"); } Utils.copyJarContents("resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME, ApplicationMaster.class.getProtectionDomain().getCodeSource().getLocation().getPath()); JobManager jm; { String pathToNepheleConfig = currDir + "/stratosphere-conf-modified.yaml"; String[] args = { "-executionMode", "cluster", "-configDir", pathToNepheleConfig }; // start the job manager jm = JobManager.initialize(args); // Start info server for jobmanager jm.startInfoServer(); } AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager LOG.info("registering ApplicationMaster"); rmClient.registerApplicationMaster(applicationMasterHost, 0, "http://" + applicationMasterHost + ":" + GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, "undefined")); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memoryPerTaskManager); capability.setVirtualCores(coresPerTaskManager); // Make container requests to ResourceManager for (int i = 0; i < taskManagerCount; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); LOG.info("Requesting TaskManager container " + i); rmClient.addContainerRequest(containerAsk); } LocalResource stratosphereJar = Records.newRecord(LocalResource.class); LocalResource stratosphereConf = Records.newRecord(LocalResource.class); // register Stratosphere Jar with remote HDFS final Path remoteJarPath = new Path(remoteStratosphereJarPath); Utils.registerLocalResource(fs, remoteJarPath, stratosphereJar); // register conf with local fs. Path remoteConfPath = Utils.setupLocalResource(conf, fs, appId, new Path("file://" + currDir + "/stratosphere-conf-modified.yaml"), stratosphereConf, new Path(clientHomeDir)); LOG.info("Prepared localresource for modified yaml: " + stratosphereConf); boolean hasLog4j = new File(currDir + "/log4j.properties").exists(); // prepare the files to ship LocalResource[] remoteShipRsc = null; String[] remoteShipPaths = shipListString.split(","); if (!shipListString.isEmpty()) { remoteShipRsc = new LocalResource[remoteShipPaths.length]; { // scope for i int i = 0; for (String remoteShipPathStr : remoteShipPaths) { if (remoteShipPathStr == null || remoteShipPathStr.isEmpty()) { continue; } remoteShipRsc[i] = Records.newRecord(LocalResource.class); Path remoteShipPath = new Path(remoteShipPathStr); Utils.registerLocalResource(fs, remoteShipPath, remoteShipRsc[i]); i++; } } } // respect custom JVM options in the YAML file final String javaOpts = GlobalConfiguration.getString(ConfigConstants.STRATOSPHERE_JVM_OPTIONS, ""); // Obtain allocated containers and launch int allocatedContainers = 0; int completedContainers = 0; while (allocatedContainers < taskManagerCount) { AllocateResponse response = rmClient.allocate(0); for (Container container : response.getAllocatedContainers()) { LOG.info("Got new Container for TM " + container.getId() + " on host " + container.getNodeId().getHost()); ++allocatedContainers; // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); String tmCommand = "$JAVA_HOME/bin/java -Xmx" + heapLimit + "m " + javaOpts; if (hasLog4j) { tmCommand += " -Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-log4j.log\" -Dlog4j.configuration=file:log4j.properties"; } tmCommand += " eu.stratosphere.yarn.YarnTaskManagerRunner -configDir . " + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stdout.log" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stderr.log"; ctx.setCommands(Collections.singletonList(tmCommand)); LOG.info("Starting TM with command=" + tmCommand); // copy resources to the TaskManagers. Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2); localResources.put("stratosphere.jar", stratosphereJar); localResources.put("stratosphere-conf.yaml", stratosphereConf); // add ship resources if (!shipListString.isEmpty()) { Preconditions.checkNotNull(remoteShipRsc); for (int i = 0; i < remoteShipPaths.length; i++) { localResources.put(new Path(remoteShipPaths[i]).getName(), remoteShipRsc[i]); } } ctx.setLocalResources(localResources); // Setup CLASSPATH for Container (=TaskTracker) Map<String, String> containerEnv = new HashMap<String, String>(); Utils.setupEnv(conf, containerEnv); //add stratosphere.jar to class path. containerEnv.put(Client.ENV_CLIENT_USERNAME, yarnClientUsername); ctx.setEnvironment(containerEnv); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try { Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); ctx.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } LOG.info("Launching container " + allocatedContainers); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { ++completedContainers; LOG.info("Completed container (while allocating) " + status.getContainerId() + ". Total Completed:" + completedContainers); LOG.info("Diagnostics " + status.getDiagnostics()); } Thread.sleep(100); } // Now wait for containers to complete while (completedContainers < taskManagerCount) { AllocateResponse response = rmClient.allocate(completedContainers / taskManagerCount); for (ContainerStatus status : response.getCompletedContainersStatuses()) { ++completedContainers; LOG.info("Completed container " + status.getContainerId() + ". Total Completed:" + completedContainers); LOG.info("Diagnostics " + status.getDiagnostics()); } Thread.sleep(5000); } LOG.info("Shutting down JobManager"); jm.shutdown(); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }
From source file:eu.stratosphere.yarn.Utils.java
License:Apache License
public static void setTokensFor(ContainerLaunchContext amContainer, Path[] paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS/*from w ww . j a va2s . c o m*/ TokenCache.obtainTokensForNamenodes(credentials, paths, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); }
From source file:FormatStorage.Unit.java
License:Open Source License
public void transfer(long newOffset) throws Exception { long adjust = newOffset - offset; boolean VAR = segment.formatData.isVar(); if (VAR) {// w w w .jav a 2 s . c om if (!compressed) { int tnum = ((DataOutputBuffer) metasBuffer).getLength() / ConstVar.Sizeof_Long; if (tnum != recordNum) { throw new SEException.InnerException("tnum != recordNum"); } DataOutputBuffer tmpOuputBuffer = new DataOutputBuffer(); DataInputBuffer tmpinput = new DataInputBuffer(); tmpinput.reset(((DataOutputBuffer) metasBuffer).getData(), 0, ((DataOutputBuffer) metasBuffer).getLength()); for (int i = 0; i < recordNum; i++) { long value = tmpinput.readLong() + adjust; tmpOuputBuffer.writeLong(value); } tmpinput.reset(tmpOuputBuffer.getData(), 0, tmpOuputBuffer.getLength()); ((DataOutputBuffer) metasBuffer).reset(); for (int i = 0; i < recordNum; i++) { ((DataOutputBuffer) metasBuffer).writeLong(tmpinput.readLong()); } tmpOuputBuffer = null; tmpinput = null; } else { compressedMetasOutput.finish(); InputStream tmpMetasInputStream = new DataInputBuffer(); ((DataInputBuffer) tmpMetasInputStream).reset(((DataOutputBuffer) metasBuffer).getData(), 0, ((DataOutputBuffer) metasBuffer).getLength()); CompressionInputStream tmpCompressedMetasInput = codec.createInputStream(tmpMetasInputStream); DataOutputBuffer tmpOutputBuffer = new DataOutputBuffer(); for (int i = 0; i < recordNum; i++) { int count = 0; try { count = tmpCompressedMetasInput.read(metaOffsetBytes, 0, ConstVar.Sizeof_Long); long meta = Util.bytes2long(metaOffsetBytes, 0, ConstVar.Sizeof_Long) + adjust; tmpOutputBuffer.writeLong(meta); } catch (Exception e) { e.printStackTrace(); System.out.println("i:" + i + ",count:" + count); throw e; } } ((DataOutputBuffer) metasBuffer).reset(); compressedMetasOutput.resetState(); DataInputBuffer tmpInputBuffer = new DataInputBuffer(); tmpInputBuffer.reset(tmpOutputBuffer.getData(), 0, tmpOutputBuffer.getLength()); for (int i = 0; i < recordNum; i++) { long newMeta = tmpInputBuffer.readLong(); Util.long2bytes(metaOffsetBytes, newMeta); compressedMetasOutput.write(metaOffsetBytes, 0, ConstVar.Sizeof_Long); } } } metaOffset += adjust; setOffset(newOffset); }