List of usage examples for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer
public DataOutputBuffer()
From source file:me.haosdent.noya.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws org.apache.hadoop.yarn.exceptions.YarnException * @throws java.io.IOException/* www . ja v a 2 s . c om*/ */ @SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Setup local RPC Server to accept status requests directly from clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info("Received " + previousAMRunningContainers.size() + " previous AM's running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainersToRequest); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } }
From source file:me.haosdent.noya.Client.java
License:Apache License
/** * Main run function for the client/*from w ww . ja v a2 s. c om*/ * * @return true if application completed successfully * * @throws java.io.IOException * @throws org.apache.hadoop.yarn.exceptions.YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); //appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.toString() + "/" + SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } if (!shellCommand.isEmpty()) { addToLocalResources(fs, null, shellCommandPath, appId.toString(), localResources, shellCommand); } if (shellArgs.length > 0) { addToLocalResources(fs, null, shellArgsPath, appId.toString(), localResources, StringUtils.join(shellArgs, " ")); } // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, ApplicationConstants.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); //commands.add(command.toString()); commands.add("echo 'hello' >/tmp/yarn_test"); amContainer.setCommands(commands); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); capability.setVirtualCores(amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return monitorApplication(appId); }
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * Populate allTokens with the tokens received *///from w ww .j a va 2s . c om private void getAllTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (LOG.isDebugEnabled()) { LOG.debug("Token type : {}", token.getKind()); } if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } this.allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); }
From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java
License:Apache License
/** * Set delegation tokens for AM container * //from ww w . j a va 2 s . c o m * @param amContainer * AM container */ private void setToken(ContainerLaunchContext amContainer) throws IOException { // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } FileSystem fs = FileSystem.get(getConf()); // For now, only getting tokens for the default file-system. final Token<?>[] tokens = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } }
From source file:net.sf.katta.lib.lucene.LuceneServer.java
License:Apache License
@Override public HitsMapWritable search(QueryWritable query, DocumentFrequencyWritable freqs, String[] shards, final long timeout, int count, SortWritable sortWritable, FilterWritable filterWritable) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("You are searching with the query: '" + query.getQuery() + "'"); }//from ww w. j a va2 s. co m Query luceneQuery = query.getQuery(); if (LOG.isDebugEnabled()) { LOG.debug("Lucene query: " + luceneQuery.toString()); } long completeSearchTime = 0; final HitsMapWritable result = new net.sf.katta.lib.lucene.HitsMapWritable(getNodeName()); long start = 0; if (LOG.isDebugEnabled()) { start = System.currentTimeMillis(); } Sort sort = null; if (sortWritable != null) { sort = sortWritable.getSort(); } Filter filter = null; if (filterWritable != null) { filter = filterWritable.getFilter(); } if (_filterCache != null && filter != null) { CachingWrapperFilter cachedFilter = _filterCache.getIfPresent(filter); if (cachedFilter == null) { cachedFilter = new CachingWrapperFilter(filter); _filterCache.put(filter, cachedFilter); } filter = cachedFilter; } search(luceneQuery, freqs, shards, result, count, sort, timeout, filter); if (LOG.isDebugEnabled()) { final long end = System.currentTimeMillis(); LOG.debug("Search took " + (end - start) / 1000.0 + "sec."); completeSearchTime += (end - start); } if (LOG.isDebugEnabled()) { LOG.debug("Complete search took " + completeSearchTime / 1000.0 + "sec."); final DataOutputBuffer buffer = new DataOutputBuffer(); result.write(buffer); LOG.debug("Result size to transfer: " + buffer.getLength()); } return result; }
From source file:net.sf.katta.zk.ZKClient.java
License:Apache License
private byte[] writableToByteArray(final Writable writable) throws KattaException { byte[] data = new byte[0]; if (writable != null) { final DataOutputBuffer out = new DataOutputBuffer(); try {//ww w .j a v a 2 s .c om writable.write(out); } catch (final IOException e) { throw new KattaException("unable to serialize writable", e); } data = out.getData(); } return data; }
From source file:net.shun.mapreduce.lib.input.XmlRecordReader.java
License:Apache License
public boolean nextKeyValue() throws IOException { if (key == null) key = new LongWritable(); key.set(pos);//from w w w. j a va 2 s. c o m if (value == null) value = new Text(); if (pos >= end) return false; DataOutputBuffer buf = new DataOutputBuffer(); //read until begin if (!readUntilMatch(beginMark, false, null)) { return false; } //read until end if (!readUntilMatch(endMark, true, buf)) { return false; } byte[] record = new byte[buf.getLength()]; System.arraycopy(buf.getData(), 0, record, 0, record.length); value.set(record); return true; }
From source file:org.anarres.lzo.LzopOutputStream.java
License:Open Source License
/** * Writes an lzop-compatible header to the OutputStream provided. *///from w ww. ja v a 2 s .com protected void writeLzopHeader() throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); try { dob.writeShort(LzopConstants.LZOP_VERSION); dob.writeShort(LzoVersion.LZO_LIBRARY_VERSION); dob.writeShort(LzopConstants.LZOP_COMPAT_VERSION); switch (getAlgorithm()) { case LZO1X: // case LZO1X_1: dob.writeByte(LzopConstants.M_LZO1X_1); dob.writeByte(5); break; /* case LZO1X_15: dob.writeByte(LzopConstants.M_LZO1X_1_15); dob.writeByte(1); break; case LZO1X_999: dob.writeByte(LzopConstants.M_LZO1X_999); dob.writeByte(9); break; */ default: throw new IOException("Incompatible lzop algorithm " + getAlgorithm()); } long mask = LzopConstants.F_ADLER32_C | LzopConstants.F_ADLER32_D; mask = mask | LzopConstants.F_CRC32_C | LzopConstants.F_CRC32_D; dob.writeInt((int) (flags & mask & 0xFFFFFFFF)); // all flags 0 dob.writeInt(33188); // mode dob.writeInt((int) (System.currentTimeMillis() / 1000)); // mtime dob.writeInt(0); // gmtdiff ignored dob.writeByte(0); // no filename Adler32 headerChecksum = new Adler32(); headerChecksum.update(dob.getData(), 0, dob.getLength()); int hc = (int) headerChecksum.getValue(); dob.writeInt(hc); out.write(LzopConstants.LZOP_MAGIC); out.write(dob.getData(), 0, dob.getLength()); } finally { dob.close(); } }
From source file:org.apache.accumulo.core.replication.ReplicationTarget.java
License:Apache License
/** * Convenience method to serialize a ReplicationTarget to {@link Text} using the {@link Writable} methods without caring about performance penalties due to * excessive object creation// w w w . j av a 2 s. com * * @return The serialized representation of the object */ public Text toText() { DataOutputBuffer buffer = new DataOutputBuffer(); try { this.write(buffer); } catch (IOException e) { throw new RuntimeException(e); } Text t = new Text(); // Throw it in a text for the mutation t.set(buffer.getData(), 0, buffer.getLength()); return t; }
From source file:org.apache.accumulo.core.replication.ReplicationTargetTest.java
License:Apache License
@Test public void writableOut() throws Exception { ReplicationTarget expected = new ReplicationTarget("foo", "bar", "1"); DataOutputBuffer buffer = new DataOutputBuffer(); expected.write(buffer);//from w w w .j a v a 2s.co m DataInputBuffer input = new DataInputBuffer(); input.reset(buffer.getData(), buffer.getLength()); ReplicationTarget actual = new ReplicationTarget(); actual.readFields(input); }