List of usage examples for org.apache.hadoop.io DataOutputBuffer getLength
public int getLength()
From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java
License:Apache License
/** * Create the common {@link ContainerLaunchContext} for all attempts. * * @param applicationACLs//from ww w . j ava 2s .c o m */ private static ContainerLaunchContext createCommonContainerLaunchContext( Map<ApplicationAccessType, String> applicationACLs, Credentials credentials, Map<String, LocalResource> localResources) { // Application environment Map<String, String> environment = new HashMap<String, String>(); // Service data Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>(); // Tokens // Setup up task credentials buffer ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[] {}); try { Credentials containerCredentials = new Credentials(); // All Credentials need to be set so that YARN can localize the resources // correctly, even though they may not be used by all tasks which will run // on this container. LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #" + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container"); containerCredentials.addAll(credentials); DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); containerCredentials.writeTokenStorageToStream(containerTokens_dob); containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength()); // Add shuffle token LOG.info("Putting shuffle token in serviceData"); serviceData.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, serializeServiceData(TokenCache.getSessionToken(containerCredentials))); } catch (IOException e) { throw new TezUncheckedException(e); } // Construct the actual Container // The null fields are per-container and will be constructed for each // container separately. ContainerLaunchContext container = ContainerLaunchContext.newInstance(localResources, environment, null, serviceData, containerCredentialsBuffer, applicationACLs); return container; }
From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java
License:Apache License
/** * A helper function to serialize the JobTokenIdentifier to be sent to the * ShuffleHandler as ServiceData.//from www . j av a 2 s . c om * * *NOTE* This is a copy of what is done by the MapReduce ShuffleHandler. Not using that directly * to avoid a dependency on mapreduce. * * @param jobToken * the job token to be used for authentication of shuffle data * requests. * @return the serialized version of the jobToken. */ private static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException { // TODO these bytes should be versioned DataOutputBuffer jobToken_dob = new DataOutputBuffer(); jobToken.write(jobToken_dob); return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength()); }
From source file:org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl.java
License:Apache License
private ByteBuffer serializeCredentials(Credentials credentials) throws IOException { Credentials containerCredentials = new Credentials(); containerCredentials.addAll(credentials); DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); containerCredentials.writeTokenStorageToStream(containerTokens_dob); ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength()); return containerCredentialsBuffer; }
From source file:org.apache.tez.runtime.api.impl.TestTezEvent.java
License:Apache License
@Test public void testSerialization() throws IOException { ArrayList<TezEvent> events = new ArrayList<TezEvent>(); Configuration conf = new Configuration(true); String confVal = RandomStringUtils.random(10000, true, true); conf.set("testKey", confVal); UserPayload payload = TezUtils.createUserPayloadFromConf(conf); TezTaskAttemptID srcTAID = TezTaskAttemptID .getInstance(TezTaskID.fromString("task_1454468251169_866787_1_02_000000"), 1000); TezTaskAttemptID destTAID = TezTaskAttemptID .getInstance(TezTaskID.fromString("task_1454468251169_866787_1_02_000000"), 2000); EventMetaData srcInfo = new EventMetaData(EventProducerConsumerType.OUTPUT, "v1", "v2", srcTAID); EventMetaData destInfo = new EventMetaData(EventProducerConsumerType.OUTPUT, "v3", "v4", destTAID); // Case of size less than 4K and parsing skipped during deserialization events.add(new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.PROCESSOR, "v1", "v2", srcTAID))); TezEvent dmeEvent = new TezEvent(DataMovementEvent.create(1000, 3, 1, payload.getPayload()), srcInfo, System.currentTimeMillis()); dmeEvent.setDestinationInfo(destInfo); events.add(dmeEvent);/* w w w . j av a2 s . co m*/ // Different code path events.add(new TezEvent(new TaskStatusUpdateEvent(null, 0.1f, null, false), new EventMetaData(EventProducerConsumerType.PROCESSOR, "v5", "v6", srcTAID))); // Serialize to different types of DataOutput // One that implements OutputStream and one that does not DataOutputBuffer dataout = new DataOutputBuffer(); ByteArrayDataOutput bout = ByteStreams.newDataOutput(); serializeEvents(events, dataout); serializeEvents(events, bout); // Deserialize from different types of DataInput // One with DataInputBuffer and another different implementation DataInputBuffer datain = new DataInputBuffer(); datain.reset(dataout.getData(), dataout.getLength()); DataInputStream dis = new DataInputStream( new ByteArrayInputStream(dataout.getData(), 0, dataout.getLength())); ArrayList<TezEvent> actual1 = deserializeEvents(datain); ArrayList<TezEvent> actual2 = deserializeEvents(dis); assertEventEquals(events, actual1); assertEventEquals(events, actual2); byte[] serializedBytes = bout.toByteArray(); datain.reset(serializedBytes, serializedBytes.length); dis = new DataInputStream(new ByteArrayInputStream(serializedBytes)); actual1 = deserializeEvents(datain); actual2 = deserializeEvents(dis); assertEventEquals(events, actual1); assertEventEquals(events, actual2); }
From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestShuffleInputEventHandlerImpl.java
License:Apache License
private InputContext createInputContext() throws IOException { DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(PORT);//from w w w.java 2s . c om final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength()); ExecutionContext executionContext = mock(ExecutionContext.class); doReturn(HOST).when(executionContext).getHostName(); InputContext inputContext = mock(InputContext.class); doReturn(new TezCounters()).when(inputContext).getCounters(); doReturn("sourceVertex").when(inputContext).getSourceVertexName(); doReturn(shuffleMetaData).when(inputContext) .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID); doReturn(executionContext).when(inputContext).getExecutionContext(); return inputContext; }
From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestShuffleManager.java
License:Apache License
private InputContext createInputContext() throws IOException { DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(PORT);/*from www. j a va2 s. c o m*/ final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength()); ExecutionContext executionContext = mock(ExecutionContext.class); doReturn(FETCHER_HOST).when(executionContext).getHostName(); InputContext inputContext = mock(InputContext.class); doReturn(new TezCounters()).when(inputContext).getCounters(); doReturn("sourceVertex").when(inputContext).getSourceVertexName(); doReturn(shuffleMetaData).when(inputContext) .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID); doReturn(executionContext).when(inputContext).getExecutionContext(); return inputContext; }
From source file:org.apache.tez.runtime.library.common.sort.impl.TestIFile.java
License:Apache License
@Test(timeout = 5000) //test with sorted data and repeat keys public void testWithRLEMarker() throws IOException { //Test with append(Object, Object) FSDataOutputStream out = localFs.create(outputPath); IFile.Writer writer = new IFile.Writer(defaultConf, out, Text.class, IntWritable.class, codec, null, null, true);//from ww w . j av a 2 s. com Text key = new Text("key0"); IntWritable value = new IntWritable(0); writer.append(key, value); //same key (RLE should kick in) key = new Text("key0"); writer.append(key, value); assertTrue(writer.sameKey); //Different key key = new Text("key1"); writer.append(key, value); assertFalse(writer.sameKey); writer.close(); out.close(); //Test with append(DataInputBuffer key, DataInputBuffer value) byte[] kvbuffer = "key1Value1key1Value2key3Value3".getBytes(); int keyLength = 4; int valueLength = 6; int pos = 0; out = localFs.create(outputPath); writer = new IFile.Writer(defaultConf, out, Text.class, IntWritable.class, codec, null, null, true); DataInputBuffer kin = new DataInputBuffer(); kin.reset(kvbuffer, pos, keyLength); DataInputBuffer vin = new DataInputBuffer(); DataOutputBuffer vout = new DataOutputBuffer(); (new IntWritable(0)).write(vout); vin.reset(vout.getData(), vout.getLength()); //Write initial KV pair writer.append(kin, vin); assertFalse(writer.sameKey); pos += (keyLength + valueLength); //Second key is similar to key1 (RLE should kick in) kin.reset(kvbuffer, pos, keyLength); (new IntWritable(0)).write(vout); vin.reset(vout.getData(), vout.getLength()); writer.append(kin, vin); assertTrue(writer.sameKey); pos += (keyLength + valueLength); //Next key (key3) is different (RLE should not kick in) kin.reset(kvbuffer, pos, keyLength); (new IntWritable(0)).write(vout); vin.reset(vout.getData(), vout.getLength()); writer.append(kin, vin); assertFalse(writer.sameKey); writer.close(); out.close(); }
From source file:org.apache.tez.runtime.library.common.sort.impl.TestIFile.java
License:Apache License
private void populateData(KVPair kvp, DataInputBuffer key, DataInputBuffer value) throws IOException { DataOutputBuffer k = new DataOutputBuffer(); DataOutputBuffer v = new DataOutputBuffer(); kvp.getKey().write(k);/*from w ww . j a v a2s. com*/ kvp.getvalue().write(v); key.reset(k.getData(), 0, k.getLength()); value.reset(v.getData(), 0, v.getLength()); }
From source file:org.apache.tez.runtime.library.common.TestValuesIterator.java
License:Apache License
/** * create inmemory segments/*from w w w.java2 s .co m*/ * * @return * @throws IOException */ public List<TezMerger.Segment> createInMemStreams() throws IOException { int numberOfStreams = Math.max(2, rnd.nextInt(10)); LOG.info("No of streams : " + numberOfStreams); SerializationFactory serializationFactory = new SerializationFactory(conf); Serializer keySerializer = serializationFactory.getSerializer(keyClass); Serializer valueSerializer = serializationFactory.getSerializer(valClass); LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); InputContext context = createTezInputContext(); MergeManager mergeManager = new MergeManager(conf, fs, localDirAllocator, context, null, null, null, null, null, 1024 * 1024 * 10, null, false, -1); DataOutputBuffer keyBuf = new DataOutputBuffer(); DataOutputBuffer valBuf = new DataOutputBuffer(); DataInputBuffer keyIn = new DataInputBuffer(); DataInputBuffer valIn = new DataInputBuffer(); keySerializer.open(keyBuf); valueSerializer.open(valBuf); List<TezMerger.Segment> segments = new LinkedList<TezMerger.Segment>(); for (int i = 0; i < numberOfStreams; i++) { BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024); InMemoryWriter writer = new InMemoryWriter(bout); Map<Writable, Writable> data = createData(); //write data for (Map.Entry<Writable, Writable> entry : data.entrySet()) { keySerializer.serialize(entry.getKey()); valueSerializer.serialize(entry.getValue()); keyIn.reset(keyBuf.getData(), 0, keyBuf.getLength()); valIn.reset(valBuf.getData(), 0, valBuf.getLength()); writer.append(keyIn, valIn); originalData.put(entry.getKey(), entry.getValue()); keyBuf.reset(); valBuf.reset(); keyIn.reset(); valIn.reset(); } IFile.Reader reader = new InMemoryReader(mergeManager, null, bout.getBuffer(), 0, bout.getBuffer().length); segments.add(new TezMerger.Segment(reader, true)); data.clear(); writer.close(); } return segments; }
From source file:org.apache.tez.runtime.library.utils.BufferUtils.java
License:Apache License
public static int compare(DataOutputBuffer buf1, DataOutputBuffer buf2) { byte[] b1 = buf1.getData(); byte[] b2 = buf2.getData(); int s1 = 0;// w w w . ja va 2 s . c om int s2 = 0; int l1 = buf1.getLength(); int l2 = buf2.getLength(); return FastByteComparisons.compareTo(b1, s1, l1, b2, s2, l2); }