Example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer

List of usage examples for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. DataOutputBuffer
  6. DataOutputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer.

Prototype

public DataOutputBuffer()

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:org.apache.tez.http.HttpConnection.java

License:Apache License

/**
 * Cleanup the error stream if any, for keepAlive connections
 *
 * @param errorStream/* w w  w.j  a  va 2s  .co m*/
 */
private void readErrorStream(InputStream errorStream) {
    if (errorStream == null) {
        return;
    }
    try {
        DataOutputBuffer errorBuffer = new DataOutputBuffer();
        IOUtils.copyBytes(errorStream, errorBuffer, 4096);
        IOUtils.closeStream(errorBuffer);
        IOUtils.closeStream(errorStream);
    } catch (IOException ioe) {
        // ignore
    }
}

From source file:org.apache.tez.runtime.api.impl.TestTezEvent.java

License:Apache License

@Test
public void testSerialization() throws IOException {

    ArrayList<TezEvent> events = new ArrayList<TezEvent>();

    Configuration conf = new Configuration(true);
    String confVal = RandomStringUtils.random(10000, true, true);
    conf.set("testKey", confVal);
    UserPayload payload = TezUtils.createUserPayloadFromConf(conf);
    TezTaskAttemptID srcTAID = TezTaskAttemptID
            .getInstance(TezTaskID.fromString("task_1454468251169_866787_1_02_000000"), 1000);
    TezTaskAttemptID destTAID = TezTaskAttemptID
            .getInstance(TezTaskID.fromString("task_1454468251169_866787_1_02_000000"), 2000);
    EventMetaData srcInfo = new EventMetaData(EventProducerConsumerType.OUTPUT, "v1", "v2", srcTAID);
    EventMetaData destInfo = new EventMetaData(EventProducerConsumerType.OUTPUT, "v3", "v4", destTAID);

    // Case of size less than 4K and parsing skipped during deserialization
    events.add(new TezEvent(new TaskAttemptCompletedEvent(),
            new EventMetaData(EventProducerConsumerType.PROCESSOR, "v1", "v2", srcTAID)));
    TezEvent dmeEvent = new TezEvent(DataMovementEvent.create(1000, 3, 1, payload.getPayload()), srcInfo,
            System.currentTimeMillis());
    dmeEvent.setDestinationInfo(destInfo);
    events.add(dmeEvent);/*from  w  w w  .j  a  v a 2s .  c o  m*/
    // Different code path
    events.add(new TezEvent(new TaskStatusUpdateEvent(null, 0.1f, null, false),
            new EventMetaData(EventProducerConsumerType.PROCESSOR, "v5", "v6", srcTAID)));

    // Serialize to different types of DataOutput
    // One that implements OutputStream and one that does not
    DataOutputBuffer dataout = new DataOutputBuffer();
    ByteArrayDataOutput bout = ByteStreams.newDataOutput();
    serializeEvents(events, dataout);
    serializeEvents(events, bout);

    // Deserialize from different types of DataInput
    // One with DataInputBuffer and another different implementation
    DataInputBuffer datain = new DataInputBuffer();
    datain.reset(dataout.getData(), dataout.getLength());
    DataInputStream dis = new DataInputStream(
            new ByteArrayInputStream(dataout.getData(), 0, dataout.getLength()));
    ArrayList<TezEvent> actual1 = deserializeEvents(datain);
    ArrayList<TezEvent> actual2 = deserializeEvents(dis);
    assertEventEquals(events, actual1);
    assertEventEquals(events, actual2);

    byte[] serializedBytes = bout.toByteArray();
    datain.reset(serializedBytes, serializedBytes.length);
    dis = new DataInputStream(new ByteArrayInputStream(serializedBytes));
    actual1 = deserializeEvents(datain);
    actual2 = deserializeEvents(dis);
    assertEventEquals(events, actual1);
    assertEventEquals(events, actual2);

}

From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestShuffleInputEventHandlerImpl.java

License:Apache License

private InputContext createInputContext() throws IOException {
    DataOutputBuffer port_dob = new DataOutputBuffer();
    port_dob.writeInt(PORT);//from  w w w .j a va2 s . c o m
    final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());

    ExecutionContext executionContext = mock(ExecutionContext.class);
    doReturn(HOST).when(executionContext).getHostName();

    InputContext inputContext = mock(InputContext.class);
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("sourceVertex").when(inputContext).getSourceVertexName();
    doReturn(shuffleMetaData).when(inputContext)
            .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
    doReturn(executionContext).when(inputContext).getExecutionContext();
    return inputContext;
}

From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestShuffleInputEventHandlerImpl.java

License:Apache License

@SuppressWarnings("unchecked")
private ShuffleManager createShuffleManager(InputContext inputContext) throws IOException {
    Path outDirBase = new Path(".", "outDir");
    String[] outDirs = new String[] { outDirBase.toString() };
    doReturn(outDirs).when(inputContext).getWorkDirs();
    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, inputContext.getWorkDirs());

    DataOutputBuffer out = new DataOutputBuffer();
    Token<JobTokenIdentifier> token = new Token(new JobTokenIdentifier(), new JobTokenSecretManager(null));
    token.write(out);/*from w  w w  . j a  va2 s  .c  o  m*/
    doReturn(ByteBuffer.wrap(out.getData())).when(inputContext)
            .getServiceConsumerMetaData(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID);

    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleManager realShuffleManager = new ShuffleManager(inputContext, conf, 2, 1024, false, -1, null,
            inputAllocator);
    ShuffleManager shuffleManager = spy(realShuffleManager);
    return shuffleManager;
}

From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestShuffleManager.java

License:Apache License

private InputContext createInputContext() throws IOException {
    DataOutputBuffer port_dob = new DataOutputBuffer();
    port_dob.writeInt(PORT);/* www  .ja  v  a 2s . c o  m*/
    final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());

    ExecutionContext executionContext = mock(ExecutionContext.class);
    doReturn(FETCHER_HOST).when(executionContext).getHostName();

    InputContext inputContext = mock(InputContext.class);
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("sourceVertex").when(inputContext).getSourceVertexName();
    doReturn(shuffleMetaData).when(inputContext)
            .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
    doReturn(executionContext).when(inputContext).getExecutionContext();
    return inputContext;
}

From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestShuffleManager.java

License:Apache License

@SuppressWarnings("unchecked")
private ShuffleManagerForTest createShuffleManager(InputContext inputContext, int expectedNumOfPhysicalInputs)
        throws IOException {
    Path outDirBase = new Path(".", "outDir");
    String[] outDirs = new String[] { outDirBase.toString() };
    doReturn(outDirs).when(inputContext).getWorkDirs();
    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, inputContext.getWorkDirs());

    DataOutputBuffer out = new DataOutputBuffer();
    Token<JobTokenIdentifier> token = new Token(new JobTokenIdentifier(), new JobTokenSecretManager(null));
    token.write(out);/*from www . j  a va 2  s.  com*/
    doReturn(ByteBuffer.wrap(out.getData())).when(inputContext)
            .getServiceConsumerMetaData(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID);

    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    return new ShuffleManagerForTest(inputContext, conf, expectedNumOfPhysicalInputs, 1024, false, -1, null,
            inputAllocator);
}

From source file:org.apache.tez.runtime.library.common.shuffle.TestShuffleUtils.java

License:Apache License

private OutputContext createTezOutputContext() throws IOException {
    ApplicationId applicationId = ApplicationId.newInstance(1, 1);
    OutputContext outputContext = mock(OutputContext.class);

    ExecutionContextImpl executionContext = mock(ExecutionContextImpl.class);
    doReturn("localhost").when(executionContext).getHostName();
    doReturn(executionContext).when(outputContext).getExecutionContext();

    DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
    serviceProviderMetaData.writeInt(80);
    doReturn(ByteBuffer.wrap(serviceProviderMetaData.getData())).when(outputContext)
            .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);

    doReturn(1).when(outputContext).getTaskVertexIndex();
    doReturn(1).when(outputContext).getOutputIndex();
    doReturn(0).when(outputContext).getDAGAttemptNumber();
    doReturn("destVertex").when(outputContext).getDestinationVertexName();

    when(outputContext.getCounters()).thenReturn(new TezCounters());
    return outputContext;
}

From source file:org.apache.tez.runtime.library.common.sort.impl.dflt.TestDefaultSorter.java

License:Apache License

private OutputContext createTezOutputContext() throws IOException {
    String[] workingDirs = { workingDir.toString() };
    UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
    DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
    serviceProviderMetaData.writeInt(PORT);

    TezCounters counters = new TezCounters();

    OutputContext context = mock(OutputContext.class);
    doReturn(counters).when(context).getCounters();
    doReturn(workingDirs).when(context).getWorkDirs();
    doReturn(payLoad).when(context).getUserPayload();
    doReturn(5 * 1024 * 1024l).when(context).getTotalMemoryAvailableToTask();
    doReturn(UniqueID).when(context).getUniqueIdentifier();
    doReturn("v1").when(context).getDestinationVertexName();
    doReturn(ByteBuffer.wrap(serviceProviderMetaData.getData())).when(context)
            .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
    doAnswer(new Answer() {
        @Override/*w  w w  .j av a  2 s.co  m*/
        public Object answer(InvocationOnMock invocation) throws Throwable {
            long requestedSize = (Long) invocation.getArguments()[0];
            MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation.getArguments()[1];
            callback.memoryAssigned(requestedSize);
            return null;
        }
    }).when(context).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
    return context;
}

From source file:org.apache.tez.runtime.library.common.sort.impl.TestIFile.java

License:Apache License

@Test(timeout = 5000)
//test with sorted data and repeat keys
public void testWithRLEMarker() throws IOException {
    //Test with append(Object, Object)
    FSDataOutputStream out = localFs.create(outputPath);
    IFile.Writer writer = new IFile.Writer(defaultConf, out, Text.class, IntWritable.class, codec, null, null,
            true);/*w ww  . ja  v  a2s  . co m*/

    Text key = new Text("key0");
    IntWritable value = new IntWritable(0);
    writer.append(key, value);

    //same key (RLE should kick in)
    key = new Text("key0");
    writer.append(key, value);
    assertTrue(writer.sameKey);

    //Different key
    key = new Text("key1");
    writer.append(key, value);
    assertFalse(writer.sameKey);
    writer.close();
    out.close();

    //Test with append(DataInputBuffer key, DataInputBuffer value)
    byte[] kvbuffer = "key1Value1key1Value2key3Value3".getBytes();
    int keyLength = 4;
    int valueLength = 6;
    int pos = 0;
    out = localFs.create(outputPath);
    writer = new IFile.Writer(defaultConf, out, Text.class, IntWritable.class, codec, null, null, true);

    DataInputBuffer kin = new DataInputBuffer();
    kin.reset(kvbuffer, pos, keyLength);

    DataInputBuffer vin = new DataInputBuffer();
    DataOutputBuffer vout = new DataOutputBuffer();
    (new IntWritable(0)).write(vout);
    vin.reset(vout.getData(), vout.getLength());

    //Write initial KV pair
    writer.append(kin, vin);
    assertFalse(writer.sameKey);
    pos += (keyLength + valueLength);

    //Second key is similar to key1 (RLE should kick in)
    kin.reset(kvbuffer, pos, keyLength);
    (new IntWritable(0)).write(vout);
    vin.reset(vout.getData(), vout.getLength());
    writer.append(kin, vin);
    assertTrue(writer.sameKey);
    pos += (keyLength + valueLength);

    //Next key (key3) is different (RLE should not kick in)
    kin.reset(kvbuffer, pos, keyLength);
    (new IntWritable(0)).write(vout);
    vin.reset(vout.getData(), vout.getLength());
    writer.append(kin, vin);
    assertFalse(writer.sameKey);

    writer.close();
    out.close();
}

From source file:org.apache.tez.runtime.library.common.sort.impl.TestIFile.java

License:Apache License

private void populateData(KVPair kvp, DataInputBuffer key, DataInputBuffer value) throws IOException {
    DataOutputBuffer k = new DataOutputBuffer();
    DataOutputBuffer v = new DataOutputBuffer();
    kvp.getKey().write(k);//from   w  ww.  j a  v  a 2s .co m
    kvp.getvalue().write(v);
    key.reset(k.getData(), 0, k.getLength());
    value.reset(v.getData(), 0, v.getLength());
}