Example usage for org.apache.hadoop.io BoundedByteArrayOutputStream BoundedByteArrayOutputStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io BoundedByteArrayOutputStream BoundedByteArrayOutputStream.

Prototype

public BoundedByteArrayOutputStream(int capacity) 

Source Link

Document

Create a BoundedByteArrayOutputStream with the specified capacity

Usage

From source file:org.apache.tez.engine.common.shuffle.impl.MapOutput.java

License:Apache License

MapOutput(TezTaskAttemptID mapId, MergeManager merger, int size, boolean primaryMapOutput) {
    this.id = ID.incrementAndGet();
    this.mapId = mapId;
    this.merger = merger;

    type = Type.MEMORY;//from w  ww. j  a va  2  s.  c  o m
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();

    this.size = size;

    localFS = null;
    disk = null;
    outputPath = null;
    tmpOutputPath = null;

    this.primaryMapOutput = primaryMapOutput;
}

---

From source file:org.apache.tez.runtime.library.common.shuffle.impl.MapOutput.java

License:Apache License

MapOutput(InputAttemptIdentifier attemptIdentifier, MergeManager merger, int size, boolean primaryMapOutput) {
    this.id = ID.incrementAndGet();
    this.attemptIdentifier = attemptIdentifier;
    this.merger = merger;

    type = Type.MEMORY;/*w  ww. jav  a 2  s  .  co  m*/
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();

    this.size = size;

    localFS = null;
    disk = null;
    outputPath = null;
    tmpOutputPath = null;

    this.primaryMapOutput = primaryMapOutput;
}

---

From source file:org.apache.tez.runtime.library.common.shuffle.MemoryFetchedInput.java

License:Apache License

public MemoryFetchedInput(long actualSize, long compressedSize, InputAttemptIdentifier inputAttemptIdentifier,
        FetchedInputCallback callbackHandler) {
    super(Type.MEMORY, actualSize, compressedSize, inputAttemptIdentifier, callbackHandler);
    this.byteStream = new BoundedByteArrayOutputStream((int) actualSize);
}

---

From source file:org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput.java

License:Apache License

private MapOutput(Type type, InputAttemptIdentifier attemptIdentifier, MergeManager merger, long size,
        Path outputPath, long offset, boolean primaryMapOutput, FileSystem fs, Path tmpOutputPath) {
    this.id = ID.incrementAndGet();
    this.type = type;
    this.attemptIdentifier = attemptIdentifier;
    this.merger = merger;
    this.primaryMapOutput = primaryMapOutput;

    this.localFS = fs;
    this.size = size;

    // Other type specific values

    if (type == Type.MEMORY) {
        // since we are passing an int from createMemoryMapOutput, its safe to cast to int
        this.byteStream = new BoundedByteArrayOutputStream((int) size);
        this.memory = byteStream.getBuffer();
    } else {//ww  w .  ja va 2  s . c  o m
        this.byteStream = null;
        this.memory = null;
    }

    this.tmpOutputPath = tmpOutputPath;
    this.disk = null;

    if (type == Type.DISK || type == Type.DISK_DIRECT) {
        if (type == Type.DISK_DIRECT) {
            this.outputPath = new FileChunk(outputPath, offset, size, true, attemptIdentifier);
        } else {
            this.outputPath = new FileChunk(outputPath, offset, size, false, attemptIdentifier);
        }
    } else {
        this.outputPath = null;
    }

}

---

From source file:org.apache.tez.runtime.library.common.sort.impl.TestIFile.java

License:Apache License

@Test(timeout = 5000)
//Test InMemoryWriter
public void testInMemoryWriter() throws IOException {
    InMemoryWriter writer = null;// w  w w. j  a v a 2 s  .  c o  m
    BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);

    List<KVPair> data = KVDataGen.generateTestData(true, 10);

    //No RLE, No RepeatKeys, no compression
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, false, false, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);

    //No RLE, RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, false, true, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);

    //RLE, No RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, true, false, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);

    //RLE, RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, true, true, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);
}

---

From source file:org.apache.tez.runtime.library.common.TestValuesIterator.java

License:Apache License

/**
 * create inmemory segments//from w w  w  .j  a v a 2s  .  co m
 *
 * @return
 * @throws IOException
 */
public List<TezMerger.Segment> createInMemStreams() throws IOException {
    int numberOfStreams = Math.max(2, rnd.nextInt(10));
    LOG.info("No of streams : " + numberOfStreams);

    SerializationFactory serializationFactory = new SerializationFactory(conf);
    Serializer keySerializer = serializationFactory.getSerializer(keyClass);
    Serializer valueSerializer = serializationFactory.getSerializer(valClass);

    LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
    InputContext context = createTezInputContext();
    MergeManager mergeManager = new MergeManager(conf, fs, localDirAllocator, context, null, null, null, null,
            null, 1024 * 1024 * 10, null, false, -1);

    DataOutputBuffer keyBuf = new DataOutputBuffer();
    DataOutputBuffer valBuf = new DataOutputBuffer();
    DataInputBuffer keyIn = new DataInputBuffer();
    DataInputBuffer valIn = new DataInputBuffer();
    keySerializer.open(keyBuf);
    valueSerializer.open(valBuf);

    List<TezMerger.Segment> segments = new LinkedList<TezMerger.Segment>();
    for (int i = 0; i < numberOfStreams; i++) {
        BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);
        InMemoryWriter writer = new InMemoryWriter(bout);
        Map<Writable, Writable> data = createData();
        //write data
        for (Map.Entry<Writable, Writable> entry : data.entrySet()) {
            keySerializer.serialize(entry.getKey());
            valueSerializer.serialize(entry.getValue());
            keyIn.reset(keyBuf.getData(), 0, keyBuf.getLength());
            valIn.reset(valBuf.getData(), 0, valBuf.getLength());
            writer.append(keyIn, valIn);
            originalData.put(entry.getKey(), entry.getValue());
            keyBuf.reset();
            valBuf.reset();
            keyIn.reset();
            valIn.reset();
        }
        IFile.Reader reader = new InMemoryReader(mergeManager, null, bout.getBuffer(), 0,
                bout.getBuffer().length);
        segments.add(new TezMerger.Segment(reader, true));

        data.clear();
        writer.close();
    }
    return segments;
}

---