Example usage for org.apache.hadoop.io IntWritable write

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable write.

Prototype

@Override
    public void write(DataOutput out) throws IOException 

Source Link

Usage

From source file:com.digitalpebble.behemoth.BehemothDocument.java

License:Apache License

protected void writeAnnotation(Annotation annot, DataOutput out, List<String> atypes) throws IOException {
    int typePos = atypes.indexOf(annot.getType());
    IntWritable intStringPool = new IntWritable(typePos);
    intStringPool.write(out);
    WritableUtils.writeVLong(out, annot.getStart());
    WritableUtils.writeVLong(out, annot.getEnd());
    out.writeInt(annot.getFeatureNum());

    if (annot.getFeatures() != null) {
        Iterator<String> featNameIter = annot.getFeatures().keySet().iterator();
        while (featNameIter.hasNext()) {
            String fname = featNameIter.next();
            int fnamePos = atypes.indexOf(fname);
            intStringPool.set(fnamePos);
            intStringPool.write(out);/*  www.  j  a v a 2s .  c  o  m*/
            WritableUtils.writeString(out, annot.getFeatures().get(fname));
        }
    }
}

---

From source file:com.marcolotz.MRComponents.SerializerConverter.java

License:Creative Commons License

/**
 * Writes an int to the output.// w ww. ja v  a 2  s.c  o  m
 * @param outputInt
 * @param out
 * @throws IOException
 */
public static void writeInt(int outputInt, DataOutput out) throws IOException {
    IntWritable writtenInt = new IntWritable(outputInt);
    writtenInt.write(out);
}

---

From source file:org.apache.giraph.comm.RequestTest.java

License:Apache License

@Test
public void sendWorkerIndividualMessagesRequest() throws IOException {
    // Data to send
    ByteArrayOneMessageToManyIds<IntWritable, IntWritable> dataToSend = new ByteArrayOneMessageToManyIds<>(
            new TestMessageValueFactory<>(IntWritable.class));
    dataToSend.setConf(conf);//from   ww w  .ja  va2  s.  com
    dataToSend.initialize();
    ExtendedDataOutput output = conf.createExtendedDataOutput();
    for (int i = 1; i <= 7; ++i) {
        IntWritable vertexId = new IntWritable(i);
        vertexId.write(output);
    }
    dataToSend.add(output.getByteArray(), output.getPos(), 7, new IntWritable(1));

    // Send the request
    SendWorkerOneMessageToManyRequest<IntWritable, IntWritable> request = new SendWorkerOneMessageToManyRequest<>(
            dataToSend, conf);
    client.sendWritableRequest(workerInfo.getTaskId(), request);
    client.waitAllRequests();

    // Stop the service
    client.stop();
    server.stop();

    // Check the output
    Iterable<IntWritable> vertices = serverData.getIncomingMessageStore().getPartitionDestinationVertices(0);
    int keySum = 0;
    int messageSum = 0;
    for (IntWritable vertexId : vertices) {
        keySum += vertexId.get();
        Iterable<IntWritable> messages = serverData.<IntWritable>getIncomingMessageStore()
                .getVertexMessages(vertexId);
        synchronized (messages) {
            for (IntWritable message : messages) {
                messageSum += message.get();
            }
        }
    }
    assertEquals(28, keySum);
    assertEquals(7, messageSum);
}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testDirectByteBufferOutput() throws Exception {

    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);//from  w  ww . j  a va 2  s  . c o m
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }

    stream.close();

    buffer.flip();
    for (int i = 0; i < 100; ++i) {
        assertEquals(i, buffer.getInt());
    }

    try {
        buffer.getInt();
        assertTrue(false);
    } catch (Exception e) {
        assertTrue(true);
    }

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testDirectByteBufferInput() throws Exception {
    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);//from ww  w.j  a  v  a 2s  . c o  m
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    intWritable.write(stream);

    stream.close();

    buffer.flip();

    DirectByteBufferInputStream inStream = new DirectByteBufferInputStream();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));
    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();

    // push in another buffer and check if the unmarked data could be read.

    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    buffer = ByteBuffer.allocateDirect(2048);
    stream.setBuffer(buffer);

    for (int i = 0; i < 400; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));

    // Read previous data
    intWritable.readFields(inStream);
    assertEquals(99, intWritable.get());

    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();

    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));

    // Read previous data with resized intermediate buffer
    for (int i = 100; i < 400; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertFalse(inStream.hasUnmarkData());

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

/**
 * //from www.  j ava2 s.  c  o  m
 * @throws Exception
 */
public void testReusableByteBufferIter() throws Exception {

    ReusableByteBuffer<IntWritable> reuseByteBuffer = new ReusableByteBuffer<IntWritable>(new IntWritable());

    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    intWritable.write(stream);
    stream.close();
    buffer.flip();
    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 400));

    Iterator<IntWritable> iter = reuseByteBuffer.iterator();
    int j = 0;
    while (iter.hasNext()) {
        assertEquals(iter.next().get(), j++);
    }
    assertEquals(j, 100);
    reuseByteBuffer.prepareForNext();

    buffer.clear();

    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);

    for (int i = 0; i < 101; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 404));
    iter = reuseByteBuffer.iterator();
    assertEquals(iter.next().get(), 99);

    j = 0;
    while (iter.hasNext()) {
        assertEquals(iter.next().get(), j++);
    }
    buffer.clear();
}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testCombineProcessor() throws Exception {
    String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
            + new BigInteger(128, new SecureRandom()).toString(32);

    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);/* w w  w.  ja  v a  2s.  c om*/
    IntWritable intWritable = new IntWritable(1);
    int sum = 0;
    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
        sum += i;
    }
    intWritable.write(stream);
    stream.close();
    buffer.flip();

    Configuration conf = new HamaConfiguration();

    conf.setClass(Constants.MESSAGE_CLASS, IntWritable.class, Writable.class);
    conf.setClass(Constants.COMBINER_CLASS, SumCombiner.class, Combiner.class);

    CombineSpilledDataProcessor<IntWritable> processor = new CombineSpilledDataProcessor<IntWritable>(fileName);
    assertTrue(processor.init(conf));
    File f = new File(fileName);
    try {
        assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer, 400)));
        buffer.flip();
        assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer, 400)));
        assertTrue(processor.close());

        assertTrue(f.exists());
        assertEquals(f.length(), 8);

        RandomAccessFile raf = new RandomAccessFile(fileName, "r");
        FileChannel fileChannel = raf.getChannel();
        ByteBuffer readBuff = ByteBuffer.allocateDirect(16);
        fileChannel.read(readBuff);
        readBuff.flip();
        assertEquals(readBuff.getInt(), sum);
        assertEquals(readBuff.getInt(), sum + 99);
        raf.close();
    } finally {
        assertTrue(f.delete());
    }

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testSyncFlushByteBufferOutputStream() throws Exception {

    File f = null;/*from   ww  w .j a  v  a 2s . c o m*/
    try {
        String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
                + "testSyncFlushByteBufferOutputStream.txt";
        SyncFlushByteBufferOutputStream stream = new SyncFlushByteBufferOutputStream(fileName);
        DirectByteBufferOutputStream syncFlushStream = new DirectByteBufferOutputStream(stream);
        ByteBuffer buffer = ByteBuffer.allocateDirect(512);
        syncFlushStream.setBuffer(buffer);
        IntWritable intWritable = new IntWritable(1);

        for (int i = 0; i < 200; ++i) {
            intWritable.set(i);
            intWritable.write(syncFlushStream);
        }
        intWritable.write(syncFlushStream);
        syncFlushStream.close();

        f = new File(fileName);
        assertTrue(f.exists());
        assertTrue(f.length() == 804);
        assertTrue(f.delete());
    } finally {
        if (f != null) {
            f.delete();
        }
    }

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testSyncFlushBufferInputStream() throws Exception {
    File f = null;/*from w  w  w .  j  av a2 s  .c  o m*/
    try {
        String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
                + "testSyncFlushBufferInputStream.txt";
        SyncFlushByteBufferOutputStream stream = new SyncFlushByteBufferOutputStream(fileName);
        DirectByteBufferOutputStream syncFlushStream = new DirectByteBufferOutputStream(stream);
        ByteBuffer buffer = ByteBuffer.allocateDirect(512);
        syncFlushStream.setBuffer(buffer);
        IntWritable intWritable = new IntWritable(1);

        for (int i = 0; i < 200; ++i) {
            intWritable.set(i);
            intWritable.write(syncFlushStream);
        }
        intWritable.write(syncFlushStream);
        syncFlushStream.close();

        f = new File(fileName);
        assertTrue(f.exists());
        assertEquals(f.length(), 804);

        SyncReadByteBufferInputStream syncReadStream = new SyncReadByteBufferInputStream(stream.isSpilled(),
                fileName);
        DirectByteBufferInputStream inStream = new DirectByteBufferInputStream(syncReadStream);
        buffer.clear();
        inStream.setBuffer(buffer);

        for (int i = 0; i < 200; ++i) {
            intWritable.readFields(inStream);
            assertEquals(intWritable.get(), i);
        }

        intWritable.readFields(inStream);
        assertEquals(intWritable.get(), 199);

        try {
            intWritable.readFields(inStream);
            assertFalse(true);
        } catch (Exception e) {
            assertTrue(true);
        }

        inStream.close();
        syncFlushStream.close();

    } finally {
        if (f != null) {
            f.delete();
        }
    }
}

---

From source file:org.apache.mahout.common.IntTuple.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    out.writeInt(tuple.size());/*w ww.  j  a  v a2s.c o m*/
    IntWritable value = new IntWritable();
    for (int entry : tuple.elements()) {
        value.set(entry);
        value.write(out);
    }
}

---

• «
• 1
• 2
• »