Example usage for org.apache.hadoop.io IntWritable set

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable set.

Prototype

public void set(int value) 

Source Link

Document

Set the value of this IntWritable.

Usage

From source file:org.apache.giraph.types.ops.IntTypeOps.java

License:Apache License

@Override
public void set(IntWritable to, IntWritable from) {
    to.set(from.get());
}

---

From source file:org.apache.giraph.types.ShortToIntWritableWrapper.java

License:Apache License

@Override
public void wrap(Short javaValue, IntWritable writableValue) {
    writableValue.set(javaValue);
}

---

From source file:org.apache.hama.bsp.Counters.java

License:Apache License

private static String getBlock(String str, char open, char close, IntWritable index) throws ParseException {
    StringBuilder split = new StringBuilder();
    int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split);
    split.setLength(0); // clear the buffer
    if (next >= 0) {
        ++next; // move over '('

        next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split);
        if (next >= 0) {
            ++next; // move over ')'
            index.set(next);
            return split.toString(); // found a block
        } else {/*from   w  ww.j  av a 2s .co  m*/
            throw new ParseException("Unexpected end of block", next);
        }
    }
    return null; // found nothing
}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testDirectByteBufferOutput() throws Exception {

    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);/*from ww w . j a v  a2 s .  c  o  m*/
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }

    stream.close();

    buffer.flip();
    for (int i = 0; i < 100; ++i) {
        assertEquals(i, buffer.getInt());
    }

    try {
        buffer.getInt();
        assertTrue(false);
    } catch (Exception e) {
        assertTrue(true);
    }

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testDirectByteBufferInput() throws Exception {
    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);/*  w w  w  .j  av a2s .c  om*/
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    intWritable.write(stream);

    stream.close();

    buffer.flip();

    DirectByteBufferInputStream inStream = new DirectByteBufferInputStream();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));
    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();

    // push in another buffer and check if the unmarked data could be read.

    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    buffer = ByteBuffer.allocateDirect(2048);
    stream.setBuffer(buffer);

    for (int i = 0; i < 400; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));

    // Read previous data
    intWritable.readFields(inStream);
    assertEquals(99, intWritable.get());

    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();

    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));

    // Read previous data with resized intermediate buffer
    for (int i = 100; i < 400; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertFalse(inStream.hasUnmarkData());

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

/**
 * //from  ww  w .j a  v  a 2s. c o m
 * @throws Exception
 */
public void testReusableByteBufferIter() throws Exception {

    ReusableByteBuffer<IntWritable> reuseByteBuffer = new ReusableByteBuffer<IntWritable>(new IntWritable());

    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    intWritable.write(stream);
    stream.close();
    buffer.flip();
    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 400));

    Iterator<IntWritable> iter = reuseByteBuffer.iterator();
    int j = 0;
    while (iter.hasNext()) {
        assertEquals(iter.next().get(), j++);
    }
    assertEquals(j, 100);
    reuseByteBuffer.prepareForNext();

    buffer.clear();

    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);

    for (int i = 0; i < 101; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 404));
    iter = reuseByteBuffer.iterator();
    assertEquals(iter.next().get(), 99);

    j = 0;
    while (iter.hasNext()) {
        assertEquals(iter.next().get(), j++);
    }
    buffer.clear();
}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testCombineProcessor() throws Exception {
    String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
            + new BigInteger(128, new SecureRandom()).toString(32);

    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);/*from w w w . ja va2 s .  c om*/
    IntWritable intWritable = new IntWritable(1);
    int sum = 0;
    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
        sum += i;
    }
    intWritable.write(stream);
    stream.close();
    buffer.flip();

    Configuration conf = new HamaConfiguration();

    conf.setClass(Constants.MESSAGE_CLASS, IntWritable.class, Writable.class);
    conf.setClass(Constants.COMBINER_CLASS, SumCombiner.class, Combiner.class);

    CombineSpilledDataProcessor<IntWritable> processor = new CombineSpilledDataProcessor<IntWritable>(fileName);
    assertTrue(processor.init(conf));
    File f = new File(fileName);
    try {
        assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer, 400)));
        buffer.flip();
        assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer, 400)));
        assertTrue(processor.close());

        assertTrue(f.exists());
        assertEquals(f.length(), 8);

        RandomAccessFile raf = new RandomAccessFile(fileName, "r");
        FileChannel fileChannel = raf.getChannel();
        ByteBuffer readBuff = ByteBuffer.allocateDirect(16);
        fileChannel.read(readBuff);
        readBuff.flip();
        assertEquals(readBuff.getInt(), sum);
        assertEquals(readBuff.getInt(), sum + 99);
        raf.close();
    } finally {
        assertTrue(f.delete());
    }

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testSyncFlushByteBufferOutputStream() throws Exception {

    File f = null;//from  www. ja  v a 2 s  .c  om
    try {
        String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
                + "testSyncFlushByteBufferOutputStream.txt";
        SyncFlushByteBufferOutputStream stream = new SyncFlushByteBufferOutputStream(fileName);
        DirectByteBufferOutputStream syncFlushStream = new DirectByteBufferOutputStream(stream);
        ByteBuffer buffer = ByteBuffer.allocateDirect(512);
        syncFlushStream.setBuffer(buffer);
        IntWritable intWritable = new IntWritable(1);

        for (int i = 0; i < 200; ++i) {
            intWritable.set(i);
            intWritable.write(syncFlushStream);
        }
        intWritable.write(syncFlushStream);
        syncFlushStream.close();

        f = new File(fileName);
        assertTrue(f.exists());
        assertTrue(f.length() == 804);
        assertTrue(f.delete());
    } finally {
        if (f != null) {
            f.delete();
        }
    }

}

---

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testSyncFlushBufferInputStream() throws Exception {
    File f = null;/*  w  w w. j a  va 2s .co m*/
    try {
        String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
                + "testSyncFlushBufferInputStream.txt";
        SyncFlushByteBufferOutputStream stream = new SyncFlushByteBufferOutputStream(fileName);
        DirectByteBufferOutputStream syncFlushStream = new DirectByteBufferOutputStream(stream);
        ByteBuffer buffer = ByteBuffer.allocateDirect(512);
        syncFlushStream.setBuffer(buffer);
        IntWritable intWritable = new IntWritable(1);

        for (int i = 0; i < 200; ++i) {
            intWritable.set(i);
            intWritable.write(syncFlushStream);
        }
        intWritable.write(syncFlushStream);
        syncFlushStream.close();

        f = new File(fileName);
        assertTrue(f.exists());
        assertEquals(f.length(), 804);

        SyncReadByteBufferInputStream syncReadStream = new SyncReadByteBufferInputStream(stream.isSpilled(),
                fileName);
        DirectByteBufferInputStream inStream = new DirectByteBufferInputStream(syncReadStream);
        buffer.clear();
        inStream.setBuffer(buffer);

        for (int i = 0; i < 200; ++i) {
            intWritable.readFields(inStream);
            assertEquals(intWritable.get(), i);
        }

        intWritable.readFields(inStream);
        assertEquals(intWritable.get(), 199);

        try {
            intWritable.readFields(inStream);
            assertFalse(true);
        } catch (Exception e) {
            assertTrue(true);
        }

        inStream.close();
        syncFlushStream.close();

    } finally {
        if (f != null) {
            f.delete();
        }
    }
}

---

From source file:org.apache.hama.bsp.sync.TestSyncServiceFactory.java

License:Apache License

@Test
public void testZKSyncStore() throws Exception {
    Configuration conf = new Configuration();
    int zkPort = BSPNetUtils.getFreePort(21811);
    conf.set("bsp.local.dir", "/tmp/hama-test");
    conf.set("bsp.output.dir", "/tmp/hama-test_out");
    conf.setInt(Constants.PEER_PORT, zkPort);
    conf.set(Constants.ZOOKEEPER_QUORUM, "localhost");
    conf.setInt(Constants.ZOOKEEPER_CLIENT_PORT, zkPort);
    System.setProperty("user.dir", "/tmp");
    // given null, should return zookeeper
    final SyncServer syncServer = SyncServiceFactory.getSyncServer(conf);
    syncServer.init(conf);/*from  ww  w .  ja  v a  2  s  .com*/
    assertTrue(syncServer instanceof ZooKeeperSyncServerImpl);

    ZKServerThread serverThread = new ZKServerThread(syncServer);
    ExecutorService executorService = Executors.newFixedThreadPool(1);
    executorService.submit(serverThread);

    executorService.awaitTermination(10, TimeUnit.SECONDS);

    final PeerSyncClient syncClient = SyncServiceFactory.getPeerSyncClient(conf);
    assertTrue(syncClient instanceof ZooKeeperSyncClientImpl);
    BSPJobID jobId = new BSPJobID("abc", 1);
    TaskAttemptID taskId = new TaskAttemptID(new TaskID(jobId, 1), 1);
    syncClient.init(conf, jobId, taskId);

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                syncServer.stopServer();

            } catch (Exception e) {
                // too late to log!
            }
        }
    });

    Thread.sleep(3000);

    IntWritable data = new IntWritable(5);

    syncClient.storeInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"), data, true, null);

    ListenerTest listenerTest = new ListenerTest();

    syncClient.registerListener(syncClient.constructKey(jobId, String.valueOf(1L), "test"),
            ZKSyncEventFactory.getValueChangeEvent(), listenerTest);

    IntWritable valueHolder = new IntWritable();
    boolean result = syncClient.getInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"),
            valueHolder);
    assertTrue(result);
    int intVal = valueHolder.get();
    assertTrue(intVal == data.get());

    data.set(6);
    syncClient.storeInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"), data, true, null);
    valueHolder = new IntWritable();
    result = syncClient.getInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"), valueHolder);

    assertTrue(result);
    intVal = valueHolder.get();
    assertTrue(intVal == data.get());

    Thread.sleep(5000);

    assertEquals(true, listenerTest.getValue().equals("Changed"));

    syncServer.stopServer();

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• »