Example usage for org.apache.hadoop.io IntWritable get

List of usage examples for org.apache.hadoop.io IntWritable get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. IntWritable
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable get.

Prototype

public int get()

Source Link

Document

Return the value of this IntWritable.

Usage

From source file:org.apache.hama.bsp.message.TestMessageBundle.java

License:Apache License

public void testPOJOWritableMessageBundle() {

    POJOMessageBundle<IntWritable> messageBundle = new POJOMessageBundle<IntWritable>();
    for (int i = 0; i < 100; ++i) {
        messageBundle.addMessage(new IntWritable(i));
    }/* w  w w.  j  a  v  a 2s  .co  m*/
    assertEquals(100, messageBundle.getSize());
    assertEquals(100, messageBundle.getNumElements());

    int i = 0;
    for (IntWritable writable : messageBundle) {
        assertEquals(i++, writable.get());
    }

}

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testDirectByteBufferInput() throws Exception {
    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    DirectByteBufferOutputStream stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);/* w w  w  . jav a 2  s.  c  om*/
    IntWritable intWritable = new IntWritable(1);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    intWritable.write(stream);

    stream.close();

    buffer.flip();

    DirectByteBufferInputStream inStream = new DirectByteBufferInputStream();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));
    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();

    // push in another buffer and check if the unmarked data could be read.

    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    buffer = ByteBuffer.allocateDirect(2048);
    stream.setBuffer(buffer);

    for (int i = 0; i < 400; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));

    // Read previous data
    intWritable.readFields(inStream);
    assertEquals(99, intWritable.get());

    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();

    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);

    for (int i = 0; i < 100; ++i) {
        intWritable.set(i);
        intWritable.write(stream);
    }
    stream.close();
    buffer.flip();

    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));

    // Read previous data with resized intermediate buffer
    for (int i = 100; i < 400; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    for (int i = 0; i < 100; ++i) {
        intWritable.readFields(inStream);
        assertEquals(i, intWritable.get());
    }

    assertFalse(inStream.hasDataToRead());
    assertFalse(inStream.hasUnmarkData());

}

From source file:org.apache.hama.bsp.message.TestMessageIO.java

License:Apache License

public void testSyncFlushBufferInputStream() throws Exception {
    File f = null;/*from   ww w.  j  a va2 s  .c om*/
    try {
        String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
                + "testSyncFlushBufferInputStream.txt";
        SyncFlushByteBufferOutputStream stream = new SyncFlushByteBufferOutputStream(fileName);
        DirectByteBufferOutputStream syncFlushStream = new DirectByteBufferOutputStream(stream);
        ByteBuffer buffer = ByteBuffer.allocateDirect(512);
        syncFlushStream.setBuffer(buffer);
        IntWritable intWritable = new IntWritable(1);

        for (int i = 0; i < 200; ++i) {
            intWritable.set(i);
            intWritable.write(syncFlushStream);
        }
        intWritable.write(syncFlushStream);
        syncFlushStream.close();

        f = new File(fileName);
        assertTrue(f.exists());
        assertEquals(f.length(), 804);

        SyncReadByteBufferInputStream syncReadStream = new SyncReadByteBufferInputStream(stream.isSpilled(),
                fileName);
        DirectByteBufferInputStream inStream = new DirectByteBufferInputStream(syncReadStream);
        buffer.clear();
        inStream.setBuffer(buffer);

        for (int i = 0; i < 200; ++i) {
            intWritable.readFields(inStream);
            assertEquals(intWritable.get(), i);
        }

        intWritable.readFields(inStream);
        assertEquals(intWritable.get(), 199);

        try {
            intWritable.readFields(inStream);
            assertFalse(true);
        } catch (Exception e) {
            assertTrue(true);
        }

        inStream.close();
        syncFlushStream.close();

    } finally {
        if (f != null) {
            f.delete();
        }
    }
}

From source file:org.apache.hama.bsp.sync.TestSyncServiceFactory.java

License:Apache License

@Test
public void testZKSyncStore() throws Exception {
    Configuration conf = new Configuration();
    int zkPort = BSPNetUtils.getFreePort(21811);
    conf.set("bsp.local.dir", "/tmp/hama-test");
    conf.set("bsp.output.dir", "/tmp/hama-test_out");
    conf.setInt(Constants.PEER_PORT, zkPort);
    conf.set(Constants.ZOOKEEPER_QUORUM, "localhost");
    conf.setInt(Constants.ZOOKEEPER_CLIENT_PORT, zkPort);
    System.setProperty("user.dir", "/tmp");
    // given null, should return zookeeper
    final SyncServer syncServer = SyncServiceFactory.getSyncServer(conf);
    syncServer.init(conf);/* w w  w  . j a va 2 s .  c o  m*/
    assertTrue(syncServer instanceof ZooKeeperSyncServerImpl);

    ZKServerThread serverThread = new ZKServerThread(syncServer);
    ExecutorService executorService = Executors.newFixedThreadPool(1);
    executorService.submit(serverThread);

    executorService.awaitTermination(10, TimeUnit.SECONDS);

    final PeerSyncClient syncClient = SyncServiceFactory.getPeerSyncClient(conf);
    assertTrue(syncClient instanceof ZooKeeperSyncClientImpl);
    BSPJobID jobId = new BSPJobID("abc", 1);
    TaskAttemptID taskId = new TaskAttemptID(new TaskID(jobId, 1), 1);
    syncClient.init(conf, jobId, taskId);

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                syncServer.stopServer();

            } catch (Exception e) {
                // too late to log!
            }
        }
    });

    Thread.sleep(3000);

    IntWritable data = new IntWritable(5);

    syncClient.storeInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"), data, true, null);

    ListenerTest listenerTest = new ListenerTest();

    syncClient.registerListener(syncClient.constructKey(jobId, String.valueOf(1L), "test"),
            ZKSyncEventFactory.getValueChangeEvent(), listenerTest);

    IntWritable valueHolder = new IntWritable();
    boolean result = syncClient.getInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"),
            valueHolder);
    assertTrue(result);
    int intVal = valueHolder.get();
    assertTrue(intVal == data.get());

    data.set(6);
    syncClient.storeInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"), data, true, null);
    valueHolder = new IntWritable();
    result = syncClient.getInformation(syncClient.constructKey(jobId, String.valueOf(1L), "test"), valueHolder);

    assertTrue(result);
    intVal = valueHolder.get();
    assertTrue(intVal == data.get());

    Thread.sleep(5000);

    assertEquals(true, listenerTest.getValue().equals("Changed"));

    syncServer.stopServer();

}

From source file:org.apache.hama.bsp.TestBSPMasterGroomServer.java

License:Apache License

public static void checkOutput(FileSystem fileSys, Configuration conf, int tasks) throws Exception {
    FileStatus[] listStatus = fileSys.globStatus(new Path(OUTPUT_PATH + "/part-*"));

    assertEquals(listStatus.length, tasks);
    for (FileStatus status : listStatus) {
        if (!status.isDir()) {
            SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, status.getPath(), conf);
            int superStep = 0;
            int taskstep = 0;
            IntWritable key = new IntWritable();
            Text value = new Text();
            /*/*from w  w  w  .  j  a  v a 2  s .c o m*/
             * The serialize printing task should write in each superstep
             * "tasks"-times its superstep, along with the hostname.
             */
            while (reader.next(key, value)) {
                assertEquals(superStep, key.get());
                taskstep++;
                if (taskstep % tasks == 0) {
                    superStep++;
                }
            }
            reader.close();
            // the maximum should be the number of supersteps defined in the task
            assertEquals(superStep, ClassSerializePrinting.NUM_SUPERSTEPS);
        }
    }

    fileSys.delete(new Path(TMP_OUTPUT), true);
}

From source file:org.apache.hama.examples.InlinkCount.java

License:Apache License

@Override
public void compute(Iterable<IntWritable> messages) throws IOException {

    if (getSuperstepCount() == 0L) {
        setValue(new IntWritable(0));
        sendMessageToNeighbors(new IntWritable(1));
    } else {/* w w  w .j  a v a  2  s . c  o  m*/
        for (IntWritable msg : messages) {
            this.setValue(new IntWritable(this.getValue().get() + msg.get()));
        }
        voteToHalt();
    }
}

From source file:org.apache.hama.examples.SpMV.java

License:Apache License

/**
 * SpMV produces a file, which contains result dense vector in format of pairs
 * of integer and double. The aim of this method is to convert SpMV output to
 * format usable in subsequent computation - dense vector. It can be usable
 * for iterative solvers. IMPORTANT: currently it is used in SpMV. It can be a
 * bottle neck, because all input needs to be stored in memory.
 * /*w ww .j av a2s.  c  o m*/
 * @param SpMVoutputPathString output path, which represents directory with
 *          part files.
 * @param conf configuration
 * @return path to output vector.
 * @throws IOException
 */
public static String convertSpMVOutputToDenseVector(String SpMVoutputPathString, HamaConfiguration conf)
        throws IOException {
    List<Integer> indeces = new ArrayList<Integer>();
    List<Double> values = new ArrayList<Double>();

    FileSystem fs = FileSystem.get(conf);
    Path SpMVOutputPath = new Path(SpMVoutputPathString);
    Path resultOutputPath = SpMVOutputPath.getParent().suffix("/result");
    FileStatus[] stats = fs.listStatus(SpMVOutputPath);
    for (FileStatus stat : stats) {
        String filePath = stat.getPath().toUri().getPath();
        SequenceFile.Reader reader = null;
        fs.open(new Path(filePath));
        try {
            reader = new SequenceFile.Reader(fs, new Path(filePath), conf);
            IntWritable key = new IntWritable();
            DoubleWritable value = new DoubleWritable();
            while (reader.next(key, value)) {
                indeces.add(key.get());
                values.add(value.get());
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        } finally {
            if (reader != null)
                reader.close();
        }
    }
    DenseVectorWritable result = new DenseVectorWritable();
    result.setSize(indeces.size());
    for (int i = 0; i < indeces.size(); i++)
        result.addCell(indeces.get(i), values.get(i));
    writeToFile(resultOutputPath.toString(), result, conf);
    return resultOutputPath.toString();
}

From source file:org.apache.hama.examples.util.WritableUtil.java

License:Apache License

/**
 * SpMV produces a file, which contains result dense vector in format of pairs
 * of integer and double. The aim of this method is to convert SpMV output to
 * format usable in subsequent computation - dense vector. It can be usable
 * for iterative solvers. IMPORTANT: currently it is used in SpMV. It can be a
 * bottle neck, because all input needs to be stored in memory.
 * //ww  w .ja va 2  s .  c om
 * @param SpMVoutputPathString
 *          output path, which represents directory with part files.
 * @param conf
 *          configuration
 * @return path to output vector.
 * @throws IOException
 */
public static String convertSpMVOutputToDenseVector(String SpMVoutputPathString, Configuration conf)
        throws IOException {
    List<Integer> indeces = new ArrayList<Integer>();
    List<Double> values = new ArrayList<Double>();

    FileSystem fs = FileSystem.get(conf);
    Path SpMVOutputPath = new Path(SpMVoutputPathString);
    Path resultOutputPath = SpMVOutputPath.getParent().suffix("/result");
    FileStatus[] stats = fs.listStatus(SpMVOutputPath);
    for (FileStatus stat : stats) {
        String filePath = stat.getPath().toUri().getPath();
        SequenceFile.Reader reader = null;
        fs.open(new Path(filePath));
        try {
            reader = new SequenceFile.Reader(fs, new Path(filePath), conf);
            IntWritable key = new IntWritable();
            DoubleWritable value = new DoubleWritable();
            while (reader.next(key, value)) {
                indeces.add(key.get());
                values.add(value.get());
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        } finally {
            if (reader != null)
                reader.close();
        }
    }
    DenseVectorWritable result = new DenseVectorWritable();
    result.setSize(indeces.size());
    for (int i = 0; i < indeces.size(); i++)
        result.addCell(indeces.get(i), values.get(i));
    writeToFile(resultOutputPath.toString(), result, conf);
    return resultOutputPath.toString();
}

From source file:org.apache.hama.graph.AggregationRunner.java

License:Apache License

/**
 * Receives aggregated values from a master task.
 * // w ww .  ja va 2  s.c  o  m
 * @return always true if no aggregators are defined, false if aggregators say
 *         we haven't seen any messages anymore.
 */
public boolean receiveAggregatedValues(MapWritable updatedValues, long iteration)
        throws IOException, SyncException, InterruptedException {
    // map is the first value that is in the queue
    for (int i = 0; i < aggregators.length; i++) {
        globalAggregatorResult[i] = updatedValues.get(aggregatorValueFlag[i]);
        globalAggregatorIncrement[i] = (IntWritable) updatedValues.get(aggregatorIncrementFlag[i]);
    }
    IntWritable count = (IntWritable) updatedValues.get(GraphJobRunner.FLAG_MESSAGE_COUNTS);
    if (count != null && count.get() == Integer.MIN_VALUE) {
        return false;
    }
    return true;
}

From source file:org.apache.hama.graph.MaxAggregator.java

License:Apache License

@Override
public void aggregate(IntWritable value) {
    if (value.get() > max) {
        max = value.get();
    }
}