Example usage for org.apache.hadoop.io IntWritable set

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable set.

Prototype

public void set(int value) 

Source Link

Document

Set the value of this IntWritable.

Usage

From source file:org.apache.accumulo.tserver.log.RecoveryLogsReaderTest.java

License:Apache License

@Test
public void testMultiReader() throws IOException {
    Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
    RecoveryLogReader reader = new RecoveryLogReader(fs, manyMaps);
    IntWritable key = new IntWritable();
    BytesWritable value = new BytesWritable();

    for (int i = 0; i < 1000; i++) {
        if (i == 10)
            continue;
        assertTrue(reader.next(key, value));
        assertEquals(i, key.get());/*w ww.j a va  2  s. c  o  m*/
    }
    assertEquals(value.compareTo(new BytesWritable("someValue".getBytes())), 0);
    assertFalse(reader.next(key, value));

    key.set(500);
    assertTrue(reader.seek(key));
    scan(reader, 500);
    key.set(10);
    assertFalse(reader.seek(key));
    scan(reader, 10);
    key.set(1000);
    assertFalse(reader.seek(key));
    assertFalse(reader.next(key, value));
    key.set(-1);
    assertFalse(reader.seek(key));
    key.set(0);
    assertTrue(reader.next(key, value));
    assertEquals(0, key.get());
    reader.close();

    fs.deleteRecursively(new Path(manyMaps, "even"));
    reader = new RecoveryLogReader(fs, manyMaps);
    key.set(501);
    assertTrue(reader.seek(key));
    scanOdd(reader, 501);
    key.set(1000);
    assertFalse(reader.seek(key));
    assertFalse(reader.next(key, value));
    key.set(-1);
    assertFalse(reader.seek(key));
    key.set(1);
    assertTrue(reader.next(key, value));
    assertEquals(1, key.get());
    reader.close();

}

---

From source file:org.apache.blur.mapreduce.lib.v2.DirectIndexingDriverTest.java

License:Apache License

public static void createInputDocument(FileSystem fileSystem, Configuration configuration, Path path)
        throws IOException {
    Writer writer = SequenceFile.createWriter(fileSystem, configuration, new Path(path, "data"),
            IntWritable.class, DocumentWritable.class);
    IntWritable docId = new IntWritable();
    DocumentWritable documentWritable = new DocumentWritable();
    int numberOfFields = 10;
    Random random = new Random();
    for (int i = 0; i < 100; i++) {
        docId.set(i);
        documentWritable.clear();//from   w  w w. j a v  a  2 s. c o m
        populate(numberOfFields, random, documentWritable);
        writer.append(docId, documentWritable);
    }
    writer.close();
}

---

From source file:org.apache.camel.component.hdfs.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadInt() throws Exception {
    if (!canTest()) {
        return;/* w  ww.j a  v  a2 s .c  o m*/
    }

    final Path file = new Path(new File("target/test/test-camel-int").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs1 = FileSystem.get(file.toUri(), conf);
    SequenceFile.Writer writer = createWriter(fs1, conf, file, NullWritable.class, IntWritable.class);
    NullWritable keyWritable = NullWritable.get();
    IntWritable valueWritable = new IntWritable();
    int value = 314159265;
    valueWritable.set(value);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0")
                    .to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

---

From source file:org.apache.camel.component.hdfs2.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadInt() throws Exception {
    if (!canTest()) {
        return;/*from  ww  w  .  ja v  a 2 s.  c o m*/
    }

    final Path file = new Path(new File("target/test/test-camel-int").getAbsolutePath());
    Configuration conf = new Configuration();
    SequenceFile.Writer writer = createWriter(conf, file, NullWritable.class, IntWritable.class);
    NullWritable keyWritable = NullWritable.get();
    IntWritable valueWritable = new IntWritable();
    int value = 314159265;
    valueWritable.set(value);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs2:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0")
                    .to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

---

From source file:org.apache.giraph.block_app.framework.piece.global_comm.ReduceUtilsObject.java

License:Apache License

public void reduceInt(ReducerHandle<IntWritable, ?> reduceHandle, int value) {
    IntWritable tmp = reusableInt;
    tmp.set(value);
    reduceHandle.reduce(tmp);/* w w w . j av  a  2s.c  om*/
}

---

From source file:org.apache.giraph.block_app.library.algo.BreadthFirstSearch.java

License:Apache License

/**
 * Default block for computing breadth-first search distances given functions
 * isVertexInSeedSet, getDistance and setDistance. This BFS computation block
 * computes only the shortest distance to seed vertices and does not compute
 * the closest seed.//ww  w  .ja  v  a2s .c om
 */
public static <I extends WritableComparable, V extends Writable> Block bfs(
        SupplierFromVertex<I, V, Writable, Boolean> isVertexInSeedSet,
        SupplierFromVertex<I, V, Writable, IntWritable> getDistance,
        ConsumerWithVertex<I, V, Writable, IntWritable> setDistance) {
    ObjectTransfer<Boolean> converged = new ObjectTransfer<>();
    ObjectTransfer<ByteWritable> vertexUpdatedDistance = new ObjectTransfer<>();

    IntWritable reusableInt = new IntWritable();
    ByteWritable emptyByteWritable = new ByteWritable();
    SupplierFromVertex<I, V, Writable, ByteWritable> initializeVertex = (vertex) -> {
        if (isVertexInSeedSet.get(vertex)) {
            reusableInt.set(0);
            setDistance.apply(vertex, reusableInt);
            return emptyByteWritable;
        } else {
            reusableInt.set(-1);
            setDistance.apply(vertex, reusableInt);
            return null;
        }
    };

    IntWritable notReachableVertex = new IntWritable(-1);
    TripleFunction<Vertex<I, V, Writable>, IntWritable, Iterator<ByteWritable>, ByteWritable> traverseVertex = (
            vertex, distance, messageIter) -> {
        if (getDistance.get(vertex).compareTo(notReachableVertex) == 0
                || getDistance.get(vertex).compareTo(distance) > 0) {
            setDistance.apply(vertex, distance);
            return emptyByteWritable;
        } else {
            return null;
        }
    };

    Class<ByteWritable> messageClass = ByteWritable.class;
    return new SequenceBlock(createInitializePiece(vertexUpdatedDistance, initializeVertex),
            RepeatUntilBlock.unlimited(createPropagateConnectedComponentsPiece(messageClass,
                    vertexUpdatedDistance, vertexUpdatedDistance, converged, traverseVertex,
                    VertexSuppliers.vertexEdgesSupplier()), converged));
}

---

From source file:org.apache.giraph.block_app.library.algo.MultiSeedBreadthFirstSearchBlockFactory.java

License:Apache License

private static TripleFunction<Vertex<LongWritable, MultiSeedBreadthFirstSearchVertexValue, Writable>, IntWritable, Iterator<IntWritable>, IntWritable> traverseVertex() {
    IntWritable notReachableVertex = new IntWritable(-1);
    IntWritable vertexValue = new IntWritable();
    IntWritable reservoirValue = new IntWritable();
    TransientRandom random = new TransientRandom();
    IntWritable reusableMessage = new IntWritable();
    // Reservoir sampling to select the seed from the set of messages received
    return (vertex, distance, messageIter) -> {
        vertexValue.set(vertex.getValue().getDistance());
        if (vertexValue.compareTo(notReachableVertex) == 0 || vertexValue.compareTo(distance) > 0) {
            reservoirValue.set(messageIter.next().get());
            int messageIndex = 1;
            while (messageIter.hasNext()) {
                if (random.nextInt(messageIndex + 1) < 1) {
                    reservoirValue.set(messageIter.next().get());
                } else {
                    messageIter.next();/*from w  w  w.j a va2 s  . c  o m*/
                }
                messageIndex++;
            }
            vertex.getValue().setSourceIndex(reservoirValue.get());
            vertex.getValue().setDistance(distance.get());
            reusableMessage.set(vertex.getValue().getSourceIndex());
            return reusableMessage;
        } else {
            return null;
        }
    };
}

---

From source file:org.apache.giraph.block_app.library.algo.MultiSeedBreadthFirstSearchBlockFactory.java

License:Apache License

@Override
public Block createBlock(GiraphConfiguration conf) {
    Long2IntOpenHashMap seeds = new Long2IntOpenHashMap();
    Piece pickSeedVertices = null;//from  www . j  a v a2 s  .  c  o  m
    if (RANDOM_SEED_COUNT.get(conf) > 0) {
        TransientRandom random = new TransientRandom();
        pickSeedVertices = Pieces.reduce("SeedSelection",
                new TopNReduce<VertexLongPair<LongWritable>>(RANDOM_SEED_COUNT.get(conf)), (vertex) -> {
                    return new VertexLongPair<LongWritable>((LongWritable) vertex.getId(),
                            random.get().nextLong());
                }, (result) -> {
                    PriorityQueue<VertexLongPair<LongWritable>> queue = result.get();
                    int index = 0;
                    while (!queue.isEmpty()) {
                        VertexLongPair<LongWritable> nextPair = queue.poll();
                        seeds.put(nextPair.getVertex().get(), index++);
                    }
                });
    } else {
        String[] sepStr = SEED_LIST.get(conf).split(", ");
        for (int ii = 0; ii < sepStr.length; ++ii) {
            seeds.put(Long.parseLong(sepStr[ii]), ii);
        }
    }

    IntWritable reusableMessage = new IntWritable();
    SupplierFromVertex<LongWritable, MultiSeedBreadthFirstSearchVertexValue, Writable, IntWritable> initializeVertex = (
            vertex) -> {
        if (seeds.containsKey(vertex.getId().get())) {
            vertex.getValue().setDistance(0);
            vertex.getValue().setSourceIndex(seeds.get(vertex.getId().get()));
            reusableMessage.set(vertex.getValue().getSourceIndex());
            return reusableMessage;
        } else {
            vertex.getValue().setDistance(-1);
            vertex.getValue().setSourceIndex(-1);
            return null;
        }
    };

    if (RANDOM_SEED_COUNT.get(conf) > 0) {
        return new SequenceBlock(pickSeedVertices,
                BreadthFirstSearch.bfs(IntWritable.class, initializeVertex, traverseVertex()));
    } else {
        return BreadthFirstSearch.bfs(IntWritable.class, initializeVertex, traverseVertex());
    }
}

---

From source file:org.apache.giraph.block_app.library.ReusableSuppliers.java

License:Apache License

/**
 * Transforms primitive int supplier into
 * IntWritable supplier, with object being reused,
 * to minimize GC overhead.//from www  .ja v  a2  s.  c  om
 */
public static <I extends WritableComparable, V extends Writable, E extends Writable> SupplierFromVertex<I, V, E, IntWritable> fromInt(
        Obj2IntFunction<Vertex<I, V, E>> supplier) {
    IntWritable reusable = new IntWritable();
    return (vertex) -> {
        reusable.set(supplier.apply(vertex));
        return reusable;
    };
}

---

From source file:org.apache.giraph.combiner.MinimumIntCombiner.java

License:Apache License

@Override
public void combine(IntWritable vertexIndex, IntWritable originalMessage, IntWritable messageToCombine) {
    if (originalMessage.get() > messageToCombine.get()) {
        originalMessage.set(messageToCombine.get());
    }//from ww w  . j  av a 2  s  .c  om
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• »