Example usage for org.apache.hadoop.io IntWritable get

List of usage examples for org.apache.hadoop.io IntWritable get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. IntWritable
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable get.

Prototype

public int get()

Source Link

Document

Return the value of this IntWritable.

Usage

From source file:org.apache.flink.streaming.connectors.fs.bucketing.BucketingSinkTest.java

License:Apache License

/**
 * This tests {@link SequenceFileWriter}
 * with non-rolling output and without compression.
 *///from w w  w . j  av  a  2  s . c o m
@Test
public void testNonRollingSequenceFileWithoutCompressionWriter() throws Exception {
    final String outPath = hdfsURI + "/seq-no-comp-non-rolling-out";

    final int numElements = 20;

    BucketingSink<Tuple2<IntWritable, Text>> sink = new BucketingSink<Tuple2<IntWritable, Text>>(outPath)
            .setWriter(new SequenceFileWriter<IntWritable, Text>())
            .setBucketer(new BasePathBucketer<Tuple2<IntWritable, Text>>()).setPartPrefix(PART_PREFIX)
            .setPendingPrefix("").setPendingSuffix("");

    sink.setInputType(TypeInformation.of(new TypeHint<Tuple2<IntWritable, Text>>() {
    }), new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<IntWritable, Text>, Object> testHarness = createTestSink(sink, 1,
            0);

    testHarness.setProcessingTime(0L);

    testHarness.setup();
    testHarness.open();

    for (int i = 0; i < numElements; i++) {
        testHarness.processElement(
                new StreamRecord<>(Tuple2.of(new IntWritable(i), new Text("message #" + Integer.toString(i)))));
    }

    testHarness.close();

    FSDataInputStream inStream = dfs.open(new Path(outPath + "/" + PART_PREFIX + "-0-0"));

    SequenceFile.Reader reader = new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());

    IntWritable intWritable = new IntWritable();
    Text txt = new Text();

    for (int i = 0; i < numElements; i++) {
        reader.next(intWritable, txt);
        Assert.assertEquals(i, intWritable.get());
        Assert.assertEquals("message #" + i, txt.toString());
    }

    reader.close();
    inStream.close();
}

From source file:org.apache.flink.streaming.connectors.fs.RollingSinkITCase.java

License:Apache License

/**
 * This tests {@link SequenceFileWriter}
 * with non-rolling output and without compression.
 *///  w  w  w .ja  va 2 s  .co m
@Test
public void testNonRollingSequenceFileWithoutCompressionWriter() throws Exception {
    final int NUM_ELEMENTS = 20;
    final int PARALLELISM = 2;
    final String outPath = hdfsURI + "/seq-no-comp-non-rolling-out";
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(PARALLELISM);

    DataStream<Tuple2<Integer, String>> source = env.addSource(new TestSourceFunction(NUM_ELEMENTS)).broadcast()
            .filter(new OddEvenFilter());

    DataStream<Tuple2<IntWritable, Text>> mapped = source
            .map(new MapFunction<Tuple2<Integer, String>, Tuple2<IntWritable, Text>>() {
                private static final long serialVersionUID = 1L;

                @Override
                public Tuple2<IntWritable, Text> map(Tuple2<Integer, String> value) throws Exception {
                    return Tuple2.of(new IntWritable(value.f0), new Text(value.f1));
                }
            });

    RollingSink<Tuple2<IntWritable, Text>> sink = new RollingSink<Tuple2<IntWritable, Text>>(outPath)
            .setWriter(new SequenceFileWriter<IntWritable, Text>()).setBucketer(new NonRollingBucketer())
            .setPartPrefix("part").setPendingPrefix("").setPendingSuffix("");

    mapped.addSink(sink);

    env.execute("RollingSink String Write Test");

    FSDataInputStream inStream = dfs.open(new Path(outPath + "/part-0-0"));

    SequenceFile.Reader reader = new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());

    IntWritable intWritable = new IntWritable();
    Text txt = new Text();

    for (int i = 0; i < NUM_ELEMENTS; i += 2) {
        reader.next(intWritable, txt);
        Assert.assertEquals(i, intWritable.get());
        Assert.assertEquals("message #" + i, txt.toString());
    }

    reader.close();
    inStream.close();

    inStream = dfs.open(new Path(outPath + "/part-1-0"));

    reader = new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());

    for (int i = 1; i < NUM_ELEMENTS; i += 2) {
        reader.next(intWritable, txt);
        Assert.assertEquals(i, intWritable.get());
        Assert.assertEquals("message #" + i, txt.toString());
    }

    reader.close();
    inStream.close();
}

From source file:org.apache.flink.streaming.connectors.fs.RollingSinkITCase.java

License:Apache License

/**
 * This tests {@link SequenceFileWriter}
 * with non-rolling output but with compression.
 *//*from ww  w . jav a 2  s . com*/
@Test
public void testNonRollingSequenceFileWithCompressionWriter() throws Exception {
    final int NUM_ELEMENTS = 20;
    final int PARALLELISM = 2;
    final String outPath = hdfsURI + "/seq-non-rolling-out";
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(PARALLELISM);

    DataStream<Tuple2<Integer, String>> source = env.addSource(new TestSourceFunction(NUM_ELEMENTS)).broadcast()
            .filter(new OddEvenFilter());

    DataStream<Tuple2<IntWritable, Text>> mapped = source
            .map(new MapFunction<Tuple2<Integer, String>, Tuple2<IntWritable, Text>>() {
                private static final long serialVersionUID = 1L;

                @Override
                public Tuple2<IntWritable, Text> map(Tuple2<Integer, String> value) throws Exception {
                    return Tuple2.of(new IntWritable(value.f0), new Text(value.f1));
                }
            });

    RollingSink<Tuple2<IntWritable, Text>> sink = new RollingSink<Tuple2<IntWritable, Text>>(outPath)
            .setWriter(new SequenceFileWriter<IntWritable, Text>("Default", SequenceFile.CompressionType.BLOCK))
            .setBucketer(new NonRollingBucketer()).setPartPrefix("part").setPendingPrefix("")
            .setPendingSuffix("");

    mapped.addSink(sink);

    env.execute("RollingSink String Write Test");

    FSDataInputStream inStream = dfs.open(new Path(outPath + "/part-0-0"));

    SequenceFile.Reader reader = new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());

    IntWritable intWritable = new IntWritable();
    Text txt = new Text();

    for (int i = 0; i < NUM_ELEMENTS; i += 2) {
        reader.next(intWritable, txt);
        Assert.assertEquals(i, intWritable.get());
        Assert.assertEquals("message #" + i, txt.toString());
    }

    reader.close();
    inStream.close();

    inStream = dfs.open(new Path(outPath + "/part-1-0"));

    reader = new SequenceFile.Reader(inStream, 1000, 0, 100000, new Configuration());

    for (int i = 1; i < NUM_ELEMENTS; i += 2) {
        reader.next(intWritable, txt);
        Assert.assertEquals(i, intWritable.get());
        Assert.assertEquals("message #" + i, txt.toString());
    }

    reader.close();
    inStream.close();
}

From source file:org.apache.flink.tez.runtime.output.SimplePartitioner.java

License:Apache License

@Override
public int getPartition(Object key, Object value, int numPartitions) {
    if (!(key instanceof IntWritable)) {
        throw new IllegalStateException("Partitioning key should be int");
    }//  w w w.  jav  a  2s  . co  m
    IntWritable channel = (IntWritable) key;
    return channel.get();
}

From source file:org.apache.giraph.aggregators.IntMaxAggregator.java

License:Apache License

@Override
public void aggregate(IntWritable value) {
    getAggregatedValue().set(Math.max(getAggregatedValue().get(), value.get()));
}

From source file:org.apache.giraph.aggregators.IntMinAggregator.java

License:Apache License

@Override
public void aggregate(IntWritable value) {
    getAggregatedValue().set(Math.min(getAggregatedValue().get(), value.get()));
}

From source file:org.apache.giraph.aggregators.IntOverwriteAggregator.java

License:Apache License

@Override
public void aggregate(IntWritable value) {
    getAggregatedValue().set(value.get());
}

From source file:org.apache.giraph.aggregators.IntProductAggregator.java

License:Apache License

@Override
public void aggregate(IntWritable value) {
    getAggregatedValue().set(getAggregatedValue().get() * value.get());
}

From source file:org.apache.giraph.aggregators.IntSumAggregator.java

License:Apache License

@Override
public void aggregate(IntWritable value) {
    getAggregatedValue().set(getAggregatedValue().get() + value.get());
}

From source file:org.apache.giraph.block_app.library.algo.MultiSeedBreadthFirstSearchBlockFactory.java

License:Apache License

private static TripleFunction<Vertex<LongWritable, MultiSeedBreadthFirstSearchVertexValue, Writable>, IntWritable, Iterator<IntWritable>, IntWritable> traverseVertex() {
    IntWritable notReachableVertex = new IntWritable(-1);
    IntWritable vertexValue = new IntWritable();
    IntWritable reservoirValue = new IntWritable();
    TransientRandom random = new TransientRandom();
    IntWritable reusableMessage = new IntWritable();
    // Reservoir sampling to select the seed from the set of messages received
    return (vertex, distance, messageIter) -> {
        vertexValue.set(vertex.getValue().getDistance());
        if (vertexValue.compareTo(notReachableVertex) == 0 || vertexValue.compareTo(distance) > 0) {
            reservoirValue.set(messageIter.next().get());
            int messageIndex = 1;
            while (messageIter.hasNext()) {
                if (random.nextInt(messageIndex + 1) < 1) {
                    reservoirValue.set(messageIter.next().get());
                } else {
                    messageIter.next();//from w  w  w  .  ja  va 2 s .  c om
                }
                messageIndex++;
            }
            vertex.getValue().setSourceIndex(reservoirValue.get());
            vertex.getValue().setDistance(distance.get());
            reusableMessage.set(vertex.getValue().getSourceIndex());
            return reusableMessage;
        } else {
            return null;
        }
    };
}