Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.conversantmedia.mapreduce.example.avro.AvroMultipleOutputExampleTest.java

License:Apache License

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testAvroMultipleOut() {

    ReduceDriver<Text, LongWritable, AvroKey<AvroExample>, NullWritable> driver = this.getTestDriver();
    LongWritable ONE = new LongWritable(1);
    List<String> words = new ArrayList<>();
    words.add("10");
    words.add("20");
    words.add("30");
    words.add("40");

    for (String word : words) {
        List<LongWritable> counts = new ArrayList<>();
        for (int i = 0; i < Long.parseLong(word); i++) {
            counts.add(ONE);/*from  w ww . j  a  va  2 s  .c  o m*/
        }
        driver.addInput(new Text(word), counts);
    }

    try {
        List<Pair<AvroKey<AvroExample>, NullWritable>> results = driver.run();
        assertThat(results.size(), equalTo(words.size()));
        for (Pair<AvroKey<AvroExample>, NullWritable> e : results) {
            validateRecords(e.getFirst().datum());
        }

        AvroMultipleOutputs avroOut = this.getAvroNamedOutput();
        ArgumentCaptor<AvroKey> recordArg = ArgumentCaptor.forClass(AvroKey.class);
        try {
            verify(avroOut, times(words.size())).write(recordArg.capture(), eq(NullWritable.get()),
                    eq("avroMulti"));
            for (AvroKey<AvroExample> record : recordArg.getAllValues()) {
                validateRecords(record.datum());
            }
        } catch (InterruptedException e) {
            fail(e.getMessage());
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

---

From source file:com.conversantmedia.mapreduce.example.distribute.DistributedWordCountMapReduceTest.java

License:Apache License

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void runMyTest() {

    List<Pair<LongWritable, Text>> inputs = new ArrayList<>();
    inputs.add(new Pair<>(new LongWritable(1), new Text("the quick brown fox jumped over the lazy dog.")));

    MapReduceDriver driver = getTestDriver();
    driver.addAll(inputs);// w w w .j  a  v a  2 s .  c o  m

    try {
        driver.runTest();
    } catch (IOException e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

---

From source file:com.conversantmedia.mapreduce.example.distribute.NamedOutputsExampleTest.java

License:Apache License

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testNamedOutputs() {

    String lineStr = "a b c d e f a";

    List<Pair<LongWritable, Text>> inputs = new ArrayList<>();
    LongWritable offset = new LongWritable(0);
    Text line = new Text(lineStr);
    inputs.add(new Pair<>(offset, line));

    LongWritable ONE = new LongWritable(1l);
    List<Pair<Text, LongWritable>> outputs = new ArrayList<>();
    outputs.add(new Pair<>(new Text("a"), new LongWritable(2l)));
    outputs.add(new Pair<>(new Text("b"), ONE));
    outputs.add(new Pair<>(new Text("c"), ONE));
    outputs.add(new Pair<>(new Text("d"), ONE));
    outputs.add(new Pair<>(new Text("e"), ONE));
    outputs.add(new Pair<>(new Text("f"), ONE));

    MapReduceDriver driver = getTestDriver();
    driver.addAll(inputs);//from www.j av a 2s .  c  o  m
    driver.addAllOutput(outputs);

    try {
        driver.runTest();

        // Check that our DEBUG line was written to the multiout
        verifyNamedOutput("DEBUG", new Text(lineStr.length() + ":"), line);

        // Example of how we can grab the (mock) MultipleOutput directly if needed
        MultipleOutputs multiOut = this.getNamedOutput();
        assertNotNull(multiOut);

    } catch (IOException e) {
        e.printStackTrace();
        fail(e.getMessage());
    }

}

---

From source file:com.conversantmedia.mapreduce.example.WordCountReducer.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    int sum = 0;//from   ww w  .  j a v a 2s. com
    for (LongWritable value : values) {
        sum += value.get();
    }
    context.write(key, new LongWritable(sum));
}

---

From source file:com.conversantmedia.mapreduce.example.WordCountReducerWithMinimum.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    int sum = 0;/*from  w  ww.  java 2 s .  c om*/
    for (LongWritable value : values) {
        sum += value.get();
    }

    // Ensure we're above the minimum
    if (sum >= minimum) {
        context.write(key, new LongWritable(sum));
    }
}

---

From source file:com.conversantmedia.mapreduce.io.CompositeSortKeyTest.java

License:Apache License

private void setupInputs(int offset) {
    for (int i = 0; i < offset; i++) {
        int i2 = i + offset;
        int i3 = i + offset * 2;
        driver.addInput(new LongWritable(i3), new Text("B," + i3 + ",B" + i3));
        driver.addInput(new LongWritable(i), new Text("B," + i + ",B" + i));
        driver.addInput(new LongWritable(i2), new Text("A," + i2 + ",A" + i2));
        driver.addInput(new LongWritable(i), new Text("A," + i + ",A" + i));
        driver.addInput(new LongWritable(i3), new Text("A," + i3 + ",A" + i3));
        driver.addInput(new LongWritable(i2), new Text("B," + i2 + ",B" + i2));
    }/*from   w  w  w  .  jav a2  s .c om*/
}

---

From source file:com.csiro.hadoop.WritableTest.java

public static void main(String[] args) {
    System.out.println("*** Primitive Writable ***");

    BooleanWritable bool1 = new BooleanWritable(true);
    ByteWritable byte1 = new ByteWritable((byte) 3);
    System.out.printf("Boolean:%s Byte:%d\n", bool1, byte1.get());

    IntWritable int1 = new IntWritable(5);
    IntWritable int2 = new IntWritable(17);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    int1.set(int2.get());
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    Integer int3 = new Integer(23);
    int1.set(int3);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    System.out.println("*** Array Writable ***");

    ArrayWritable a = new ArrayWritable(IntWritable.class);
    a.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] values = (IntWritable[]) a.get();
    for (IntWritable i : values) {
        System.out.println(i);/*from   www  .ja  va  2s  .  c  om*/
    }

    IntArrayWritable ia = new IntArrayWritable();
    ia.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] ivalues = (IntWritable[]) ia.get();

    ia.set((new LongWritable[] { new LongWritable(10001) }));

    System.out.println("*** Map Writables ***");

    MapWritable m = new MapWritable();
    IntWritable key1 = new IntWritable(5);
    NullWritable value1 = NullWritable.get();

    m.put(key1, value1);
    System.out.println(m.containsKey(key1));
    System.out.println(m.get(key1));
    m.put(new LongWritable(100000000), key1);
    Set<Writable> keys = m.keySet();

    for (Writable k : keys)
        System.out.println(k.getClass());

}

---

From source file:com.dasasian.chok.util.WritableType.java

License:Apache License

/**
 * Convert a java primitive type wrapper (like String, Integer, Float, etc...)
 * to the corresponding hadoop {@link WritableComparable}.
 * @param object the object to convert/*from   ww  w. j ava2  s.  com*/
 * @return the writable comparable
 */
public WritableComparable convertComparable(Object object) {
    switch (this) {
    case TEXT:
        return new Text((String) object);
    case BYTE:
        return new ByteWritable((Byte) object);
    case INT:
        return new IntWritable((Integer) object);
    case LONG:
        return new LongWritable(((Long) object));
    case FLOAT:
        return new FloatWritable((Float) object);
    case DOUBLE:
        return new DoubleWritable((Double) object);
    }
    throw getUnhandledTypeException();
}

---

From source file:com.datasalt.pangool.utils.test.AbstractHadoopTestLibrary.java

License:Apache License

public Writable writable(Object obj) {
    if (obj instanceof Integer) {
        return new IntWritable((Integer) obj);
    } else if (obj instanceof Double) {
        return new DoubleWritable((Double) obj);
    } else if (obj instanceof Long) {
        return new LongWritable((Long) obj);
    } else if (obj instanceof String) {
        return new Text((String) obj);
    } else if (obj instanceof Float) {
        return new FloatWritable((Float) obj);
    } else if (obj instanceof Boolean) {
        return new BooleanWritable((Boolean) obj);
    }/*from  w  ww . j av  a2s . co  m*/
    return null;
}

---

From source file:com.ebay.nest.io.sede.lazy.LazyLong.java

License:Apache License

public LazyLong(LazyLong copy) {
    super(copy);
    data = new LongWritable(copy.data.get());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »