Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void filterPredicateTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/* w  w w.j  a  v  a2  s.c  o m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://schema.org/p1")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://context/1")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s1")),
                    with(new TextMatcher("<http://schema.org/p1> \"o1\" <http://context/1> .")));
            one(mrContext).write(with(new TextMatcher("http://schema.org/p2")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://context/1")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s2")),
                    with(new TextMatcher("<http://schema.org/p2> \"o2\" <http://context/1> .")));
            one(mrContext).write(with(new TextMatcher("http://schema.org/p4")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://o4")), with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://context/2")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s3")),
                    with(new TextMatcher("<http://schema.org/p4> <http://o4> <http://context/2> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();

    RegexTupleFilter filter = new RegexTupleFilter();
    filter.setPredicateRegex("schema\\.org");
    mapper.setFilter(filter);

    mapper.map(new LongWritable(5l), new Text("<http://s1> <http://schema.org/p1> \"o1\" <http://context/1> ."),
            mrContext);
    mapper.map(new LongWritable(6l), new Text("<http://s2> <http://schema.org/p2> \"o2\" <http://context/1> ."),
            mrContext);
    mapper.map(new LongWritable(7l),
            new Text("<http://s2> <http://nothing.org/p3> \"o3\" <http://context/1> ."), mrContext);
    mapper.map(new LongWritable(8l),
            new Text("<http://s3> <http://schema.org/p4> <http://o4> <http://context/2> ."), mrContext);
    mapper.map(new LongWritable(9l),
            new Text("<http://s3> <http://nothing.org/p5> <http://o5> <http://context/2> ."), mrContext);

    context.assertIsSatisfied();
}

---

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void partial1ModeStringValuesExplicitParameters() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, intConstantInspector,
            floatConstantInspector, longConstantInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkIntermediateResultInspector(resultInspector);

    final long seed = 1;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] { new Text("a"), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });
    eval.iterate(state, new Object[] { new Text("b"), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);//from  w  w w  .  j  av  a 2 s. co  m
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), 8);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), seed);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()),
            seed);
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);

    // check if seed is correct in the result
    Union union = SetOperation.builder().setSeed(seed).buildUnion();
    // this must fail if the seed is incompatible
    union.update(resultSketch);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void partial2Mode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, new ObjectInspector[] { structInspector });
    checkIntermediateResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);// ww w  .j a v a 2 s.c o  m
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), DEFAULT_NOMINAL_ENTRIES);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), DEFAULT_UPDATE_SEED);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void finalMode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.FINAL, new ObjectInspector[] { structInspector });
    checkFinalResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);/*from   w  w  w .ja va2 s . co m*/
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void completeModeDoubleValuesExplicitParameters() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, intConstantInspector,
            floatConstantInspector, longConstantInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    final long seed = 2;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] { new DoubleWritable(1), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });
    eval.iterate(state, new Object[] { new DoubleWritable(2), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);//from   ww  w. j a v a 2s.  c  o m
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()), seed);
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.UnionEvaluator.java

License:Apache License

@Override
public Object terminatePartial(final @SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    final UnionState state = (UnionState) agg;
    final Sketch intermediate = state.getResult();
    if (intermediate == null)
        return null;
    final byte[] bytes = intermediate.toByteArray();
    // sampling probability is not relevant for merging
    return Arrays.asList(new IntWritable(state.getNominalEntries()), new LongWritable(state.getSeed()),
            new BytesWritable(bytes));
}

---

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void partial1ModeExplicitParams() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, intInspector, longInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    DataToSketchUDAFTest.checkIntermediateResultInspector(resultInspector);

    final int nomEntries = 8;
    final long seed = 1;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().setSeed(seed).build();
    sketch1.update(1);/*from  w  ww .ja  v a 2  s  .com*/
    eval.iterate(state, new Object[] { new BytesWritable(sketch1.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    UpdateSketch sketch2 = UpdateSketch.builder().setSeed(seed).build();
    sketch2.update(2);
    eval.iterate(state, new Object[] { new BytesWritable(sketch2.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), nomEntries);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), seed);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()),
            seed);
    Assert.assertFalse(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void partial2Mode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, new ObjectInspector[] { structInspector });
    DataToSketchUDAFTest.checkIntermediateResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);//from  w ww .  j  a  v  a2s  .c o  m
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), DEFAULT_NOMINAL_ENTRIES);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), DEFAULT_UPDATE_SEED);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void finalMode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.FINAL, new ObjectInspector[] { structInspector });
    DataToSketchUDAFTest.checkFinalResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);/* w  w  w .  j a v a  2 s.c o  m*/
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

---

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void completeModeExplicitSizeAndSeed() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, intInspector, longInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    DataToSketchUDAFTest.checkFinalResultInspector(resultInspector);

    final int nomEntries = 8;
    final long seed = 1;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().setSeed(seed).build();
    sketch1.update(1);//  w  w w . j ava 2s  .co m
    eval.iterate(state, new Object[] { new BytesWritable(sketch1.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    UpdateSketch sketch2 = UpdateSketch.builder().setSeed(seed).build();
    sketch2.update(2);
    eval.iterate(state, new Object[] { new BytesWritable(sketch2.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()), seed);
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »