Example usage for org.apache.hadoop.io LongWritable LongWritable

List of usage examples for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void filterPredicateTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/* w  w w.j  a  v  a2  s.c  o m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://schema.org/p1")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://context/1")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s1")),
                    with(new TextMatcher("<http://schema.org/p1> \"o1\" <http://context/1> .")));
            one(mrContext).write(with(new TextMatcher("http://schema.org/p2")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://context/1")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s2")),
                    with(new TextMatcher("<http://schema.org/p2> \"o2\" <http://context/1> .")));
            one(mrContext).write(with(new TextMatcher("http://schema.org/p4")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://o4")), with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://context/2")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s3")),
                    with(new TextMatcher("<http://schema.org/p4> <http://o4> <http://context/2> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();

    RegexTupleFilter filter = new RegexTupleFilter();
    filter.setPredicateRegex("schema\\.org");
    mapper.setFilter(filter);

    mapper.map(new LongWritable(5l), new Text("<http://s1> <http://schema.org/p1> \"o1\" <http://context/1> ."),
            mrContext);
    mapper.map(new LongWritable(6l), new Text("<http://s2> <http://schema.org/p2> \"o2\" <http://context/1> ."),
            mrContext);
    mapper.map(new LongWritable(7l),
            new Text("<http://s2> <http://nothing.org/p3> \"o3\" <http://context/1> ."), mrContext);
    mapper.map(new LongWritable(8l),
            new Text("<http://s3> <http://schema.org/p4> <http://o4> <http://context/2> ."), mrContext);
    mapper.map(new LongWritable(9l),
            new Text("<http://s3> <http://nothing.org/p5> <http://o5> <http://context/2> ."), mrContext);

    context.assertIsSatisfied();
}

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void partial1ModeStringValuesExplicitParameters() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, intConstantInspector,
            floatConstantInspector, longConstantInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkIntermediateResultInspector(resultInspector);

    final long seed = 1;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] { new Text("a"), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });
    eval.iterate(state, new Object[] { new Text("b"), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);//from  w  w w  .  j  av  a 2 s. co  m
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), 8);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), seed);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()),
            seed);
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);

    // check if seed is correct in the result
    Union union = SetOperation.builder().setSeed(seed).buildUnion();
    // this must fail if the seed is incompatible
    union.update(resultSketch);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void partial2Mode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, new ObjectInspector[] { structInspector });
    checkIntermediateResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);// ww w  .j a v a 2 s.c o  m
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), DEFAULT_NOMINAL_ENTRIES);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), DEFAULT_UPDATE_SEED);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void finalMode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { intInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.FINAL, new ObjectInspector[] { structInspector });
    checkFinalResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);/*from   w  w  w .ja va2 s . co m*/
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.DataToSketchUDAFTest.java

License:Apache License

@Test
public void completeModeDoubleValuesExplicitParameters() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, intConstantInspector,
            floatConstantInspector, longConstantInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    final long seed = 2;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] { new DoubleWritable(1), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });
    eval.iterate(state, new Object[] { new DoubleWritable(2), new IntWritable(8), new FloatWritable(0.99f),
            new LongWritable(seed) });

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);//from   ww  w. j a v a 2s.  c  o m
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()), seed);
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.UnionEvaluator.java

License:Apache License

@Override
public Object terminatePartial(final @SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    final UnionState state = (UnionState) agg;
    final Sketch intermediate = state.getResult();
    if (intermediate == null)
        return null;
    final byte[] bytes = intermediate.toByteArray();
    // sampling probability is not relevant for merging
    return Arrays.asList(new IntWritable(state.getNominalEntries()), new LongWritable(state.getSeed()),
            new BytesWritable(bytes));
}

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void partial1ModeExplicitParams() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, intInspector, longInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    DataToSketchUDAFTest.checkIntermediateResultInspector(resultInspector);

    final int nomEntries = 8;
    final long seed = 1;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().setSeed(seed).build();
    sketch1.update(1);/*from  w  ww .ja  v a 2  s  .com*/
    eval.iterate(state, new Object[] { new BytesWritable(sketch1.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    UpdateSketch sketch2 = UpdateSketch.builder().setSeed(seed).build();
    sketch2.update(2);
    eval.iterate(state, new Object[] { new BytesWritable(sketch2.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), nomEntries);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), seed);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()),
            seed);
    Assert.assertFalse(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void partial2Mode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL2, new ObjectInspector[] { structInspector });
    DataToSketchUDAFTest.checkIntermediateResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);//from  w ww .  j  a  v  a2s  .c o  m
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) (r.get(0))).get(), DEFAULT_NOMINAL_ENTRIES);
    Assert.assertEquals(((LongWritable) (r.get(1))).get(), DEFAULT_UPDATE_SEED);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) (r.get(2))).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void finalMode() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.FINAL, new ObjectInspector[] { structInspector });
    DataToSketchUDAFTest.checkFinalResultInspector(resultInspector);

    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().build();
    sketch1.update(1);/* w  w  w .  j a v a  2 s.c o  m*/
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch1.compact().toByteArray())));

    UpdateSketch sketch2 = UpdateSketch.builder().build();
    sketch2.update(2);
    eval.merge(state, Arrays.asList(new IntWritable(DEFAULT_NOMINAL_ENTRIES),
            new LongWritable(DEFAULT_UPDATE_SEED), new BytesWritable(sketch2.compact().toByteArray())));

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()));
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}

From source file:com.yahoo.sketches.hive.theta.UnionSketchUDAFTest.java

License:Apache License

@Test
public void completeModeExplicitSizeAndSeed() throws Exception {
    ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, intInspector, longInspector };
    GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false);
    GenericUDAFEvaluator eval = new UnionSketchUDAF().getEvaluator(info);
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    DataToSketchUDAFTest.checkFinalResultInspector(resultInspector);

    final int nomEntries = 8;
    final long seed = 1;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();

    UpdateSketch sketch1 = UpdateSketch.builder().setSeed(seed).build();
    sketch1.update(1);//  w  w w . j ava 2s  .co m
    eval.iterate(state, new Object[] { new BytesWritable(sketch1.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    UpdateSketch sketch2 = UpdateSketch.builder().setSeed(seed).build();
    sketch2.update(2);
    eval.iterate(state, new Object[] { new BytesWritable(sketch2.compact().toByteArray()),
            new IntWritable(nomEntries), new LongWritable(seed) });

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.heapifySketch(new NativeMemory(((BytesWritable) result).getBytes()), seed);
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);

    eval.close();
}