List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:com.kylinolap.cube.measure.LongMaxAggregator.java
License:Apache License
@Override public void aggregate(LongWritable value) { if (max == null) max = new LongWritable(value.get()); else if (max.get() < value.get()) max.set(value.get());/*w w w .j a v a 2 s .c o m*/ }
From source file:com.kylinolap.cube.measure.LongMinAggregator.java
License:Apache License
@Override public void aggregate(LongWritable value) { if (min == null) min = new LongWritable(value.get()); else if (min.get() > value.get()) min.set(value.get());//from w w w .j a v a 2s .com }
From source file:com.kylinolap.cube.measure.MeasureCodecTest.java
License:Apache License
@Test public void basicTest() { MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16") }; MeasureCodec codec = new MeasureCodec(descs); DoubleWritable d = new DoubleWritable(1.0); LongWritable l = new LongWritable(2); BigDecimal b = new BigDecimal("333.1234567"); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); Object values[] = new Object[] { d, l, b, hllc, hllc2 }; ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(values, buf);// w w w. j a v a2s .c o m buf.flip(); System.out.println("size: " + buf.limit()); Object copy[] = new Object[values.length]; codec.decode(buf, copy); assertTrue(Arrays.equals(values, copy)); }
From source file:com.kylinolap.job.hadoop.cardinality.ColumnCardinalityReducer.java
License:Apache License
@Override protected void cleanup(Context context) throws IOException, InterruptedException { List<Integer> keys = new ArrayList<Integer>(); Iterator<Integer> it = hllcMap.keySet().iterator(); while (it.hasNext()) { keys.add(it.next());//from ww w . j a va2 s .co m } Collections.sort(keys); it = keys.iterator(); while (it.hasNext()) { int key = it.next(); HyperLogLogPlusCounter hllc = hllcMap.get(key); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); buf.clear(); hllc.writeRegisters(buf); buf.flip(); context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate())); // context.write(new Text("ErrorRate_" + key), new // LongWritable((long)hllc.getErrorRate())); } }
From source file:com.kylinolap.job.hadoop.cube.CubeReducerTest.java
License:Apache License
private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) { Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) }; buf.clear();//from w ww .j ava 2s . c o m codec.encode(values, buf); Text t = new Text(); t.set(buf.array(), 0, buf.position()); return t; }
From source file:com.kylinolap.job.tools.ColumnCardinalityMapperTest.java
License:Apache License
@SuppressWarnings({ "unchecked" })
@Test/*from w w w . jav a 2s . com*/
@Ignore
public void testMapperOn177() throws IOException {
mapDriver.clearInput();
File file = new File("src/test/resources/data/test_cal_dt/part-r-00000");
FileReader reader = new FileReader(file);
BufferedReader breader = new BufferedReader(reader);
String s = breader.readLine();
int i = 0;
while (s != null) {
LongWritable inputKey = new LongWritable(i++);
mapDriver.addInput(inputKey, new Text(s));
s = breader.readLine();
}
// breader.close();
mapDriver.getConfiguration().set(HiveColumnCardinalityJob.KEY_INPUT_DELIM, "\20");
List<Pair<IntWritable, BytesWritable>> result = mapDriver.run();
breader.close();
assertEquals(9, result.size());
int key1 = result.get(0).getFirst().get();
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
hllc.readRegisters(ByteBuffer.wrap(bytes));
assertTrue(key1 > 0);
assertEquals(8, hllc.getCountEstimate());
}
From source file:com.kylinolap.job.tools.ColumnCardinalityMapperTest.java
License:Apache License
@SuppressWarnings("unchecked") @Test//from www . j av a 2s.c o m public void testMapperOnComma() throws IOException { mapDriver.clearInput(); LongWritable inputKey1 = new LongWritable(1); LongWritable inputKey2 = new LongWritable(2); LongWritable inputKey3 = new LongWritable(3); LongWritable inputKey4 = new LongWritable(4); LongWritable inputKey5 = new LongWritable(5); LongWritable inputKey6 = new LongWritable(6); LongWritable inputKey7 = new LongWritable(7); mapDriver.addInput(inputKey1, new Text()); mapDriver.addInput(inputKey2, new Text(strArr)); mapDriver.addInput(inputKey3, new Text(strArr)); mapDriver.addInput(inputKey4, new Text(strArr)); mapDriver.addInput(inputKey5, new Text(strArr)); mapDriver.addInput(inputKey6, new Text(strArr)); mapDriver.addInput(inputKey7, new Text(strArr)); List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); assertEquals(9, result.size()); int key1 = result.get(0).getFirst().get(); BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); hllc.readRegisters(ByteBuffer.wrap(bytes)); System.out.println("ab\177ab".length()); assertTrue(key1 > 0); assertEquals(1, hllc.getCountEstimate()); }
From source file:com.kylinolap.storage.hbase.coprocessor.AggregateRegionObserverTest.java
License:Apache License
private Cell newCell(byte[] key, HCol col, String decimal, int number) { Object[] values = number == Integer.MIN_VALUE ? // new Object[] { new BigDecimal(decimal) } // : new Object[] { new BigDecimal(decimal), new LongWritable(number) }; buf.clear();/* w w w . ja v a2s .com*/ col.measureCodec.encode(values, buf); Cell keyValue = new KeyValue(key, 0, key.length, // col.family, 0, col.family.length, // col.qualifier, 0, col.qualifier.length, // HConstants.LATEST_TIMESTAMP, Type.Put, // buf.array(), 0, buf.position()); return keyValue; }
From source file:com.mapred.JoinReducer.java
protected void reduce(DepDatePair key, Iterable<MapWritable> values, Context context) throws IOException, InterruptedException { Iterator valuesIterator = values.iterator(); if (valuesIterator.hasNext()) { this.index = 1L; MapWritable depMap = (MapWritable) valuesIterator.next(); this.departmentName = depMap.get(MRUtils.DEPARTMENT_NAME).toString(); if (this.departmentName == null) { this.departmentName = "DEP#NAME#ERROR"; }//ww w. ja va2 s .c o m while (valuesIterator.hasNext()) { MapWritable map = (MapWritable) valuesIterator.next(); Set keySet = map.keySet(); for (Object singleKey : keySet) { this.builder.append(map.get((Text) singleKey)); this.builder.append(","); } this.builder.append(this.departmentName); this.builder.append(";"); this.outputValue.set(this.builder.toString()); context.write(new LongWritable(this.index++), outputValue); this.builder.delete(0, this.builder.length()); } } }
From source file:com.marklogic.mapreduce.ValueReader.java
License:Apache License
@SuppressWarnings("unchecked") @Override//from w w w . j a v a 2 s. c o m protected boolean nextResult(ResultItem result) { if (key == null) { key = new LongWritable(getCount()); } else { key.set(getCount()); } if (value == null) { value = (VALUEIN) ReflectionUtils.newInstance(valueClass, getConf()); } InternalUtilities.assignResultValue(valueClass, result, value); return true; }