List of usage examples for org.apache.hadoop.io DoubleWritable get
public double get()
From source file:main.okapi.graphs.SingleSourceShortestPaths.java
License:Apache License
@Override public void compute(Vertex<LongWritable, DoubleWritable, FloatWritable> vertex, Iterable<DoubleWritable> messages) { if (getSuperstep() == 0) { vertex.setValue(new DoubleWritable(Double.MAX_VALUE)); }/*from w ww . j a v a 2s .c o m*/ // In directed graphs, vertices that have no outgoing edges will be created // in the 1st superstep as a result of messages sent to them. if (getSuperstep() == 1 && vertex.getNumEdges() == 0) { vertex.setValue(new DoubleWritable(Double.MAX_VALUE)); } double minDist = isSource(vertex) ? 0d : Double.MAX_VALUE; for (DoubleWritable message : messages) { minDist = Math.min(minDist, message.get()); } if (LOG.isDebugEnabled()) { LOG.debug("Vertex " + vertex.getId() + " got minDist = " + minDist + " vertex value = " + vertex.getValue()); } if (minDist < vertex.getValue().get()) { vertex.setValue(new DoubleWritable(minDist)); for (Edge<LongWritable, FloatWritable> edge : vertex.getEdges()) { double distance = minDist + edge.getValue().get(); if (LOG.isDebugEnabled()) { LOG.debug( "Vertex " + vertex.getId() + " sent to " + edge.getTargetVertexId() + " = " + distance); } sendMessage(edge.getTargetVertexId(), new DoubleWritable(distance)); } } vertex.voteToHalt(); }
From source file:mapreducesentiment.SentimentKeyWritableComparable.java
public void setScore(DoubleWritable _score) { score.set(_score.get()); }
From source file:mapreducesentiment.SentimentReducer.java
private double getScore(DoubleWritable expected, Long got) { Long distance = Math.abs((got + 1) - Math.round(expected.get())); return distance == 0 ? 1.0 : distance == 1 ? 0.75 : 0.0; }
From source file:ml.shifu.shifu.core.posttrain.FeatureImportanceReducer.java
License:Apache License
@Override protected void reduce(IntWritable key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException { double sum = 0d; for (DoubleWritable dw : values) { sum += dw.get(); }/*from ww w . j a v a2 s .c o m*/ this.variableStatsMap.put(key.get(), sum); }
From source file:net.java.jatextmining.lib.CountReducer.java
License:Apache License
@Override public final void reduce(Text key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); double sum = 0.0; if (conf.getBoolean("weighting", false)) { String stringKey = key.toString(); if (dfMap.containsKey(stringKey)) { double df = dfMap.get(stringKey); for (DoubleWritable val : values) { sum += val.get() / df; }//from w ww . ja v a 2 s .c o m } } else { for (DoubleWritable val : values) { sum += val.get(); } } if (sum >= countMinNum) { value.set(sum); context.write(key, value); } }
From source file:net.thecubic.mockbi.MockBILogReducer.java
License:Apache License
protected void reduce(MockBISummaryKey key, Iterable<DoubleWritable> dataPoints, Context context) throws IOException { RollingNumberSummary rns = new RollingNumberSummary(); for (DoubleWritable dataPoint : dataPoints) { rns.push(dataPoint.get()); }// w w w.jav a 2 s . c o m Put record; try { record = new Put(key.getBytes()); } catch (IOException e) { context.getCounter(getClass().getName(), "record.IOException").increment(1L); e.printStackTrace(); // well, i tried; i gave up return; } MockBIHBase mbitc = new MockBIHBase(); byte[] columnFamily = key.getColumnFamily(); mbitc.addSummaryColumn(columnFamily); record.add(columnFamily, MockBI.countKey, Integer.toString(rns.n).getBytes()); record.add(columnFamily, MockBI.minKey, Double.toString(rns.min).getBytes()); record.add(columnFamily, MockBI.maxKey, Double.toString(rns.max).getBytes()); record.add(columnFamily, MockBI.meanKey, Double.toString(rns.mean).getBytes()); record.add(columnFamily, MockBI.stddevKey, Double.toString(rns.getSD()).getBytes()); try { context.write(new ImmutableBytesWritable(columnFamily), record); } catch (IOException e) { context.getCounter(getClass().getName(), "context.IOException").increment(1L); e.printStackTrace(); } catch (InterruptedException e) { context.getCounter(getClass().getName(), "context.InterruptedException").increment(1L); e.printStackTrace(); } }
From source file:nl.tudelft.graphalytics.giraph.algorithms.pr.PageRankComputation.java
License:Apache License
@Override public void compute(Vertex<LongWritable, DoubleWritable, NullWritable> vertex, Iterable<DoubleWritable> messages) throws IOException { if (getSuperstep() == 0) { vertex.getValue().set(1.0 / getTotalNumVertices()); } else {/*from www .j a v a2 s.c o m*/ double sum = this.<PageRankWorkerContext>getWorkerContext().getLastDanglingNodeSum() / getTotalNumVertices(); for (DoubleWritable message : messages) { sum += message.get(); } vertex.getValue().set((1.0 - dampingFactor) / getTotalNumVertices() + dampingFactor * sum); } if (getSuperstep() < numberOfIterations) { if (vertex.getNumEdges() == 0) { aggregate(DANGLING_NODE_SUM, vertex.getValue()); } else { msgObject.set(vertex.getValue().get() / vertex.getNumEdges()); sendMessageToAllEdges(vertex, msgObject); } } else { vertex.voteToHalt(); } }
From source file:nl.tudelft.graphalytics.giraph.algorithms.sssp.SingleSourceShortestPathComputation.java
License:Apache License
@Override public void compute(Vertex<LongWritable, DoubleWritable, DoubleWritable> vertex, Iterable<DoubleWritable> messages) throws IOException { // New distance of this vertex boolean informNeighbors = false; // In the first superstep, the source vertex sets its distance to 0.0 if (getSuperstep() == 0) { if (vertex.getId().get() == sourceVertexId) { vertex.getValue().set(0.0);/*w w w . ja va2 s . c om*/ informNeighbors = true; } else { vertex.getValue().set(Double.POSITIVE_INFINITY); } } // In subsequent supersteps, vertices need to find the minimum // value from the messages sent by their neighbors else { double minDist = Double.POSITIVE_INFINITY; // find minimum for (DoubleWritable message : messages) { if (message.get() < minDist) { minDist = message.get(); } } // if smaller, set new distance and update neighbors if (minDist < vertex.getValue().get()) { vertex.getValue().set(minDist); informNeighbors = true; } } // Send messages to neighbors to inform them of new distance if (informNeighbors) { double dist = vertex.getValue().get(); for (Edge<LongWritable, DoubleWritable> edge : vertex.getEdges()) { LongWritable id = edge.getTargetVertexId(); double value = edge.getValue().get(); msg.set(dist + value); sendMessage(id, msg); } } // Always halt so the compute method is only executed for those vertices // that have an incoming message vertex.voteToHalt(); }
From source file:org.apache.crunch.io.avro.AvroWritableIT.java
License:Apache License
@Test public void testAvroBasedWritablePipeline() throws Exception { String customersInputPath = tmpDir.copyResourceFileName("customers.txt"); Pipeline pipeline = new MRPipeline(AvroWritableIT.class, tmpDir.getDefaultConfiguration()); pipeline.enableDebug();//from w ww .j a v a 2 s .com PCollection<String> customerLines = pipeline.readTextFile(customersInputPath); Map<Integer, DoubleWritable> outputMap = customerLines .parallelDo(new MapFn<String, Pair<Integer, DoubleWritable>>() { @Override public Pair<Integer, DoubleWritable> map(String input) { int len = input.length(); return Pair.of(len, new DoubleWritable(len)); } }, tableOf(ints(), writables(DoubleWritable.class))).groupByKey() .combineValues(new CombineFn<Integer, DoubleWritable>() { @Override public void process(Pair<Integer, Iterable<DoubleWritable>> input, Emitter<Pair<Integer, DoubleWritable>> emitter) { double sum = 0.0; for (DoubleWritable dw : input.second()) { sum += dw.get(); } emitter.emit(Pair.of(input.first(), new DoubleWritable(sum))); } }).materializeToMap(); Map<Integer, DoubleWritable> expectedMap = Maps.newHashMap(); expectedMap.put(17, new DoubleWritable(17.0)); expectedMap.put(16, new DoubleWritable(16.0)); expectedMap.put(12, new DoubleWritable(24.0)); assertEquals(expectedMap, outputMap); pipeline.done(); }
From source file:org.apache.eagle.storage.hbase.it.CoprocessorITSuite.java
License:Apache License
private void logGroupbyKeyValue(List<GroupbyKeyValue> keyValues) { for (GroupbyKeyValue keyValue : keyValues) { GroupbyKey key = keyValue.getKey(); List<String> keys = new ArrayList<>(); for (BytesWritable bytes : key.getValue()) { keys.add(new String(bytes.copyBytes())); }//from w w w. j a v a 2 s . c o m List<Double> vals = new ArrayList<>(); GroupbyValue val = keyValue.getValue(); for (DoubleWritable dw : val.getValue()) { vals.add(dw.get()); } if (LOG.isDebugEnabled()) LOG.debug("KEY: " + keys + ", VALUE: " + vals); } }