List of usage examples for org.apache.hadoop.io DoubleWritable get
public double get()
From source file:org.apache.mahout.math.stats.entropy.CalculateEntropyReducer.java
License:Apache License
@Override protected void reduce(NullWritable key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException { double entropy = 0.0; for (DoubleWritable value : values) { entropy += value.get(); }//from w w w . j av a 2 s.c om result.set((Math.log(numberItems) - entropy / numberItems) / LOG_2); context.write(key, result); }
From source file:org.apache.mahout.math.stats.entropy.DoubleSumReducer.java
License:Apache License
@Override protected void reduce(Writable key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException { double sum = 0.0; for (DoubleWritable value : values) { sum += value.get(); }/*from w w w . j av a 2s. com*/ result.set(sum); context.write(key, result); }
From source file:org.apache.mahout.utils.eval.ParallelFactorizationEvaluator.java
License:Apache License
protected double computeRmse(Path errors) { RunningAverage average = new FullRunningAverage(); for (Pair<DoubleWritable, NullWritable> entry : new SequenceFileDirIterable<DoubleWritable, NullWritable>( errors, PathType.LIST, getConf())) { DoubleWritable error = entry.getFirst(); average.addDatum(error.get() * error.get()); }//from w w w .j a v a 2 s .c om return Math.sqrt(average.getAverage()); }
From source file:org.apache.sysml.runtime.matrix.sort.ReadWithZeros.java
License:Apache License
public boolean readNextKeyValuePairs(DoubleWritable readKey, IntWritable readValue) throws IOException { boolean ret = true; try {//ww w . jav a 2 s.co m if (contain0s && justFound0) { readKey.set(keyAfterZero.get()); readValue.set(valueAfterZero.get()); contain0s = false; } else { readKey.readFields(currentStream); readValue.readFields(currentStream); } } catch (EOFException e) { // case in which zero is the maximum value in the matrix. // The zero value from the last entry is not present in the input sorted matrix, but needs to be accounted for. if (contain0s && !justFound0) { justFound0 = true; readKey.set(0); readValue.set((int) numZeros); } else { ret = false; } } if (contain0s && !justFound0 && readKey.get() >= 0) { justFound0 = true; keyAfterZero.set(readKey.get()); valueAfterZero.set(readValue.get()); readKey.set(0); readValue.set((int) numZeros); } return ret; }
From source file:org.apache.sysml.runtime.util.MapReduceTool.java
License:Apache License
public static double[] pickValueWeight(String dir, MetaDataNumItemsByEachReducer metadata, double p, boolean average) throws IOException { long[] counts = metadata.getNumItemsArray(); long[] ranges = new long[counts.length]; ranges[0] = counts[0];// www . j av a 2 s . co m for (int i = 1; i < counts.length; i++) ranges[i] = ranges[i - 1] + counts[i]; long total = ranges[ranges.length - 1]; // do averaging only if it is asked for; and sum_wt is even average = average && (total % 2 == 0); int currentPart = 0; double cum_weight = 0; long pos = (long) Math.ceil(total * p); while (ranges[currentPart] < pos) { currentPart++; cum_weight += ranges[currentPart]; } int offset; if (currentPart > 0) offset = (int) (pos - ranges[currentPart - 1] - 1); else offset = (int) pos - 1; Path path = new Path(dir); FileSystem fs = IOUtilFunctions.getFileSystem(path); FileStatus[] files = fs.listStatus(path); Path fileToRead = null; for (FileStatus file : files) if (file.getPath().toString().endsWith(Integer.toString(currentPart))) { fileToRead = file.getPath(); break; } if (fileToRead == null) throw new RuntimeException("cannot read partition " + currentPart); int buffsz = 64 * 1024; DoubleWritable readKey = new DoubleWritable(); IntWritable readValue = new IntWritable(); FSDataInputStream currentStream = null; double ret = -1; try { currentStream = fs.open(fileToRead, buffsz); boolean contain0s = false; long numZeros = 0; if (currentPart == metadata.getPartitionOfZero()) { contain0s = true; numZeros = metadata.getNumberOfZero(); } ReadWithZeros reader = new ReadWithZeros(currentStream, contain0s, numZeros); int numRead = 0; while (numRead <= offset) { reader.readNextKeyValuePairs(readKey, readValue); numRead += readValue.get(); cum_weight += readValue.get(); } ret = readKey.get(); if (average) { if (numRead <= offset + 1) { reader.readNextKeyValuePairs(readKey, readValue); cum_weight += readValue.get(); ret = (ret + readKey.get()) / 2; } } } finally { IOUtilFunctions.closeSilently(currentStream); } return new double[] { ret, (average ? -1 : readValue.get()), (average ? -1 : cum_weight) }; }
From source file:org.archive.giraph.PageRankComputation.java
License:Apache License
@Override public void compute(Vertex<LongWritable, DoubleWritable, FloatWritable> vertex, Iterable<DoubleWritable> messages) { if (getSuperstep() >= 1) { double sum = 0; for (DoubleWritable message : messages) { sum += message.get(); }/*w ww .ja v a2s . co m*/ // add in the dangling factor sum += this.<DoubleWritable>getAggregatedValue(DANGLING_SUM_AGG).get(); float jumpProbability = JUMP_PROBABILITY.get(getConf()); DoubleWritable vertexValue = new DoubleWritable(jumpProbability + (1 - jumpProbability) * sum); vertex.setValue(vertexValue); aggregate(NUMVERTICES_SUM_AGG, new LongWritable(1)); } if (getSuperstep() < MAX_SUPERSTEPS.get(getConf())) { long edges = vertex.getNumEdges(); double vertexValue = vertex.getValue().get(); //dangling nodes -- transfer score evenly to all nodes if (0 == edges) { aggregate(DANGLING_SUM_AGG, new DoubleWritable(vertexValue / getTotalNumVertices())); } else { sendMessageToAllEdges(vertex, new DoubleWritable(vertexValue / edges)); } } else { vertex.voteToHalt(); } }
From source file:org.archive.giraph.PageRankVertex.java
License:Apache License
public void compute(Iterable<DoubleWritable> messages) { if (getSuperstep() >= 1) { double sum = 0; for (DoubleWritable message : messages) { sum += message.get(); }/*from w w w . j a va 2s . com*/ // add in the dangling factor sum += this.<DoubleWritable>getAggregatedValue(DANGLING_SUM_AGG).get(); float jump_probability = JUMP_PROBABILITY.get(getConf()); DoubleWritable vertexValue = new DoubleWritable(jump_probability + (1 - jump_probability) * sum); setValue(vertexValue); aggregate(NUMVERTICES_SUM_AGG, new LongWritable(1)); } if (getSuperstep() < MAX_SUPERSTEPS.get(getConf())) { long edges = getNumEdges(); double vertexValue = getValue().get(); //dangling nodes -- transfer score evenly to all nodes if (0 == edges) aggregate(DANGLING_SUM_AGG, new DoubleWritable(vertexValue / getTotalNumVertices())); else sendMessageToAllEdges(new DoubleWritable(vertexValue / edges)); } else { voteToHalt(); } }
From source file:org.archive.giraph.WeightedPageRankComputation.java
License:Apache License
@Override public void compute(Vertex<Text, DoubleWritable, FloatWritable> vertex, Iterable<DoubleWritable> messages) { if (getSuperstep() >= 1) { double sum = 0; for (DoubleWritable message : messages) { sum += message.get(); }/* w w w .jav a 2s. c o m*/ // add in the dangling factor sum += this.<DoubleWritable>getAggregatedValue(DANGLING_SUM_AGG).get(); float jumpProbability = JUMP_PROBABILITY.get(getConf()); DoubleWritable vertexValue = new DoubleWritable(jumpProbability + (1 - jumpProbability) * sum); vertex.setValue(vertexValue); aggregate(NUMVERTICES_SUM_AGG, new LongWritable(1)); } if (getSuperstep() < MAX_SUPERSTEPS.get(getConf())) { long edges = vertex.getNumEdges(); double vertexValue = vertex.getValue().get(); //dangling nodes -- transfer score evenly to all nodes if (0 == edges) { aggregate(DANGLING_SUM_AGG, new DoubleWritable(vertexValue / getTotalNumVertices())); } else { //Pass 1: Sum up all neighbor weights float totalEdgeWeight = 0; for (Edge<Text, FloatWritable> edge : vertex.getEdges()) { totalEdgeWeight += edge.getValue().get(); } boolean sumEdgeWeightsFlag = SUM_EDGE_WEIGHTS_FLAG.get(getConf()); if (!sumEdgeWeightsFlag) { totalEdgeWeight = 1; } //Pass 2: send weighted PR value to each neighbor if (totalEdgeWeight > 0) { for (Edge<Text, FloatWritable> edge : vertex.getEdges()) { sendMessage(edge.getTargetVertexId(), new DoubleWritable((vertexValue * edge.getValue().get()) / totalEdgeWeight)); } } } } else { vertex.voteToHalt(); } }
From source file:org.archive.giraph.WeightedPageRankVertex.java
License:Apache License
public void compute(Iterable<DoubleWritable> messages) { if (getSuperstep() >= 1) { double sum = 0; for (DoubleWritable message : messages) { sum += message.get(); }/* www .ja va 2s . c o m*/ // add in the dangling factor sum += this.<DoubleWritable>getAggregatedValue(DANGLING_SUM_AGG).get(); float jump_probability = JUMP_PROBABILITY.get(getConf()); DoubleWritable vertexValue = new DoubleWritable(jump_probability + (1 - jump_probability) * sum); setValue(vertexValue); aggregate(NUMVERTICES_SUM_AGG, new LongWritable(1)); } if (getSuperstep() < MAX_SUPERSTEPS.get(getConf())) { long edges = getNumEdges(); double vertexValue = getValue().get(); //dangling nodes -- transfer score evenly to all nodes if (0 == edges) aggregate(DANGLING_SUM_AGG, new DoubleWritable(vertexValue / getTotalNumVertices())); else { //Pass 1: Sum up all neighbor weights float totalEdgeWeight = 0; for (Edge<Text, FloatWritable> edge : getEdges()) totalEdgeWeight += edge.getValue().get(); //Pass 2: send weighted PR value to each neighbor if (totalEdgeWeight > 0) { for (Edge<Text, FloatWritable> edge : getEdges()) { sendMessage(edge.getTargetVertexId(), new DoubleWritable((vertexValue * edge.getValue().get()) / totalEdgeWeight)); } } } } else { voteToHalt(); } }
From source file:org.commoncrawl.util.JoinValue.java
License:Open Source License
public JoinValue(TextBytes tag, DoubleWritable value) { _tag = tag; _type = DOUBLE_TYPE_JOIN_VALUE; _doubleValue = value.get(); }