List of usage examples for org.apache.hadoop.io DoubleWritable DoubleWritable
public DoubleWritable()
From source file:org.apache.hama.graph.TestSubmitGraphJob.java
License:Apache License
private void verifyResult() throws IOException { double sum = 0.0; FileStatus[] globStatus = fs.globStatus(new Path(OUTPUT + "/part-*")); for (FileStatus fts : globStatus) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, fts.getPath(), configuration); Text key = new Text(); DoubleWritable value = new DoubleWritable(); while (reader.next(key, value)) { sum += value.get();/*from ww w .j a v a 2 s . c o m*/ } reader.close(); } LOG.info("Sum is: " + sum); assertTrue("Sum was: " + sum, sum > 0.9d && sum <= 1.1d); }
From source file:org.apache.hama.ml.regression.GradientDescentBSP.java
License:Apache License
private int getXSize( BSPPeer<VectorWritable, DoubleWritable, VectorWritable, DoubleWritable, VectorWritable> peer) throws IOException { VectorWritable key = new VectorWritable(); DoubleWritable value = new DoubleWritable(); peer.readNext(key, value);/*from w w w . j a v a 2 s . c o m*/ peer.reopenInput(); // reset input to start if (key.getVector() == null) { throw new IOException("cannot read input vector size"); } return key.getVector().getDimension(); }
From source file:org.apache.hama.pipes.TestPipes.java
License:Apache License
static void verifyOutput(HamaConfiguration conf, Path outputPath, double expectedResult, double delta) throws IOException { FileStatus[] listStatus = fs.listStatus(outputPath); for (FileStatus status : listStatus) { if (!status.isDir()) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, status.getPath(), conf); NullWritable key = NullWritable.get(); DoubleWritable value = new DoubleWritable(); if (reader.next(key, value)) { LOG.info("Output File: " + status.getPath()); LOG.info("key: '" + key + "' value: '" + value + "' expected: '" + expectedResult + "'"); assertEquals("Expected value: '" + expectedResult + "' != '" + value + "'", expectedResult, value.get(), delta); }// w ww . j a va2 s.com reader.close(); } } }
From source file:org.apache.jena.grande.giraph.pagerank.PageRankVertex.java
License:Apache License
@Override public void compute(Iterable<DoubleWritable> msgIterator) { log.debug("{}#{} compute() vertexValue={}", new Object[] { getId(), getSuperstep(), getValue() }); if (getConf() != null) { numIterations = getConf().getInt("giraph.pagerank.iterations", DEFAULT_NUM_ITERATIONS); tolerance = getConf().getFloat("giraph.pagerank.tolerance", DEFAULT_TOLERANCE); } else {//w w w . j ava 2 s. c o m log.warn("{}#{} compute() getConf() is null!", getId(), getSuperstep()); } if (getSuperstep() == 0) { log.debug( "{}#{} compute(): sending fake messages to count vertices, including 'implicit' dangling ones", getId(), getSuperstep()); sendMessageToAllEdges(new DoubleWritable()); } else if (getSuperstep() == 1) { log.debug("{}#{} compute(): counting vertices including 'implicit' dangling ones", getId(), getSuperstep()); aggregate("vertices-count", new LongWritable(1L)); aggregate("error-current", new DoubleWritable(Double.MAX_VALUE)); } else if (getSuperstep() == 2) { long numVertices = ((LongWritable) getAggregatedValue("vertices-count")).get(); aggregate("error-current", new DoubleWritable(Double.MAX_VALUE)); log.debug("{}#{} compute(): initializing pagerank scores to 1/N, N={}", new Object[] { getId(), getSuperstep(), numVertices }); DoubleWritable vertexValue = new DoubleWritable(1.0 / numVertices); setValue(vertexValue); log.debug("{}#{} compute() vertexValue <-- {}", new Object[] { getId(), getSuperstep(), getValue() }); sendMessages(); } else if (getSuperstep() > 2) { long numVertices = ((LongWritable) getAggregatedValue("vertices-count")).get(); double sum = 0; for (DoubleWritable msgValue : msgIterator) { log.debug("{}#{} compute() <-- {}", new Object[] { getId(), getSuperstep(), msgValue }); sum += msgValue.get(); } double danglingNodesContribution = ((DoubleWritable) getAggregatedValue("dangling-previous")).get(); DoubleWritable vertexValue = new DoubleWritable( (0.15f / numVertices) + 0.85f * (sum + danglingNodesContribution / numVertices)); aggregate("error-current", new DoubleWritable(Math.abs(vertexValue.get() - getValue().get()))); setValue(vertexValue); log.debug("{}#{} compute() vertexValue <-- {}", new Object[] { getId(), getSuperstep(), getValue() }); sendMessages(); } }
From source file:org.apache.mahout.classifier.bayes.io.SequenceFileModelReader.java
License:Apache License
public static void loadWeightMatrix(InMemoryBayesDatastore datastore, FileSystem fs, Path pathPattern, Configuration conf) throws IOException { StringTuple key = new StringTuple(); DoubleWritable value = new DoubleWritable(); FileStatus[] outputFiles = fs.globStatus(pathPattern); for (FileStatus fileStatus : outputFiles) { Path path = fileStatus.getPath(); log.info("{}", path); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); // the key is label,feature while (reader.next(key, value)) { datastore.loadFeatureWeight(key.stringAt(2), key.stringAt(1), value.get()); }/*from w ww . ja va 2 s .c om*/ } }
From source file:org.apache.mahout.classifier.bayes.io.SequenceFileModelReader.java
License:Apache License
public static void loadFeatureWeights(InMemoryBayesDatastore datastore, FileSystem fs, Path pathPattern, Configuration conf) throws IOException { StringTuple key = new StringTuple(); DoubleWritable value = new DoubleWritable(); FileStatus[] outputFiles = fs.globStatus(pathPattern); for (FileStatus fileStatus : outputFiles) { Path path = fileStatus.getPath(); log.info("{}", path); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); // the key is either _label_ or label,feature long count = 0; while (reader.next(key, value)) { // Sum of weights for a Feature if (key.stringAt(0).equals(BayesConstants.FEATURE_SUM)) { datastore.setSumFeatureWeight(key.stringAt(1), value.get()); count++;/* w w w . j a va 2 s.co m*/ if (count % 50000 == 0) { log.info("Read {} feature weights", count); } } } } }
From source file:org.apache.mahout.classifier.bayes.io.SequenceFileModelReader.java
License:Apache License
public static void loadLabelWeights(InMemoryBayesDatastore datastore, FileSystem fs, Path pathPattern, Configuration conf) throws IOException { StringTuple key = new StringTuple(); DoubleWritable value = new DoubleWritable(); FileStatus[] outputFiles = fs.globStatus(pathPattern); for (FileStatus fileStatus : outputFiles) { Path path = fileStatus.getPath(); log.info("{}", path); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); long count = 0; while (reader.next(key, value)) { // Sum of weights in a Label if (key.stringAt(0).equals(BayesConstants.LABEL_SUM)) { datastore.setSumLabelWeight(key.stringAt(1), value.get()); count++;/* w w w . j av a 2s .c o m*/ if (count % 10000 == 0) { log.info("Read {} label weights", count); } } } } }
From source file:org.apache.mahout.classifier.bayes.io.SequenceFileModelReader.java
License:Apache License
public static void loadThetaNormalizer(InMemoryBayesDatastore datastore, FileSystem fs, Path pathPattern, Configuration conf) throws IOException { StringTuple key = new StringTuple(); DoubleWritable value = new DoubleWritable(); FileStatus[] outputFiles = fs.globStatus(pathPattern); for (FileStatus fileStatus : outputFiles) { Path path = fileStatus.getPath(); log.info("{}", path); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); long count = 0; while (reader.next(key, value)) { // Sum of weights in a Label if (key.stringAt(0).equals(BayesConstants.LABEL_THETA_NORMALIZER)) { datastore.setThetaNormalizer(key.stringAt(1), value.get()); count++;/*from ww w . j a v a 2 s. c o m*/ if (count % 50000 == 0) { log.info("Read {} theta norms", count); } } } } }
From source file:org.apache.mahout.classifier.bayes.io.SequenceFileModelReader.java
License:Apache License
public static void loadSumWeight(InMemoryBayesDatastore datastore, FileSystem fs, Path pathPattern, Configuration conf) throws IOException { StringTuple key = new StringTuple(); DoubleWritable value = new DoubleWritable(); FileStatus[] outputFiles = fs.globStatus(pathPattern); for (FileStatus fileStatus : outputFiles) { Path path = fileStatus.getPath(); log.info("{}", path); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); // the key is _label while (reader.next(key, value)) { if (key.stringAt(0).equals(BayesConstants.TOTAL_SUM)) { // Sum of // weights for // all Features and all Labels datastore.setSigmaJSigmaK(value.get()); log.info("{}", value.get()); }/*from w ww . ja va 2 s. c o m*/ } } }
From source file:org.apache.mahout.classifier.bayes.io.SequenceFileModelReader.java
License:Apache License
public static Map<String, Double> readLabelSums(FileSystem fs, Path pathPattern, Configuration conf) throws IOException { Map<String, Double> labelSum = new HashMap<String, Double>(); StringTuple key = new StringTuple(); DoubleWritable value = new DoubleWritable(); FileStatus[] outputFiles = fs.globStatus(pathPattern); for (FileStatus fileStatus : outputFiles) { Path path = fileStatus.getPath(); SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); // the key is either _label_ or label,feature while (reader.next(key, value)) { if (key.stringAt(0).equals(BayesConstants.LABEL_SUM)) { // Sum of counts // of labels labelSum.put(key.stringAt(1), value.get()); }/*from w w w. ja v a 2 s. c om*/ } } return labelSum; }