List of usage examples for org.apache.hadoop.io FloatWritable FloatWritable
public FloatWritable(float value)
From source file:com.facebook.hive.orc.lazy.OrcLazyFloat.java
License:Open Source License
public OrcLazyFloat(OrcLazyFloat copy) { super(copy);/*from w w w.j a v a2 s .c om*/ if (copy.previous != null) { previous = new FloatWritable(((FloatWritable) copy.previous).get()); } }
From source file:com.github.ygf.pagerank.PageRankIterationCombiner.java
License:Apache License
@Override protected void reduce(ShortWritable inKey, Iterable<FloatArrayWritable> inValues, Context context) throws IOException, InterruptedException { // This task sums all the partial results for one stripe of the vector // v_k. It is a separate class since PageRankIterationReducer also adds // the teleportation factor. FloatWritable[] vi = null;/*from ww w . j av a2 s .co m*/ for (FloatArrayWritable inValue : inValues) { Writable[] partialVi = inValue.get(); if (vi == null) { // vi is initialized here in order to know the correct size of // the stripe (the last stripe can be incomplete). vi = new FloatWritable[partialVi.length]; for (int k = 0; k < vi.length; k++) { vi[k] = new FloatWritable(0); } } // Sum the partial results. for (int k = 0; k < vi.length; k++) { vi[k].set(vi[k].get() + ((FloatWritable) partialVi[k]).get()); } } context.write(inKey, new FloatArrayWritable(vi)); }
From source file:com.github.ygf.pagerank.PageRankIterationMapper.java
License:Apache License
@Override public void map(ShortArrayWritable inKey, MatrixBlockWritable inValue, Context context) throws IOException, InterruptedException { // This task gets each block M_{i,j}, loads the corresponding stripe j // of the vector v_{k-1} and produces the partial result of the stripe i // of the vector v_k. Configuration conf = context.getConfiguration(); int iter = Integer.parseInt(conf.get("pagerank.iteration")); int numPages = Integer.parseInt(conf.get("pagerank.num_pages")); short blockSize = Short.parseShort(conf.get("pagerank.block_size")); Writable[] blockIndexes = inKey.get(); short i = ((ShortWritable) blockIndexes[0]).get(); short j = ((ShortWritable) blockIndexes[1]).get(); int vjSize = (j > numPages / blockSize) ? (numPages % blockSize) : blockSize; FloatWritable[] vj = new FloatWritable[vjSize]; if (iter == 1) { // Initial PageRank vector with 1/n for all pages. for (int k = 0; k < vj.length; k++) { vj[k] = new FloatWritable(1.0f / numPages); }// w w w .jav a2 s. co m } else { // Load the stripe j of the vector v_{k-1} from the MapFiles. Path outputDir = MapFileOutputFormat.getOutputPath(context).getParent(); Path vjDir = new Path(outputDir, "v" + (iter - 1)); MapFile.Reader[] readers = MapFileOutputFormat.getReaders(vjDir, conf); Partitioner<ShortWritable, FloatArrayWritable> partitioner = new HashPartitioner<ShortWritable, FloatArrayWritable>(); ShortWritable key = new ShortWritable(j); FloatArrayWritable value = new FloatArrayWritable(); MapFileOutputFormat.getEntry(readers, partitioner, key, value); Writable[] writables = value.get(); for (int k = 0; k < vj.length; k++) { vj[k] = (FloatWritable) writables[k]; } for (MapFile.Reader reader : readers) { reader.close(); } } // Initialize the partial result i of the vector v_k. int viSize = (i > numPages / blockSize) ? (numPages % blockSize) : blockSize; FloatWritable[] vi = new FloatWritable[viSize]; for (int k = 0; k < vi.length; k++) { vi[k] = new FloatWritable(0); } // Multiply M_{i,j} by the stripe j of the vector v_{k-1} to obtain the // partial result i of the vector v_k. Writable[][] blockColumns = inValue.get(); for (int k = 0; k < blockColumns.length; k++) { Writable[] blockColumn = blockColumns[k]; if (blockColumn.length > 0) { int vDegree = ((ShortWritable) blockColumn[0]).get(); for (int columnIndex = 1; columnIndex < blockColumn.length; columnIndex++) { int l = ((ShortWritable) blockColumn[columnIndex]).get(); vi[l].set(vi[l].get() + (1.0f / vDegree) * vj[k].get()); } } } context.write(new ShortWritable(i), new FloatArrayWritable(vi)); }
From source file:com.github.ygf.pagerank.PageRankIterationReducer.java
License:Apache License
@Override protected void reduce(ShortWritable inKey, Iterable<FloatArrayWritable> inValues, Context context) throws IOException, InterruptedException { // This task sums all the partial results for one stripe of the vector // v_k and adds the teleportation factor. Configuration conf = context.getConfiguration(); int numPages = Integer.parseInt(conf.get("pagerank.num_pages")); float beta = Float.parseFloat(conf.get("pagerank.damping_factor")); FloatWritable[] vi = null;//from w w w . java 2 s . c om for (FloatArrayWritable inValue : inValues) { Writable[] partialVi = inValue.get(); if (vi == null) { // vi is initialized here in order to know the correct size of // the stripe (the last stripe can be incomplete). vi = new FloatWritable[partialVi.length]; for (int k = 0; k < vi.length; k++) { vi[k] = new FloatWritable(0); } } // Sum the partial results. for (int k = 0; k < vi.length; k++) { vi[k].set(vi[k].get() + ((FloatWritable) partialVi[k]).get()); } } // Add the teleportation factor. for (int k = 0; k < vi.length; k++) { vi[k].set(beta * vi[k].get() + (1 - beta) / numPages); } context.write(inKey, new FloatArrayWritable(vi)); }
From source file:com.github.ygf.pagerank.PageRankTopNMapper.java
License:Apache License
@Override protected void cleanup(Context context) throws IOException, InterruptedException { // The mapper outputs the top N pages by PageRank in the partition. for (Map.Entry<Float, Integer> entry : topN) { context.write(new FloatWritable(entry.getKey()), new IntWritable(entry.getValue())); }// w ww. j a va 2s. c o m }
From source file:com.github.ygf.pagerank.PageRankTopNReducer.java
License:Apache License
@Override protected void cleanup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); Path titlesDir = new Path(conf.get("pagerank.titles_dir")); MapFile.Reader[] readers = MapFileOutputFormat.getReaders(titlesDir, conf); Partitioner<IntWritable, Text> partitioner = new HashPartitioner<IntWritable, Text>(); IntWritable page = new IntWritable(); Text title = new Text(); float[] pageRanks = new float[topN.size()]; String[] titles = new String[topN.size()]; // The order of the entries is reversed. The priority queue is in // non-decreasing order and we want the highest PageRank first. for (int i = pageRanks.length - 1; i >= 0; i--) { Map.Entry<Float, Integer> entry = topN.poll(); // Get the title of the page from the title index. page.set(entry.getValue());/*from w w w. j a v a2 s .c o m*/ MapFileOutputFormat.getEntry(readers, partitioner, page, title); pageRanks[i] = entry.getKey(); titles[i] = title.toString(); } for (MapFile.Reader reader : readers) { reader.close(); } for (int i = 0; i < pageRanks.length; i++) { context.write(new FloatWritable(pageRanks[i]), new Text(titles[i])); } }
From source file:com.gotometrics.orderly.TestFloatRowKey.java
License:Apache License
@Override public int compareTo(Object o1, Object o2) { if (o1 == null || o2 == null) return super.compareTo(o1, o2); return super.compareTo(new FloatWritable((Float) o1), new FloatWritable((Float) o2)); }
From source file:com.gotometrics.orderly.TestFloatWritableRowKey.java
License:Apache License
@Override public Object createObject() { if (r.nextInt(128) == 0) return null; float f;//ww w. j a v a2s . c o m switch (r.nextInt(128)) { case 0: f = +0.0f; break; case 1: f = -0.0f; break; case 2: f = Float.POSITIVE_INFINITY; break; case 3: f = Float.NEGATIVE_INFINITY; break; case 4: f = Float.NaN; break; default: f = r.nextFloat(); break; } return new FloatWritable(f); }
From source file:com.hotels.corc.DefaultConverterFactoryTest.java
License:Apache License
@Test public void floatJava() throws UnexpectedTypeException { Converter converter = factory.newConverter(PrimitiveObjectInspectorFactory.javaFloatObjectInspector); assertThat(converter.toJavaObject(new FloatWritable(1.0F)), is((Object) 1.0F)); }
From source file:com.hotels.corc.DefaultConverterFactoryTest.java
License:Apache License
@Test public void floatWritable() throws UnexpectedTypeException { Converter converter = factory.newConverter(PrimitiveObjectInspectorFactory.javaFloatObjectInspector); assertThat(converter.toWritableObject(1.0F), is((Object) new FloatWritable(1.0F))); }