List of usage examples for org.apache.mahout.math VectorWritable VectorWritable
public VectorWritable(Vector vector)
From source file:Vectors.java
License:Apache License
public static void write(Vector vector, Path path, Configuration conf, boolean laxPrecision) throws IOException { FileSystem fs = FileSystem.get(path.toUri(), conf); FSDataOutputStream out = fs.create(path); try {//from ww w .j a v a 2s. c o m VectorWritable vectorWritable = new VectorWritable(vector); vectorWritable.setWritesLaxPrecision(laxPrecision); vectorWritable.write(out); } finally { Closeables.closeQuietly(out); } }
From source file:DisplayClustering.java
License:Apache License
/** * Generate random samples and add them to the sampleData * * @param num/* ww w . j av a2 s . c o m*/ * int number of samples to generate * @param mx * double x-value of the sample mean * @param my * double y-value of the sample mean * @param sd * double standard deviation of the samples */ protected static void generateSamples(int num, double mx, double my, double sd) { double[] params = { mx, my, sd, sd }; SAMPLE_PARAMS.add(new DenseVector(params)); log.info("Generating {} samples m=[{}, {}] sd={}", num, mx, my, sd); for (int i = 0; i < num; i++) { SAMPLE_DATA.add(new VectorWritable(new DenseVector( new double[] { UncommonDistributions.rNorm(mx, sd), UncommonDistributions.rNorm(my, sd) }))); } }
From source file:DisplayClustering.java
License:Apache License
/** * Generate random samples and add them to the sampleData * * @param num//w ww .j a va 2 s.co m * int number of samples to generate * @param mx * double x-value of the sample mean * @param my * double y-value of the sample mean * @param sdx * double x-value standard deviation of the samples * @param sdy * double y-value standard deviation of the samples */ protected static void generate2dSamples(int num, double mx, double my, double sdx, double sdy) { double[] params = { mx, my, sdx, sdy }; SAMPLE_PARAMS.add(new DenseVector(params)); log.info("Generating {} samples m=[{}, {}] sd=[{}, {}]", num, mx, my, sdx, sdy); for (int i = 0; i < num; i++) { SAMPLE_DATA.add(new VectorWritable(new DenseVector( new double[] { UncommonDistributions.rNorm(mx, sdx), UncommonDistributions.rNorm(my, sdy) }))); } }
From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java
License:Apache License
public static void writeDistributedRowMatrix(Configuration conf, double[][] matrix, int rows, int columns, Path path, boolean saveTransposed) throws Exception { SequenceFile.Writer writer = null; try {//ww w . ja v a 2 s . c o m FileSystem fs = FileSystem.get(conf); writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class); if (saveTransposed) { // Transpose Matrix before saving double[][] transposed = new double[columns][rows]; for (int i = 0; i < rows; i++) { for (int j = 0; j < columns; j++) { transposed[j][i] = matrix[i][j]; } } matrix = transposed; } for (int i = 0; i < matrix.length; i++) { DenseVector rowVector = new DenseVector(matrix[i]); writer.append(new IntWritable(i), new VectorWritable(rowVector)); } } catch (IOException e) { e.printStackTrace(); } finally { if (writer != null) { try { writer.close(); } catch (IOException e) { e.printStackTrace(); } } } }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.compositeinput.cpu.MatrixMultiplicationBSPCpu.java
License:Apache License
@Override public void cleanup(BSPPeer<IntWritable, TupleWritable, IntWritable, VectorWritable, MatrixRowMessage> peer) throws IOException { // MasterTask accumulates result if (peer.getPeerName().equals(masterTask)) { // SortedMap because the final matrix rows should be in order SortedMap<Integer, Vector> accumlatedRows = new TreeMap<Integer, Vector>(); MatrixRowMessage currentMatrixRowMessage = null; // Collect messages while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) { int rowIndex = currentMatrixRowMessage.getRowIndex(); Vector rowValues = currentMatrixRowMessage.getRowValues().get(); if (isDebuggingEnabled) { logger.writeChars("bsp,gotMsg,key=" + rowIndex + ",value=" + rowValues.toString() + "\n"); }/*from w w w . j a v a2s.c om*/ if (accumlatedRows.containsKey(rowIndex)) { accumlatedRows.get(rowIndex).assign(rowValues, Functions.PLUS); } else { accumlatedRows.put(rowIndex, new RandomAccessSparseVector(rowValues)); } } // Write accumulated results for (Map.Entry<Integer, Vector> row : accumlatedRows.entrySet()) { if (isDebuggingEnabled) { logger.writeChars( "bsp,write,key=" + row.getKey() + ",value=" + row.getValue().toString() + "\n"); } peer.write(new IntWritable(row.getKey()), new VectorWritable(row.getValue())); } } }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.compositeinput.gpu.MatrixMultiplicationBSPGpu.java
License:Apache License
@Override public void bsp(BSPPeer<IntWritable, TupleWritable, IntWritable, VectorWritable, MatrixRowMessage> peer) throws IOException, SyncException, InterruptedException { IntWritable key = new IntWritable(); TupleWritable value = new TupleWritable(); while (peer.readNext(key, value)) { // Logging if (isDebuggingEnabled) { for (int i = 0; i < value.size(); i++) { Vector vector = ((VectorWritable) value.get(i)).get(); logger.writeChars("bsp,input,key=" + key + ",value=" + vector.toString() + "\n"); }/* w ww .j a va2s. c om*/ } Vector firstVector = ((VectorWritable) value.get(0)).get(); Vector secondVector = ((VectorWritable) value.get(1)).get(); // outCardinality is resulting column size n // (l x m) * (m x n) = (l x n) boolean firstIsOutFrag = secondVector.size() == outCardinality; // outFrag is Matrix which has the resulting column cardinality // (matrixB) Vector outFrag = firstIsOutFrag ? secondVector : firstVector; // multiplier is Matrix which has the resulting row count // (transposed matrixA) Vector multiplier = firstIsOutFrag ? firstVector : secondVector; if (isDebuggingEnabled) { logger.writeChars("bsp,firstIsOutFrag=" + firstIsOutFrag + "\n"); logger.writeChars("bsp,outFrag=" + outFrag + "\n"); logger.writeChars("bsp,multiplier=" + multiplier + "\n"); } // outFrag to double[] double[] outFragArray = new double[outFrag.size()]; int i = 0; for (Vector.Element e : outFrag.all()) { outFragArray[i] = e.get(); i++; } // One map task consists of multiple kernels within one block // Each kernel computes a scalar multiplication blockSize = multiplier.size(); gridSize++; for (int j = 0; j < blockSize; j++) { kernels.add(new MatrixMultiplicationBSPKernel(j, multiplier.get(j), outFragArray)); } // Run GPU Kernels Rootbeer rootbeer = new Rootbeer(); Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start(); // blockSize = rows of Matrix A (multiplier) // gridSize = cols of Matrix B (for each row a scalar multiplication // has to be made) rootbeer.run(kernels, new ThreadConfig(blockSize, gridSize, kernels.size()), context); watch.stop(); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { System.out.println(" StatsRow:\n"); System.out.println(" serial time: " + row.getSerializationTime() + "\n"); System.out.println(" exec time: " + row.getExecutionTime() + "\n"); System.out.println(" deserial time: " + row.getDeserializationTime() + "\n"); System.out.println(" num blocks: " + row.getNumBlocks() + "\n"); System.out.println(" num threads: " + row.getNumThreads() + "\n"); } if (isDebuggingEnabled) { logger.writeChars( "bsp,KernelCount=" + kernels.size() + ",GPUTime=" + watch.elapsedTimeMillis() + "ms\n"); logger.writeChars("bps,blockSize=" + blockSize + ",gridSize=" + gridSize + "\n"); logger.flush(); } // Collect results of GPU kernels for (Kernel kernel : kernels) { MatrixMultiplicationBSPKernel bspKernel = (MatrixMultiplicationBSPKernel) kernel; if (isDebuggingEnabled) { logger.writeChars("bsp,thread_idxx=" + bspKernel.thread_idxx + ",multiplier=" + bspKernel.multiplierVal + ",vector=" + Arrays.toString(bspKernel.vectorVal) + "\n"); } peer.send(masterTask, new MatrixRowMessage(bspKernel.row, new VectorWritable(new DenseVector(bspKernel.results)))); if (isDebuggingEnabled) { logger.writeChars("bsp,send,key=" + bspKernel.row + ",value=" + Arrays.toString(bspKernel.results) + "\n"); } } } peer.sync(); }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.compositeinput.util.MatrixRowMessage.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { rowIndex = in.readInt();// w w w . ja v a 2 s. c o m rowValues = new VectorWritable(VectorWritable.readVector(in)); }
From source file:com.ctrip.gs.recommendation.itemcf.mapreduce.VectorSumReducer.java
License:Apache License
@Override protected void reduce(WritableComparable<?> key, Iterable<VectorWritable> values, Context ctx) throws IOException, InterruptedException { ctx.write(key, new VectorWritable(Vectors.sum(values.iterator()))); }
From source file:com.elex.dmp.lda.CachingCVB0Mapper.java
License:Apache License
@Override protected void cleanup(Context context) throws IOException, InterruptedException { log.info("Stopping model trainer"); modelTrainer.stop();/*from w w w . j a va 2 s. co m*/ log.info("Writing model"); TopicModel model = modelTrainer.getReadModel(); for (MatrixSlice topic : model) { context.write(new Text(Integer.toString(topic.index())), new VectorWritable(topic.vector())); } }
From source file:com.elex.dmp.lda.CVB0DocInferenceMapper.java
License:Apache License
@Override public void map(Text docId, VectorWritable doc, Context context) throws IOException, InterruptedException { int numTopics = getNumTopics(); Vector docTopics = new DenseVector(new double[numTopics]).assign(1.0 / numTopics); Matrix docModel = new SparseRowMatrix(numTopics, doc.get().size()); int maxIters = getMaxIters(); ModelTrainer modelTrainer = getModelTrainer(); for (int i = 0; i < maxIters; i++) { modelTrainer.getReadModel().trainDocTopicModel(doc.get(), docTopics, docModel); }/*from ww w. ja v a 2s.co m*/ context.write(docId, new VectorWritable(docTopics)); }