List of usage examples for org.apache.hadoop.io IntWritable IntWritable
public IntWritable()
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBSP.java
License:Apache License
@Override public void bsp( BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer) throws IOException, SyncException, InterruptedException { IntWritable aKey = new IntWritable(); PipesVectorWritable aVector = new PipesVectorWritable(); // while for each row of matrix A while (peer.readNext(aKey, aVector)) { // Logging if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,input,key=" + aKey + ",value=" + aVector.getVector().toString() + "\n"); }// w ww.j ava2 s. c om DenseDoubleVector outVector = null; // for each col of matrix B (cause by transposed B) for (KeyValuePair<Integer, DoubleVector> bVectorRow : m_bColumns) { if (outVector == null) { outVector = new DenseDoubleVector(bVectorRow.getValue().getDimension()); } double dot = aVector.getVector().dot(bVectorRow.getValue()); outVector.set(bVectorRow.getKey(), dot); } peer.send(m_masterTask, new MatrixRowMessage(aKey.get(), outVector)); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,send,key=" + aKey.get() + ",value=" + outVector.toString() + "\n"); m_logger.flush(); } } peer.sync(); // MasterTask accumulates result if (peer.getPeerName().equals(m_masterTask)) { MatrixRowMessage currentMatrixRowMessage = null; // Collect messages while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) { int rowIndex = currentMatrixRowMessage.getRowIndex(); DoubleVector rowValues = currentMatrixRowMessage.getRowValues(); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,write,key=" + rowIndex + ",value=" + rowValues.toString() + "\n"); } peer.write(new IntWritable(rowIndex), new PipesVectorWritable(rowValues)); } } }
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.util.DistributedRowMatrix.java
License:Apache License
public static DenseDoubleMatrix readDistributedRowMatrix(Configuration conf, Path path) { // System.out.println("readDistributedRowMatrix: " + path); List<DoubleVector> matrix = new ArrayList<DoubleVector>(); SequenceFile.Reader reader = null; try {// w w w .j a v a 2 s .c o m FileSystem fs = FileSystem.get(conf); reader = new SequenceFile.Reader(fs, path, conf); IntWritable key = new IntWritable(); PipesVectorWritable vector = new PipesVectorWritable(); while (reader.next(key, vector)) { // System.out.println("readDistributedRowMatrix: key: " + key // + Arrays.toString(vector.getVector().toArray())); matrix.add(vector.getVector()); } reader.close(); if (matrix.size() > 0) { DoubleVector list[] = new DoubleVector[matrix.size()]; DenseDoubleMatrix result = new DenseDoubleMatrix(matrix.toArray(list)); return result; } return null; } catch (IOException e) { e.printStackTrace(); } finally { if (reader != null) { try { reader.close(); } catch (IOException e) { e.printStackTrace(); } } } return null; }
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.DistributedRowMatrix.java
License:Apache License
public static DenseDoubleMatrix readDistributedRowMatrix(Configuration conf, Path path) { // System.out.println("readDistributedRowMatrix: " + path); List<DoubleVector> matrix = new ArrayList<DoubleVector>(); SequenceFile.Reader reader = null; try {/*from ww w. j a va2 s .c o m*/ FileSystem fs = FileSystem.get(conf); reader = new SequenceFile.Reader(fs, path, conf); IntWritable key = new IntWritable(); VectorWritable vector = new VectorWritable(); while (reader.next(key, vector)) { // System.out.println("readDistributedRowMatrix: key: " + key // + Arrays.toString(vector.getVector().toArray())); matrix.add(vector.getVector()); } reader.close(); if (matrix.size() > 0) { DoubleVector list[] = new DoubleVector[matrix.size()]; DenseDoubleMatrix result = new DenseDoubleMatrix(matrix.toArray(list)); return result; } return null; } catch (IOException e) { e.printStackTrace(); } finally { if (reader != null) { try { reader.close(); } catch (IOException e) { e.printStackTrace(); } } } return null; }
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java
License:Apache License
/********************************* CPU *********************************/ @Override/*from w w w .j av a2 s. c o m*/ public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer) throws IOException { HamaConfiguration conf = peer.getConfiguration(); this.m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false); // used by GPU only m_tileWidth = conf.getInt(CONF_TILE_WIDTH, 32); // Choose one as a master, who sorts the matrix rows at the end // this.m_masterTask = peer.getPeerName(peer.getNumPeers() / 2); // TODO // task must be 0 otherwise write out does NOT work! this.m_masterTask = peer.getPeerName(0); // Init logging if (m_isDebuggingEnabled) { try { FileSystem fs = FileSystem.get(conf); m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); } catch (IOException e) { e.printStackTrace(); } } // Load transposed Matrix B SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), new Path(conf.get(CONF_MATRIX_B_PATH)), conf); IntWritable transposedMatrixBRowId = new IntWritable(); VectorWritable transposedMatrixBRow = new VectorWritable(); // for each col of matrix B (cause by transposed B) while (reader.next(transposedMatrixBRowId, transposedMatrixBRow)) { m_transposedMatrixB.add(new KeyValuePair<Integer, DoubleVector>(transposedMatrixBRowId.get(), transposedMatrixBRow.getVector())); if (m_isDebuggingEnabled) { m_logger.writeChars("setup,read,transposedMatrixB,key=" + transposedMatrixBRowId.get() + ",value=" + transposedMatrixBRow.getVector().toString() + "\n"); } } reader.close(); }
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java
License:Apache License
@Override public void bsp(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer) throws IOException, SyncException, InterruptedException { IntWritable matrixARowId = new IntWritable(); VectorWritable matrixARow = new VectorWritable(); // while for each row of matrix A while (peer.readNext(matrixARowId, matrixARow)) { // Logging if (m_isDebuggingEnabled) { m_logger.writeChars(/*from www . jav a2 s . c o m*/ "bsp,input,key=" + matrixARowId + ",value=" + matrixARow.getVector().toString() + "\n"); } DenseDoubleVector outVector = null; // for each column of matrix B (cause by transposed matrix B) for (KeyValuePair<Integer, DoubleVector> bVectorRow : m_transposedMatrixB) { if (outVector == null) { // init outVector only once outVector = new DenseDoubleVector(m_transposedMatrixB.size()); } double dot = matrixARow.getVector().dot(bVectorRow.getValue()); outVector.set(bVectorRow.getKey(), dot); } peer.send(m_masterTask, new MatrixRowMessage(matrixARowId.get(), outVector)); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,send,key=" + matrixARowId.get() + ",value=" + outVector.toString() + "\n"); m_logger.flush(); } } peer.sync(); // the master task writes out the incoming messages if (peer.getPeerName().equals(m_masterTask)) { MatrixRowMessage currentMatrixRowMessage = null; // Collect messages while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) { int rowIndex = currentMatrixRowMessage.getRowIndex(); DoubleVector rowValues = currentMatrixRowMessage.getRowValues(); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,write,key=" + rowIndex + ",value=" + rowValues.toString() + "\n"); } peer.write(new IntWritable(rowIndex), new VectorWritable(rowValues)); } } }
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java
License:Apache License
@Override public void bspGpu(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer, Rootbeer rootbeer) throws IOException, SyncException, InterruptedException { // Fetch inputs List<KeyValuePair<Integer, DoubleVector>> matrixA = new ArrayList<KeyValuePair<Integer, DoubleVector>>(); final IntWritable matrixARowId = new IntWritable(); final VectorWritable matrixARow = new VectorWritable(); while (peer.readNext(matrixARowId, matrixARow)) { matrixA.add(new KeyValuePair<Integer, DoubleVector>(matrixARowId.get(), matrixARow.getVector())); if (m_isDebuggingEnabled) { m_logger.writeChars("bspGpu,matrixA,key=" + matrixARowId.get() + ",value=" + matrixARow.getVector().toString() + "\n"); }//w w w . j a v a2 s. c o m } // Convert data for GPU // n - rows of matrix A int n = matrixA.size(); // m - cols of matrix A and rows of matrix B int m = matrixA.get(0).getValue().getDimension(); // l - cols of matrix B int l = m_transposedMatrixB.size(); double[] transposedmatrixA = new double[m * n]; double[] matrixB = new double[m * l]; double[] matrixC = new double[n * l]; // Convert matrixA rows to double[] int i = 0; for (KeyValuePair<Integer, DoubleVector> row : matrixA) { for (int j = 0; j < m; j++) { // store row column wise to get a transposed matrix A transposedmatrixA[(j * n) + i] = row.getValue().get(j); } i++; } // Convert transposedMatrixB rows to double[] i = 0; for (KeyValuePair<Integer, DoubleVector> row : m_transposedMatrixB) { for (int j = 0; j < m; j++) { // store row column wise to get a normal matrix B matrixB[(j * l) + i] = row.getValue().get(j); } i++; } // DEBUG if (m_isDebuggingEnabled) { m_logger.writeChars("transposedmatrixA: \n"); printMatrix(transposedmatrixA, m, n); m_logger.writeChars("\nmatrixB:\n"); printMatrix(matrixB, m, l); m_logger.writeChars("\n"); } int subMatrixSize = m_tileWidth * m_tileWidth; int numberOfSubMatrices = divup(n * l, subMatrixSize); int gridSize = numberOfSubMatrices; int blockSize = subMatrixSize; // int subMatrixSize = tileWidth * tileWidth; // rows of A and cols of B per block int subMatricesPerThread = divup(m, m_tileWidth); if (m_isDebuggingEnabled) { m_logger.writeChars("bspGpu,tileWidth: " + m_tileWidth + "\n"); m_logger.writeChars("bspGpu,gridSize: " + gridSize + "\n"); m_logger.writeChars("bspGpu,blockSize: " + blockSize + "\n"); m_logger.writeChars("bspGpu,n: " + n + "\n"); m_logger.writeChars("bspGpu,m: " + m + "\n"); m_logger.writeChars("bspGpu,l: " + l + "\n"); m_logger.writeChars("bspGpu,subMatricesPerThread: " + subMatricesPerThread + "\n"); } MatrixMultiplicationHybridKernel kernel = new MatrixMultiplicationHybridKernel(transposedmatrixA, matrixB, matrixC, n, m, l, gridSize, blockSize, m_tileWidth, subMatricesPerThread); // Run GPU kernel Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); context.init((long) 1024 * 1024 * 1024); // 1GB watch.start(); rootbeer.run(kernel, new ThreadConfig(blockSize, gridSize, blockSize * gridSize), context); watch.stop(); if (m_isDebuggingEnabled) { List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { m_logger.writeChars(" StatsRow:\n"); m_logger.writeChars(" serial time: " + row.getSerializationTime() + "\n"); m_logger.writeChars(" exec time: " + row.getExecutionTime() + "\n"); m_logger.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); m_logger.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); m_logger.writeChars(" num threads: " + row.getNumThreads() + "\n"); } m_logger.writeChars("bspGpu,GPUTime=" + watch.elapsedTimeMillis() + "ms\n"); m_logger.flush(); } // Send results of GPU kernels DenseDoubleVector resultRow = new DenseDoubleVector(l); for (int x = 0; x < n; x++) { for (int y = 0; y < l; y++) { // submit in col-wise order resultRow.set(y, matrixC[(x * l) + y]); } peer.send(m_masterTask, new MatrixRowMessage(x, resultRow)); if (m_isDebuggingEnabled) { m_logger.writeChars("bspGpu,send,key=" + x + ",value=" + resultRow.toString() + "\n"); m_logger.flush(); } } // Global barrier synchronization peer.sync(); // the master task writes out the incoming messages if (peer.getPeerName().equals(m_masterTask)) { MatrixRowMessage currentMatrixRowMessage = null; // Collect messages while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) { int rowIndex = currentMatrixRowMessage.getRowIndex(); DoubleVector rowValues = currentMatrixRowMessage.getRowValues(); if (m_isDebuggingEnabled) { m_logger.writeChars("bspGpu,write,key=" + rowIndex + ",value=" + rowValues.toString() + "\n"); } peer.write(new IntWritable(rowIndex), new VectorWritable(rowValues)); } } }
From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java
License:Apache License
public static void main(String[] args) throws Exception { // Defaults// ww w . j av a 2 s . co m int numBspTask = 1; // CPU + GPU tasks int numGpuBspTask = 1; // GPU tasks int blockSize = BLOCK_SIZE; int gridSize = GRID_SIZE; int maxIteration = 3; // 150; int matrixRank = 3; int skipCount = 1; double alpha = ALPHA; int userCount = 0; int itemCount = 0; int percentNonZeroValues = 0; int GPUPercentage = 20; boolean useTestExampleInput = true; boolean isDebugging = true; String inputFile = ""; String separator = "\\t"; Configuration conf = new HamaConfiguration(); FileSystem fs = FileSystem.get(conf); // Set numBspTask to maxTasks // BSPJobClient jobClient = new BSPJobClient(conf); // ClusterStatus cluster = jobClient.getClusterStatus(true); // numBspTask = cluster.getMaxTasks(); if (args.length > 0) { if (args.length >= 14) { numBspTask = Integer.parseInt(args[0]); numGpuBspTask = Integer.parseInt(args[1]); blockSize = Integer.parseInt(args[2]); gridSize = Integer.parseInt(args[3]); maxIteration = Integer.parseInt(args[4]); matrixRank = Integer.parseInt(args[5]); skipCount = Integer.parseInt(args[6]); alpha = Double.parseDouble(args[7]); userCount = Integer.parseInt(args[8]); itemCount = Integer.parseInt(args[9]); percentNonZeroValues = Integer.parseInt(args[10]); GPUPercentage = Integer.parseInt(args[11]); useTestExampleInput = Boolean.parseBoolean(args[12]); isDebugging = Boolean.parseBoolean(args[13]); // optional parameters if (args.length > 14) { inputFile = args[14]; } if (args.length > 15) { separator = args[15]; } } else { System.out.println("Wrong argument size!"); System.out.println(" Argument1=numBspTask"); System.out.println(" Argument2=numGpuBspTask"); System.out.println(" Argument3=blockSize"); System.out.println(" Argument4=gridSize"); System.out.println( " Argument5=maxIterations | Number of maximal iterations (" + maxIteration + ")"); System.out.println(" Argument6=matrixRank | matrixRank (" + matrixRank + ")"); System.out.println(" Argument7=skipCount | skipCount (" + skipCount + ")"); System.out.println(" Argument8=alpha | alpha (" + alpha + ")"); System.out.println(" Argument9=userCount | userCount (" + userCount + ")"); System.out.println(" Argument10=itemCount | itemCount (" + itemCount + ")"); System.out.println(" Argument11=percentNonZeroValues | percentNonZeroValues (" + percentNonZeroValues + ")"); System.out.println(" Argument12=GPUPercentage (percentage of input)"); System.out.println(" Argument13=testExample | Use testExample input (true|false=default)"); System.out.println(" Argument14=debug | Enable debugging (true|false=default)"); System.out.println(" Argument15=inputFile (optional) | MovieLens inputFile"); System.out.println(" Argument16=separator (optional) | default '" + separator + "' "); return; } } // Check if inputFile exists if ((!inputFile.isEmpty()) && (!new File(inputFile).exists())) { System.out.println("Error: inputFile: " + inputFile + " does not exist!"); return; } // Check parameters if ((inputFile.isEmpty()) && (!useTestExampleInput) && (userCount <= 0) && (itemCount <= 0) && (percentNonZeroValues <= 0)) { System.out.println("Invalid parameter: userCount: " + userCount + " itemCount: " + itemCount + " percentNonZeroValues: " + percentNonZeroValues); return; } // Check if blockSize < matrixRank when using GPU if ((numGpuBspTask > 0) && (blockSize < matrixRank)) { System.out.println("Error: BlockSize < matrixRank"); return; } // Check GPUPercentage if ((GPUPercentage < 0) && (GPUPercentage > 100)) { System.out.println("Error: GPUPercentage must be between 0 and 100 percent"); return; } // Set config variables conf.setBoolean(CONF_DEBUG, isDebugging); conf.setBoolean("hama.pipes.logging", isDebugging); // Set CPU tasks conf.setInt("bsp.peers.num", numBspTask); // Set GPU tasks conf.setInt("bsp.peers.gpu.num", numGpuBspTask); // Set GPU blockSize and gridSize conf.set(CONF_BLOCKSIZE, "" + blockSize); conf.set(CONF_GRIDSIZE, "" + gridSize); conf.setInt(OnlineCF.CONF_ITERATION_COUNT, maxIteration); conf.setInt(OnlineCF.CONF_MATRIX_RANK, matrixRank); conf.setInt(OnlineCF.CONF_SKIP_COUNT, skipCount); // Debug output LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0)); LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0)); LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum")); LOG.info("BlockSize: " + conf.get(CONF_BLOCKSIZE)); LOG.info("GridSize: " + conf.get(CONF_GRIDSIZE)); LOG.info("GPUPercentage: " + GPUPercentage); LOG.info("isDebugging: " + isDebugging); LOG.info("useTestExampleInput: " + useTestExampleInput); LOG.info("inputPath: " + CONF_INPUT_DIR); LOG.info("outputPath: " + CONF_OUTPUT_DIR); LOG.info("maxIteration: " + maxIteration); LOG.info("matrixRank: " + matrixRank); LOG.info("skipCount: " + skipCount); LOG.info("alpha: " + alpha); LOG.info("userCount: " + userCount); LOG.info("itemCount: " + itemCount); LOG.info("percentNonZeroValues: " + percentNonZeroValues); if (!inputFile.isEmpty()) { LOG.info("inputFile: " + inputFile); LOG.info("separator: " + separator); } // prepare Input int maxTestPrefs = 10; Path preferencesIn = new Path(CONF_INPUT_DIR, "preferences_in.seq"); List<Preference<Long, Long>> testPrefs = null; if (useTestExampleInput) { testPrefs = prepareTestInputData(conf, fs, CONF_INPUT_DIR, preferencesIn); } else if (inputFile.isEmpty()) { testPrefs = generateRandomInputData(conf, fs, CONF_INPUT_DIR, numBspTask, numGpuBspTask, userCount, itemCount, percentNonZeroValues, GPUPercentage, maxTestPrefs); } else if (!inputFile.isEmpty()) { // parse inputFile and return first entries for testing testPrefs = convertInputData(conf, fs, CONF_INPUT_DIR, preferencesIn, inputFile, separator, maxTestPrefs); } // Generate Job config BSPJob job = createOnlineCFTrainHybridBSPConf(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR); // Execute Job long startTime = System.currentTimeMillis(); if (job.waitForCompletion(true)) { LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); // Load Job results for testing OnlineCF recommender = new OnlineCF(); recommender.load(CONF_OUTPUT_DIR.toString(), false); // Test results int error = 0; double totalError = 0; for (Preference<Long, Long> test : testPrefs) { double expected = test.getValue().get(); double estimated = recommender.estimatePreference(test.getUserId(), test.getItemId()); if (testPrefs.size() <= 20) { LOG.info("(" + test.getUserId() + ", " + test.getItemId() + ", " + expected + "): " + estimated + " error: " + Math.abs(expected - estimated)); } totalError += Math.abs(expected - estimated); error += (Math.abs(expected - estimated) < 0.5) ? 1 : 0; } LOG.info("totalError: " + totalError); LOG.info("assertEquals(expected: " + (testPrefs.size() * 0.75) + " == " + error + " actual) with delta: 1"); if (isDebugging) { printOutput(conf, fs, ".log", new IntWritable(), new PipesVectorWritable()); } } }
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
@Override public void bsp(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer) throws IOException, SyncException, InterruptedException { long startTime = System.currentTimeMillis(); // test input int[] input = new int[CONF_BLOCK_SIZE * CONF_GRID_SIZE]; IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); while (peer.readNext(key, value)) { input[key.get()] = value.get();//from ww w . j ava 2s . c om } String peerName = peer.getPeerName(); String[] allPeerNames = peer.getAllPeerNames(); long stopTime = System.currentTimeMillis(); // Debug output BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("TestRootbeerHybridBSP.bsp executed on CPU!\n"); outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + (stopTime - startTime) + " ms\n"); outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + ((stopTime - startTime) / 1000.0) + " seconds\n"); outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + peerName + "'\n"); outStream.writeChars("TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(allPeerNames) + "'\n"); // outStream.writeChars("TestRootbeerHybridBSP,input: '" // + Arrays.toString(input) + "'\n"); // Verify input peer.reopenInput(); while (peer.readNext(key, value)) { Assert.assertEquals(value.get(), input[key.get()]); } outStream.writeChars("TestRootbeerHybridBSP.bsp: input verified!'\n"); outStream.close(); }
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
@Override public void bspGpu(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer, Rootbeer rootbeer) throws IOException, SyncException, InterruptedException { TestRootbeerKernel kernel = new TestRootbeerKernel(CONF_BLOCK_SIZE * CONF_GRID_SIZE); // Run GPU Kernels Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start();/*w ww . j ava 2 s . co m*/ rootbeer.run(kernel, new ThreadConfig(CONF_BLOCK_SIZE, CONF_GRID_SIZE, CONF_BLOCK_SIZE * CONF_GRID_SIZE), context); watch.stop(); // Debug output BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("TestRootbeerHybridBSP.bspGpu executed on GPU!\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.writeChars("TestRootbeerHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n"); outStream .writeChars("TestRootbeerHybridBSP,GPUTime=" + (watch.elapsedTimeMillis() / 1000.0) + " seconds\n"); outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + kernel.peerName + "'\n"); outStream.writeChars( "TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(kernel.allPeerNames) + "'\n"); // outStream.writeChars("TestRootbeerHybridBSP,input: '" // + Arrays.toString(kernel.input) + "'\n"); // Verify input peer.reopenInput(); IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); while (peer.readNext(key, value)) { Assert.assertEquals(value.get(), kernel.input[key.get()]); } outStream.writeChars("TestRootbeerHybridBSP.bspGpu: input verified!'\n"); outStream.close(); }
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
private static void prepareInput(Configuration conf, Path inputPath, int n, int maxVal) throws IOException { FileSystem fs = inputPath.getFileSystem(conf); // Create input file writers depending on bspTaskNum int bspTaskNum = conf.getInt("bsp.peers.num", 1); SequenceFile.Writer[] inputWriters = new SequenceFile.Writer[bspTaskNum]; for (int i = 0; i < bspTaskNum; i++) { Path inputFile = new Path(inputPath, "input" + i + ".seq"); LOG.info("inputFile: " + inputFile.toString()); inputWriters[i] = SequenceFile.createWriter(fs, conf, inputFile, IntWritable.class, IntWritable.class, CompressionType.NONE);/*from w w w.j ava 2 s.com*/ } // Write random values to input files IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); Random r = new Random(); for (int i = 0; i < n; i++) { key.set(i); value.set(r.nextInt(maxVal)); for (int j = 0; j < inputWriters.length; j++) { inputWriters[j].append(key, value); } } // Close file writers for (int j = 0; j < inputWriters.length; j++) { inputWriters[j].close(); } }