List of usage examples for org.apache.hadoop.fs FileSystem get
public static FileSystem get(Configuration conf) throws IOException
From source file:at.illecker.hama.hybrid.examples.testglobalgpusync.TestGlobalGpuSyncHybridBSP.java
License:Apache License
public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException { // Defaults/*from w ww.ja va2 s . c o m*/ int numBspTask = 1; int numGpuBspTask = 1; int blockSize = BLOCK_SIZE; int gridSize = GRID_SIZE; boolean isDebugging = false; Configuration conf = new HamaConfiguration(); if (args.length > 0) { if (args.length == 5) { numBspTask = Integer.parseInt(args[0]); numGpuBspTask = Integer.parseInt(args[1]); blockSize = Integer.parseInt(args[2]); gridSize = Integer.parseInt(args[3]); isDebugging = Boolean.parseBoolean(args[4]); } else { System.out.println("Wrong argument size!"); System.out.println(" Argument1=numBspTask"); System.out.println(" Argument2=numGpuBspTask"); System.out.println(" Argument3=blockSize"); System.out.println(" Argument4=gridSize"); System.out.println(" Argument5=debug | Enable debugging (true|false=default)"); return; } } // Set config variables conf.setBoolean("hama.pipes.logging", isDebugging); // Set CPU tasks conf.setInt("bsp.peers.num", numBspTask); // Set GPU tasks conf.setInt("bsp.peers.gpu.num", numGpuBspTask); // Set GPU blockSize and gridSize conf.set(CONF_BLOCK_SIZE, "" + blockSize); conf.set(CONF_GRID_SIZE, "" + gridSize); conf.set(CONF_TMP_DIR, TMP_DIR.toString()); LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0)); LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0)); LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum")); LOG.info("BlockSize: " + conf.get(CONF_BLOCK_SIZE)); LOG.info("GridSize: " + conf.get(CONF_GRID_SIZE)); LOG.info("TempDir: " + conf.get(CONF_TMP_DIR)); LOG.info("isDebugging: " + conf.getBoolean("hama.pipes.logging", false)); BSPJob job = createTestGlobalGpuSyncHybridBSPConf(conf); long startTime = System.currentTimeMillis(); if (job.waitForCompletion(true)) { LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); printOutput(job, FileSystem.get(conf), new Path(conf.get(CONF_TMP_DIR))); } }
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
@Override public void bsp(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer) throws IOException, SyncException, InterruptedException { long startTime = System.currentTimeMillis(); // test input int[] input = new int[CONF_BLOCK_SIZE * CONF_GRID_SIZE]; IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); while (peer.readNext(key, value)) { input[key.get()] = value.get();// w w w . ja v a 2 s.c o m } String peerName = peer.getPeerName(); String[] allPeerNames = peer.getAllPeerNames(); long stopTime = System.currentTimeMillis(); // Debug output BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("TestRootbeerHybridBSP.bsp executed on CPU!\n"); outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + (stopTime - startTime) + " ms\n"); outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + ((stopTime - startTime) / 1000.0) + " seconds\n"); outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + peerName + "'\n"); outStream.writeChars("TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(allPeerNames) + "'\n"); // outStream.writeChars("TestRootbeerHybridBSP,input: '" // + Arrays.toString(input) + "'\n"); // Verify input peer.reopenInput(); while (peer.readNext(key, value)) { Assert.assertEquals(value.get(), input[key.get()]); } outStream.writeChars("TestRootbeerHybridBSP.bsp: input verified!'\n"); outStream.close(); }
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
@Override public void bspGpu(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer, Rootbeer rootbeer) throws IOException, SyncException, InterruptedException { TestRootbeerKernel kernel = new TestRootbeerKernel(CONF_BLOCK_SIZE * CONF_GRID_SIZE); // Run GPU Kernels Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start();/*from w w w .j av a2s. c o m*/ rootbeer.run(kernel, new ThreadConfig(CONF_BLOCK_SIZE, CONF_GRID_SIZE, CONF_BLOCK_SIZE * CONF_GRID_SIZE), context); watch.stop(); // Debug output BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("TestRootbeerHybridBSP.bspGpu executed on GPU!\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.writeChars("TestRootbeerHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n"); outStream .writeChars("TestRootbeerHybridBSP,GPUTime=" + (watch.elapsedTimeMillis() / 1000.0) + " seconds\n"); outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + kernel.peerName + "'\n"); outStream.writeChars( "TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(kernel.allPeerNames) + "'\n"); // outStream.writeChars("TestRootbeerHybridBSP,input: '" // + Arrays.toString(kernel.input) + "'\n"); // Verify input peer.reopenInput(); IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); while (peer.readNext(key, value)) { Assert.assertEquals(value.get(), kernel.input[key.get()]); } outStream.writeChars("TestRootbeerHybridBSP.bspGpu: input verified!'\n"); outStream.close(); }
From source file:at.illecker.hama.rootbeer.examples.hellorootbeer.HelloRootbeerGpuBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer) throws IOException, SyncException, InterruptedException { // Generate GPU Kernels List<Kernel> kernels = new ArrayList<Kernel>(); for (int i = 0; i < m_kernelCount; i++) { kernels.add(new HelloRootbeerKernel(m_iterations)); }// ww w. j a v a2 s.c o m // Run GPU Kernels Rootbeer rootbeer = new Rootbeer(); Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start(); rootbeer.run(kernels, context); watch.stop(); // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("KernelCount: " + m_kernelCount + "\n"); outStream.writeChars("Iterations: " + m_iterations + "\n"); outStream.writeChars("GPU time: " + watch.elapsedTimeMillis() + " ms\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.close(); // Send result to MasterTask for (int i = 0; i < m_kernelCount; i++) { HelloRootbeerKernel kernel = (HelloRootbeerKernel) kernels.get(i); peer.send(m_masterTask, new DoubleWritable(kernel.m_result)); } peer.sync(); }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.compositeinput.cpu.MatrixMultiplicationBSPCpu.java
License:Apache License
@Override public void setup(BSPPeer<IntWritable, TupleWritable, IntWritable, VectorWritable, MatrixRowMessage> peer) throws IOException { Configuration conf = peer.getConfiguration(); outCardinality = conf.getInt(OUT_CARD, Integer.MAX_VALUE); isDebuggingEnabled = conf.getBoolean(DEBUG, false); // Choose one as a master, who sorts the matrix rows at the end this.masterTask = peer.getPeerName(peer.getNumPeers() / 2); // Init logging if (isDebuggingEnabled) { try {/*from w ww.j av a2s.co m*/ FileSystem fs = FileSystem.get(conf); logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); logger.writeChars("bsp,setup,outCardinality=" + outCardinality + "\n"); } catch (IOException e) { e.printStackTrace(); } } }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.cpu.MatrixMultiplicationBSPCpu.java
License:Apache License
@Override public void setup( BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer) throws IOException { Configuration conf = peer.getConfiguration(); m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false); // Choose one as a master, who sorts the matrix rows at the end // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2); // TODO//from w w w . ja va 2s . c o m // task must be 0 otherwise write out does NOT work! m_masterTask = peer.getPeerName(0); // Init logging if (m_isDebuggingEnabled) { try { FileSystem fs = FileSystem.get(conf); m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); } catch (IOException e) { e.printStackTrace(); } } // Receive transposed Matrix B SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf); IntWritable bKey = new IntWritable(); PipesVectorWritable bVector = new PipesVectorWritable(); // for each col of matrix B (cause by transposed B) while (reader.next(bKey, bVector)) { m_bColumns.add(new KeyValuePair<Integer, DoubleVector>(bKey.get(), bVector.getVector())); if (m_isDebuggingEnabled) { m_logger.writeChars("setup,read,transposedMatrixB,key=" + bKey.get() + ",value=" + bVector.getVector().toString() + "\n"); } } reader.close(); }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.gpu.MatrixMultiplicationBSPGpu.java
License:Apache License
@Override public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, NullWritable> peer) throws IOException { Configuration conf = peer.getConfiguration(); m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false); // Choose one as a master, who sorts the matrix rows at the end // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2); // TODO/*from w w w .j a va 2 s . co m*/ // task must be 0 otherwise write out does NOT work! m_masterTask = peer.getPeerName(0); this.m_blockSize = Integer.parseInt(peer.getConfiguration().get(CONF_BLOCKSIZE)); this.m_gridSize = Integer.parseInt(peer.getConfiguration().get(CONF_GRIDSIZE)); // Init logging if (m_isDebuggingEnabled) { try { FileSystem fs = FileSystem.get(conf); m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); } catch (IOException e) { e.printStackTrace(); } } // Load matrixB SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf); List<DoubleVector> matrixB = new ArrayList<DoubleVector>(); IntWritable bKey = new IntWritable(); VectorWritable bVector = new VectorWritable(); // for each row of matrix B while (reader.next(bKey, bVector)) { matrixB.add(bVector.getVector()); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,setup,MatrixB (" + bKey.get() + "," + bVector.getVector() + ")\n"); } } reader.close(); // Convert matrixB to double array for GPU kernels m_matrixBArr = toArray(matrixB); if (m_isDebuggingEnabled) { for (int i = 0; i < m_matrixBArr.length; i++) { m_logger.writeChars("bsp,setup,MatrixBArr (" + i + "," + Arrays.toString(m_matrixBArr[i]) + ")\n"); } } // threadSliceSize defines how much multipliers // of column B has to be multiplied with column A m_threadSliceSize = divup(m_matrixBArr.length, m_blockSize); // blockSliceSize defines the column slice amount // columns of B per blockIters m_blockSliceSize = divup(m_matrixBArr[0].length, m_gridSize); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,setup,blockSize=" + m_blockSize + ",gridSize=" + m_gridSize + ",threadSliceSize=" + m_threadSliceSize + ",blockSliceSize=" + m_blockSliceSize + "\n"); } }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.MatrixMultiplicationBenchmark.java
License:Apache License
@Override protected void setUp() throws Exception { m_conf = new Configuration(); // Try to load Hadoop configuration String HADOOP_HOME = System.getenv("HADOOP_HOME"); String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL"); if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) { String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL); m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml")); m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml")); m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml")); m_conf.addResource(new Path(HADOOP, "conf/core-site.xml")); m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml")); m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml")); // System.out.println("Loaded Hadoop configuration from " + HADOOP); try {//from www .ja v a 2 s . c om // Connect to HDFS Filesystem FileSystem.get(m_conf); } catch (Exception e) { // HDFS not reachable run Benchmark locally m_conf = new Configuration(); m_runLocally = true; } // System.out.println("Run Benchmark local: " + m_runLocally); } // Try to load Hama configuration String HAMA_HOME = System.getenv("HAMA_HOME"); String HAMA_INSTALL = System.getenv("HAMA_INSTALL"); if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) { String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL); m_conf.addResource(new Path(HAMA, "conf/hama-default.xml")); m_conf.addResource(new Path(HAMA, "conf/hama-site.xml")); // System.out.println("Loaded Hama configuration from " + HAMA); } // Setup outputs m_OUTPUT_DIR_PATH = new Path(OUTPUT_DIR + "/bench_" + System.currentTimeMillis()); System.out.println("OUTPUT_DIR_PATH: " + m_OUTPUT_DIR_PATH); m_MATRIX_A_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixA.seq"); m_MATRIX_B_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixB.seq"); m_MATRIX_C_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixC.seq"); m_MATRIX_D_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixD.seq"); m_blockSize = MatrixMultiplicationBSPGpu.BLOCK_SIZE; m_gridSize = MatrixMultiplicationBSPGpu.GRID_SIZE; System.out.println("Benchmark MatrixMultiplication " + type + " [blockSize=" + m_blockSize + ",gridSize=" + m_gridSize + "] " + n + " x " + n + " matrix"); // Create random DistributedRowMatrix DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(42L), m_MATRIX_A_PATH, false); DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(1337L), m_MATRIX_B_PATH, (type == CalcType.CPU) ? true : false); // Load DistributedRowMatrix a and b m_matrixA = new DistributedRowMatrix(m_MATRIX_A_PATH, m_OUTPUT_DIR_PATH, n, n); m_matrixB = new DistributedRowMatrix(m_MATRIX_B_PATH, m_OUTPUT_DIR_PATH, n, n); m_matrixA.setConf(m_conf); m_matrixB.setConf(m_conf); }
From source file:at.illecker.hama.rootbeer.examples.piestimator.cpu.PiEstimatorCpuBSP.java
License:Apache License
@Override public void cleanup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException { // MasterTask writes out results if (peer.getPeerName().equals(m_masterTask)) { int numMessages = peer.getNumCurrentMessages(); long totalHits = 0; LongWritable received;//w w w .j a v a2s . co m while ((received = peer.getCurrentMessage()) != null) { totalHits += received.get(); } double pi = 4.0 * totalHits / (m_calculationsPerBspTask * numMessages); // DEBUG if (m_isDebuggingEnabled) { // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("BSP=PiEstimatorCpuBSP,Iterations=" + m_iterations + "\n"); outStream.writeChars("totalHits: " + totalHits + "\n"); outStream.writeChars("numMessages: " + numMessages + "\n"); outStream.writeChars("calculationsPerBspTask: " + m_calculationsPerBspTask + "\n"); outStream.writeChars("calculationsTotal: " + (m_calculationsPerBspTask * numMessages) + "\n"); outStream.close(); } peer.write( new Text("Estimated value of PI(3,14159265) using " + (m_calculationsPerBspTask * numMessages) // + (peer.getNumPeers() * m_threadCount * m_iterations) + " points is"), new DoubleWritable(pi)); } }
From source file:at.illecker.hama.rootbeer.examples.piestimator.gpu.PiEstimatorGpuBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException, SyncException, InterruptedException { PiEstimatorKernel kernel = new PiEstimatorKernel(m_calculationsPerThread, System.currentTimeMillis()); // Run GPU Kernels Rootbeer rootbeer = new Rootbeer(); Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start();// ww w. j a v a 2 s .co m rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context); watch.stop(); // Get GPU results long totalHits = 0; Result[] resultList = kernel.resultList.getList(); for (Result result : resultList) { if (result == null) { // break at end of list break; } totalHits += result.hits; } // DEBUG if (m_isDebuggingEnabled) { // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("BSP=PiEstimatorGpuBSP,Iterations=" + m_iterations + ",GPUTime=" + watch.elapsedTimeMillis() + "ms\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.writeChars("totalHits: " + totalHits + "\n"); outStream.writeChars("calculationsPerThread: " + m_calculationsPerThread + "\n"); outStream.writeChars("calculationsTotal: " + m_calculationsPerThread * m_blockSize * m_gridSize + "\n"); outStream.close(); } // Send result to MasterTask peer.send(m_masterTask, new LongWritable(totalHits)); peer.sync(); }