Example usage for org.apache.hadoop.fs FileSystem get

List of usage examples for org.apache.hadoop.fs FileSystem get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem get.

Prototype

public static FileSystem get(Configuration conf) throws IOException 

Source Link

Document

Returns the configured FileSystem implementation.

Usage

From source file:at.illecker.hama.hybrid.examples.testglobalgpusync.TestGlobalGpuSyncHybridBSP.java

License:Apache License

public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException {

    // Defaults/*from w  ww.ja va2  s  .  c  o  m*/
    int numBspTask = 1;
    int numGpuBspTask = 1;
    int blockSize = BLOCK_SIZE;
    int gridSize = GRID_SIZE;
    boolean isDebugging = false;

    Configuration conf = new HamaConfiguration();

    if (args.length > 0) {
        if (args.length == 5) {
            numBspTask = Integer.parseInt(args[0]);
            numGpuBspTask = Integer.parseInt(args[1]);
            blockSize = Integer.parseInt(args[2]);
            gridSize = Integer.parseInt(args[3]);
            isDebugging = Boolean.parseBoolean(args[4]);
        } else {
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            System.out.println("    Argument2=numGpuBspTask");
            System.out.println("    Argument3=blockSize");
            System.out.println("    Argument4=gridSize");
            System.out.println("    Argument5=debug | Enable debugging (true|false=default)");
            return;
        }
    }

    // Set config variables
    conf.setBoolean("hama.pipes.logging", isDebugging);
    // Set CPU tasks
    conf.setInt("bsp.peers.num", numBspTask);
    // Set GPU tasks
    conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    // Set GPU blockSize and gridSize
    conf.set(CONF_BLOCK_SIZE, "" + blockSize);
    conf.set(CONF_GRID_SIZE, "" + gridSize);
    conf.set(CONF_TMP_DIR, TMP_DIR.toString());

    LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0));
    LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0));
    LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum"));
    LOG.info("BlockSize: " + conf.get(CONF_BLOCK_SIZE));
    LOG.info("GridSize: " + conf.get(CONF_GRID_SIZE));
    LOG.info("TempDir: " + conf.get(CONF_TMP_DIR));
    LOG.info("isDebugging: " + conf.getBoolean("hama.pipes.logging", false));

    BSPJob job = createTestGlobalGpuSyncHybridBSPConf(conf);

    long startTime = System.currentTimeMillis();
    if (job.waitForCompletion(true)) {
        LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
        printOutput(job, FileSystem.get(conf), new Path(conf.get(CONF_TMP_DIR)));
    }
}

From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer)
        throws IOException, SyncException, InterruptedException {

    long startTime = System.currentTimeMillis();

    // test input
    int[] input = new int[CONF_BLOCK_SIZE * CONF_GRID_SIZE];
    IntWritable key = new IntWritable();
    IntWritable value = new IntWritable();
    while (peer.readNext(key, value)) {
        input[key.get()] = value.get();// w w w  .  ja  v  a  2  s.c  o  m
    }

    String peerName = peer.getPeerName();
    String[] allPeerNames = peer.getAllPeerNames();

    long stopTime = System.currentTimeMillis();

    // Debug output
    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("TestRootbeerHybridBSP.bsp executed on CPU!\n");
    outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + (stopTime - startTime) + " ms\n");
    outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + ((stopTime - startTime) / 1000.0) + " seconds\n");
    outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + peerName + "'\n");
    outStream.writeChars("TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(allPeerNames) + "'\n");
    // outStream.writeChars("TestRootbeerHybridBSP,input: '"
    // + Arrays.toString(input) + "'\n");

    // Verify input
    peer.reopenInput();
    while (peer.readNext(key, value)) {
        Assert.assertEquals(value.get(), input[key.get()]);
    }

    outStream.writeChars("TestRootbeerHybridBSP.bsp: input verified!'\n");
    outStream.close();
}

From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    TestRootbeerKernel kernel = new TestRootbeerKernel(CONF_BLOCK_SIZE * CONF_GRID_SIZE);

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();/*from w w  w  .j av  a2s.  c o  m*/
    rootbeer.run(kernel, new ThreadConfig(CONF_BLOCK_SIZE, CONF_GRID_SIZE, CONF_BLOCK_SIZE * CONF_GRID_SIZE),
            context);
    watch.stop();

    // Debug output
    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("TestRootbeerHybridBSP.bspGpu executed on GPU!\n");
    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("TestRootbeerHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n");
    outStream
            .writeChars("TestRootbeerHybridBSP,GPUTime=" + (watch.elapsedTimeMillis() / 1000.0) + " seconds\n");
    outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + kernel.peerName + "'\n");
    outStream.writeChars(
            "TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(kernel.allPeerNames) + "'\n");
    // outStream.writeChars("TestRootbeerHybridBSP,input: '"
    // + Arrays.toString(kernel.input) + "'\n");

    // Verify input
    peer.reopenInput();
    IntWritable key = new IntWritable();
    IntWritable value = new IntWritable();
    while (peer.readNext(key, value)) {
        Assert.assertEquals(value.get(), kernel.input[key.get()]);
    }

    outStream.writeChars("TestRootbeerHybridBSP.bspGpu: input verified!'\n");
    outStream.close();
}

From source file:at.illecker.hama.rootbeer.examples.hellorootbeer.HelloRootbeerGpuBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
        throws IOException, SyncException, InterruptedException {

    // Generate GPU Kernels
    List<Kernel> kernels = new ArrayList<Kernel>();
    for (int i = 0; i < m_kernelCount; i++) {
        kernels.add(new HelloRootbeerKernel(m_iterations));
    }// ww  w. j a v a2 s.c  o m

    // Run GPU Kernels
    Rootbeer rootbeer = new Rootbeer();
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();
    rootbeer.run(kernels, context);
    watch.stop();

    // Write log to dfs
    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("KernelCount: " + m_kernelCount + "\n");
    outStream.writeChars("Iterations: " + m_iterations + "\n");
    outStream.writeChars("GPU time: " + watch.elapsedTimeMillis() + " ms\n");
    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }
    outStream.close();

    // Send result to MasterTask
    for (int i = 0; i < m_kernelCount; i++) {
        HelloRootbeerKernel kernel = (HelloRootbeerKernel) kernels.get(i);
        peer.send(m_masterTask, new DoubleWritable(kernel.m_result));
    }
    peer.sync();
}

From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.compositeinput.cpu.MatrixMultiplicationBSPCpu.java

License:Apache License

@Override
public void setup(BSPPeer<IntWritable, TupleWritable, IntWritable, VectorWritable, MatrixRowMessage> peer)
        throws IOException {

    Configuration conf = peer.getConfiguration();

    outCardinality = conf.getInt(OUT_CARD, Integer.MAX_VALUE);
    isDebuggingEnabled = conf.getBoolean(DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    this.masterTask = peer.getPeerName(peer.getNumPeers() / 2);

    // Init logging
    if (isDebuggingEnabled) {
        try {/*from   w  ww.j av a2s.co  m*/
            FileSystem fs = FileSystem.get(conf);
            logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

            logger.writeChars("bsp,setup,outCardinality=" + outCardinality + "\n");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}

From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.cpu.MatrixMultiplicationBSPCpu.java

License:Apache License

@Override
public void setup(
        BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer)
        throws IOException {

    Configuration conf = peer.getConfiguration();
    m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO//from  w w w .  ja va  2s  .  c  o m
    // task must be 0 otherwise write out does NOT work!
    m_masterTask = peer.getPeerName(0);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Receive transposed Matrix B
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf);

    IntWritable bKey = new IntWritable();
    PipesVectorWritable bVector = new PipesVectorWritable();
    // for each col of matrix B (cause by transposed B)
    while (reader.next(bKey, bVector)) {
        m_bColumns.add(new KeyValuePair<Integer, DoubleVector>(bKey.get(), bVector.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("setup,read,transposedMatrixB,key=" + bKey.get() + ",value="
                    + bVector.getVector().toString() + "\n");
        }
    }
    reader.close();
}

From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.gpu.MatrixMultiplicationBSPGpu.java

License:Apache License

@Override
public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, NullWritable> peer)
        throws IOException {

    Configuration conf = peer.getConfiguration();
    m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO/*from  w  w w  .j  a  va 2  s . co m*/
    // task must be 0 otherwise write out does NOT work!
    m_masterTask = peer.getPeerName(0);

    this.m_blockSize = Integer.parseInt(peer.getConfiguration().get(CONF_BLOCKSIZE));

    this.m_gridSize = Integer.parseInt(peer.getConfiguration().get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Load matrixB
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf);

    List<DoubleVector> matrixB = new ArrayList<DoubleVector>();
    IntWritable bKey = new IntWritable();
    VectorWritable bVector = new VectorWritable();
    // for each row of matrix B
    while (reader.next(bKey, bVector)) {
        matrixB.add(bVector.getVector());

        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bsp,setup,MatrixB (" + bKey.get() + "," + bVector.getVector() + ")\n");
        }
    }
    reader.close();

    // Convert matrixB to double array for GPU kernels
    m_matrixBArr = toArray(matrixB);

    if (m_isDebuggingEnabled) {
        for (int i = 0; i < m_matrixBArr.length; i++) {
            m_logger.writeChars("bsp,setup,MatrixBArr (" + i + "," + Arrays.toString(m_matrixBArr[i]) + ")\n");
        }
    }

    // threadSliceSize defines how much multipliers
    // of column B has to be multiplied with column A
    m_threadSliceSize = divup(m_matrixBArr.length, m_blockSize);

    // blockSliceSize defines the column slice amount
    // columns of B per blockIters
    m_blockSliceSize = divup(m_matrixBArr[0].length, m_gridSize);

    if (m_isDebuggingEnabled) {
        m_logger.writeChars("bsp,setup,blockSize=" + m_blockSize + ",gridSize=" + m_gridSize
                + ",threadSliceSize=" + m_threadSliceSize + ",blockSliceSize=" + m_blockSliceSize + "\n");
    }
}

From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.MatrixMultiplicationBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {//from www  .ja v  a  2 s . c  om
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // Setup outputs
    m_OUTPUT_DIR_PATH = new Path(OUTPUT_DIR + "/bench_" + System.currentTimeMillis());
    System.out.println("OUTPUT_DIR_PATH: " + m_OUTPUT_DIR_PATH);

    m_MATRIX_A_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixA.seq");
    m_MATRIX_B_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixB.seq");
    m_MATRIX_C_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixC.seq");
    m_MATRIX_D_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixD.seq");

    m_blockSize = MatrixMultiplicationBSPGpu.BLOCK_SIZE;
    m_gridSize = MatrixMultiplicationBSPGpu.GRID_SIZE;

    System.out.println("Benchmark MatrixMultiplication " + type + " [blockSize=" + m_blockSize + ",gridSize="
            + m_gridSize + "] " + n + " x " + n + " matrix");

    // Create random DistributedRowMatrix
    DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(42L), m_MATRIX_A_PATH,
            false);

    DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(1337L), m_MATRIX_B_PATH,
            (type == CalcType.CPU) ? true : false);

    // Load DistributedRowMatrix a and b
    m_matrixA = new DistributedRowMatrix(m_MATRIX_A_PATH, m_OUTPUT_DIR_PATH, n, n);
    m_matrixB = new DistributedRowMatrix(m_MATRIX_B_PATH, m_OUTPUT_DIR_PATH, n, n);
    m_matrixA.setConf(m_conf);
    m_matrixB.setConf(m_conf);
}

From source file:at.illecker.hama.rootbeer.examples.piestimator.cpu.PiEstimatorCpuBSP.java

License:Apache License

@Override
public void cleanup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer)
        throws IOException {

    // MasterTask writes out results
    if (peer.getPeerName().equals(m_masterTask)) {

        int numMessages = peer.getNumCurrentMessages();

        long totalHits = 0;
        LongWritable received;//w  w  w  .j a  v a2s  .  co m
        while ((received = peer.getCurrentMessage()) != null) {
            totalHits += received.get();
        }

        double pi = 4.0 * totalHits / (m_calculationsPerBspTask * numMessages);

        // DEBUG
        if (m_isDebuggingEnabled) {
            // Write log to dfs
            BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
            FileSystem fs = FileSystem.get(peer.getConfiguration());
            FSDataOutputStream outStream = fs
                    .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

            outStream.writeChars("BSP=PiEstimatorCpuBSP,Iterations=" + m_iterations + "\n");

            outStream.writeChars("totalHits: " + totalHits + "\n");
            outStream.writeChars("numMessages: " + numMessages + "\n");
            outStream.writeChars("calculationsPerBspTask: " + m_calculationsPerBspTask + "\n");
            outStream.writeChars("calculationsTotal: " + (m_calculationsPerBspTask * numMessages) + "\n");
            outStream.close();
        }

        peer.write(
                new Text("Estimated value of PI(3,14159265) using " + (m_calculationsPerBspTask * numMessages)
                // + (peer.getNumPeers() * m_threadCount * m_iterations)
                        + " points is"),
                new DoubleWritable(pi));
    }
}

From source file:at.illecker.hama.rootbeer.examples.piestimator.gpu.PiEstimatorGpuBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer)
        throws IOException, SyncException, InterruptedException {

    PiEstimatorKernel kernel = new PiEstimatorKernel(m_calculationsPerThread, System.currentTimeMillis());

    // Run GPU Kernels
    Rootbeer rootbeer = new Rootbeer();
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();//  ww w. j a v a  2  s .co m
    rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context);
    watch.stop();

    // Get GPU results
    long totalHits = 0;
    Result[] resultList = kernel.resultList.getList();
    for (Result result : resultList) {
        if (result == null) { // break at end of list
            break;
        }
        totalHits += result.hits;
    }

    // DEBUG
    if (m_isDebuggingEnabled) {
        // Write log to dfs
        BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
        FileSystem fs = FileSystem.get(peer.getConfiguration());
        FSDataOutputStream outStream = fs
                .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

        outStream.writeChars("BSP=PiEstimatorGpuBSP,Iterations=" + m_iterations + ",GPUTime="
                + watch.elapsedTimeMillis() + "ms\n");
        List<StatsRow> stats = context.getStats();
        for (StatsRow row : stats) {
            outStream.writeChars("  StatsRow:\n");
            outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
            outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
            outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
            outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
            outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
        }

        outStream.writeChars("totalHits: " + totalHits + "\n");
        outStream.writeChars("calculationsPerThread: " + m_calculationsPerThread + "\n");
        outStream.writeChars("calculationsTotal: " + m_calculationsPerThread * m_blockSize * m_gridSize + "\n");
        outStream.close();
    }

    // Send result to MasterTask
    peer.send(m_masterTask, new LongWritable(totalHits));
    peer.sync();
}