Example usage for org.apache.hadoop.fs FileSystem get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem get.

Prototype

public static FileSystem get(Configuration conf) throws IOException 

Source Link

Document

Returns the configured FileSystem implementation.

Usage

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {

    // printOutput(m_conf, CONF_OUTPUT_DIR);

    // Cleanup/*  ww  w.  j av a2  s  .  c om*/
    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);
}

---

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBenchmark.java

License:Apache License

static void printOutput(Configuration conf, Path path) throws IOException {
    System.out.println("printOutput: " + path);
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] files = fs.listStatus(path);
    if (files != null) {
        for (int i = 0; i < files.length; i++) {
            if (files[i].getLen() > 0) {
                System.out.println("File " + files[i].getPath());
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, conf, false);
                in.close();/*from www.  j a v a 2s  .c  o  m*/
            }
        }
    } else {
        System.out.println("No directory available: " + path);
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override/*from  w  ww . j  a v a2s .co m*/
public void setup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer)
        throws IOException {

    m_conf = peer.getConfiguration();
    this.m_timeMeasurement = m_conf.getBoolean(CONF_TIME, false);
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }

    int totalTaskNum = peer.getNumPeers();
    int cpuTaskNum = totalTaskNum - m_conf.getInt("bsp.peers.gpu.num", 0);

    // Choose one as a master
    this.m_masterTask = peer.getPeerName(totalTaskNum / 2);

    this.m_iterations = Long.parseLong(m_conf.get(CONF_ITERATIONS));
    this.m_GPUPercentage = Integer.parseInt(m_conf.get(CONF_GPU_PERCENTAGE));
    this.m_blockSize = Integer.parseInt(m_conf.get(CONF_BLOCKSIZE));
    this.m_gridSize = Integer.parseInt(m_conf.get(CONF_GRIDSIZE));

    // Compute work distributions
    long iterationsPerGPUTask = 0;
    long iterationsPerCPU = 0;
    if ((m_GPUPercentage > 0) && (m_GPUPercentage <= 100)) {
        iterationsPerGPUTask = (m_iterations * m_GPUPercentage) / 100;
        iterationsPerCPU = m_iterations - iterationsPerGPUTask;
    } else {
        iterationsPerCPU = m_iterations;
    }
    if (cpuTaskNum > 0) {
        m_iterationsPerCPUTask = iterationsPerCPU / cpuTaskNum;
    }
    m_iterationsPerGPUThread = iterationsPerGPUTask / (m_blockSize * m_gridSize);

    long stopTime = 0;
    if (m_timeMeasurement) {
        stopTime = System.currentTimeMillis();
        LOG.info("# setupTime: " + ((stopTime - startTime) / 1000.0) + " sec");
    }

    // DEBUG
    if (m_isDebuggingEnabled) {
        m_logger.writeChars("PiEstimatorHybrid,GPUPercentage=" + m_GPUPercentage + "\n");
        m_logger.writeChars("PiEstimatorHybrid,cpuTaskNum=" + cpuTaskNum + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerCPU=" + iterationsPerCPU + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerCPUTask=" + m_iterationsPerCPUTask + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerGPUTask=" + iterationsPerGPUTask + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerGPUThread=" + m_iterationsPerGPUThread + "\n");
        if (m_timeMeasurement) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,setupTime: " + ((stopTime - startTime) / 1000.0) + " sec\n\n");
        }
    }

}

---

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBSP.java

License:Apache License

static void printOutput(BSPJob job) throws IOException {
    FileSystem fs = FileSystem.get(job.getConfiguration());
    FileStatus[] files = fs.listStatus(FileOutputFormat.getOutputPath(job));
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            FSDataInputStream in = fs.open(files[i].getPath());
            IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
            in.close();/* ww  w  .  java 2s. c om*/
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<Text, Text, Text, DoubleWritable, DoubleWritable> peer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs//from  w w w.  j  a  v a  2 s .  c  om
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("SummationBSP.bsp executed on CPU!\n");

    double intermediateSum = 0.0;
    Text key = new Text();
    Text value = new Text();

    while (peer.readNext(key, value)) {
        outStream.writeChars("SummationBSP.bsp key: " + key + " value: " + value + "\n");
        intermediateSum += Double.parseDouble(value.toString());
    }

    outStream.writeChars("SummationBSP.bsp send intermediateSum: " + intermediateSum + "\n");

    peer.send(m_masterTask, new DoubleWritable(intermediateSum));
    peer.sync();

    // Consume messages
    if (peer.getPeerName().equals(m_masterTask)) {
        outStream.writeChars("SummationBSP.bsp consume messages...\n");

        double sum = 0.0;
        int msg_count = peer.getNumCurrentMessages();

        for (int i = 0; i < msg_count; i++) {
            DoubleWritable msg = peer.getCurrentMessage();
            outStream.writeChars("SummationBSP.bsp message: " + msg.get() + "\n");
            sum += msg.get();
        }

        outStream.writeChars("SummationBSP.bsp write Sum: " + sum + "\n");
        peer.write(new Text("Sum"), new DoubleWritable(sum));
    }
    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<Text, Text, Text, DoubleWritable, DoubleWritable> peer, Rootbeer rootbeer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs// www  . j av  a  2  s . c  o  m
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("SummationBSP.bspGpu executed on GPU!\n");

    SummationKernel kernel = new SummationKernel(m_masterTask);

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();
    // 1 Kernel within 1 Block
    rootbeer.run(kernel, new ThreadConfig(1, 1, 1), context);
    watch.stop();

    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("SummationBSP,GPUTime=" + watch.elapsedTimeMillis() + "ms\n");
    outStream.writeChars("SummationBSP,peerName: '" + kernel.m_peerName + "'\n");
    outStream.writeChars("SummationBSP,numPeers: '" + kernel.m_numPeers + "'\n");
    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

static void printOutput(BSPJob job, BigDecimal sum) throws IOException {
    FileSystem fs = FileSystem.get(job.getConfiguration());
    FileStatus[] listStatus = fs.listStatus(FileOutputFormat.getOutputPath(job));
    for (FileStatus status : listStatus) {
        if (!status.isDir()) {
            try {
                SequenceFile.Reader reader = new SequenceFile.Reader(fs, status.getPath(),
                        job.getConfiguration());

                Text key = new Text();
                DoubleWritable value = new DoubleWritable();

                if (reader.next(key, value)) {
                    LOG.info("Output File: " + status.getPath());
                    LOG.info("key: '" + key + "' value: '" + value + "' expected: '" + sum.doubleValue() + "'");
                    Assert.assertEquals("Expected value: '" + sum.doubleValue() + "' != '" + value + "'",
                            sum.doubleValue(), value.get(), Math.pow(10, (DOUBLE_PRECISION * -1)));
                }//from   ww w .j a v a2 s  .c o  m
                reader.close();

            } catch (IOException e) {
                if (status.getLen() > 0) {
                    System.out.println("Output File " + status.getPath());
                    FSDataInputStream in = fs.open(status.getPath());
                    IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
                    in.close();
                }
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException {

    HamaConfiguration conf = new HamaConfiguration();
    BSPJob job = getSummationJob(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR);

    if (args.length > 0) {
        if (args.length == 1) {
            job.setNumBspTask(Integer.parseInt(args[0]));
        } else {/*from w  w w  . ja v a  2 s .  c  om*/
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            return;
        }
    } else {
        // job.setNumBspTask(cluster.getMaxTasks());
        job.setNumBspTask(2); // 1 CPU and 1 GPU
    }
    job.setNumBspGpuTask(1);

    // Generate Summation input
    FileSystem fs = FileSystem.get(conf);
    BigDecimal sum = writeSummationInputFile(fs, CONF_INPUT_DIR, job.getNumBspTask());

    LOG.info("NumBspTask: " + job.getNumBspTask());
    LOG.info("NumBspGpuTask: " + job.getNumBspGpuTask());
    LOG.info("inputPath: " + CONF_INPUT_DIR);
    LOG.info("outputPath: " + CONF_OUTPUT_DIR);

    long startTime = System.currentTimeMillis();
    if (job.waitForCompletion(true)) {
        System.out.println("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");

        printOutput(job, sum);
    }
}

---

From source file:at.illecker.hama.hybrid.examples.testglobalgpusync.TestGlobalGpuSyncHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable, IntWritable> peer)
        throws IOException, SyncException, InterruptedException {

    // Debug output
    HamaConfiguration conf = peer.getConfiguration();
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs.create(new Path(conf.get(CONF_TMP_DIR), peer.getTaskId() + ".log"));
    outStream.writeChars("TestGlobalGpuSycHybridBSP.bsp executed on CPU!\n");

    peer.send(m_masterTask, new IntWritable(peer.getPeerIndex()));

    peer.sync();//from  w  w  w . ja  va2 s  .  c o m

    // If master, fetch messages
    if (peer.getPeerName().equals(m_masterTask)) {
        peer.getNumCurrentMessages();

        int msgCount = peer.getNumCurrentMessages();
        for (int i = 0; i < msgCount; i++) {
            int id = peer.getCurrentMessage().get();
            outStream.writeChars(id + "\n");
        }
    }

    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.testglobalgpusync.TestGlobalGpuSyncHybridBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable, IntWritable> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    HamaConfiguration conf = peer.getConfiguration();
    int blockSize = Integer.parseInt(conf.get(CONF_BLOCK_SIZE));
    int gridSize = Integer.parseInt(conf.get(CONF_GRID_SIZE));

    TestGlobalGpuSyncKernel kernel = new TestGlobalGpuSyncKernel(m_masterTask);

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();/*www . j a v  a 2 s .  co m*/
    rootbeer.run(kernel, new ThreadConfig(blockSize, gridSize, blockSize * gridSize), context);
    watch.stop();

    // Debug output
    FileSystem fs = FileSystem.get(conf);
    FSDataOutputStream outStream = fs.create(new Path(conf.get(CONF_TMP_DIR), peer.getTaskId() + ".log"));

    outStream.writeChars("TestGlobalGpuSycHybridBSP.bspGpu executed on GPU!\n");
    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("TestGlobalGpuSycHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,BlockSize=" + blockSize + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,GridSize=" + gridSize + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,TotalThreads=" + (blockSize * gridSize) + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,MessageCount=" + kernel.messageCount + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,MessageSum=" + kernel.messageSum + "\n");

    Assert.assertEquals((blockSize * gridSize), kernel.messageCount);
    int n = (blockSize * gridSize) - 1;
    Assert.assertEquals((n * (n + 1)) / 2, kernel.messageSum);
    outStream.writeChars("TestGlobalGpuSycHybridBSP.bspGpu: messages verified!'\n");
    outStream.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »