Example usage for org.apache.hadoop.fs Path Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

@Override
public void bspGpu(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }/*from  ww  w  .j  a  v  a 2s . co  m*/

    // Fetch inputs
    final List<DoubleVector> inputs = new ArrayList<DoubleVector>();
    final PipesVectorWritable key = new PipesVectorWritable();
    final NullWritable nullValue = NullWritable.get();
    while (peer.readNext(key, nullValue)) {
        inputs.add(key.getVector());
    }
    // Convert inputs to double[][]
    double[][] inputsArr = new double[inputs.size()][inputs.get(0).getLength()];
    for (int i = 0; i < inputs.size(); i++) {
        double[] vector = inputs.get(i).toArray();
        for (int j = 0; j < vector.length; j++) {
            inputsArr[i][j] = vector[j];
        }
    }

    // Logging
    if (m_isDebuggingEnabled) {
        m_logger.writeChars("KMeansHybrid.bspGpu executed on GPU!\n");
        m_logger.writeChars(
                "KMeansHybrid.bspGpu blockSize: " + m_blockSize + " gridSize: " + m_gridSize + "\n");
        m_logger.writeChars("KMeansHybrid.bspGpu inputSize: " + inputs.size() + "\n");
    }

    KMeansHybridKernel kernel = new KMeansHybridKernel(inputsArr, m_centers_gpu,
            m_conf.getInt(CONF_MAX_ITERATIONS, 0), peer.getAllPeerNames());

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();
    rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context);
    watch.stop();

    // Output inputs with corresponding new center id
    for (int i = 0; i < inputs.size(); i++) {
        peer.write(new IntWritable(kernel.m_input_centers[i]), new PipesVectorWritable(inputs.get(i)));
    }

    // Output new Centers only on first task
    // to prevent collisions
    if (peer.getPeerName().equals(peer.getPeerName(0))) {
        String pathString = m_conf.get(CONF_CENTER_OUT_PATH);
        if (pathString != null) {
            final SequenceFile.Writer dataWriter = SequenceFile.createWriter(FileSystem.get(m_conf), m_conf,
                    new Path(pathString), PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);

            for (int i = 0; i < kernel.m_centers.length; i++) {
                dataWriter.append(new PipesVectorWritable(new DenseDoubleVector(kernel.m_centers[i])),
                        nullValue);
            }
            dataWriter.close();
        }
    }

    long stopTime = System.currentTimeMillis();
    if (m_timeMeasurement) {
        LOG.info("# bspGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec");

        if (m_isDebuggingEnabled) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,bspGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }

    // Logging
    if (m_isDebuggingEnabled) {
        List<StatsRow> stats = context.getStats();
        for (StatsRow row : stats) {
            m_logger.writeChars("  StatsRow:\n");
            m_logger.writeChars("    serial time: " + row.getSerializationTime() + "\n");
            m_logger.writeChars("    exec time: " + row.getExecutionTime() + "\n");
            m_logger.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
            m_logger.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
            m_logger.writeChars("    num threads: " + row.getNumThreads() + "\n");
            m_logger.writeChars("GPUTime: " + watch.elapsedTimeMillis() + " ms" + "\n");
        }

        m_logger.close();
    }
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBenchmark.java

License:Apache License

static void printOutput(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] files = fs.listStatus(new Path(OUTPUT_DIR));
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            FSDataInputStream in = fs.open(files[i].getPath());
            IOUtils.copyBytes(in, System.out, conf, false);
            in.close();/*from w  ww .j  a  v  a  2s .c om*/
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}