Example usage for org.apache.hadoop.fs FileSystem get

List of usage examples for org.apache.hadoop.fs FileSystem get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem get.

Prototype

public static FileSystem get(Configuration conf) throws IOException

Source Link

Document

Returns the configured FileSystem implementation.

Usage

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.MatrixMultiplicationBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {
    // verify();//from  w  w  w . j a va2 s .com

    // Cleanup
    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);

    // printOutput(m_conf);
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.MatrixMultiplicationBenchmark.java

License:Apache License

private void printOutput(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] files = fs.listStatus(CONF_OUTPUT_DIR);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            FSDataInputStream in = fs.open(files[i].getPath());
            IOUtils.copyBytes(in, System.out, conf, false);
            in.close();//from ww w . j a v a  2 s. c om
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<IntWritable, NullWritable, IntWritable, NullWritable, NullWritable> peer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs/*from   w  ww.  j a  v a2 s .  c  o m*/
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("HelloHybrid.bsp executed on CPU!\n");

    ArrayList<Integer> summation = new ArrayList<Integer>();

    // test input
    IntWritable key = new IntWritable();
    NullWritable nullValue = NullWritable.get();

    while (peer.readNext(key, nullValue)) {
        outStream.writeChars("input: key: '" + key.get() + "'\n");
        summation.add(key.get());
    }

    // test sequenceFileReader
    Path example = new Path(peer.getConfiguration().get(CONF_EXAMPLE_PATH));
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, example, peer.getConfiguration());

        int i = 0;
        while (reader.next(key, nullValue)) {
            outStream.writeChars("sequenceFileReader: key: '" + key.get() + "'\n");
            if (i < summation.size()) {
                summation.set(i, summation.get(i) + key.get());
            }
            i++;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    // test output
    for (Integer i : summation) {
        key.set(i);
        outStream.writeChars("output: key: '" + key.get() + "'\n");
        peer.write(key, nullValue);
    }

    // test getAllPeerNames
    outStream.writeChars("getAllPeerNames: '" + Arrays.toString(peer.getAllPeerNames()) + "'\n");

    // test String.split
    String splitString = "boo:and:foo";
    String[] splits;

    outStream.writeChars("splitString: '" + splitString + "'\n");

    splits = splitString.split(":");
    outStream.writeChars("split(\":\") len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", 2);
    outStream.writeChars(
            "split(\":\",2) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", 5);
    outStream.writeChars(
            "split(\":\",5) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", -2);
    outStream.writeChars(
            "split(\":\",-2) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(";");
    outStream.writeChars("split(\";\") len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    outStream.close();
}

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<IntWritable, NullWritable, IntWritable, NullWritable, NullWritable> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs/*from   w  ww.j  a  v  a  2 s .c  om*/
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("HelloHybrid.bspGpu executed on GPU!\n");

    HelloHybridKernel kernel = new HelloHybridKernel(peer.getConfiguration().get(CONF_EXAMPLE_PATH), CONF_N,
            "boo:and:foo", ":");

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();
    // 1 Kernel within 1 Block
    rootbeer.run(kernel, new ThreadConfig(1, 1, 1), context);
    watch.stop();

    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("HelloHybridKernel,GPUTime=" + watch.elapsedTimeMillis() + "ms\n");
    outStream.writeChars("HelloHybridKernel,peerName: '" + kernel.peerName + "'\n");
    outStream.writeChars("HelloHybridKernel,numPeers: '" + kernel.numPeers + "'\n");
    outStream.writeChars("HelloHybridKernel,summation: '" + Arrays.toString(kernel.summation) + "'\n");
    outStream.writeChars("HelloHybridKernel,getAllPeerNames: '" + Arrays.toString(kernel.allPeerNames) + "'\n");

    // test String.split
    outStream.writeChars("HelloHybridKernel,splitString: '" + kernel.splitString + "'\n");
    outStream.writeChars("HelloHybridKernel,split(\"" + kernel.delimiter + "\") len: " + kernel.splits1.length
            + " values: '" + Arrays.toString(kernel.splits1) + "'\n");
    outStream.writeChars("HelloHybridKernel,split(\"" + kernel.delimiter + "\",2) len: " + kernel.splits2.length
            + " values: '" + Arrays.toString(kernel.splits2) + "'\n");
    outStream.writeChars("HelloHybridKernel,split(\"" + kernel.delimiter + "\",5) len: " + kernel.splits3.length
            + " values: '" + Arrays.toString(kernel.splits3) + "'\n");
    outStream.writeChars("HelloHybridKernel,split(\"" + kernel.delimiter + "\",-2) len: "
            + kernel.splits4.length + " values: '" + Arrays.toString(kernel.splits4) + "'\n");
    outStream.writeChars("HelloHybridKernel,split(\";\") len: " + kernel.splits5.length + " values: '"
            + Arrays.toString(kernel.splits5) + "'\n");

    outStream.close();
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {//from  ww  w .j av  a2s .c  o m
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // Setup KMeans config variables
    m_conf.setBoolean(KMeansHybridBSP.CONF_DEBUG, false);
    m_conf.setBoolean("hama.pipes.logging", false);
    m_conf.setBoolean(KMeansHybridBSP.CONF_TIME, false);

    // Set GPU blockSize and gridSize
    m_conf.set(KMeansHybridBSP.CONF_BLOCKSIZE, "" + BLOCK_SIZE);
    m_conf.set(KMeansHybridBSP.CONF_GRIDSIZE, "" + GRID_SIZE);
    // Set maxIterations for KMeans
    m_conf.setInt(KMeansHybridBSP.CONF_MAX_ITERATIONS, maxIteration);
    // Set n for KMeans
    m_conf.setLong(KMeansHybridBSP.CONF_N, n);
    // Set GPUPercentage
    m_conf.setInt(KMeansHybridBSP.CONF_GPU_PERCENTAGE, GPUWorkload);

    Path centerIn = new Path(CONF_CENTER_DIR, "center_in.seq");
    Path centerOut = new Path(CONF_CENTER_DIR, "center_out.seq");
    m_conf.set(KMeansHybridBSP.CONF_CENTER_IN_PATH, centerIn.toString());
    m_conf.set(KMeansHybridBSP.CONF_CENTER_OUT_PATH, centerOut.toString());

    // CPU vs GPU benchmark
    // Plot 1 and 2
    int numGpuBspTask = 0;
    // if (type == CalcType.GPU) {
    // bspTaskNum = 1;
    // numGpuBspTask = 1;
    // GPUWorkload = 100;
    // }

    // CPU + GPU Hybrid benchmark
    // Plot 3
    if (bspTaskNum == maxTaskNum) {
        numGpuBspTask = 1;
        GPUWorkload = 75;
    } else {
        numGpuBspTask = 0;
    }

    // Set CPU tasks
    m_conf.setInt("bsp.peers.num", bspTaskNum);
    // Set GPU tasks
    m_conf.setInt("bsp.peers.gpu.num", numGpuBspTask);

    // Generate input data
    KMeansHybridBSP.prepareInputData(m_conf, FileSystem.get(m_conf), CONF_INPUT_DIR, centerIn, bspTaskNum,
            numGpuBspTask, n, k, vectorDimension, null, GPUWorkload);

    // Debug output
    // System.out.println("CalcType: " + type);
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
    System.out.println("NumBspTask: " + m_conf.getInt("bsp.peers.num", 0) + " NumGpuBspTask: "
            + m_conf.getInt("bsp.peers.gpu.num", 0));
    System.out.println("n: " + n + " k: " + k + " vectorDimension: " + vectorDimension + " maxIteration: "
            + maxIteration + " GPUWorkload: " + GPUWorkload + "%");
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {
    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override/*w ww .  jav a  2 s .  c o m*/
public void setup(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException {

    this.m_conf = peer.getConfiguration();
    this.m_timeMeasurement = m_conf.getBoolean(CONF_TIME, false);
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);
    this.m_maxIterations = m_conf.getInt(CONF_MAX_ITERATIONS, -1);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }

    // Init center vectors
    Path centroids = new Path(m_conf.get(CONF_CENTER_IN_PATH));
    FileSystem fs = FileSystem.get(m_conf);

    final ArrayList<DoubleVector> centers = new ArrayList<DoubleVector>();
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, centroids, m_conf);
        PipesVectorWritable key = new PipesVectorWritable();
        NullWritable value = NullWritable.get();
        while (reader.next(key, value)) {
            DoubleVector center = key.getVector();
            centers.add(center);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    Preconditions.checkArgument(centers.size() > 0, "Centers file must contain at least a single center!");

    this.m_centers_cpu = centers.toArray(new DoubleVector[centers.size()]);

    long stopTime = 0;
    if (m_timeMeasurement) {
        stopTime = System.currentTimeMillis();
        LOG.info("# setupTime: " + ((stopTime - startTime) / 1000.0) + " sec");
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("PiEstimatorHybrid,setupTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

private void recalculateAssignmentsAndWrite(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException {

    IntWritable keyWrite = new IntWritable();
    for (DoubleVector v : m_cache) {
        final int lowestDistantCenter = getNearestCenter(v);
        keyWrite.set(lowestDistantCenter);
        peer.write(keyWrite, new PipesVectorWritable(v));
    }/* ww  w. ja va2  s  .c o  m*/

    // just on the first task write the centers to filesystem to prevent
    // collisions
    if (peer.getPeerName().equals(peer.getPeerName(0))) {
        String pathString = m_conf.get(CONF_CENTER_OUT_PATH);
        if (pathString != null) {
            final SequenceFile.Writer dataWriter = SequenceFile.createWriter(FileSystem.get(m_conf), m_conf,
                    new Path(pathString), PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);
            final NullWritable value = NullWritable.get();

            for (DoubleVector center : m_centers_cpu) {
                dataWriter.append(new PipesVectorWritable(center), value);
            }
            dataWriter.close();
        }
    }
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

/********************************* GPU *********************************/
@Override/* w ww. j  a  v a 2s.c  om*/
public void setupGpu(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException, SyncException, InterruptedException {

    this.m_conf = peer.getConfiguration();
    this.m_timeMeasurement = m_conf.getBoolean(CONF_TIME, false);
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);
    this.m_maxIterations = m_conf.getInt(CONF_MAX_ITERATIONS, -1);
    this.m_blockSize = Integer.parseInt(this.m_conf.get(CONF_BLOCKSIZE));
    this.m_gridSize = Integer.parseInt(this.m_conf.get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }

    // Init center vectors
    Path centroids = new Path(m_conf.get(CONF_CENTER_IN_PATH));
    FileSystem fs = FileSystem.get(m_conf);

    final List<double[]> centers = new ArrayList<double[]>();
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, centroids, m_conf);
        PipesVectorWritable key = new PipesVectorWritable();
        NullWritable value = NullWritable.get();
        while (reader.next(key, value)) {
            centers.add(key.getVector().toArray());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    Preconditions.checkArgument(centers.size() > 0, "Centers file must contain at least a single center!");

    // build centers_gpu double[][]
    this.m_centers_gpu = new double[centers.size()][centers.get(0).length];
    for (int i = 0; i < centers.size(); i++) {
        double[] vector = centers.get(i);
        for (int j = 0; j < vector.length; j++) {
            this.m_centers_gpu[i][j] = vector[j];
        }
    }

    long stopTime = 0;
    if (m_timeMeasurement) {
        stopTime = System.currentTimeMillis();
        LOG.info("# setupGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec");
        if (m_isDebuggingEnabled) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,setupGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

@Override
public void bspGpu(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }//from w ww  .j  ava 2 s.  c  o  m

    // Fetch inputs
    final List<DoubleVector> inputs = new ArrayList<DoubleVector>();
    final PipesVectorWritable key = new PipesVectorWritable();
    final NullWritable nullValue = NullWritable.get();
    while (peer.readNext(key, nullValue)) {
        inputs.add(key.getVector());
    }
    // Convert inputs to double[][]
    double[][] inputsArr = new double[inputs.size()][inputs.get(0).getLength()];
    for (int i = 0; i < inputs.size(); i++) {
        double[] vector = inputs.get(i).toArray();
        for (int j = 0; j < vector.length; j++) {
            inputsArr[i][j] = vector[j];
        }
    }

    // Logging
    if (m_isDebuggingEnabled) {
        m_logger.writeChars("KMeansHybrid.bspGpu executed on GPU!\n");
        m_logger.writeChars(
                "KMeansHybrid.bspGpu blockSize: " + m_blockSize + " gridSize: " + m_gridSize + "\n");
        m_logger.writeChars("KMeansHybrid.bspGpu inputSize: " + inputs.size() + "\n");
    }

    KMeansHybridKernel kernel = new KMeansHybridKernel(inputsArr, m_centers_gpu,
            m_conf.getInt(CONF_MAX_ITERATIONS, 0), peer.getAllPeerNames());

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();
    rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context);
    watch.stop();

    // Output inputs with corresponding new center id
    for (int i = 0; i < inputs.size(); i++) {
        peer.write(new IntWritable(kernel.m_input_centers[i]), new PipesVectorWritable(inputs.get(i)));
    }

    // Output new Centers only on first task
    // to prevent collisions
    if (peer.getPeerName().equals(peer.getPeerName(0))) {
        String pathString = m_conf.get(CONF_CENTER_OUT_PATH);
        if (pathString != null) {
            final SequenceFile.Writer dataWriter = SequenceFile.createWriter(FileSystem.get(m_conf), m_conf,
                    new Path(pathString), PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);

            for (int i = 0; i < kernel.m_centers.length; i++) {
                dataWriter.append(new PipesVectorWritable(new DenseDoubleVector(kernel.m_centers[i])),
                        nullValue);
            }
            dataWriter.close();
        }
    }

    long stopTime = System.currentTimeMillis();
    if (m_timeMeasurement) {
        LOG.info("# bspGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec");

        if (m_isDebuggingEnabled) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,bspGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }

    // Logging
    if (m_isDebuggingEnabled) {
        List<StatsRow> stats = context.getStats();
        for (StatsRow row : stats) {
            m_logger.writeChars("  StatsRow:\n");
            m_logger.writeChars("    serial time: " + row.getSerializationTime() + "\n");
            m_logger.writeChars("    exec time: " + row.getExecutionTime() + "\n");
            m_logger.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
            m_logger.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
            m_logger.writeChars("    num threads: " + row.getNumThreads() + "\n");
            m_logger.writeChars("GPUTime: " + watch.elapsedTimeMillis() + " ms" + "\n");
        }

        m_logger.close();
    }
}