Example usage for org.apache.hadoop.fs FileSystem get

List of usage examples for org.apache.hadoop.fs FileSystem get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem get.

Prototype

public static FileSystem get(Configuration conf) throws IOException

Source Link

Document

Returns the configured FileSystem implementation.

Usage

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.DistributedRowMatrix.java

License:Apache License

public static List<Path> writeDistributedRowMatrix(Configuration conf, double[][] matrix, int rows, int columns,
        Path path, int numBspTask, int numGPUBspTask, int GPUPercentage) throws IOException {

    List<Path> splittedFiles = new ArrayList<Path>();

    // Compute work distributions
    int cpuTaskNum = numBspTask - numGPUBspTask;
    int inputVectorsPerGPUTask = 0;
    int inputVectorsPerCPU = 0;
    int inputVectorsPerCPUTask = 0;
    if ((numGPUBspTask > 0) && (GPUPercentage > 0) && (GPUPercentage <= 100)) {
        inputVectorsPerGPUTask = (rows * GPUPercentage) / 100;
        inputVectorsPerCPU = rows - inputVectorsPerGPUTask;
    } else {//from   w  ww.java 2  s.  c om
        inputVectorsPerCPU = rows;
    }
    if (cpuTaskNum > 0) {
        inputVectorsPerCPUTask = inputVectorsPerCPU / cpuTaskNum;
    }

    for (int part = 0; part < numBspTask; part++) {

        Path partIn = new Path(path, "part" + part + ".seq");
        splittedFiles.add(partIn);
        FileSystem fs = FileSystem.get(conf);
        final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf, partIn, IntWritable.class,
                VectorWritable.class, CompressionType.NONE);

        int interval = 0;
        if (part > cpuTaskNum) {
            interval = inputVectorsPerGPUTask;
        } else {
            interval = inputVectorsPerCPUTask;
        }
        int start = interval * part;
        int end = start + interval;
        if ((numBspTask - 1) == part) {
            end = rows; // set to totalRows
        }
        LOG.info("Partition " + part + " file " + partIn.getParent().getName() + "/" + partIn.getName()
                + " from " + start + " to " + (end - 1));

        for (int i = start; i < end; i++) {
            DenseDoubleVector rowVector = new DenseDoubleVector(matrix[i]);
            dataWriter.append(new IntWritable(i), new VectorWritable(rowVector));
        }
        dataWriter.close();
    }

    return splittedFiles;
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {//w  ww  .  j  a  v a 2s . c om
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // CPU vs GPU benchmark
    int numGpuBspTask = 0;
    // if (type == CalcType.GPU) {
    // bspTaskNum = 1;
    // numGpuBspTask = 1;
    // GPUWorkload = 100;
    // }

    // CPU + GPU Hybrid benchmark
    if (bspTaskNum == maxTaskNum) {
        numGpuBspTask = 1;
        GPUWorkload = 95;
    } else {
        numGpuBspTask = 0;
    }

    // Set CPU tasks
    m_conf.setInt("bsp.peers.num", bspTaskNum);
    m_numBspTask = bspTaskNum;
    // Set GPU tasks
    m_conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    m_numGpuBspTask = numGpuBspTask;

    m_conf.setBoolean("hama.pipes.logging", false);

    // Generate input matrix A and transposed matrix B
    prepareInput();

    // Debug output
    // System.out.println("CalcType: " + type);
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
    System.out.println("NumBspTask: " + m_conf.getInt("bsp.peers.num", 0) + " NumGpuBspTask: "
            + m_conf.getInt("bsp.peers.gpu.num", 0));
    System.out.println("n: " + n + " GPUWorkload: " + GPUWorkload + "%");
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {
    // skip verification
    // verify();//from   ww w .  java  2  s. c om

    // Cleanup
    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);

    // printOutput(m_conf);
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override//ww w.  j  av a 2s .  c  o m
public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer)
        throws IOException {

    HamaConfiguration conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // used by GPU only
    m_tileWidth = conf.getInt(CONF_TILE_WIDTH, 32);

    // Choose one as a master, who sorts the matrix rows at the end
    // this.m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO
    // task must be 0 otherwise write out does NOT work!
    this.m_masterTask = peer.getPeerName(0);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Load transposed Matrix B
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_B_PATH)), conf);

    IntWritable transposedMatrixBRowId = new IntWritable();
    VectorWritable transposedMatrixBRow = new VectorWritable();

    // for each col of matrix B (cause by transposed B)
    while (reader.next(transposedMatrixBRowId, transposedMatrixBRow)) {
        m_transposedMatrixB.add(new KeyValuePair<Integer, DoubleVector>(transposedMatrixBRowId.get(),
                transposedMatrixBRow.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("setup,read,transposedMatrixB,key=" + transposedMatrixBRowId.get() + ",value="
                    + transposedMatrixBRow.getVector().toString() + "\n");
        }
    }
    reader.close();
}

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {/*from   www.j  a  v a2s .c  o m*/
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // Setup OnlineCF config variables
    m_conf.setBoolean(OnlineCFTrainHybridBSP.CONF_DEBUG, false);
    m_conf.setBoolean("hama.pipes.logging", false);

    // Set GPU blockSize and gridSize
    m_conf.set(OnlineCFTrainHybridBSP.CONF_BLOCKSIZE, "" + BLOCK_SIZE);
    m_conf.set(OnlineCFTrainHybridBSP.CONF_GRIDSIZE, "" + GRID_SIZE);

    int numGpuBspTask = 0;

    // CPU vs GPU iterations benchmark
    // Plot 1 and 2
    // if (type == CalcType.GPU) {
    // bspTaskNum = 1;
    // numGpuBspTask = 1;
    // GPUWorkload = 100;
    // }

    // CPU + GPU Hybrid benchmark
    // Plot 3
    if (bspTaskNum == maxTaskNum) {
        numGpuBspTask = 1;
        // GPUWorkload = 75;
    } else {
        numGpuBspTask = 0;
    }

    // Set CPU tasks
    m_conf.setInt("bsp.peers.num", bspTaskNum);
    // Set GPU tasks
    m_conf.setInt("bsp.peers.gpu.num", numGpuBspTask);

    m_conf.setInt(OnlineCF.CONF_ITERATION_COUNT, iteration);
    m_conf.setInt(OnlineCF.CONF_MATRIX_RANK, matrixRank);
    m_conf.setInt(OnlineCF.CONF_SKIP_COUNT, skipCount);

    Path preferencesIn = new Path(CONF_INPUT_DIR, "preferences_in.seq");

    if (!m_useInputFile) {
        // Generate random input data
        m_testPrefs = generateRandomInputData(m_conf, FileSystem.get(m_conf), CONF_INPUT_DIR, bspTaskNum,
                numGpuBspTask, n, m, percentNonZeroValues, GPUWorkload, m_maxTestPrefs);
    } else {
        // Convert MovieLens input data
        // parse inputFile and return first entries for testing
        m_testPrefs = convertInputData(m_conf, FileSystem.get(m_conf), CONF_INPUT_DIR, preferencesIn,
                m_movieLensInputFile, "::", m_maxTestPrefs);
    }

    // Debug output
    // System.out.println("CalcType: " + type);
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
    System.out.println("NumBspTask: " + m_conf.getInt("bsp.peers.num", 0) + " NumGpuBspTask: "
            + m_conf.getInt("bsp.peers.gpu.num", 0));
    if (!m_useInputFile) {
        System.out.println("n: " + n + " m: " + m + " percentNonZeroValues: " + percentNonZeroValues);
    } else {
        System.out.println("Use inputFile: " + m_movieLensInputFile);
    }
    System.out.println("matrixRank: " + matrixRank + " iterations: " + iteration);
}

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFHybridBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {

    verify();

    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);
}

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
// **********************************************************************
// setup//  w  w w. j  a v a 2  s .  co  m
// **********************************************************************
@Override
public void setup(BSPPeer<LongWritable, PipesVectorWritable, Text, PipesVectorWritable, ItemMessage> peer)
        throws IOException {

    long startTime = System.currentTimeMillis();

    this.m_conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);

    this.m_maxIterations = m_conf.getInt(OnlineCF.CONF_ITERATION_COUNT, OnlineCF.DFLT_ITERATION_COUNT);

    this.m_matrixRank = m_conf.getInt(OnlineCF.CONF_MATRIX_RANK, OnlineCF.DFLT_MATRIX_RANK);

    this.m_skipCount = m_conf.getInt(OnlineCF.CONF_SKIP_COUNT, OnlineCF.DFLT_SKIP_COUNT);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    this.m_setupTimeCpu = System.currentTimeMillis() - startTime;
}

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

/********************************* GPU *********************************/
// **********************************************************************
// setupGpu/*w ww  . j  a v  a 2  s . c o  m*/
// **********************************************************************
@Override
public void setupGpu(BSPPeer<LongWritable, PipesVectorWritable, Text, PipesVectorWritable, ItemMessage> peer)
        throws IOException, SyncException, InterruptedException {

    long startTime = System.currentTimeMillis();

    this.m_conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);

    this.m_maxIterations = m_conf.getInt(OnlineCF.CONF_ITERATION_COUNT, OnlineCF.DFLT_ITERATION_COUNT);

    this.m_matrixRank = m_conf.getInt(OnlineCF.CONF_MATRIX_RANK, OnlineCF.DFLT_MATRIX_RANK);

    this.m_skipCount = m_conf.getInt(OnlineCF.CONF_SKIP_COUNT, OnlineCF.DFLT_SKIP_COUNT);

    this.m_blockSize = Integer.parseInt(this.m_conf.get(CONF_BLOCKSIZE));
    this.m_gridSize = Integer.parseInt(this.m_conf.get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    this.m_setupTimeGpu = System.currentTimeMillis() - startTime;
}

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

public static void main(String[] args) throws Exception {
    // Defaults/*from w w  w . ja va2  s  . com*/
    int numBspTask = 1; // CPU + GPU tasks
    int numGpuBspTask = 1; // GPU tasks
    int blockSize = BLOCK_SIZE;
    int gridSize = GRID_SIZE;

    int maxIteration = 3; // 150;
    int matrixRank = 3;
    int skipCount = 1;

    double alpha = ALPHA;
    int userCount = 0;
    int itemCount = 0;
    int percentNonZeroValues = 0;

    int GPUPercentage = 20;

    boolean useTestExampleInput = true;
    boolean isDebugging = true;
    String inputFile = "";
    String separator = "\\t";

    Configuration conf = new HamaConfiguration();
    FileSystem fs = FileSystem.get(conf);

    // Set numBspTask to maxTasks
    // BSPJobClient jobClient = new BSPJobClient(conf);
    // ClusterStatus cluster = jobClient.getClusterStatus(true);
    // numBspTask = cluster.getMaxTasks();

    if (args.length > 0) {
        if (args.length >= 14) {
            numBspTask = Integer.parseInt(args[0]);
            numGpuBspTask = Integer.parseInt(args[1]);
            blockSize = Integer.parseInt(args[2]);
            gridSize = Integer.parseInt(args[3]);

            maxIteration = Integer.parseInt(args[4]);
            matrixRank = Integer.parseInt(args[5]);
            skipCount = Integer.parseInt(args[6]);

            alpha = Double.parseDouble(args[7]);
            userCount = Integer.parseInt(args[8]);
            itemCount = Integer.parseInt(args[9]);
            percentNonZeroValues = Integer.parseInt(args[10]);

            GPUPercentage = Integer.parseInt(args[11]);

            useTestExampleInput = Boolean.parseBoolean(args[12]);
            isDebugging = Boolean.parseBoolean(args[13]);

            // optional parameters
            if (args.length > 14) {
                inputFile = args[14];
            }
            if (args.length > 15) {
                separator = args[15];
            }

        } else {
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            System.out.println("    Argument2=numGpuBspTask");
            System.out.println("    Argument3=blockSize");
            System.out.println("    Argument4=gridSize");
            System.out.println(
                    "    Argument5=maxIterations | Number of maximal iterations (" + maxIteration + ")");
            System.out.println("    Argument6=matrixRank | matrixRank (" + matrixRank + ")");
            System.out.println("    Argument7=skipCount | skipCount (" + skipCount + ")");
            System.out.println("    Argument8=alpha | alpha (" + alpha + ")");
            System.out.println("    Argument9=userCount | userCount (" + userCount + ")");
            System.out.println("    Argument10=itemCount | itemCount (" + itemCount + ")");
            System.out.println("    Argument11=percentNonZeroValues | percentNonZeroValues ("
                    + percentNonZeroValues + ")");
            System.out.println("    Argument12=GPUPercentage (percentage of input)");
            System.out.println("    Argument13=testExample | Use testExample input (true|false=default)");
            System.out.println("    Argument14=debug | Enable debugging (true|false=default)");
            System.out.println("    Argument15=inputFile (optional) | MovieLens inputFile");
            System.out.println("    Argument16=separator (optional) | default '" + separator + "' ");
            return;
        }
    }

    // Check if inputFile exists
    if ((!inputFile.isEmpty()) && (!new File(inputFile).exists())) {
        System.out.println("Error: inputFile: " + inputFile + " does not exist!");
        return;
    }

    // Check parameters
    if ((inputFile.isEmpty()) && (!useTestExampleInput) && (userCount <= 0) && (itemCount <= 0)
            && (percentNonZeroValues <= 0)) {
        System.out.println("Invalid parameter: userCount: " + userCount + " itemCount: " + itemCount
                + " percentNonZeroValues: " + percentNonZeroValues);
        return;
    }

    // Check if blockSize < matrixRank when using GPU
    if ((numGpuBspTask > 0) && (blockSize < matrixRank)) {
        System.out.println("Error: BlockSize < matrixRank");
        return;
    }

    // Check GPUPercentage
    if ((GPUPercentage < 0) && (GPUPercentage > 100)) {
        System.out.println("Error: GPUPercentage must be between 0 and 100 percent");
        return;
    }

    // Set config variables
    conf.setBoolean(CONF_DEBUG, isDebugging);
    conf.setBoolean("hama.pipes.logging", isDebugging);
    // Set CPU tasks
    conf.setInt("bsp.peers.num", numBspTask);
    // Set GPU tasks
    conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    // Set GPU blockSize and gridSize
    conf.set(CONF_BLOCKSIZE, "" + blockSize);
    conf.set(CONF_GRIDSIZE, "" + gridSize);

    conf.setInt(OnlineCF.CONF_ITERATION_COUNT, maxIteration);
    conf.setInt(OnlineCF.CONF_MATRIX_RANK, matrixRank);
    conf.setInt(OnlineCF.CONF_SKIP_COUNT, skipCount);

    // Debug output
    LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0));
    LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0));
    LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum"));
    LOG.info("BlockSize: " + conf.get(CONF_BLOCKSIZE));
    LOG.info("GridSize: " + conf.get(CONF_GRIDSIZE));
    LOG.info("GPUPercentage: " + GPUPercentage);

    LOG.info("isDebugging: " + isDebugging);
    LOG.info("useTestExampleInput: " + useTestExampleInput);
    LOG.info("inputPath: " + CONF_INPUT_DIR);
    LOG.info("outputPath: " + CONF_OUTPUT_DIR);

    LOG.info("maxIteration: " + maxIteration);
    LOG.info("matrixRank: " + matrixRank);
    LOG.info("skipCount: " + skipCount);

    LOG.info("alpha: " + alpha);
    LOG.info("userCount: " + userCount);
    LOG.info("itemCount: " + itemCount);
    LOG.info("percentNonZeroValues: " + percentNonZeroValues);

    if (!inputFile.isEmpty()) {
        LOG.info("inputFile: " + inputFile);
        LOG.info("separator: " + separator);
    }

    // prepare Input
    int maxTestPrefs = 10;
    Path preferencesIn = new Path(CONF_INPUT_DIR, "preferences_in.seq");
    List<Preference<Long, Long>> testPrefs = null;
    if (useTestExampleInput) {

        testPrefs = prepareTestInputData(conf, fs, CONF_INPUT_DIR, preferencesIn);

    } else if (inputFile.isEmpty()) {

        testPrefs = generateRandomInputData(conf, fs, CONF_INPUT_DIR, numBspTask, numGpuBspTask, userCount,
                itemCount, percentNonZeroValues, GPUPercentage, maxTestPrefs);

    } else if (!inputFile.isEmpty()) {
        // parse inputFile and return first entries for testing
        testPrefs = convertInputData(conf, fs, CONF_INPUT_DIR, preferencesIn, inputFile, separator,
                maxTestPrefs);
    }

    // Generate Job config
    BSPJob job = createOnlineCFTrainHybridBSPConf(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR);

    // Execute Job
    long startTime = System.currentTimeMillis();
    if (job.waitForCompletion(true)) {

        LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");

        // Load Job results for testing
        OnlineCF recommender = new OnlineCF();
        recommender.load(CONF_OUTPUT_DIR.toString(), false);

        // Test results
        int error = 0;
        double totalError = 0;
        for (Preference<Long, Long> test : testPrefs) {
            double expected = test.getValue().get();
            double estimated = recommender.estimatePreference(test.getUserId(), test.getItemId());

            if (testPrefs.size() <= 20) {
                LOG.info("(" + test.getUserId() + ", " + test.getItemId() + ", " + expected + "): " + estimated
                        + " error: " + Math.abs(expected - estimated));
            }
            totalError += Math.abs(expected - estimated);
            error += (Math.abs(expected - estimated) < 0.5) ? 1 : 0;
        }

        LOG.info("totalError: " + totalError);
        LOG.info("assertEquals(expected: " + (testPrefs.size() * 0.75) + " == " + error
                + " actual) with delta: 1");

        if (isDebugging) {
            printOutput(conf, fs, ".log", new IntWritable(), new PipesVectorWritable());
        }
    }

}

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {/*from   w ww .  j  a  va2  s .co  m*/
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // calculate total sampling size
    m_totalIterations = (long) 1024 * (long) 14 * (long) 1000 * n;

    int numGpuBspTask = 0;

    // Used only for Plot 1 - CPU vs GPU comparison
    // if (type == CalcType.GPU) {
    // bspTaskNum = 1;
    // numGpuBspTask = 1;
    // GPUWorkload = 100;
    // }

    // Used only for Plot 2 - CPU + GPU Hybrid benchmark
    if (bspTaskNum == maxBspTaskNum) {
        numGpuBspTask = 1;
        GPUWorkload = 95;
    } else {
        numGpuBspTask = 0;
    }

    // Set CPU tasks
    m_conf.setInt("bsp.peers.num", bspTaskNum);
    // Set GPU tasks
    m_conf.setInt("bsp.peers.gpu.num", numGpuBspTask);

    m_conf.setInt(PiEstimatorHybridBSP.CONF_BLOCKSIZE, m_blockSize);
    m_conf.setInt(PiEstimatorHybridBSP.CONF_GRIDSIZE, m_gridSize);
    m_conf.setLong(PiEstimatorHybridBSP.CONF_ITERATIONS, m_totalIterations);
    m_conf.setInt(PiEstimatorHybridBSP.CONF_GPU_PERCENTAGE, GPUWorkload);
    m_conf.setBoolean(PiEstimatorHybridBSP.CONF_DEBUG, false);
    m_conf.setBoolean(PiEstimatorHybridBSP.CONF_TIME, false);

    // Debug output
    System.out.println("Benchmark PiEstimatorHybridBSP[blockSize=" + m_blockSize + ",gridSize=" + m_gridSize
            + "] n=" + n + ",bspTaskNum=" + bspTaskNum + ",GpuBspTaskNum=" + numGpuBspTask + ",GPUWorkload="
            + GPUWorkload + ",totalSamples=" + m_totalIterations);
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
}