Example usage for org.apache.hadoop.fs FileSystem open

List of usage examples for org.apache.hadoop.fs FileSystem open

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

public static double readDMLScalar(String filePath) {
    FileSystem fs;
    try {//  w  w  w.  j av a 2s.co m
        double d = Double.NaN;
        fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) { // only 1 scalar value in file
                d = Double.parseDouble(line);
            }
            outIn.close();
        }
        return d;
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }
    return Double.NaN;
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

public static boolean readDMLBoolean(String filePath) {
    FileSystem fs;
    try {//from   ww w . j  a  v  a  2  s  . co  m
        Boolean b = null;
        fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) { // only 1 scalar value in file
                b = Boolean.valueOf(Boolean.parseBoolean(line));
            }
            outIn.close();
        }
        return b.booleanValue();
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }
    return _AssertOccured;
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

public static String readDMLString(String filePath) {
    FileSystem fs;
    try {// w w w  .  j a v a  2  s.  c o  m
        String s = null;
        fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) { // only 1 scalar value in file
                s = line;
            }
            outIn.close();
        }
        return s;
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }
    return null;
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//from   w w w  . j a va 2  s .  c  o m
 * Compares a dml matrix file in HDFS with a file in normal file system
 * generated by R
 * </p>
 * 
 * @param rFile
 *            file with values calculated by R
 * @param hdfsDir
 *            file with actual values calculated by DML
 * @param epsilon
 *            tolerance for value comparison
 */
public static void compareDMLHDFSFileWithRFile(String rFile, String hdfsDir, double epsilon) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(hdfsDir);
        BufferedReader compareIn = new BufferedReader(new FileReader(rFile));
        HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>();
        HashMap<CellIndex, Double> actualValues = new HashMap<CellIndex, Double>();
        String line;
        /** skip both R header lines */
        compareIn.readLine();
        compareIn.readLine();
        while ((line = compareIn.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, " ");
            int i = Integer.parseInt(st.nextToken());
            int j = Integer.parseInt(st.nextToken());
            double v = Double.parseDouble(st.nextToken());
            expectedValues.put(new CellIndex(i, j), v);
        }
        compareIn.close();

        FileStatus[] outFiles = fs.listStatus(outDirectory);

        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, " ");
                int i = Integer.parseInt(st.nextToken());
                int j = Integer.parseInt(st.nextToken());
                double v = Double.parseDouble(st.nextToken());
                actualValues.put(new CellIndex(i, j), v);
            }
            outIn.close();
        }

        int countErrors = 0;
        for (CellIndex index : expectedValues.keySet()) {
            Double expectedValue = expectedValues.get(index);
            Double actualValue = actualValues.get(index);
            if (expectedValue == null)
                expectedValue = 0.0;
            if (actualValue == null)
                actualValue = 0.0;

            if (!compareCellValue(expectedValue, actualValue, epsilon, false))
                countErrors++;
        }
        assertTrue("for file " + hdfsDir + " " + countErrors + " values are not in equal", countErrors == 0);
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from w  w  w .  ja va  2 s.  c  om*/
 * Checks a matrix read from a file in text format against a number of
 * specifications.
 * </p>
 * 
 * @param outDir
 *            directory containing the matrix
 * @param rows
 *            number of rows
 * @param cols
 *            number of columns
 * @param min
 *            minimum value
 * @param max
 *            maximum value
 */
public static void checkMatrix(String outDir, long rows, long cols, double min, double max) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(outDir);
        assertTrue(outDir + " does not exist", fs.exists(outDirectory));

        if (fs.getFileStatus(outDirectory).isDirectory()) {
            FileStatus[] outFiles = fs.listStatus(outDirectory);
            for (FileStatus file : outFiles) {
                FSDataInputStream fsout = fs.open(file.getPath());
                BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

                String line;
                while ((line = outIn.readLine()) != null) {
                    String[] rcv = line.split(" ");
                    long row = Long.parseLong(rcv[0]);
                    long col = Long.parseLong(rcv[1]);
                    double value = Double.parseDouble(rcv[2]);
                    assertTrue("invalid row index", (row > 0 && row <= rows));
                    assertTrue("invlaid column index", (col > 0 && col <= cols));
                    assertTrue("invalid value", ((value >= min && value <= max) || value == 0));
                }
                outIn.close();
            }
        } else {
            FSDataInputStream fsout = fs.open(outDirectory);
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            String line;
            while ((line = outIn.readLine()) != null) {
                String[] rcv = line.split(" ");
                long row = Long.parseLong(rcv[0]);
                long col = Long.parseLong(rcv[1]);
                double value = Double.parseDouble(rcv[2]);
                assertTrue("invalid row index", (row > 0 && row <= rows));
                assertTrue("invlaid column index", (col > 0 && col <= cols));
                assertTrue("invalid value", ((value >= min && value <= max) || value == 0));
            }
            outIn.close();
        }
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>// ww w .  ja  v  a  2  s  .com
 * Checks for matrix in directory existence.
 * </p>
 * 
 * @param outDir
 *            directory
 */
public static void checkForOutputExistence(String outDir) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(outDir);
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        assertEquals("number of files in directory not 1", 1, outFiles.length);
        FSDataInputStream fsout = fs.open(outFiles[0].getPath());
        BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

        String outLine = outIn.readLine();
        outIn.close();
        assertNotNull("file is empty", outLine);
        assertTrue("file is empty", outLine.length() > 0);
    } catch (IOException e) {
        fail("unable to read " + outDir + ": " + e.getMessage());
    }
}

From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java

License:Open Source License

/**
 * /*from w w w  .  j  a  va  2s.co  m*/
 * @param conf
 * @param yconf
 * @param appId
 * @return
 */
private String readMessageToHDFSWorkingDir(DMLConfig conf, YarnConfiguration yconf, ApplicationId appId) {
    String ret = null;

    //construct working directory (consistent with client)
    String hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(conf, appId);
    Path msgPath = new Path(hdfsWD, DMLYarnClient.DML_STOPMSG_NAME);

    //write given message to hdfs
    try {
        FileSystem fs = FileSystem.get(yconf);
        if (fs.exists(msgPath)) {
            FSDataInputStream fin = fs.open(msgPath);
            BufferedReader br = new BufferedReader(new InputStreamReader(fin));
            ret = br.readLine();
            fin.close();
            LOG.debug("Stop message read from HDFS file " + msgPath + ": " + ret);
        }
    } catch (Exception ex) {
        LOG.error("Failed to read stop message from HDFS file: " + msgPath, ex);
    }

    return ret;
}

From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java

License:Apache License

public void readSequentialDirect() throws Exception {
    System.out.println("reading sequential file in direct mode " + path);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    FileStatus status = fs.getFileStatus(path);
    FSDataInputStream instream = fs.open(path);
    ByteBuffer buf = ByteBuffer.allocateDirect(size);
    buf.clear();//www  . j  ava2 s . co m
    double sumbytes = 0;
    double ops = 0;
    System.out.println("file capacity " + status.getLen());
    System.out.println("read size " + size);
    System.out.println("operations " + loop);

    long start = System.currentTimeMillis();
    while (ops < loop) {
        buf.clear();
        double ret = (double) instream.read(buf);
        if (ret > 0) {
            sumbytes = sumbytes + ret;
            ops = ops + 1.0;
        } else {
            ops = ops + 1.0;
            if (instream.getPos() == 0) {
                break;
            } else {
                instream.seek(0);
            }
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1024.0 / 1024.0;
        latency = 1000000.0 * executionTime / ops;
    }
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    System.out.println("closing stream");
    instream.close();
    fs.close();
}

From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java

License:Apache License

public void readSequentialHeap() throws Exception {
    System.out.println("reading sequential file in heap mode " + path);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    FileStatus status = fs.getFileStatus(path);
    FSDataInputStream instream = fs.open(path);
    byte[] buf = new byte[size];
    double sumbytes = 0;
    double ops = 0;
    System.out.println("file capacity " + status.getLen());
    System.out.println("read size " + size);
    System.out.println("operations " + loop);

    long start = System.currentTimeMillis();
    while (ops < loop) {
        double ret = (double) this.read(instream, buf);
        if (ret > 0) {
            sumbytes = sumbytes + ret;/* ww w .  j  a  v  a  2  s  .  c o m*/
            ops = ops + 1.0;
        } else {
            ops = ops + 1.0;
            if (instream.getPos() == 0) {
                break;
            } else {
                instream.seek(0);
            }
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1024.0 / 1024.0;
        latency = 1000000.0 * executionTime / ops;
    }
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    System.out.println("closing stream");
    instream.close();
    fs.close();
}

From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java

License:Apache License

public void readRandomDirect() throws Exception {
    System.out.println("reading random file in direct mode " + path);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    FileStatus status = fs.getFileStatus(path);
    FSDataInputStream instream = fs.open(path);
    ByteBuffer buf = ByteBuffer.allocateDirect(size);
    buf.clear();/*from  w ww.j a va  2  s.co  m*/
    double sumbytes = 0;
    double ops = 0;
    long _range = status.getLen() - ((long) buf.capacity());
    double range = (double) _range;
    Random random = new Random();

    System.out.println("file capacity " + status.getLen());
    System.out.println("read size " + size);
    System.out.println("operations " + loop);
    long start = System.currentTimeMillis();
    while (ops < loop) {
        buf.clear();
        double _offset = range * random.nextDouble();
        long offset = (long) _offset;
        instream.seek(offset);
        double ret = (double) instream.read(buf);
        if (ret > 0) {
            sumbytes = sumbytes + ret;
            ops = ops + 1.0;
        } else {
            break;
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1024.0 / 1024.0;
        latency = 1000000.0 * executionTime / ops;
    }

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    System.out.println("closing stream");
    instream.close();
    fs.close();
}