Example usage for org.apache.hadoop.fs FileStatus getPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getPath.

Prototype

public Path getPath() 

Source Link

Usage

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//  w  ww . j  av a2s  . c  o m
 * Compares the expected values calculated in Java by testcase and which are
 * in the normal filesystem, with those calculated by SystemML located in
 * HDFS
 * </p>
 * 
 * @param expectedFile
 *            file with expected values, which is located in OS filesystem
 * @param actualDir
 *            file with actual values, which is located in HDFS
 * @param epsilon
 *            tolerance for value comparison
 */
public static void compareDMLMatrixWithJavaMatrix(String expectedFile, String actualDir, double epsilon) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(actualDir);
        Path compareFile = new Path(expectedFile);
        FSDataInputStream fsin = fs.open(compareFile);
        BufferedReader compareIn = new BufferedReader(new InputStreamReader(fsin));

        HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>();
        String line;
        while ((line = compareIn.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, " ");
            int i = Integer.parseInt(st.nextToken());
            int j = Integer.parseInt(st.nextToken());
            double v = Double.parseDouble(st.nextToken());
            expectedValues.put(new CellIndex(i, j), v);
        }
        compareIn.close();

        HashMap<CellIndex, Double> actualValues = new HashMap<CellIndex, Double>();

        FileStatus[] outFiles = fs.listStatus(outDirectory);

        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, " ");
                int i = Integer.parseInt(st.nextToken());
                int j = Integer.parseInt(st.nextToken());
                double v = Double.parseDouble(st.nextToken());
                actualValues.put(new CellIndex(i, j), v);
            }
            outIn.close();
        }

        int countErrors = 0;
        for (CellIndex index : expectedValues.keySet()) {
            Double expectedValue = expectedValues.get(index);
            Double actualValue = actualValues.get(index);
            if (expectedValue == null)
                expectedValue = 0.0;
            if (actualValue == null)
                actualValue = 0.0;

            //   System.out.println("actual value: "+actualValue+", expected value: "+expectedValue);

            if (!compareCellValue(expectedValue, actualValue, epsilon, false)) {
                System.out.println(expectedFile + ": " + index + " mismatch: expected " + expectedValue
                        + ", actual " + actualValue);
                countErrors++;
            }
        }
        assertTrue("for file " + actualDir + " " + countErrors + " values are not equal", countErrors == 0);
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * Reads values from a matrix file in HDFS in DML format
 * //from w  ww . j  a  v a2s .  c o  m
 * @deprecated You should not use this method, it is recommended to use the
 *             corresponding method in AutomatedTestBase
 * @param filePath
 * @return
 */
public static HashMap<CellIndex, Double> readDMLMatrixFromHDFS(String filePath) {
    HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>();

    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;

        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream outIn = fs.open(file.getPath());
            BufferedReader reader = new BufferedReader(new InputStreamReader(outIn));
            while ((line = reader.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, " ");
                int i = Integer.parseInt(st.nextToken());
                int j = Integer.parseInt(st.nextToken());
                double v = Double.parseDouble(st.nextToken());
                expectedValues.put(new CellIndex(i, j), v);
            }
            outIn.close();
        }
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }

    return expectedValues;
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

public static double readDMLScalar(String filePath) {
    FileSystem fs;// w w w . j av  a2  s  .  c om
    try {
        double d = Double.NaN;
        fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) { // only 1 scalar value in file
                d = Double.parseDouble(line);
            }
            outIn.close();
        }
        return d;
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }
    return Double.NaN;
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

public static boolean readDMLBoolean(String filePath) {
    FileSystem fs;//from  w w w  .  j a  v  a 2 s .  co m
    try {
        Boolean b = null;
        fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) { // only 1 scalar value in file
                b = Boolean.valueOf(Boolean.parseBoolean(line));
            }
            outIn.close();
        }
        return b.booleanValue();
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }
    return _AssertOccured;
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

public static String readDMLString(String filePath) {
    FileSystem fs;/* w  w  w  . j ava2  s.c  o m*/
    try {
        String s = null;
        fs = FileSystem.get(conf);
        Path outDirectory = new Path(filePath);
        String line;
        FileStatus[] outFiles = fs.listStatus(outDirectory);
        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) { // only 1 scalar value in file
                s = line;
            }
            outIn.close();
        }
        return s;
    } catch (IOException e) {
        assertTrue("could not read from file " + filePath, false);
    }
    return null;
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from   ww  w.  ja va2 s.c o m*/
 * Compares a dml matrix file in HDFS with a file in normal file system
 * generated by R
 * </p>
 * 
 * @param rFile
 *            file with values calculated by R
 * @param hdfsDir
 *            file with actual values calculated by DML
 * @param epsilon
 *            tolerance for value comparison
 */
public static void compareDMLHDFSFileWithRFile(String rFile, String hdfsDir, double epsilon) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(hdfsDir);
        BufferedReader compareIn = new BufferedReader(new FileReader(rFile));
        HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>();
        HashMap<CellIndex, Double> actualValues = new HashMap<CellIndex, Double>();
        String line;
        /** skip both R header lines */
        compareIn.readLine();
        compareIn.readLine();
        while ((line = compareIn.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, " ");
            int i = Integer.parseInt(st.nextToken());
            int j = Integer.parseInt(st.nextToken());
            double v = Double.parseDouble(st.nextToken());
            expectedValues.put(new CellIndex(i, j), v);
        }
        compareIn.close();

        FileStatus[] outFiles = fs.listStatus(outDirectory);

        for (FileStatus file : outFiles) {
            FSDataInputStream fsout = fs.open(file.getPath());
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            while ((line = outIn.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, " ");
                int i = Integer.parseInt(st.nextToken());
                int j = Integer.parseInt(st.nextToken());
                double v = Double.parseDouble(st.nextToken());
                actualValues.put(new CellIndex(i, j), v);
            }
            outIn.close();
        }

        int countErrors = 0;
        for (CellIndex index : expectedValues.keySet()) {
            Double expectedValue = expectedValues.get(index);
            Double actualValue = actualValues.get(index);
            if (expectedValue == null)
                expectedValue = 0.0;
            if (actualValue == null)
                actualValue = 0.0;

            if (!compareCellValue(expectedValue, actualValue, epsilon, false))
                countErrors++;
        }
        assertTrue("for file " + hdfsDir + " " + countErrors + " values are not in equal", countErrors == 0);
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//ww w. j a v a 2 s  . co  m
 * Checks a matrix read from a file in text format against a number of
 * specifications.
 * </p>
 * 
 * @param outDir
 *            directory containing the matrix
 * @param rows
 *            number of rows
 * @param cols
 *            number of columns
 * @param min
 *            minimum value
 * @param max
 *            maximum value
 */
public static void checkMatrix(String outDir, long rows, long cols, double min, double max) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(outDir);
        assertTrue(outDir + " does not exist", fs.exists(outDirectory));

        if (fs.getFileStatus(outDirectory).isDirectory()) {
            FileStatus[] outFiles = fs.listStatus(outDirectory);
            for (FileStatus file : outFiles) {
                FSDataInputStream fsout = fs.open(file.getPath());
                BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

                String line;
                while ((line = outIn.readLine()) != null) {
                    String[] rcv = line.split(" ");
                    long row = Long.parseLong(rcv[0]);
                    long col = Long.parseLong(rcv[1]);
                    double value = Double.parseDouble(rcv[2]);
                    assertTrue("invalid row index", (row > 0 && row <= rows));
                    assertTrue("invlaid column index", (col > 0 && col <= cols));
                    assertTrue("invalid value", ((value >= min && value <= max) || value == 0));
                }
                outIn.close();
            }
        } else {
            FSDataInputStream fsout = fs.open(outDirectory);
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            String line;
            while ((line = outIn.readLine()) != null) {
                String[] rcv = line.split(" ");
                long row = Long.parseLong(rcv[0]);
                long col = Long.parseLong(rcv[1]);
                double value = Double.parseDouble(rcv[2]);
                assertTrue("invalid row index", (row > 0 && row <= rows));
                assertTrue("invlaid column index", (col > 0 && col <= cols));
                assertTrue("invalid value", ((value >= min && value <= max) || value == 0));
            }
            outIn.close();
        }
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//from  w  w w . jav a  2 s . co  m
 * Clears a complete directory.
 * </p>
 * 
 * @param directory
 *            directory
 */
public static void clearDirectory(String directory) {
    try {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] directoryContent = fs.listStatus(new Path(directory));
        for (FileStatus content : directoryContent) {
            fs.delete(content.getPath(), true);
        }
    } catch (IOException e) {
    }
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//from   ww w.  ja  v  a2  s . c o  m
 * Removes all temporary files and directories in the current working
 * directory.
 * </p>
 */
public static void removeTemporaryFiles() {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path workingDir = new Path(".");
        FileStatus[] files = fs.listStatus(workingDir);
        for (FileStatus file : files) {
            String fileName = file.getPath().toString()
                    .substring(file.getPath().getParent().toString().length() + 1);
            if (fileName.contains("temp"))
                fs.delete(file.getPath(), false);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to remove temporary files: " + e.getMessage());
    }
}

---

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from  w  ww .j  a  v  a2  s.c  om*/
 * Checks if any temporary files or directories exist in the current working
 * directory.
 * </p>
 * 
 * @return true if temporary files or directories are available
 */
public static boolean checkForTemporaryFiles() {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path workingDir = new Path(".");
        FileStatus[] files = fs.listStatus(workingDir);
        for (FileStatus file : files) {
            String fileName = file.getPath().toString()
                    .substring(file.getPath().getParent().toString().length() + 1);
            if (fileName.contains("temp"))
                return true;
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to remove temporary files: " + e.getMessage());
    }

    return false;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »