Example usage for org.apache.hadoop.fs FileSystem exists

List of usage examples for org.apache.hadoop.fs FileSystem exists

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException

Source Link

Document

Check if a path exists.

Usage

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

protected static boolean checkValidInputFile(FileSystem fs, Path path, boolean err) throws IOException {
    // check non-existing file
    if (!fs.exists(path))
        if (err)//from  ww  w .j a  v  a2 s .c  om
            throw new IOException("File " + path.toString() + " does not exist on HDFS/LFS.");
        else
            return false;

    // check for empty file
    if (MapReduceTool.isFileEmpty(fs, path.toString()))
        if (err)
            throw new EOFException("Empty input file " + path.toString() + ".");
        else
            return false;

    return true;
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static void deleteFileIfExistOnHDFS(String dir) throws IOException {
    Path outpath = new Path(dir);
    FileSystem fs = FileSystem.get(_rJob);
    if (fs.exists(outpath)) {
        //System.err.println("Deleting " + outpath + " ... ");
        fs.delete(outpath, true);/*from   ww  w.  j av  a2 s  . com*/
    }
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static void renameFileOnHDFS(String originalDir, String newDir) throws IOException {
    Path originalpath = new Path(originalDir);

    deleteFileIfExistOnHDFS(newDir);//from w  ww . jav  a2  s  .  com
    Path newpath = new Path(newDir);

    FileSystem fs = FileSystem.get(_rJob);
    if (fs.exists(originalpath)) {
        fs.rename(originalpath, newpath);
    } else {
        throw new FileNotFoundException(originalDir);
    }
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static void copyFileOnHDFS(String originalDir, String newDir) throws IOException {
    Path originalPath = new Path(originalDir);
    Path newPath = new Path(newDir);
    boolean deleteSource = false;
    boolean overwrite = true;

    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    FileSystem fs = FileSystem.get(job);
    if (fs.exists(originalPath)) {
        FileUtil.copy(fs, originalPath, fs, newPath, deleteSource, overwrite, job);
    }/* www .jav a2  s  .c o  m*/
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static MatrixCharacteristics[] processDimsFiles(String dir, MatrixCharacteristics[] stats)
        throws IOException {
    Path pt = new Path(dir);
    FileSystem fs = FileSystem.get(_rJob);

    if (!fs.exists(pt))
        return stats;

    FileStatus fstat = fs.getFileStatus(pt);

    if (fstat.isDirectory()) {
        FileStatus[] files = fs.listStatus(pt);
        for (int i = 0; i < files.length; i++) {
            Path filePath = files[i].getPath();
            //System.out.println("Processing dims file: " + filePath.toString());
            BufferedReader br = setupInputFile(filePath.toString());

            String line = "";
            while ((line = br.readLine()) != null) {
                String[] parts = line.split(" ");
                int resultIndex = Integer.parseInt(parts[0]);
                long maxRows = Long.parseLong(parts[1]);
                long maxCols = Long.parseLong(parts[2]);

                stats[resultIndex].setDimension(
                        (stats[resultIndex].getRows() < maxRows ? maxRows : stats[resultIndex].getRows()),
                        (stats[resultIndex].getCols() < maxCols ? maxCols : stats[resultIndex].getCols()));
            }//from w ww .j  av a  2  s  . com

            br.close();
        }
    } else {
        throw new IOException(dir + " is expected to be a folder!");
    }

    return stats;
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

/**
 * /*www.  j av a2 s .  c o m*/
 * @param dir
 * @param permissions
 * @throws IOException
 */
public static void createDirIfNotExistOnHDFS(String dir, String permissions) throws IOException {
    Path path = new Path(dir);
    try {
        FileSystem fs = FileSystem.get(_rJob);
        if (!fs.exists(path)) {
            char[] c = permissions.toCharArray();
            short sU = (short) ((c[0] - 48) * 64);
            short sG = (short) ((c[1] - 48) * 8);
            short sO = (short) ((c[2] - 48));
            short mode = (short) (sU + sG + sO);
            FsPermission perm = new FsPermission(mode);
            fs.mkdirs(path, perm);
        }
    } catch (Exception ex) {
        throw new IOException("Failed in creating a non existing dir on HDFS", ex);
    }

    //NOTE: we depend on the configured umask, setting umask in job or fspermission has no effect
    //similarly setting dfs.datanode.data.dir.perm as no effect either.
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//from   www. j  ava 2 s.  co  m
 * Checks a matrix read from a file in text format against a number of
 * specifications.
 * </p>
 * 
 * @param outDir
 *            directory containing the matrix
 * @param rows
 *            number of rows
 * @param cols
 *            number of columns
 * @param min
 *            minimum value
 * @param max
 *            maximum value
 */
public static void checkMatrix(String outDir, long rows, long cols, double min, double max) {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path outDirectory = new Path(outDir);
        assertTrue(outDir + " does not exist", fs.exists(outDirectory));

        if (fs.getFileStatus(outDirectory).isDirectory()) {
            FileStatus[] outFiles = fs.listStatus(outDirectory);
            for (FileStatus file : outFiles) {
                FSDataInputStream fsout = fs.open(file.getPath());
                BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

                String line;
                while ((line = outIn.readLine()) != null) {
                    String[] rcv = line.split(" ");
                    long row = Long.parseLong(rcv[0]);
                    long col = Long.parseLong(rcv[1]);
                    double value = Double.parseDouble(rcv[2]);
                    assertTrue("invalid row index", (row > 0 && row <= rows));
                    assertTrue("invlaid column index", (col > 0 && col <= cols));
                    assertTrue("invalid value", ((value >= min && value <= max) || value == 0));
                }
                outIn.close();
            }
        } else {
            FSDataInputStream fsout = fs.open(outDirectory);
            BufferedReader outIn = new BufferedReader(new InputStreamReader(fsout));

            String line;
            while ((line = outIn.readLine()) != null) {
                String[] rcv = line.split(" ");
                long row = Long.parseLong(rcv[0]);
                long col = Long.parseLong(rcv[1]);
                double value = Double.parseDouble(rcv[2]);
                assertTrue("invalid row index", (row > 0 && row <= rows));
                assertTrue("invlaid column index", (col > 0 && col <= cols));
                assertTrue("invalid value", ((value >= min && value <= max) || value == 0));
            }
            outIn.close();
        }
    } catch (IOException e) {
        fail("unable to read file: " + e.getMessage());
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//ww  w. j  a  v  a  2 s . c om
 * Removes all the directories specified in the array in HDFS
 * </p>
 * 
 * @param directories
 *            directories array
 */
public static void removeHDFSDirectories(String[] directories) {
    try {
        FileSystem fs = FileSystem.get(conf);
        for (String directory : directories) {
            Path dir = new Path(directory);
            if (fs.exists(dir) && fs.getFileStatus(dir).isDirectory()) {
                fs.delete(dir, true);
            }
        }
    } catch (IOException e) {
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>// w w  w . ja  va  2  s. c o m
 * Removes all the files specified in the array in HDFS
 * </p>
 * 
 * @param files
 *            files array
 */
public static void removeHDFSFiles(String[] files) {
    try {
        FileSystem fs = FileSystem.get(conf);
        for (String directory : files) {
            Path dir = new Path(directory);
            if (fs.exists(dir) && !fs.getFileStatus(dir).isDirectory()) {
                fs.delete(dir, false);
            }
        }
    } catch (IOException e) {
    }
}

From source file:com.ibm.bi.dml.udf.lib.RemoveEmptyRows.java

License:Open Source License

@Override
public void execute() {
    Matrix mat = (Matrix) this.getFunctionInput(0);
    String fnameOld = mat.getFilePath();

    HashMap<Long, Long> keyMap = new HashMap<Long, Long>(); //old,new rowID

    try {//from   w w w  .  j  a va  2s .  com
        //prepare input
        JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
        Path path = new Path(fnameOld);
        FileSystem fs = FileSystem.get(job);
        if (!fs.exists(path))
            throw new IOException("File " + fnameOld + " does not exist on HDFS.");
        FileInputFormat.addInputPath(job, path);
        TextInputFormat informat = new TextInputFormat();
        informat.configure(job);

        //prepare output
        String fnameNew = createOutputFilePathAndName(OUTPUT_FILE);
        DataOutputStream ostream = MapReduceTool.getHDFSDataOutputStream(fnameNew, true);

        //read and write if necessary
        InputSplit[] splits = informat.getSplits(job, 1);

        LongWritable key = new LongWritable();
        Text value = new Text();
        long ID = 1;

        try {
            //for obj reuse and preventing repeated buffer re-allocations
            StringBuilder sb = new StringBuilder();

            for (InputSplit split : splits) {
                RecordReader<LongWritable, Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
                try {
                    while (reader.next(key, value)) {
                        String cellStr = value.toString().trim();
                        StringTokenizer st = new StringTokenizer(cellStr, " ");
                        long row = Integer.parseInt(st.nextToken());
                        long col = Integer.parseInt(st.nextToken());
                        double lvalue = Double.parseDouble(st.nextToken());

                        if (!keyMap.containsKey(row))
                            keyMap.put(row, ID++);
                        long rowNew = keyMap.get(row);

                        sb.append(rowNew);
                        sb.append(' ');
                        sb.append(col);
                        sb.append(' ');
                        sb.append(lvalue);
                        sb.append('\n');

                        ostream.writeBytes(sb.toString());
                        sb.setLength(0);
                    }
                } finally {
                    if (reader != null)
                        reader.close();
                }
            }

            _ret = new Matrix(fnameNew, keyMap.size(), mat.getNumCols(), ValueType.Double);
        } finally {
            if (ostream != null)
                ostream.close();
        }
    } catch (Exception ex) {
        throw new RuntimeException("Unable to execute external function.", ex);
    }
}