Example usage for org.apache.hadoop.fs FileSystem exists

List of usage examples for org.apache.hadoop.fs FileSystem exists

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException

Source Link

Document

Check if a path exists.

Usage

From source file:com.iflytek.spider.util.FSUtils.java

License:Apache License

/**
 * Replaces the current path with the new path and if set removes the old
 * path. If removeOld is set to false then the old path will be set to the
 * name current.old./*from www.  java2  s .  c o  m*/
 * 
 * @param fs The FileSystem.
 * @param current The end path, the one being replaced.
 * @param replacement The path to replace with.
 * @param removeOld True if we are removing the current path.
 * 
 * @throws IOException If an error occurs during replacement.
 */
public static void replace(FileSystem fs, Path current, Path replacement, boolean removeOld)
        throws IOException {

    // rename any current path to old
    Path old = new Path(current + ".old");
    if (fs.exists(current)) {
        fs.rename(current, old);
    }

    // rename the new path to current and remove the old path if needed
    fs.rename(replacement, current);
    if (fs.exists(old) && removeOld) {
        fs.delete(old, true);
    }
}

From source file:com.iflytek.spider.util.LockUtil.java

License:Apache License

/**
 * Create a lock file./*w  ww .  j a v a  2s. c om*/
 * @param fs filesystem
 * @param lockFile name of the lock file
 * @param accept if true, and the target file exists, consider it valid. If false
 * and the target file exists, throw an IOException.
 * @throws IOException if accept is false, and the target file already exists,
 * or if it's a directory.
 */
public static void createLockFile(FileSystem fs, Path lockFile, boolean accept) throws IOException {
    if (fs.exists(lockFile)) {
        if (!accept)
            throw new IOException("lock file " + lockFile + " already exists.");
        if (fs.getFileStatus(lockFile).isDir())
            throw new IOException("lock file " + lockFile + " already exists and is a directory.");
        // do nothing - the file already exists.
    } else {
        // make sure parents exist
        fs.mkdirs(lockFile.getParent());
        fs.createNewFile(lockFile);
    }
}

From source file:com.iflytek.spider.util.LockUtil.java

License:Apache License

/**
 * Remove lock file. NOTE: applications enforce the semantics of this file -
 * this method simply removes any file with a given name.
 * @param fs filesystem//from ww w  .j  av  a 2  s .c om
 * @param lockFile lock file name
 * @return false, if the lock file doesn't exist. True, if it existed and was
 * successfully removed.
 * @throws IOException if lock file exists but it is a directory.
 */
public static boolean removeLockFile(FileSystem fs, Path lockFile) throws IOException {
    if (!fs.exists(lockFile))
        return false;
    if (fs.getFileStatus(lockFile).isDir())
        throw new IOException("lock file " + lockFile + " exists but is a directory!");
    return fs.delete(lockFile, false);
}

From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java

License:Open Source License

private Path ensureOutputDirectory(CustomMapReduceJobPojo cmr)
        throws IOException, SAXException, ParserConfigurationException {
    Configuration config = HadoopUtils.getConfiguration(prop_custom);
    Path path = HadoopUtils.getPathForJob(cmr, config, true);

    FileSystem fs = FileSystem.get(config);
    if (fs.exists(path)) { // delete it
        fs.delete(path, true); // (might be dir => recursive)
    }/*from  ww  w .  j a  v a  2s .com*/
    // (don't create the dir, this all gets sorted out by the reducer)
    return path;
}

From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java

License:Open Source License

private void bringTempOutputToFront(CustomMapReduceJobPojo cmr)
        throws IOException, SAXException, ParserConfigurationException {
    // Get the names:
    Configuration config = HadoopUtils.getConfiguration(prop_custom);
    FileSystem fs = FileSystem.get(config);
    Path pathTmp = HadoopUtils.getPathForJob(cmr, config, true);
    Path pathFinal = HadoopUtils.getPathForJob(cmr, config, false);

    // OK don't do anything if pathTmp doesn't exist...
    if (fs.exists(pathTmp)) {
        // If the final path exists, delete it

        if (!fs.exists(pathFinal)) { // create it, which guarantees the parent path also exists
            //(otherwise the rename fails sigh)
            fs.mkdirs(pathFinal);/*from   ww  w  .  jav a2s .co m*/
        }
        fs.delete(pathFinal, true);
        fs.rename(pathTmp, pathFinal);
    }
}

From source file:com.ikanow.infinit.e.processing.custom.utils.HadoopUtils.java

License:Open Source License

public static void deleteHadoopDir(CustomMapReduceJobPojo cmr)
        throws SAXException, IOException, ParserConfigurationException {
    PropertiesManager props = new PropertiesManager();
    Configuration conf = getConfiguration(props);
    Path pathDir = HadoopUtils.getPathForJob(cmr, conf, false);
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(pathDir)) {
        fs.delete(pathDir, true);/*from  w w w  .  j  a  va 2 s.c om*/
    }
}

From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java

License:Open Source License

public static Path ensureOutputDirectory(CustomMapReduceJobPojo cmr, PropertiesManager prop_custom)
        throws IOException, SAXException, ParserConfigurationException {
    Configuration config = HadoopUtils.getConfiguration(prop_custom);
    Path path = HadoopUtils.getPathForJob(cmr, config, true);

    FileSystem fs = FileSystem.get(config);
    if (fs.exists(path)) { // delete it
        fs.delete(path, true); // (might be dir => recursive)
    }/*from   w w w  . j a v a2s.  c o m*/
    // (don't create the dir, this all gets sorted out by the reducer)
    return path;
}

From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java

License:Open Source License

public static void bringTempOutputToFront(CustomMapReduceJobPojo cmr, PropertiesManager prop_custom)
        throws IOException, SAXException, ParserConfigurationException {
    // Get the names:
    Configuration config = HadoopUtils.getConfiguration(prop_custom);
    FileSystem fs = FileSystem.get(config);
    Path pathTmp = HadoopUtils.getPathForJob(cmr, config, true);
    Path pathFinal = HadoopUtils.getPathForJob(cmr, config, false);

    // OK don't do anything if pathTmp doesn't exist...
    if (fs.exists(pathTmp)) {
        // If the final path exists, delete it

        if (!fs.exists(pathFinal)) { // create it, which guarantees the parent path also exists
            //(otherwise the rename fails sigh)
            fs.mkdirs(pathFinal);//from   w  ww.  j a  v a  2 s .co  m
        }
        fs.delete(pathFinal, true);
        fs.rename(pathTmp, pathFinal);
    }
}

From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java

License:Open Source License

public static Path cacheLocalFile(String localPath, String localName, Configuration config) throws IOException {
    FileSystem fs = FileSystem.get(config);
    Path toDir = new Path("cache");
    Path destFile = new Path("cache/" + localName);
    File fromFile = new File(localPath + "/" + localName);
    if (!fromFile.exists()) {
        throw new IOException("Source file does not exist: " + fromFile.toString());
    }//from   ww w.  j a v  a  2 s  .com
    boolean needToCopyFile = true;
    if (!fs.exists(toDir)) { // (ie relative to WD)
        fs.mkdirs(toDir);
    } else {
        // Now check if the file already exists
        if (fs.exists(destFile)) {
            FileStatus fsStat = fs.getFileStatus(destFile);
            if ((fsStat.getLen() == fromFile.length())
                    && (fromFile.lastModified() <= fsStat.getModificationTime())) {
                needToCopyFile = false;
            }
        }
    }
    if (needToCopyFile) {
        fs.copyFromLocalFile(false, true, new Path(localPath + "/" + localName), destFile);
    }
    return new Path(fs.getFileStatus(destFile).getPath().toUri().getPath());
    // (apparently the path has to be in absolute format without even the hdfs:// at the front?!)
}

From source file:com.indeed.imhotep.builder.tsv.EasyIndexBuilderFromTSV.java

License:Apache License

private BufferedReader getInputFileReader(Path inputFile) {
    try {//from  w ww .j ava 2s .c  om
        final FileSystem hdfs = getHDFS(inputFile);
        final Path qualifiedInputFile = inputFile.makeQualified(hdfs);
        if (!hdfs.exists(inputFile)) {
            throw new RuntimeException("The provided input file doesn't exist " + qualifiedInputFile
                    + "\nFor hdfs files use 'hdfs:' prefix like hdfs:/tmp/file.tsv");
        }
        log.info("Reading TSV data from " + qualifiedInputFile);
        InputStream inputStream = hdfs.open(inputFile);
        if (inputFile.getName().endsWith(".gz")) {
            inputStream = new GZIPInputStream(inputStream);
        }
        return new BufferedReader(new InputStreamReader(inputStream, Charsets.UTF_8));
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}