Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException 

Source Link

Document

Check if a path exists.

Usage

From source file:com.cloudera.sqoop.TestTargetDir.java

License:Apache License

/** test target-dir contains imported files. */
public void testTargetDir() throws IOException {

    try {/*w  ww. j a v a  2s .c  om*/
        String targetDir = getWarehouseDir() + "/tempTargetDir";

        ArrayList args = getOutputArgv(true);
        args.add("--target-dir");
        args.add(targetDir);

        // delete target-dir if exists and recreate it
        FileSystem fs = FileSystem.get(getConf());
        Path outputPath = new Path(targetDir);
        if (fs.exists(outputPath)) {
            fs.delete(outputPath, true);
        }

        String[] argv = (String[]) args.toArray(new String[0]);
        runImport(argv);

        ContentSummary summ = fs.getContentSummary(outputPath);

        assertTrue("There's no new imported files in target-dir", summ.getFileCount() > 0);

    } catch (Exception e) {
        LOG.error("Got Exception: " + StringUtils.stringifyException(e));
        fail(e.toString());
    }
}

---

From source file:com.cloudera.sqoop.TestTargetDir.java

License:Apache License

/** test target-dir breaks if already existing
 * (only allowed in append mode). */
public void testExistingTargetDir() throws IOException {

    try {// w w  w  . ja  v  a  2 s  . c  om
        String targetDir = getWarehouseDir() + "/tempTargetDir";

        ArrayList args = getOutputArgv(true);
        args.add("--target-dir");
        args.add(targetDir);

        // delete target-dir if exists and recreate it
        FileSystem fs = FileSystem.get(getConf());
        Path outputPath = new Path(targetDir);
        if (!fs.exists(outputPath)) {
            fs.mkdirs(outputPath);
        }

        String[] argv = (String[]) args.toArray(new String[0]);
        runImport(argv);

        fail("Existing target-dir run without problem report");

    } catch (IOException e) {
        // expected
    }
}

---

From source file:com.cloudera.sqoop.util.AppendUtils.java

License:Apache License

/**
 * Moves the imported files from temporary directory to specified target-dir,
 * renaming partition number if appending file exists.
 *//*from ww  w .ja v  a 2 s .  c  om*/
public void append() throws IOException {

    SqoopOptions options = context.getOptions();
    FileSystem fs = FileSystem.get(options.getConf());
    Path tempDir = context.getDestination();

    // Try in this order: target-dir or warehouse-dir
    Path userDestDir = null;
    if (options.getTargetDir() != null) {
        userDestDir = new Path(options.getTargetDir());
    } else if (options.getWarehouseDir() != null) {
        userDestDir = new Path(options.getWarehouseDir(), context.getTableName());
    } else {
        userDestDir = new Path(context.getTableName());
    }

    int nextPartition = 0;

    if (!fs.exists(tempDir)) {
        // This occurs if there was no source (tmp) dir. This might happen
        // if the import was an HBase-target import, but the user specified
        // --append anyway. This is a warning, not an error.
        LOG.warn("Cannot append files to target dir; no such directory: " + tempDir);
        return;
    }

    // Create target directory.
    if (!fs.exists(userDestDir)) {
        LOG.info("Creating missing output directory - " + userDestDir.getName());
        fs.mkdirs(userDestDir);
        nextPartition = 0;
    } else {
        LOG.info("Appending to directory " + userDestDir.getName());
        // Get the right next partition for the imported files
        nextPartition = getNextPartition(fs, userDestDir);
    }

    // move files
    moveFiles(fs, tempDir, userDestDir, nextPartition);

    // delete temporary path
    LOG.debug("Deleting temporary folder " + tempDir.getName());
    fs.delete(tempDir, true);
}

---

From source file:com.cloudera.sqoop.util.AppendUtils.java

License:Apache License

/**
 * Move files from source to target using a specified starting partition.
 *///from  ww  w  . j  ava  2 s . c  o m
private void moveFiles(FileSystem fs, Path sourceDir, Path targetDir, int partitionStart) throws IOException {

    NumberFormat numpart = NumberFormat.getInstance();
    numpart.setMinimumIntegerDigits(PARTITION_DIGITS);
    numpart.setGroupingUsed(false);
    Pattern patt = Pattern.compile("part.*-([0-9][0-9][0-9][0-9][0-9]).*");
    FileStatus[] tempFiles = fs.listStatus(sourceDir);

    if (null == tempFiles) {
        // If we've already checked that the dir exists, and now it can't be
        // listed, this is a genuine error (permissions, fs integrity, or other).
        throw new IOException("Could not list files from " + sourceDir);
    }

    // Move and rename files & directories from temporary to target-dir thus
    // appending file's next partition
    for (FileStatus fileStat : tempFiles) {
        if (!fileStat.isDir()) {
            // Move imported data files
            String filename = fileStat.getPath().getName();
            Matcher mat = patt.matcher(filename);
            if (mat.matches()) {
                String name = getFilename(filename);
                String fileToMove = name.concat(numpart.format(partitionStart++));
                String extension = getFileExtension(filename);
                if (extension != null) {
                    fileToMove = fileToMove.concat(extension);
                }
                LOG.debug("Filename: " + filename + " repartitioned to: " + fileToMove);
                fs.rename(fileStat.getPath(), new Path(targetDir, fileToMove));
            }
        } else {
            // Move directories (_logs & any other)
            String dirName = fileStat.getPath().getName();
            Path path = new Path(targetDir, dirName);
            int dirNumber = 0;
            while (fs.exists(path)) {
                path = new Path(targetDir, dirName.concat("-").concat(numpart.format(dirNumber++)));
            }
            LOG.debug("Directory: " + dirName + " renamed to: " + path.getName());
            fs.rename(fileStat.getPath(), path);
        }
    }
}

---

From source file:com.cloudera.training.metrics.JobHistoryHelper.java

License:Apache License

public static JobHistory.JobInfo getJobInfoFromHdfsOutputDir(String outputDir, Configuration conf)
        throws IOException {
    Path output = new Path(outputDir);
    Path historyLogDir = new Path(output, "_logs/history");
    FileSystem fs = output.getFileSystem(conf);
    if (!fs.exists(output)) {
        throw new IOException("History directory " + historyLogDir.toString() + " does not exist");
    }//w  w  w  .  j av  a  2  s  .  co  m
    Path[] jobFiles = FileUtil.stat2Paths(fs.listStatus(historyLogDir, jobLogFileFilter));
    if (jobFiles.length == 0) {
        throw new IOException("Not a valid history directory " + historyLogDir.toString());
    }
    String[] jobDetails = JobHistory.JobInfo.decodeJobHistoryFileName(jobFiles[0].getName()).split("_");
    String jobId = jobDetails[2] + "_" + jobDetails[3] + "_" + jobDetails[4];
    JobHistory.JobInfo job = new JobHistory.JobInfo(jobId);
    DefaultJobHistoryParser.parseJobTasks(jobFiles[0].toString(), job, fs);
    return job;
}

---

From source file:com.cloudy.mapred.base.JobUtil.java

License:Apache License

public static void delete(Configuration conf, Path path) throws IOException {
    if (conf == null) {
        conf = new Configuration();
    }//from   w  w  w .  ja v a 2s . co m
    FileSystem fs = path.getFileSystem(conf);
    if (fs.exists(path)) {
        log.info("Deleting {}", path);
        fs.delete(path, true);
    }
}

---

From source file:com.collective.celos.ci.testing.fixtures.deploy.HdfsInputDeployer.java

License:Apache License

@Override
public void validate(TestRun testRun) throws Exception {
    Path pathToCheck = new Path(testRun.getHdfsPrefix(), path);
    FileSystem fileSystem = testRun.getCiContext().getFileSystem();
    if (fileSystem.exists(pathToCheck)) {
        throw new CelosCiDirtyStateException("Celos-CI temporary path still exists: " + pathToCheck);
    }/*from w w w .ja va2  s. com*/
}

---

From source file:com.congiu.load.csv.FSUtils.java

License:Open Source License

public static boolean fileExists(Configuration conf, String file) throws IOException {
    Path p = new Path(file);
    FileSystem fs = p.getFileSystem(conf);
    return fs.exists(p);
}

---

From source file:com.conversantmedia.mapreduce.tool.BaseTool.java

License:Apache License

protected List<FileStatus> getInputFiles(Path input) throws IOException {
    FileSystem fs = FileSystem.get(getConf());
    List<FileStatus> status = new ArrayList<>();
    if (fs.exists(input)) {
        FileStatus inputStatus = fs.getFileStatus(input);
        if (inputStatus.isDirectory()) {
            // Move all files under this directory
            status = Arrays.asList(fs.listStatus(input));
        } else {/*from   ww w  .j  av a2s  .c  o m*/
            status.add(inputStatus);
        }
    }
    // Must be a glob path
    else {
        FileStatus[] statusAry = fs.globStatus(input);
        status.addAll(Arrays.asList(statusAry));
    }
    return status;
}

---

From source file:com.dalabs.droop.util.password.FilePasswordLoader.java

License:Apache License

/**
 * Verify that given path leads to a file that we can read.
 *
 * @param fs Associated FileSystem/*from  w  w w  .ja  v  a2  s .co  m*/
 * @param path Path
 * @throws IOException
 */
protected void verifyPath(FileSystem fs, Path path) throws IOException {
    if (!fs.exists(path)) {
        throw new IOException("The provided password file " + path + " does not exist!");
    }

    if (!fs.isFile(path)) {
        throw new IOException("The provided password file " + path + " is a directory!");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »