Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException

Source Link

Document

Check if a path exists.

Usage

From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java

License:Apache License

private static void verifyResults() throws Exception {
    for (Path path : pathList) {
        FileSystem fs = cluster.getFileSystem();

        Path sourcePath = path.makeQualified(fs);
        Path targetPath = new Path(sourcePath.toString().replaceAll(SOURCE_PATH, TARGET_PATH));

        Assert.assertTrue(fs.exists(targetPath));
        Assert.assertEquals(fs.isFile(sourcePath), fs.isFile(targetPath));
    }/* www . j av  a 2s.c om*/
}

From source file:com.inmobi.conduit.distcp.tools.util.DistCpUtils.java

License:Apache License

/**
 * Sort sequence file containing FileStatus and Text as key and value respecitvely
 *
 * @param fs - File System/*from  www . jav  a 2 s. c om*/
 * @param conf - Configuration
 * @param sourceListing - Source listing file
 * @return Path of the sorted file. Is source file with _sorted appended to the name
 * @throws IOException - Any exception during sort.
 */
public static Path sortListing(FileSystem fs, Configuration conf, Path sourceListing) throws IOException {
    SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, Text.class, FileStatus.class, conf);
    Path output = new Path(sourceListing.toString() + "_sorted");

    if (fs.exists(output)) {
        fs.delete(output, false);
    }

    sorter.sort(sourceListing, output);
    return output;
}

From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java

License:Apache License

public static boolean checkIfFoldersAreInSync(FileSystem fs, String targetBase, String sourceBase)
        throws IOException {
    Path base = new Path(targetBase);

    Stack<Path> stack = new Stack<Path>();
    stack.push(base);/*  w  w  w  .  jav  a 2s . c  om*/
    while (!stack.isEmpty()) {
        Path file = stack.pop();
        if (!fs.exists(file))
            continue;
        FileStatus[] fStatus = fs.listStatus(file);
        if (fStatus == null || fStatus.length == 0)
            continue;

        for (FileStatus status : fStatus) {
            if (status.isDir()) {
                stack.push(status.getPath());
            }
            Assert.assertTrue(fs.exists(new Path(
                    sourceBase + "/" + DistCpUtils.getRelativePath(new Path(targetBase), status.getPath()))));
        }
    }
    return true;
}

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

private void cleanUpTmp(FileSystem fs) throws Exception {
    if (fs.exists(tmpPath)) {
        LOG.info("Deleting tmpPath recursively [" + tmpPath + "]");
        fs.delete(tmpPath, true);// w  ww.  j  a  va  2 s. com
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void createTestPurgePartitionFiles(FileSystem fs, Cluster cluster, Calendar date, Table table)
        throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String[] files = new String[NUM_OF_FILES];
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;/* w  w  w.  java2s.c o m*/
        fs.mkdirs(new Path(commitpath));
        Map<String, String> partSpec = TestHCatUtil.getPartitionMap(date);
        LOG.info("Adding partition " + partSpec + " for stream " + streamname);
        TestHCatUtil.addPartition(table, partSpec);
        for (int j = 0; j < NUM_OF_FILES; ++j) {
            files[j] = new String(cluster.getName() + "-"
                    + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j));
            {
                Path path = new Path(commitpath + File.separator + files[j]);
                LOG.info("Creating streams_local File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }
        }
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void createTestPurgefiles(FileSystem fs, Cluster cluster, Calendar date, boolean createEmptyDirs)
        throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String[] files = new String[NUM_OF_FILES];
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;/*from   w ww . j  a v a 2 s  . co  m*/
        String mergecommitpath = cluster.getFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;

        String trashpath = cluster.getTrashPath() + File.separator + CalendarHelper.getDateAsString(date)
                + File.separator;
        fs.mkdirs(new Path(commitpath));

        for (int j = 0; j < NUM_OF_FILES; ++j) {
            files[j] = new String(cluster.getName() + "-"
                    + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j));
            {
                Path path = new Path(commitpath + File.separator + files[j]);
                // LOG.info("Creating streams_local File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }
            {
                Path path = new Path(mergecommitpath + File.separator + files[j]);
                // LOG.info("Creating streams File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }

            {
                if (!createEmptyDirs) {
                    Path path = new Path(trashpath + File.separator
                            + String.valueOf(date.get(Calendar.HOUR_OF_DAY)) + File.separator + files[j]);
                    // LOG.info("Creating trash File " + path.toString());
                    FSDataOutputStream streamout = fs.create(path);
                    streamout.writeBytes("Creating Test trash data for teststream " + path.getName());
                    streamout.close();
                    Assert.assertTrue(fs.exists(path));
                }
            }

        }
        if (createEmptyDirs) {
            Path path = new Path(trashpath);
            if (!fs.exists(path))
                fs.mkdirs(path);
            Assert.assertTrue(fs.exists(path));
        }
    }

}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void verifyPurgePartitionFiles(FileSystem fs, Cluster cluster, Calendar date, boolean checkexists,
        boolean checktrashexists, Table table) throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;//from w ww.  ja v  a 2 s  .  co m
        {
            Path path = new Path(commitpath);
            LOG.info("Verifying File " + path.toString());
            Assert.assertEquals(fs.exists(path), checkexists);
            Assert.assertEquals(Hive.get().getPartitions(table).size(), 0);
        }
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void verifyPurgefiles(FileSystem fs, Cluster cluster, Calendar date, boolean checkexists,
        boolean checktrashexists) throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;//from w ww. j  ava2  s.c  o  m
        String mergecommitpath = cluster.getFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;
        String trashpath = cluster.getTrashPath() + File.separator + CalendarHelper.getDateAsString(date)
                + File.separator;
        {
            Path path = new Path(commitpath);
            LOG.info("Verifying File " + path.toString());
            Assert.assertEquals(fs.exists(path), checkexists);
        }
        {
            Path path = new Path(mergecommitpath);
            LOG.info("Verifying File " + path.toString());
            Assert.assertEquals(fs.exists(path), checkexists);
        }

        {
            Path path = new Path(trashpath + File.separator + String.valueOf(date.get(Calendar.HOUR_OF_DAY)));
            LOG.info("Verifying File " + path.toString());
            Assert.assertEquals(fs.exists(path), checktrashexists);
        }
    }
}

From source file:com.inmobi.databus.AbstractService.java

License:Apache License

protected Set<Path> publishMissingPaths(FileSystem fs, String destDir, long commitTime, String categoryName)
        throws Exception {
    Set<Path> missingDirectories = new TreeSet<Path>();
    Calendar commitTimeMinutes = new GregorianCalendar();
    commitTimeMinutes.set(Calendar.MILLISECOND, 0);
    commitTimeMinutes.set(Calendar.SECOND, 0);
    commitTime = commitTimeMinutes.getTimeInMillis();
    Long prevRuntime = new Long(-1);
    if (!prevRuntimeForCategory.containsKey(categoryName)) {
        LOG.debug("Calculating Previous Runtime from Directory Listing");
        prevRuntime = getPreviousRuntime(fs, destDir, categoryName);
    } else {//from w ww .j a v a 2  s .c  om
        LOG.debug("Reading Previous Runtime from Cache");
        prevRuntime = prevRuntimeForCategory.get(categoryName);
    }

    if (prevRuntime != -1) {
        if (isMissingPaths(commitTime, prevRuntime)) {
            LOG.debug("Previous Runtime: [" + getLogDateString(prevRuntime) + "]");
            while (isMissingPaths(commitTime, prevRuntime)) {
                String missingPath = Cluster.getDestDir(destDir, categoryName, prevRuntime);
                Path missingDir = new Path(missingPath);
                if (!fs.exists(missingDir)) {
                    LOG.debug("Creating Missing Directory [" + missingPath + "]");
                    missingDirectories.add(new Path(missingPath));
                    fs.mkdirs(missingDir);
                }
                prevRuntime += MILLISECONDS_IN_MINUTE;
            }
        }
        prevRuntimeForCategory.put(categoryName, commitTime);
    }
    return missingDirectories;
}

From source file:com.inmobi.databus.distcp.DistcpBaseService.java

License:Apache License

private void readConsumePath(FileSystem fs, Path consumePath, Set<String> minFilesSet) throws IOException {
    BufferedReader reader = null;
    try {/*w  ww. j av a 2 s . c  om*/
        FSDataInputStream fsDataInputStream = fs.open(consumePath);
        reader = new BufferedReader(new InputStreamReader(fsDataInputStream));
        String minFileName = null;
        do {
            minFileName = reader.readLine();
            if (minFileName != null) {
                /*
                * To avoid data-loss in all services we publish the paths to
                * consumers directory first before publishing on HDFS for
                * finalConsumption. In a distributed transaction failure it's
                * possible that some of these paths do not exist. Do isExistence
                * check before adding them as DISTCP input otherwise DISTCP
                * jobs can fail continously thereby blocking Merge/Mirror
                * stream to run further
                */
                Path p = new Path(minFileName);
                if (fs.exists(p)) {
                    LOG.info("Adding sourceFile [" + minFileName + "] to distcp " + "FinalList");
                    minFilesSet.add(minFileName.trim());
                } else {
                    LOG.info("Skipping [" + minFileName + "] to pull as it's an " + "INVALID PATH");
                }
            }
        } while (minFileName != null);
    } finally {
        if (reader != null)
            reader.close();
    }
}