Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException 

Source Link

Document

Check if a path exists.

Usage

From source file:com.firewallid.util.FIFile.java

public static void deleteExistHDFSPath(String fullPath) throws IOException {
    Configuration hadoopConf = new Configuration();
    FileSystem fileSystem = FileSystem.get(hadoopConf);
    Path path = new Path(fullPath);

    if (fileSystem.exists(path)) {
        fileSystem.delete(path, true);//w  ww . ja v a  2s  .  c  o  m
        LOG.info("Deleted " + fullPath);
    }
}

---

From source file:com.firewallid.util.FIFile.java

public static void writeStringToHDFSFile(String pathFile, String text) throws IOException {
    Configuration hadoopConf = new Configuration();
    FileSystem fileSystem = FileSystem.get(hadoopConf);
    Path path = new Path(pathFile);

    if (fileSystem.exists(path)) {
        fileSystem.delete(path, true);//from  w  w w . ja  v a  2 s  .c  o m
    }

    try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fileSystem.create(path)))) {
        bw.write(text);
    }
    LOG.info("Created file: " + pathFile);
}

---

From source file:com.fullcontact.sstable.index.SSTableIndexIndexer.java

License:Apache License

public void index(final Path sstablePath) throws IOException {

    final FileSystem fileSystem = FileSystem.get(URI.create(sstablePath.toString()), configuration);
    final FileStatus fileStatus = fileSystem.getFileStatus(sstablePath);

    if (fileStatus.isDir()) {
        LOG.info("SSTable Indexing directory {}", sstablePath);
        final FileStatus[] statuses = fileSystem.listStatus(sstablePath);
        for (final FileStatus childStatus : statuses) {
            index(childStatus.getPath());
        }//from   ww  w .  ja  v  a 2 s .c  o  m
    } else if (sstablePath.toString().endsWith(SST_EXTENSION)) {
        final Path sstableIndexPath = new Path(sstablePath.toString() + SSTableIndexIndex.SSTABLE_INDEX_SUFFIX);
        if (fileSystem.exists(sstableIndexPath)) {
            LOG.info("Skipping as SSTable index file already exists for {}", sstablePath);
        } else {
            // Kick a thread for the index.
            final ListenableFuture<IndexRequest> indexFuture = service.submit(new Callable<IndexRequest>() {
                @Override
                public IndexRequest call() throws Exception {
                    final long startTime = System.currentTimeMillis();
                    final long fileSize = fileStatus.getLen();

                    LOG.info("Indexing SSTABLE Indexing file {}, size {} GB...", sstablePath,
                            decimalFormat.format(fileSize / (1024.0 * 1024.0 * 1024.0)));

                    indexSingleFile(fileSystem, sstablePath);

                    return new IndexRequest(sstableIndexPath, startTime, fileSize);
                }
            });

            Futures.addCallback(indexFuture, new FutureCallback<IndexRequest>() {
                public void onSuccess(final IndexRequest indexRequest) {
                    long indexSize = 0;

                    try {
                        indexSize = fileSystem.getFileStatus(indexRequest.getIndexPath()).getLen();
                    } catch (IOException e) {
                        LOG.error("Error getting file status for index path: {}", indexRequest.getIndexPath());
                    }

                    final double elapsed = (System.currentTimeMillis() - indexRequest.getStartTime()) / 1000.0;

                    LOG.info("Completed SSTABLE Indexing in {} seconds ({} MB/s).  Index size is {} KB.",
                            decimalFormat.format(elapsed),
                            decimalFormat.format(indexRequest.getFileSize() / (1024.0 * 1024.0 * elapsed)),
                            decimalFormat.format(indexSize / 1024.0));
                }

                public void onFailure(Throwable e) {
                    LOG.error("Failed to index.", e);
                }
            });

        }
    }
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.HDFSStoreImpl.java

License:Apache License

public boolean checkFileSystemExists() throws IOException {
    try {//from   w w w.j  av a 2  s.co  m
        return fsExists.runSerially(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                FileSystem fileSystem = getCachedFileSystem();
                if (fileSystem == null) {
                    return false;
                }
                return fileSystem.exists(new Path("/"));
            }
        });
    } catch (Exception e) {
        if (e instanceof IOException) {
            throw (IOException) e;
        }
        throw new IOException(e);
    }
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.AbstractHoplogOrganizer.java

License:Apache License

/**
 * creates a expiry marker for a file on file system
 * /*  ww  w . jav  a  2 s  .  c  o m*/
 * @param hoplog
 * @throws IOException
 */
protected void addExpiryMarkerForAFile(Hoplog hoplog) throws IOException {
    FileSystem fs = store.getFileSystem();

    // TODO optimization needed here. instead of creating expired marker
    // file per file, create a meta file. the main thing to worry is
    // compaction of meta file itself
    Path expiryMarker = getExpiryMarkerPath(hoplog.getFileName());

    // uh-oh, why are we trying to expire an already expired file?
    if (ENABLE_INTEGRITY_CHECKS) {
        Assert.assertTrue(!fs.exists(expiryMarker), "Expiry marker already exists: " + expiryMarker);
    }

    FSDataOutputStream expiryMarkerFile = fs.create(expiryMarker);
    expiryMarkerFile.close();

    if (logger.isDebugEnabled())
        logger.debug("Hoplog marked expired: " + getPathStr(hoplog));
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.HdfsSortedOplogOrganizer.java

License:Apache License

/**
 * @param region//from w ww  .  ja  va  2  s.  co m
 *          Region manager instance. Instances of hdfs listener instance,
 *          stats collector, file system, etc are shared by all buckets of a
 *          region and provided by region manager instance
 * @param bucketId bucket id to be managed by this organizer
 * @throws IOException
 */
public HdfsSortedOplogOrganizer(HdfsRegionManager region, int bucketId) throws IOException {
    super(region, bucketId);

    String val = System.getProperty(HoplogConfig.COMPACTION_FILE_RATIO);
    try {
        RATIO = Float.parseFloat(val);
    } catch (Exception e) {
    }

    hoplogReadersController = new HoplogReadersController();

    // initialize with all the files in the directory
    List<Hoplog> hoplogs = identifyAndLoadSortedOplogs(true);
    if (logger.isDebugEnabled()) {
        logger.debug("{}Initializing bucket with existing hoplogs, count = " + hoplogs.size(), logPrefix);
    }
    for (Hoplog hoplog : hoplogs) {
        addSortedOplog(hoplog, false, true);
    }

    // initialize sequence to the current maximum
    sequence = new AtomicInteger(findMaxSequenceNumber(hoplogs));

    initOldTmpFiles();

    FileSystem fs = store.getFileSystem();
    Path cleanUpIntervalPath = new Path(store.getHomeDir(), HoplogConfig.CLEAN_UP_INTERVAL_FILE_NAME);
    if (!fs.exists(cleanUpIntervalPath)) {
        long intervalDurationMillis = store.getPurgeInterval() * 60 * 1000;
        HoplogUtil.exposeCleanupIntervalMillis(fs, cleanUpIntervalPath, intervalDurationMillis);
    }

    if (startCompactionOnStartup) {
        forceCompactionOnVersionUpgrade();
        if (logger.isInfoEnabled()) {
            logger.info(LocalizedStrings.HOPLOG_MAJOR_COMPACTION_SCHEDULED_FOR_BETTER_ESTIMATE);
        }
    }
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.HdfsSortedOplogOrganizer.java

License:Apache License

/**
 * Returns a list of of hoplogs present in the bucket's directory, expected to be called during
 * hoplog set initialization/*from   w  w  w  .jav a 2  s. c  o  m*/
 */
List<Hoplog> identifyAndLoadSortedOplogs(boolean countSize) throws IOException {
    FileSystem fs = store.getFileSystem();
    if (!fs.exists(bucketPath)) {
        return new ArrayList<Hoplog>();
    }

    FileStatus allFiles[] = fs.listStatus(bucketPath);
    ArrayList<FileStatus> validFiles = new ArrayList<FileStatus>();
    for (FileStatus file : allFiles) {
        // All hoplog files contribute to disk usage
        Matcher matcher = HOPLOG_NAME_PATTERN.matcher(file.getPath().getName());
        if (!matcher.matches()) {
            // not a hoplog
            continue;
        }

        // account for the disk used by this file
        if (countSize) {
            incrementDiskUsage(file.getLen());
        }

        // All valid hoplog files must match the regex
        matcher = SORTED_HOPLOG_PATTERN.matcher(file.getPath().getName());
        if (matcher.matches()) {
            validFiles.add(file);
        }
    }

    FileStatus[] markers = getExpiryMarkers();
    FileStatus[] validHoplogs = filterValidHoplogs(validFiles.toArray(new FileStatus[validFiles.size()]),
            markers);

    ArrayList<Hoplog> results = new ArrayList<Hoplog>();
    if (validHoplogs == null || validHoplogs.length == 0) {
        return results;
    }

    for (int i = 0; i < validHoplogs.length; i++) {
        // Skip directories
        if (validHoplogs[i].isDirectory()) {
            continue;
        }

        final Path p = validHoplogs[i].getPath();
        // skip empty file
        if (fs.getFileStatus(p).getLen() <= 0) {
            continue;
        }

        Hoplog hoplog = new HFileSortedOplog(store, p, store.getBlockCache(), stats, store.getStats());
        results.add(hoplog);
    }

    return results;
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.HdfsSortedOplogOrganizer.java

License:Apache License

protected FileStatus[] getExpiryMarkers() throws IOException {
    FileSystem fs = store.getFileSystem();
    if (hoplogReadersController.hoplogs == null || hoplogReadersController.hoplogs.size() == 0) {
        // there are no hoplogs in the system. May be the bucket is not existing
        // at all.
        if (!fs.exists(bucketPath)) {
            if (logger.isDebugEnabled())
                logger.debug("{}This bucket is unused, skipping expired hoplog check", logPrefix);
            return null;
        }// ww  w. ja v a 2 s . c  o m
    }

    FileStatus files[] = FSUtils.listStatus(fs, bucketPath, new PathFilter() {
        @Override
        public boolean accept(Path file) {
            // All expired hoplog end with expire extension and must match the valid file regex
            String fileName = file.getName();
            if (!fileName.endsWith(EXPIRED_HOPLOG_EXTENSION)) {
                return false;
            }
            fileName = truncateExpiryExtension(fileName);
            Matcher matcher = SORTED_HOPLOG_PATTERN.matcher(fileName);
            return matcher.find();
        }

    });
    return files;
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.HdfsSortedOplogOrganizer.java

License:Apache License

private void initOldTmpFiles() throws IOException {
    FileSystem fs = store.getFileSystem();
    if (!fs.exists(bucketPath)) {
        return;//  w  w w .  ja  v  a 2 s .c  om
    }

    oldTmpFiles = new LinkedList<FileStatus>(Arrays.asList(fs.listStatus(bucketPath, new TmpFilePathFilter())));
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.HDFSSplitIterator.java

License:Apache License

public HDFSSplitIterator(FileSystem fs, Path[] paths, long[] offsets, long[] lengths, long startTime,
        long endTime) throws IOException {
    this.fs = fs;
    this.split = new CombineFileSplit(paths, offsets, lengths, null);
    while (currentHopIndex < split.getNumPaths() && !fs.exists(split.getPath(currentHopIndex))) {
        logger.warn(LocalizedMessage.create(LocalizedStrings.HOPLOG_CLEANED_UP_BY_JANITOR,
                split.getPath(currentHopIndex)));
        currentHopIndex++;//from  w  ww.j  a va2  s. c  om
    }
    if (currentHopIndex == split.getNumPaths()) {
        this.hoplog = null;
        iterator = null;
    } else {
        this.hoplog = getHoplog(fs, split.getPath(currentHopIndex));
        iterator = hoplog.getReader().scan(split.getOffset(currentHopIndex), split.getLength(currentHopIndex));
    }
    this.startTime = startTime;
    this.endTime = endTime;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »