List of usage examples for org.apache.hadoop.fs FileStatus isFile
public boolean isFile()
From source file:com.flipkart.fdp.migration.distcp.utils.FileCountDriver.java
License:Apache License
public List<String> getAllFilePath(Path filePath, FileSystem fs, String destBasePath) throws IOException { List<String> fileList = new ArrayList<String>(); List<String> inputPaths = new ArrayList<String>(); FileStatus[] fileStatus = fs.globStatus(filePath); for (FileStatus fileStat : fileStatus) { if (fileStat.isFile()) { fileList.add(trimExtension(fileStat.getPath().toUri().getPath(), destBasePath)); } else {/*from w ww . j av a 2s .com*/ //System.out.println("Found a directory : " + fileStat.getPath().toUri().getPath()); inputPaths.add(fileStat.getPath().toUri().getPath()); } } System.out.println("InputPaths size : " + inputPaths.size()); if (inputPaths.size() > 0) { for (String path : inputPaths) { List<String> fstat = getFileStatusRecursive(new Path(path), fs, destBasePath); fileList.addAll(fstat); } } return fileList; }
From source file:com.flipkart.fdp.migration.distcp.utils.FileCountDriver.java
License:Apache License
public List<String> getFileStatusRecursive(Path path, FileSystem fs, String destBasePath) throws IOException { List<String> response = new ArrayList<String>(); FileStatus file = fs.getFileStatus(path); if (file != null && file.isFile()) { response.add(trimExtension(file.getPath().toUri().getPath(), destBasePath)); return response; }//from w ww . ja va 2s.co m FileStatus[] fstats = fs.listStatus(path); if (fstats != null && fstats.length > 0) { for (FileStatus fstat : fstats) { if (fstat.isDirectory()) { response.addAll(getFileStatusRecursive(fstat.getPath(), fs, destBasePath)); } else { response.add(trimExtension(fstat.getPath().toUri().getPath(), destBasePath)); } } } return response; }
From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.HoplogUtil.java
License:Apache License
public static Collection<Collection<FileStatus>> getBucketHoplogs(Path regionPath, FileSystem fs, String type, long start, long end) throws IOException { Collection<Collection<FileStatus>> allBuckets = new ArrayList<Collection<FileStatus>>(); // hoplog files names follow this pattern String HOPLOG_NAME_REGEX = AbstractHoplogOrganizer.HOPLOG_NAME_REGEX + type; String EXPIRED_HOPLOG_NAME_REGEX = HOPLOG_NAME_REGEX + AbstractHoplogOrganizer.EXPIRED_HOPLOG_EXTENSION; final Pattern pattern = Pattern.compile(HOPLOG_NAME_REGEX); final Pattern expiredPattern = Pattern.compile(EXPIRED_HOPLOG_NAME_REGEX); Path cleanUpIntervalPath = new Path(regionPath.getParent(), HoplogConfig.CLEAN_UP_INTERVAL_FILE_NAME); long intervalDurationMillis = readCleanUpIntervalMillis(fs, cleanUpIntervalPath); // a region directory contains directories for individual buckets. A bucket // has a integer name. FileStatus[] bucketDirs = fs.listStatus(regionPath); for (FileStatus bucket : bucketDirs) { if (!bucket.isDirectory()) { continue; }/*from w w w.j a v a 2 s .co m*/ try { Integer.valueOf(bucket.getPath().getName()); } catch (NumberFormatException e) { continue; } ArrayList<FileStatus> bucketHoplogs = new ArrayList<FileStatus>(); // identify all the flush hoplogs and seq hoplogs by visiting all the // bucket directories FileStatus[] bucketFiles = fs.listStatus(bucket.getPath()); Map<String, Long> expiredHoplogs = getExpiredHoplogs(fs, bucketFiles, expiredPattern); FileStatus oldestHopAfterEndTS = null; long oldestHopTS = Long.MAX_VALUE; long currentTimeStamp = System.currentTimeMillis(); for (FileStatus file : bucketFiles) { if (!file.isFile()) { continue; } Matcher match = pattern.matcher(file.getPath().getName()); if (!match.matches()) { continue; } long timeStamp = AbstractHoplogOrganizer.getHoplogTimestamp(match); if (start > 0 && timeStamp < start) { // this hoplog contains records less than the start time stamp continue; } if (end > 0 && timeStamp > end) { // this hoplog contains records mutated after end time stamp. Ignore // this hoplog if it is not the oldest. if (oldestHopTS > timeStamp) { oldestHopTS = timeStamp; oldestHopAfterEndTS = file; } continue; } long expiredTimeStamp = expiredTime(file, expiredHoplogs); if (expiredTimeStamp > 0 && intervalDurationMillis > 0) { if ((currentTimeStamp - expiredTimeStamp) > 0.8 * intervalDurationMillis) { continue; } } bucketHoplogs.add(file); } if (oldestHopAfterEndTS != null) { long expiredTimeStamp = expiredTime(oldestHopAfterEndTS, expiredHoplogs); if (expiredTimeStamp <= 0 || intervalDurationMillis <= 0 || (currentTimeStamp - expiredTimeStamp) <= 0.8 * intervalDurationMillis) { bucketHoplogs.add(oldestHopAfterEndTS); } } if (bucketHoplogs.size() > 0) { allBuckets.add(bucketHoplogs); } } return allBuckets; }
From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.HoplogUtil.java
License:Apache License
private static Map<String, Long> getExpiredHoplogs(FileSystem fs, FileStatus[] bucketFiles, Pattern expiredPattern) throws IOException { Map<String, Long> expiredHoplogs = new HashMap<String, Long>(); for (FileStatus file : bucketFiles) { if (!file.isFile()) { continue; }/*w w w. j av a 2 s.co m*/ String fileName = file.getPath().getName(); Matcher match = expiredPattern.matcher(fileName); if (!match.matches()) { continue; } expiredHoplogs.put(fileName, file.getModificationTime()); } return expiredHoplogs; }
From source file:com.github.joshelser.accumulo.DelimitedIngest.java
License:Apache License
private List<Path> convertInputToPaths() throws IOException { List<String> inputs = args.getInput(); List<Path> paths = new ArrayList<>(inputs.size()); for (String input : inputs) { Path p = new Path(input); FileSystem fs = p.getFileSystem(conf); FileStatus fstat = fs.getFileStatus(p); if (fstat.isFile()) { paths.add(p);/*from w w w . ja v a2s . c o m*/ } else if (fstat.isDirectory()) { for (FileStatus child : fs.listStatus(p)) { if (child.isFile()) { paths.add(child.getPath()); } } } else { throw new IllegalStateException("Unable to handle that which is not file nor directory: " + p); } } return paths; }
From source file:com.github.libsml.commons.util.HadoopUtils.java
License:Apache License
public static String readString(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); FileStatus[] statuses = fs.listStatus(path); StringBuilder re = new StringBuilder(); for (FileStatus status : statuses) { if (status.isFile() && !status.getPath().getName().equals("_SUCCESS")) { FSDataInputStream streaming = fs.open(status.getPath()); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(streaming)); re.append(bufferedReader.readLine() + System.lineSeparator()); }/*from w w w .j av a 2s . com*/ } return re.toString(); }
From source file:com.ikanow.aleph2.aleph2_rest_utils.DataStoreCrudService.java
License:Apache License
/** * Utility for checking if a file exists or not, attempts to get * the paths status and returns true if that is successful and * its either a file or dir.//from www . j a v a 2 s . co m * * If a filenotfound exception is thrown, returns false * * Lets all other exceptions raise * * This is used to get around FileContext.delete not reporting correctly if a file has been successfully deleted. * @param path * @param context * @return * @throws AccessControlException * @throws UnsupportedFileSystemException * @throws IOException */ private static boolean doesPathExist(final Path path, final FileContext context) throws AccessControlException, UnsupportedFileSystemException, IOException { try { FileStatus status = context.getFileStatus(path); return status.isFile() || status.isDirectory(); } catch (FileNotFoundException e) { return false; } }
From source file:com.jkoolcloud.tnt4j.streams.configure.state.HdfsFileStreamStateHandler.java
License:Apache License
@Override public boolean isStreamedFileAvailable() { try {// www .jav a 2s.com FileStatus fStatus = file == null ? null : fs.getFileStatus(file); return fStatus != null && fStatus.isFile(); } catch (IOException exc) { logger().log(OpLevel.ERROR, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "FileStreamStateHandler.file.error"), exc); return false; } }
From source file:com.linkedin.thirdeye.hadoop.util.ThirdeyeAvroUtils.java
License:Apache License
/** * Finds the avro file in the input folder, and returns its avro schema * @param inputPathDir// ww w . ja v a 2s . com * @return * @throws IOException */ public static Schema getSchema(String inputPathDir) throws IOException { FileSystem fs = FileSystem.get(new Configuration()); Schema avroSchema = null; for (String input : inputPathDir.split(ThirdEyeConstants.FIELD_SEPARATOR)) { Path inputPath = new Path(input); for (FileStatus fileStatus : fs.listStatus(inputPath)) { if (fileStatus.isFile() && fileStatus.getPath().getName().endsWith(ThirdEyeConstants.AVRO_SUFFIX)) { LOGGER.info("Extracting schema from {}", fileStatus.getPath()); avroSchema = extractSchemaFromAvro(fileStatus.getPath()); break; } } } return avroSchema; }
From source file:com.msd.gin.halyard.sail.HBaseSail.java
License:Apache License
@Override public synchronized long size(Resource... contexts) throws SailException { if (contexts != null && contexts.length > 0 && contexts[0] != null) { throw new SailException("Size calculation is not supported for named graphs"); }/* w w w. ja v a 2 s . c om*/ if (sizeTimestamp < 0 || (isWritable() && sizeTimestamp + STATUS_CACHING_TIMEOUT < System.currentTimeMillis())) try { long entries = 0; FileSystem fs = FileSystem.get(config); Collection<HColumnDescriptor> families = table.getTableDescriptor().getFamilies(); Set<String> familyNames = new HashSet<>(families.size()); for (HColumnDescriptor hcd : families) { familyNames.add(hcd.getNameAsString()); } Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(config), table.getName()); PathFilter dirFilter = new FSUtils.DirFilter(fs); int divider = 1; for (HRegionLocation hrl : table.getRegionLocator().getAllRegionLocations()) { HRegionInfo hri = hrl.getRegionInfo(); byte[] skey = hri.getStartKey(); if (skey.length == 0 || skey[0] == HalyardTableUtils.SPO_PREFIX) { byte[] ekey = hri.getEndKey(); if (ekey.length == 0 || ekey[0] > HalyardTableUtils.POS_PREFIX) { divider = 3; } for (FileStatus familyDir : fs.listStatus(new Path(tableDir, hri.getEncodedName()), dirFilter)) { if (familyNames.contains(familyDir.getPath().getName())) { for (FileStatus file : fs.listStatus(familyDir.getPath())) { if (file.isFile()) { try (FSDataInputStream in = fs.open(file.getPath())) { entries += FixedFileTrailer.readFromStream(in, file.getLen()) .getEntryCount(); } catch (Exception e) { LOG.log(Level.WARNING, "Exception while reading trailer from hfile: " + file.getPath(), e); } } } } } } } size = entries / divider; sizeTimestamp = System.currentTimeMillis(); } catch (IOException e) { throw new SailException(e); } return size; }