List of usage examples for org.apache.hadoop.fs FileStatus getModificationTime
public long getModificationTime()
From source file:com.nexr.rhive.util.TransformUtils.java
License:Apache License
public static String[] tranform(FileStatus fs) throws Exception { String[] datas = new String[6]; datas[0] = fs.getPermission().toString(); datas[1] = fs.getOwner();//from w ww. ja v a 2s . c o m datas[2] = fs.getGroup(); datas[3] = Long.toString(fs.getLen()); Date date = new Date(fs.getModificationTime()); datas[4] = formatter.format(date); datas[5] = fs.getPath().toUri().getPath(); return datas; }
From source file:com.ning.metrics.action.hdfs.reader.HdfsEntry.java
License:Apache License
public HdfsEntry(FileSystem fs, FileStatus status, boolean raw, RowFileContentsIteratorFactory rowFileContentsIteratorFactory) throws IOException { this.fs = fs; this.path = status.getPath(); this.modificationDate = new DateTime(status.getModificationTime()); this.blockSize = status.getBlockSize(); this.size = status.getLen(); this.replication = status.getReplication(); this.replicatedSize = status.getReplication() * status.getLen(); this.directory = status.isDir(); this.raw = raw; this.rowFileContentsIteratorFactory = rowFileContentsIteratorFactory; }
From source file:com.pinterest.secor.util.FileUtil.java
License:Apache License
public static long getModificationTimeMsRecursive(String path) throws IOException { FileSystem fs = getFileSystem(path); Path fsPath = new Path(path); FileStatus status = fs.getFileStatus(fsPath); long modificationTime = status.getModificationTime(); FileStatus[] statuses = fs.listStatus(fsPath); if (statuses != null) { for (FileStatus fileStatus : statuses) { Path statusPath = fileStatus.getPath(); String stringPath;/* w w w .j av a 2 s . c om*/ if (path.startsWith("s3://") || path.startsWith("s3n://")) { stringPath = statusPath.toUri().toString(); } else { stringPath = statusPath.toUri().getPath(); } if (!stringPath.equals(path)) { modificationTime = Math.max(modificationTime, getModificationTimeMsRecursive(stringPath)); } } } return modificationTime; }
From source file:com.pivotal.hawq.mapreduce.parquet.HAWQParquetInputFormat.java
License:Apache License
@Override protected List<FileStatus> listStatus(JobContext jobContext) throws IOException { List<FileStatus> result = Lists.newArrayList(); for (HAWQFileStatus hawqFileStatus : hawqFileStatuses) { if (hawqFileStatus.getFileLength() == 0) continue; // skip empty file Path path = new Path(hawqFileStatus.getFilePath()); FileSystem fs = path.getFileSystem(jobContext.getConfiguration()); FileStatus dfsStat = fs.getFileStatus(path); // rewrite file length because HAWQ records the logicalEOF of file, which may // be smaller than the file's actual EOF FileStatus hawqStat = new FileStatus(hawqFileStatus.getFileLength(), // rewrite to logicalEOF dfsStat.isDirectory(), dfsStat.getReplication(), dfsStat.getBlockSize(), dfsStat.getModificationTime(), dfsStat.getAccessTime(), dfsStat.getPermission(), dfsStat.getOwner(), dfsStat.getGroup(), dfsStat.getPath()); result.add(hawqStat);//from ww w . j av a 2 s.c o m } return result; }
From source file:com.quantcast.qfs.hadoop.QFSEmulationImpl.java
License:Apache License
public KfsFileAttr fullStat(Path path) throws IOException { FileStatus fs = localFS.getFileStatus(new Path(path.toUri().getPath())); KfsFileAttr fa = new KfsFileAttr(); fa.filename = fs.getPath().toUri().getPath(); fa.isDirectory = fs.isDir();//from w w w .j av a2 s .co m fa.filesize = fs.getLen(); fa.replication = fs.getReplication(); fa.modificationTime = fs.getModificationTime(); return fa; }
From source file:com.quantcast.qfs.hadoop.QFSEmulationImpl.java
License:Apache License
public long getModificationTime(String path) throws IOException { FileStatus s = localFS.getFileStatus(new Path(path)); if (s == null) { return 0; }/*w w w . j ava2 s.c o m*/ return s.getModificationTime(); }
From source file:com.redsqirl.workflow.server.OozieManager.java
License:Open Source License
/** * Clean the directory where the Job details are stored * //w w w.j a v a 2s.c o m * @param nameWf * @throws RemoteException */ public void cleanJobDirectory(final String nameWf) throws RemoteException { Path hdfsWfPath = new Path(WorkflowPrefManager.getHDFSPathJobs()); FileSystem fs = null; int numberToKeep = WorkflowPrefManager.getNbOozieDirToKeep(); try { fs = NameNodeVar.getFS(); FileStatus[] children = fs.listStatus(hdfsWfPath, new PathFilter() { @Override public boolean accept(Path arg0) { return arg0.getName().startsWith(nameWf + "_"); } }); Arrays.sort(children, 0, children.length, new Comparator<FileStatus>() { @Override public int compare(FileStatus arg0, FileStatus arg1) { return (int) ((arg0.getModificationTime() - arg1.getModificationTime()) / 10000); } }); for (int i = 0; i < children.length - numberToKeep; ++i) { fs.delete(children[i].getPath(), true); } } catch (Exception e1) { logger.error(e1); } }
From source file:com.redsqirl.workflow.server.Workflow.java
License:Open Source License
/** * Clean the backup directory/*from ww w .jav a 2 s . c om*/ * * @throws IOException */ public void cleanUpBackup() throws IOException { String path = WorkflowPrefManager.getBackupPath(); int nbBackup = WorkflowPrefManager.getNbBackup(); FileSystem fs = NameNodeVar.getFS(); // FileStatus stat = fs.getFileStatus(new Path(path)); FileStatus[] fsA = fs.listStatus(new Path(path), new PathFilter() { @Override public boolean accept(Path arg0) { return arg0.getName().matches(".*[0-9]{14}(.rs|.srs)$"); } }); logger.debug("Backup directory: " + fsA.length + " files, " + nbBackup + " to keep, " + Math.max(0, fsA.length - nbBackup) + " to remove"); if (fsA.length > nbBackup) { int numberToRemove = fsA.length - nbBackup; Map<Path, Long> pathToRemove = new HashMap<Path, Long>(); Path pathMin = null; Long min = Long.MAX_VALUE; for (FileStatus stat : fsA) { if (pathToRemove.size() < numberToRemove) { pathToRemove.put(stat.getPath(), stat.getModificationTime()); } else if (min > stat.getModificationTime()) { pathToRemove.remove(pathMin); pathToRemove.put(stat.getPath(), stat.getModificationTime()); } if (min > stat.getModificationTime()) { min = stat.getModificationTime(); pathMin = stat.getPath(); } } for (Path pathDel : pathToRemove.keySet()) { fs.delete(pathDel, false); } } // fs.close(); }
From source file:com.revolutionanalytics.hadoop.hdfs.FileUtils.java
License:Apache License
private static void ls__(FileSystem srcFS, String path, ArrayList<String> lsco, boolean dorecurse) throws IOException, FileNotFoundException { Path spath = new Path(path); FileStatus[] srcs;/* w w w. ja v a 2s . co m*/ srcs = srcFS.globStatus(spath); if (srcs == null || srcs.length == 0) { throw new FileNotFoundException("Cannot access " + path + ": No such file or directory."); } if (srcs.length == 1 && srcs[0].isDir()) srcs = srcFS.listStatus(srcs[0].getPath()); Calendar c = Calendar.getInstance(); for (FileStatus status : srcs) { StringBuilder sb = new StringBuilder(); boolean idir = status.isDir(); String x = idir ? "d" : "-"; if (dorecurse && idir) ls__(srcFS, status.getPath().toUri().getPath(), lsco, dorecurse); else { sb.append(x); sb.append(status.getPermission().toString()); sb.append(fsep); sb.append(status.getOwner()); sb.append(fsep); sb.append(status.getGroup()); sb.append(fsep); sb.append(status.getLen()); sb.append(fsep); Date d = new Date(status.getModificationTime()); sb.append(formatter.format(d)); sb.append(fsep); sb.append(status.getPath().toUri().getPath()); lsco.add(sb.toString()); } } }
From source file:com.rim.logdriver.admin.HFind.java
License:Apache License
@Override public int run(String[] args) throws Exception { final long startTime = System.currentTimeMillis(); int i = 0;/*from www. j av a 2 s . c o m*/ while (i < args.length) { if (args[i].startsWith("-")) { break; } Path path = new Path(args[i]); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] fileStatuses = fs.globStatus(path); if (fileStatuses != null) { for (FileStatus fileStatus : fileStatuses) { paths.add(fileStatus.getPath()); fileStatusCache.put(fileStatus.getPath(), fileStatus); } } i++; } while (i < args.length) { // -print action if ("-print".equals(args[i])) { actions.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { System.out.println(fileStatus.getPath()); return true; } }); } // -delete action if ("-delete".equals(args[i])) { actions.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { try { FileSystem fs = fileStatus.getPath().getFileSystem(getConf()); if (!fileStatus.isDir() || fs.listStatus(fileStatus.getPath()).length == 0) { return fs.delete(fileStatus.getPath(), true); } } catch (IOException e) { e.printStackTrace(); } return false; } }); } // -atime test else if ("-atime".equals(args[i])) { i++; if (i >= args.length) { System.err.println("Missing arguement for -atime"); System.exit(1); } String t = args[i]; if (t.charAt(0) == '+') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getAccessTime()) / (24 * 60 * 60 * 1000) > time) { return true; } else { return false; } } }); } else if (t.charAt(0) == '-') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getAccessTime()) / (24 * 60 * 60 * 1000) < time) { return true; } else { return false; } } }); } else { final long time = Long.parseLong(t); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getAccessTime()) / (24 * 60 * 60 * 1000) == time) { return true; } else { return false; } } }); } } // -mtime test else if ("-mtime".equals(args[i])) { i++; if (i >= args.length) { System.err.println("Missing arguement for -mtime"); System.exit(1); } String t = args[i]; if (t.charAt(0) == '+') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getModificationTime()) / (24 * 60 * 60 * 1000) > time) { return true; } else { return false; } } }); } else if (t.charAt(0) == '-') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getModificationTime()) / (24 * 60 * 60 * 1000) < time) { return true; } else { return false; } } }); } else { final long time = Long.parseLong(t); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getModificationTime()) / (24 * 60 * 60 * 1000) == time) { return true; } else { return false; } } }); } } // -amin test else if ("-amin".equals(args[i])) { i++; if (i >= args.length) { System.err.println("Missing arguement for -amin"); System.exit(1); } String t = args[i]; if (t.charAt(0) == '+') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getAccessTime()) / (60 * 1000) > time) { return true; } else { return false; } } }); } else if (t.charAt(0) == '-') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getAccessTime()) / (60 * 1000) < time) { return true; } else { return false; } } }); } else { final long time = Long.parseLong(t); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getAccessTime()) / (60 * 1000) == time) { return true; } else { return false; } } }); } } // -mmin test else if ("-mmin".equals(args[i])) { i++; if (i >= args.length) { System.err.println("Missing arguement for -mmin"); System.exit(1); } String t = args[i]; if (t.charAt(0) == '+') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getModificationTime()) / (60 * 1000) > time) { return true; } else { return false; } } }); } else if (t.charAt(0) == '-') { final long time = Long.parseLong(t.substring(1)); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getModificationTime()) / (60 * 1000) < time) { return true; } else { return false; } } }); } else { final long time = Long.parseLong(t); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if ((startTime - fileStatus.getModificationTime()) / (60 * 1000) == time) { return true; } else { return false; } } }); } } // -regex test else if ("-regex".equals(args[i])) { i++; if (i >= args.length) { System.err.println("Missing arguement for -regex"); System.exit(1); } final Pattern p = Pattern.compile(args[i]); tests.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { if (p.matcher(fileStatus.getPath().toString()).matches()) { return true; } else { return false; } } }); } i++; } if (actions.size() == 0) { actions.add(new FileStatusFilter() { @Override public boolean accept(FileStatus fileStatus) { System.out.println(fileStatus.getPath()); return true; } }); } search(); return 0; }