List of usage examples for org.apache.hadoop.fs FileStatus getOwner
public String getOwner()
From source file:com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemBase.java
License:Open Source License
/** * Converts the given FileStatus to its string representation. * * @param stat FileStatus to convert.// w w w. ja v a2 s .co m * @return String representation of the given FileStatus. */ private static String fileStatusToString(FileStatus stat) { assert stat != null; return String.format("path: %s, isDir: %s, len: %d, owner: %s", stat.getPath().toString(), stat.isDir(), stat.getLen(), stat.getOwner()); }
From source file:com.gruter.hadoop.customShell.CustomShell.java
License:Apache License
private int ls(FileStatus src, FileSystem srcFs, boolean recursive, boolean printHeader) throws IOException { final String cmd = recursive ? "lsr" : "ls"; final FileStatus[] items = shellListStatus(cmd, srcFs, src); if (items == null) { return 1; } else {//from www.ja v a2 s . c o m int numOfErrors = 0; if (!recursive && printHeader) { if (items.length != 0) { System.out.println("Found " + items.length + " items"); } } int maxReplication = 3, maxLen = 10, maxOwner = 0, maxGroup = 0; for (int i = 0; i < items.length; i++) { FileStatus stat = items[i]; int replication = String.valueOf(stat.getReplication()).length(); int len = String.valueOf(stat.getLen()).length(); int owner = String.valueOf(stat.getOwner()).length(); int group = String.valueOf(stat.getGroup()).length(); if (replication > maxReplication) maxReplication = replication; if (len > maxLen) maxLen = len; if (owner > maxOwner) maxOwner = owner; if (group > maxGroup) maxGroup = group; } for (int i = 0; i < items.length; i++) { FileStatus stat = items[i]; Path cur = stat.getPath(); String mdate = dateForm.format(new Date(stat.getModificationTime())); System.out.print((stat.isDir() ? "d" : "-") + stat.getPermission() + " "); System.out.printf("%" + maxReplication + "s ", (!stat.isDir() ? stat.getReplication() : "-")); if (maxOwner > 0) System.out.printf("%-" + maxOwner + "s ", stat.getOwner()); if (maxGroup > 0) System.out.printf("%-" + maxGroup + "s ", stat.getGroup()); System.out.printf("%" + maxLen + "d ", stat.getLen()); System.out.print(mdate + " "); System.out.println(cur.toUri().getPath()); if (recursive && stat.isDir()) { numOfErrors += ls(stat, srcFs, recursive, printHeader); } } return numOfErrors; } }
From source file:com.ibm.jaql.lang.expr.system.LsFn.java
License:Apache License
@Override public JsonIterator iter(final Context context) throws Exception { JsonString glob = (JsonString) exprs[0].eval(context); // Configuration conf = context.getConfiguration(); Configuration conf = new Configuration(); // TODO: get from context, incl options //URI uri;//from w ww. j a v a 2 s . co m //FileSystem fs = FileSystem.get(uri, conf); Path inpath = new Path(glob.toString()); FileSystem fs = inpath.getFileSystem(conf); //final FileStatus[] stats = fs.listStatus(path, filter); final FileStatus[] stats = fs.globStatus(inpath); if (stats == null || stats.length == 0) { return JsonIterator.EMPTY; } final MutableJsonDate accessTime = new MutableJsonDate(); final MutableJsonDate modifyTime = new MutableJsonDate(); final MutableJsonLong length = new MutableJsonLong(); final MutableJsonLong blockSize = new MutableJsonLong(); final MutableJsonLong replication = new MutableJsonLong(); final MutableJsonString path = new MutableJsonString(); final MutableJsonString owner = new MutableJsonString(); final MutableJsonString group = new MutableJsonString(); final MutableJsonString permission = new MutableJsonString(); final JsonValue[] values = new JsonValue[] { accessTime, modifyTime, length, blockSize, replication, path, owner, group, permission }; final BufferedJsonRecord rec = new BufferedJsonRecord(); rec.set(LsField.names, values, values.length, false); return new JsonIterator(rec) { int i = 0; @Override public boolean moveNext() throws Exception { if (i >= stats.length) { return false; } FileStatus stat = stats[i++]; // fs.getUri().toString(); long x = HadoopShim.getAccessTime(stat); if (x <= 0) { values[LsField.ACCESS_TIME.ordinal()] = null; } else { accessTime.set(x); values[LsField.ACCESS_TIME.ordinal()] = accessTime; } modifyTime.set(stat.getModificationTime()); length.set(stat.getLen()); blockSize.set(stat.getBlockSize()); replication.set(stat.getReplication()); path.setCopy(stat.getPath().toString()); owner.setCopy(stat.getOwner()); group.setCopy(stat.getGroup()); permission.setCopy(stat.getPermission().toString()); return true; } }; }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
private void testPreserveUserGroupImpl(boolean preserve) { try {// w w w . j a v a 2 s.c o m deleteState(); createSourceData(); changeUserGroup("Michael", "Corleone"); FileSystem fs = cluster.getFileSystem(); CopyMapper copyMapper = new CopyMapper(); StatusReporter reporter = new StubStatusReporter(); InMemoryWriter writer = new InMemoryWriter(); Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter, writer); Configuration configuration = context.getConfiguration(); EnumSet<DistCpOptions.FileAttribute> fileAttributes = EnumSet.noneOf(DistCpOptions.FileAttribute.class); if (preserve) { fileAttributes.add(DistCpOptions.FileAttribute.USER); fileAttributes.add(DistCpOptions.FileAttribute.GROUP); fileAttributes.add(DistCpOptions.FileAttribute.PERMISSION); } configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), DistCpUtils.packAttributes(fileAttributes)); copyMapper.setup(context); for (Path path : pathList) { final FileStatus fileStatus = fs.getFileStatus(path); copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus, context); } // Check that the user/group attributes are preserved // (only) as necessary. for (Path path : pathList) { final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH)); final FileStatus source = fs.getFileStatus(path); final FileStatus target = fs.getFileStatus(targetPath); if (!source.isDir()) { Assert.assertTrue(!preserve || source.getOwner().equals(target.getOwner())); Assert.assertTrue(!preserve || source.getGroup().equals(target.getGroup())); Assert.assertTrue(!preserve || source.getPermission().equals(target.getPermission())); Assert.assertTrue(preserve || !source.getOwner().equals(target.getOwner())); Assert.assertTrue(preserve || !source.getGroup().equals(target.getGroup())); Assert.assertTrue(preserve || !source.getPermission().equals(target.getPermission())); Assert.assertTrue(source.isDir() || source.getReplication() != target.getReplication()); } } } catch (Exception e) { Assert.assertTrue("Unexpected exception: " + e.getMessage(), false); e.printStackTrace(); } }
From source file:com.inmobi.conduit.distcp.tools.util.DistCpUtils.java
License:Apache License
/** * Preserve attribute on file matching that of the file status being sent * as argument. Barring the block size, all the other attributes are preserved * by this function/*w w w .j a v a 2 s.c o m*/ * * @param targetFS - File system * @param path - Path that needs to preserve original file status * @param srcFileStatus - Original file status * @param attributes - Attribute set that need to be preserved * @throws IOException - Exception if any (particularly relating to group/owner * change or any transient error) */ public static void preserve(FileSystem targetFS, Path path, FileStatus srcFileStatus, EnumSet<FileAttribute> attributes) throws IOException { FileStatus targetFileStatus = targetFS.getFileStatus(path); String group = targetFileStatus.getGroup(); String user = targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDir() && srcFileStatus.getReplication() != targetFileStatus.getReplication()) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } }
From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java
License:Apache License
@Test public void testPreserve() { try {/*from w w w .ja v a2 s . c o m*/ FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class); Path path = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); fs.mkdirs(path); fs.mkdirs(src); FileStatus srcStatus = fs.getFileStatus(src); FsPermission noPerm = new FsPermission((short) 0); fs.setPermission(path, noPerm); fs.setOwner(path, "nobody", "nobody"); DistCpUtils.preserve(fs, path, srcStatus, attributes); FileStatus target = fs.getFileStatus(path); Assert.assertEquals(target.getPermission(), noPerm); Assert.assertEquals(target.getOwner(), "nobody"); Assert.assertEquals(target.getGroup(), "nobody"); attributes.add(FileAttribute.PERMISSION); DistCpUtils.preserve(fs, path, srcStatus, attributes); target = fs.getFileStatus(path); Assert.assertEquals(target.getPermission(), srcStatus.getPermission()); Assert.assertEquals(target.getOwner(), "nobody"); Assert.assertEquals(target.getGroup(), "nobody"); attributes.add(FileAttribute.GROUP); attributes.add(FileAttribute.USER); DistCpUtils.preserve(fs, path, srcStatus, attributes); target = fs.getFileStatus(path); Assert.assertEquals(target.getPermission(), srcStatus.getPermission()); Assert.assertEquals(target.getOwner(), srcStatus.getOwner()); Assert.assertEquals(target.getGroup(), srcStatus.getGroup()); fs.delete(path, true); fs.delete(src, true); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Preserve test failure"); } }
From source file:com.jeffy.hdfs.FileMetaData.java
License:Apache License
public static void showFileStatusForFile(String path) { Configuration config = new Configuration(); try {// w ww. j a v a 2 s .co m FileSystem fs = FileSystem.get(URI.create(path), config); FileStatus stat = fs.getFileStatus(new Path(path)); System.out.println("File URI: " + stat.getPath().toUri().getPath()); System.out.println("Is directory: " + stat.isDirectory()); System.out.println("File length: " + stat.getLen()); System.out.println("Modification Time: " + new Date(stat.getModificationTime())); System.out.println("File replications: " + stat.getReplication()); System.out.println("File Block Size: " + (stat.getBlockSize() >>> 10 >>> 10) + " MB"); System.out.println("File Owner: " + stat.getOwner()); System.out.println("File Group: " + stat.getGroup()); System.out.println("File Permission: " + stat.getPermission().toString()); } catch (IOException e) { e.printStackTrace(); } }
From source file:com.nexr.rhive.util.TransformUtils.java
License:Apache License
public static String[] tranform(FileStatus fs) throws Exception { String[] datas = new String[6]; datas[0] = fs.getPermission().toString(); datas[1] = fs.getOwner(); datas[2] = fs.getGroup();/* w w w . j a v a 2s . c o m*/ datas[3] = Long.toString(fs.getLen()); Date date = new Date(fs.getModificationTime()); datas[4] = formatter.format(date); datas[5] = fs.getPath().toUri().getPath(); return datas; }
From source file:com.pinterest.hdfsbackup.distcp.DistCp.java
License:Apache License
private static void updatePermissions(FileStatus src, FileStatus dst, EnumSet<FileAttribute> preseved, FileSystem destFileSys) throws IOException { String owner = null;/*from ww w . ja v a2s . c om*/ String group = null; if (preseved.contains(FileAttribute.USER) && !src.getOwner().equals(dst.getOwner())) { owner = src.getOwner(); } if (preseved.contains(FileAttribute.GROUP) && !src.getGroup().equals(dst.getGroup())) { group = src.getGroup(); } if (owner != null || group != null) { destFileSys.setOwner(dst.getPath(), owner, group); } if (preseved.contains(FileAttribute.PERMISSION) && !src.getPermission().equals(dst.getPermission())) { destFileSys.setPermission(dst.getPath(), src.getPermission()); } }
From source file:com.pivotal.hawq.mapreduce.parquet.HAWQParquetInputFormat.java
License:Apache License
@Override protected List<FileStatus> listStatus(JobContext jobContext) throws IOException { List<FileStatus> result = Lists.newArrayList(); for (HAWQFileStatus hawqFileStatus : hawqFileStatuses) { if (hawqFileStatus.getFileLength() == 0) continue; // skip empty file Path path = new Path(hawqFileStatus.getFilePath()); FileSystem fs = path.getFileSystem(jobContext.getConfiguration()); FileStatus dfsStat = fs.getFileStatus(path); // rewrite file length because HAWQ records the logicalEOF of file, which may // be smaller than the file's actual EOF FileStatus hawqStat = new FileStatus(hawqFileStatus.getFileLength(), // rewrite to logicalEOF dfsStat.isDirectory(), dfsStat.getReplication(), dfsStat.getBlockSize(), dfsStat.getModificationTime(), dfsStat.getAccessTime(), dfsStat.getPermission(), dfsStat.getOwner(), dfsStat.getGroup(), dfsStat.getPath()); result.add(hawqStat);// w ww . j a va 2s . c o m } return result; }