List of usage examples for org.apache.hadoop.fs FileStatus getOwner
public String getOwner()
From source file:de.tiqsolutions.hdfs.HadoopFileSystem.java
License:Apache License
void checkAccess(Path path, AccessMode... modes) throws IOException { try {/*from ww w . jav a 2 s .c o m*/ FileStatus fileStatus = getFileContext().getFileStatus(((HadoopFileSystemPath) path).getPath()); if (modes == null || modes.length == 0) return; String group = fileStatus.getGroup(); String owner = fileStatus.getOwner(); UserGroupInformation userGroupInformation = getFileContext().getUgi(); boolean checkuser = false; boolean checkgroup = false; if (owner.equals(userGroupInformation.getUserName())) { checkuser = true; } else { for (String g : userGroupInformation.getGroupNames()) { if (group.equals(g)) { checkgroup = true; break; } } } PosixFileAttributeView view = provider().getFileAttributeView(path, PosixFileAttributeView.class); PosixFileAttributes attributes = view.readAttributes(); Set<PosixFilePermission> permissions = attributes.permissions(); getFileContext().getUgi().getGroupNames(); for (AccessMode accessMode : modes) { switch (accessMode) { case READ: if (!permissions.contains(checkuser ? PosixFilePermission.OWNER_READ : (checkgroup ? PosixFilePermission.GROUP_READ : PosixFilePermission.OTHERS_READ))) throw new AccessDeniedException(path.toString()); break; case WRITE: if (!permissions.contains(checkuser ? PosixFilePermission.OWNER_WRITE : (checkgroup ? PosixFilePermission.GROUP_WRITE : PosixFilePermission.OTHERS_WRITE))) throw new AccessDeniedException(path.toString()); break; case EXECUTE: if (!permissions.contains(checkuser ? PosixFilePermission.OWNER_EXECUTE : (checkgroup ? PosixFilePermission.GROUP_EXECUTE : PosixFilePermission.OTHERS_EXECUTE))) throw new AccessDeniedException(path.toString()); break; } } } catch (FileNotFoundException e) { throw new NoSuchFileException(path.toString()); } }
From source file:eagle.security.hdfs.entity.FileStatusEntity.java
License:Apache License
public FileStatusEntity(FileStatus status) throws IOException { //this.path = status.getPath(); this.length = status.getLen(); this.isdir = status.isDirectory(); this.block_replication = status.getReplication(); this.blocksize = status.getBlockSize(); this.modification_time = status.getModificationTime(); this.access_time = status.getAccessTime(); this.permission = status.getPermission(); this.owner = status.getOwner(); this.group = status.getGroup(); if (status.isSymlink()) { this.symlink = status.getSymlink(); }/*ww w . ja v a 2 s.co m*/ }
From source file:edu.umn.cs.spatialHadoop.visualization.HadoopvizServer.java
License:Open Source License
/** * Lists the contents of a directory/*from w w w.j a v a 2 s .c o m*/ * @param request * @param response */ private void handleListFiles(HttpServletRequest request, HttpServletResponse response) { try { String pathStr = request.getParameter("path"); Path path = new Path(pathStr == null || pathStr.isEmpty() ? "/" : pathStr); FileSystem fs = path.getFileSystem(commonParams); FileStatus[] fileStatuses = fs.listStatus(path, SpatialSite.NonHiddenFileFilter); Arrays.sort(fileStatuses, new Comparator<FileStatus>() { @Override public int compare(FileStatus o1, FileStatus o2) { if (o1.isDirectory() && o2.isFile()) return -1; if (o1.isFile() && o2.isDirectory()) return 1; return o1.getPath().getName().toLowerCase().compareTo(o2.getPath().getName().toLowerCase()); } }); response.setContentType("application/json;charset=utf-8"); response.setStatus(HttpServletResponse.SC_OK); PrintWriter out = response.getWriter(); out.print("{\"FileStatuses\":{"); if (pathStr.endsWith("/")) { pathStr = pathStr.substring(0, pathStr.length() - 1); } out.printf("\"BaseDir\":\"%s\",", pathStr); if (path.getParent() != null) out.printf("\"ParentDir\":\"%s\",", path.getParent()); out.print("\"FileStatus\":["); for (int i = 0; i < fileStatuses.length; i++) { FileStatus fileStatus = fileStatuses[i]; if (i != 0) out.print(','); String filename = fileStatus.getPath().getName(); int idot = filename.lastIndexOf('.'); String extension = idot == -1 ? "" : filename.substring(idot + 1); out.printf( "{\"accessTime\":%d,\"blockSize\":%d,\"childrenNum\":%d,\"fileId\":%d," + "\"group\":\"%s\",\"length\":%d,\"modificationTime\":%d," + "\"owner\":\"%s\",\"pathSuffix\":\"%s\",\"permission\":\"%s\"," + "\"replication\":%d,\"storagePolicy\":%d,\"type\":\"%s\",\"extension\":\"%s\"}", fileStatus.getAccessTime(), fileStatus.getBlockSize(), 0, 0, fileStatus.getGroup(), fileStatus.getLen(), fileStatus.getModificationTime(), fileStatus.getOwner(), fileStatus.getPath().getName(), fileStatus.getPermission(), fileStatus.getReplication(), 0, fileStatus.isDirectory() ? "DIRECTORY" : "FILE", extension.toLowerCase()); } out.print("]}"); // Check if there is an image or master file FileStatus[] metaFiles = fs.listStatus(path, new PathFilter() { @Override public boolean accept(Path path) { return path.getName().startsWith("_master") || path.getName().equals("_data.png"); } }); for (FileStatus metaFile : metaFiles) { String metaFileName = metaFile.getPath().getName(); if (metaFileName.startsWith("_master")) { out.printf(",\"MasterPath\":\"%s\"", metaFileName); String shape = OperationsParams.detectShape(fileStatuses[0].getPath(), commonParams); if (shape != null) out.printf(",\"Shape\":\"%s\"", shape); } else if (metaFileName.equals("_data.png")) out.printf(",\"ImagePath\":\"%s\"", metaFileName); } out.print("}"); out.close(); } catch (Exception e) { System.out.println("error happened"); e.printStackTrace(); try { e.printStackTrace(response.getWriter()); } catch (IOException ioe) { ioe.printStackTrace(); e.printStackTrace(); } response.setContentType("text/plain;charset=utf-8"); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } }
From source file:fr.ens.biologie.genomique.eoulsan.modules.mgmt.hadoop.DistCp.java
License:LGPL
private static void updatePermissions(final FileStatus src, final FileStatus dst, final EnumSet<FileAttribute> preseved, final FileSystem destFileSys) throws IOException { String owner = null;//from w w w .jav a2 s . c om String group = null; if (preseved.contains(FileAttribute.USER) && !src.getOwner().equals(dst.getOwner())) { owner = src.getOwner(); } if (preseved.contains(FileAttribute.GROUP) && !src.getGroup().equals(dst.getGroup())) { group = src.getGroup(); } if (owner != null || group != null) { destFileSys.setOwner(dst.getPath(), owner, group); } if (preseved.contains(FileAttribute.PERMISSION) && !src.getPermission().equals(dst.getPermission())) { destFileSys.setPermission(dst.getPath(), src.getPermission()); } }
From source file:fuse4j.hadoopfs.HdfsClientImpl.java
License:Apache License
/** * getFileInfo()/* www . j a v a 2 s . c o m*/ */ @Override public HdfsFileAttr getFileInfo(int uid, String path) { FileSystem dfs = null; try { dfs = getDfs(uid); FileStatus dfsStat = dfs.getFileStatus(new Path(path)); final boolean directory = dfsStat.isDir(); final int inode = 0; final int mode = dfsStat.getPermission().toShort(); final int uuid = userCache.getUid(dfsStat.getOwner()); final int gid = 0; // TODO: per-file block-size can't be retrieved correctly, // using default block size for now. final long size = dfsStat.getLen(); final int blocks = (int) Math.ceil(((double) size) / dfs.getDefaultBlockSize()); // modification/create-times are the same as access-time final int modificationTime = (int) (dfsStat.getModificationTime() / 1000); final int accessTime = (int) (dfsStat.getAccessTime() / 1000); HdfsFileAttr hdfsFileAttr = new HdfsFileAttr(directory, inode, mode, uuid, gid, 1); hdfsFileAttr.setSize(size, blocks); hdfsFileAttr.setTime(modificationTime, modificationTime, accessTime); // TODO Hack to set inode; hdfsFileAttr.inode = hdfsFileAttr.hashCode(); return hdfsFileAttr; } catch (Exception ioe) { // fall through to failure } // failed return null; }
From source file:fuse4j.hadoopfs.HdfsClientReal.java
License:Apache License
/** * getFileInfo()//from w ww. j ava 2s .co m */ public HdfsFileAttr getFileInfo(String path) { try { FileStatus dfsStat = dfs.getFileStatus(new Path(path)); final boolean directory = dfsStat.isDirectory(); final int inode = 0; final int mode = dfsStat.getPermission().toShort(); final int uid = userCache.getUid(dfsStat.getOwner()); final int gid = 0; // TODO: per-file block-size can't be retrieved correctly, // using default block size for now. final long size = dfsStat.getLen(); final int blocks = (int) Math.ceil(((double) size) / dfs.getDefaultBlockSize()); // modification/create-times are the same as access-time final int modificationTime = (int) (dfsStat.getModificationTime() / 1000); HdfsFileAttr hdfsFileAttr = new HdfsFileAttr(directory, inode, mode, uid, gid, 1); hdfsFileAttr.setSize(size, blocks); hdfsFileAttr.setTime(modificationTime); // TODO Hack to set inode; hdfsFileAttr.inode = hdfsFileAttr.hashCode(); return hdfsFileAttr; } catch (IOException ioe) { // fall through to failure } // failed return null; }
From source file:gobblin.data.management.copy.CopyableFileTest.java
License:Apache License
@Test public void testCopyableFileBuilderMinimumConfiguration() throws IOException { // Source/*from ww w .j ava 2 s.com*/ String datasetRootDir = "/data/databases/source"; Path datasetRoot = new Path(datasetRootDir); FileSystem originFS = FileSystem.getLocal(new Configuration()); Path originFile = new Path(datasetRootDir, "copyableFile"); FileStatus origin = new FileStatus(0l, false, 0, 0l, System.currentTimeMillis(), originFile); PreserveAttributes preserveAttributes = PreserveAttributes.fromMnemonicString("ugp"); // Target String targetRoot = "/data/databases/destination"; Path relativePath = PathUtils.relativizePath(originFile, datasetRoot); Path targetPath = new Path(targetRoot, relativePath); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher"); CopyConfiguration copyConfiguration = CopyConfiguration .builder(FileSystem.getLocal(new Configuration()), properties).preserve(preserveAttributes).build(); CopyableFile copyableFile = CopyableFile.builder(originFS, origin, datasetRoot, copyConfiguration) .destination(targetPath).ancestorsOwnerAndPermission(Lists.<OwnerAndPermission>newArrayList()) // not testing ancestors .build(); // Making sure all fields are populated correctly via CopyableFile builder // Verify preserve attribute options Assert.assertEquals(copyableFile.getPreserve().toMnemonicString(), preserveAttributes.toMnemonicString()); // Verify origin Assert.assertEquals(copyableFile.getFileSet(), ""); Assert.assertEquals(copyableFile.getOrigin(), origin); // Verify destination target, permissions and other attributes Assert.assertEquals(copyableFile.getChecksum().length, 0); Assert.assertEquals(copyableFile.getDestination().toString(), targetPath.toString()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getGroup(), origin.getGroup()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getOwner(), origin.getOwner()); Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getFsPermission(), origin.getPermission()); // Verify auto determined timestamp Assert.assertEquals(copyableFile.getOriginTimestamp(), origin.getModificationTime()); Assert.assertEquals(copyableFile.getUpstreamTimestamp(), origin.getModificationTime()); }
From source file:gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterTest.java
License:Apache License
@Test public void testWrite() throws Exception { String streamString = "testContents"; FileStatus status = fs.getFileStatus(testTempPath); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission); CopyableDatasetMetadata metadata = new CopyableDatasetMetadata( new TestCopyableDataset(new Path("/source"))); WorkUnitState state = TestUtils.createTestWorkUnitState(); state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString()); state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString()); state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5)); CopySource.serializeCopyEntity(state, cf); CopySource.serializeCopyableDataset(state, metadata); FileAwareInputStreamDataWriter dataWriter = new FileAwareInputStreamDataWriter(state, 1, 0); FileAwareInputStream fileAwareInputStream = new FileAwareInputStream(cf, StreamUtils.convertStream(IOUtils.toInputStream(streamString))); dataWriter.write(fileAwareInputStream); dataWriter.commit();//from w w w . ja va 2 s .c om Path writtenFilePath = new Path(new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), cf.getDatasetAndPartition(metadata).identifier()), cf.getDestination()); Assert.assertEquals(IOUtils.toString(new FileInputStream(writtenFilePath.toString())), streamString); }
From source file:gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterTest.java
License:Apache License
@Test public void testWriteWithEncryption() throws Exception { byte[] streamString = "testEncryptedContents".getBytes("UTF-8"); byte[] expectedContents = new byte[streamString.length]; for (int i = 0; i < streamString.length; i++) { expectedContents[i] = (byte) ((streamString[i] + 1) % 256); }//from w w w . j a v a2 s . c o m FileStatus status = fs.getFileStatus(testTempPath); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission); CopyableDatasetMetadata metadata = new CopyableDatasetMetadata( new TestCopyableDataset(new Path("/source"))); WorkUnitState state = TestUtils.createTestWorkUnitState(); state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString()); state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString()); state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5)); state.setProp("writer.encrypt." + EncryptionConfigParser.ENCRYPTION_ALGORITHM_KEY, "insecure_shift"); CopySource.serializeCopyEntity(state, cf); CopySource.serializeCopyableDataset(state, metadata); FileAwareInputStreamDataWriter dataWriter = new FileAwareInputStreamDataWriter(state, 1, 0); FileAwareInputStream fileAwareInputStream = new FileAwareInputStream(cf, StreamUtils.convertStream(new ByteArrayInputStream(streamString))); dataWriter.write(fileAwareInputStream); dataWriter.commit(); Path writtenFilePath = new Path(new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), cf.getDatasetAndPartition(metadata).identifier()), cf.getDestination()); Assert.assertTrue(writtenFilePath.getName().endsWith("insecure_shift"), "Expected encryption name to be appended to destination"); Assert.assertEquals(IOUtils.toByteArray(new FileInputStream(writtenFilePath.toString())), expectedContents); }
From source file:gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterTest.java
License:Apache License
@Test public void testCommit() throws IOException { String destinationExistingToken = "destination"; String destinationAdditionalTokens = "path"; String fileName = "file"; // Asemble destination paths Path destination = new Path(new Path(new Path("/", destinationExistingToken), destinationAdditionalTokens), fileName);//ww w. j a v a 2 s . c o m Path destinationWithoutLeadingSeparator = new Path( new Path(destinationExistingToken, destinationAdditionalTokens), fileName); // Create temp directory File tmpFile = Files.createTempDir(); tmpFile.deleteOnExit(); Path tmpPath = new Path(tmpFile.getAbsolutePath()); // create origin file Path originFile = new Path(tmpPath, fileName); this.fs.createNewFile(originFile); // create stating dir Path stagingDir = new Path(tmpPath, "staging"); this.fs.mkdirs(stagingDir); // create output dir Path outputDir = new Path(tmpPath, "output"); this.fs.mkdirs(outputDir); // create copyable file FileStatus status = this.fs.getFileStatus(originFile); FsPermission readWrite = new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.READ_WRITE); FsPermission dirReadWrite = new FsPermission(FsAction.ALL, FsAction.READ_WRITE, FsAction.READ_WRITE); OwnerAndPermission ownerAndPermission = new OwnerAndPermission(status.getOwner(), status.getGroup(), readWrite); List<OwnerAndPermission> ancestorOwnerAndPermissions = Lists.newArrayList(); ancestorOwnerAndPermissions.add(ownerAndPermission); ancestorOwnerAndPermissions.add(ownerAndPermission); ancestorOwnerAndPermissions.add(ownerAndPermission); ancestorOwnerAndPermissions.add(ownerAndPermission); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher"); CopyableFile cf = CopyableFile .fromOriginAndDestination(this.fs, status, destination, CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties) .publishDir(new Path("/target")).preserve(PreserveAttributes.fromMnemonicString("")) .build()) .destinationOwnerAndPermission(ownerAndPermission) .ancestorsOwnerAndPermission(ancestorOwnerAndPermissions).build(); // create work unit state WorkUnitState state = TestUtils.createTestWorkUnitState(); state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, stagingDir.toUri().getPath()); state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, outputDir.toUri().getPath()); state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5)); CopyableDatasetMetadata metadata = new CopyableDatasetMetadata( new TestCopyableDataset(new Path("/source"))); CopySource.serializeCopyEntity(state, cf); CopySource.serializeCopyableDataset(state, metadata); // create writer FileAwareInputStreamDataWriter writer = new FileAwareInputStreamDataWriter(state, 1, 0); // create output of writer.write Path writtenFile = writer.getStagingFilePath(cf); this.fs.mkdirs(writtenFile.getParent()); this.fs.createNewFile(writtenFile); // create existing directories in writer output Path outputRoot = FileAwareInputStreamDataWriter.getPartitionOutputRoot(outputDir, cf.getDatasetAndPartition(metadata)); Path existingOutputPath = new Path(outputRoot, destinationExistingToken); this.fs.mkdirs(existingOutputPath); FileStatus fileStatus = this.fs.getFileStatus(existingOutputPath); FsPermission existingPathPermission = fileStatus.getPermission(); // check initial state of the relevant directories Assert.assertTrue(this.fs.exists(existingOutputPath)); Assert.assertEquals(this.fs.listStatus(existingOutputPath).length, 0); writer.actualProcessedCopyableFile = Optional.of(cf); // commit writer.commit(); // check state of relevant paths after commit Path expectedOutputPath = new Path(outputRoot, destinationWithoutLeadingSeparator); Assert.assertTrue(this.fs.exists(expectedOutputPath)); fileStatus = this.fs.getFileStatus(expectedOutputPath); Assert.assertEquals(fileStatus.getOwner(), ownerAndPermission.getOwner()); Assert.assertEquals(fileStatus.getGroup(), ownerAndPermission.getGroup()); Assert.assertEquals(fileStatus.getPermission(), readWrite); // parent should have permissions set correctly fileStatus = this.fs.getFileStatus(expectedOutputPath.getParent()); Assert.assertEquals(fileStatus.getPermission(), dirReadWrite); // previously existing paths should not have permissions changed fileStatus = this.fs.getFileStatus(existingOutputPath); Assert.assertEquals(fileStatus.getPermission(), existingPathPermission); Assert.assertFalse(this.fs.exists(writer.stagingDir)); }