List of usage examples for org.apache.hadoop.fs.permission AclStatus getEntries
public List<AclEntry> getEntries()
From source file:alluxio.underfs.hdfs.acl.SupportedHdfsAclProvider.java
License:Apache License
@Override public Pair<AccessControlList, DefaultAccessControlList> getAcl(FileSystem hdfs, String path) throws IOException { AclStatus hdfsAcl; Path filePath = new Path(path); boolean isDir = hdfs.isDirectory(filePath); try {/*from ww w. j av a 2 s . c om*/ hdfsAcl = hdfs.getAclStatus(filePath); } catch (AclException e) { // When dfs.namenode.acls.enabled is false, getAclStatus throws AclException. return new Pair<>(null, null); } AccessControlList acl = new AccessControlList(); DefaultAccessControlList defaultAcl = new DefaultAccessControlList(); acl.setOwningUser(hdfsAcl.getOwner()); acl.setOwningGroup(hdfsAcl.getGroup()); defaultAcl.setOwningUser(hdfsAcl.getOwner()); defaultAcl.setOwningGroup(hdfsAcl.getGroup()); for (AclEntry entry : hdfsAcl.getEntries()) { alluxio.security.authorization.AclEntry.Builder builder = new alluxio.security.authorization.AclEntry.Builder(); builder.setType(getAclEntryType(entry)); builder.setSubject(entry.getName() == null ? "" : entry.getName()); FsAction permission = entry.getPermission(); if (permission.implies(FsAction.READ)) { builder.addAction(AclAction.READ); } else if (permission.implies(FsAction.WRITE)) { builder.addAction(AclAction.WRITE); } else if (permission.implies(FsAction.EXECUTE)) { builder.addAction(AclAction.EXECUTE); } if (entry.getScope().equals(AclEntryScope.ACCESS)) { acl.setEntry(builder.build()); } else { // default ACL, must be a directory defaultAcl.setEntry(builder.build()); } } if (isDir) { return new Pair<>(acl, defaultAcl); } else { // a null defaultACL indicates this is a file return new Pair<>(acl, null); } }
From source file:com.bigstep.datalake.JsonUtil.java
License:Apache License
/** Convert a AclStatus object to a Json string. */ public static String toJsonString(final AclStatus status) { if (status == null) { return null; }/* w w w. j a v a2 s .c o m*/ final Map<String, Object> m = new TreeMap<String, Object>(); m.put("owner", status.getOwner()); m.put("group", status.getGroup()); m.put("stickyBit", status.isStickyBit()); final List<String> stringEntries = new ArrayList<>(); for (AclEntry entry : status.getEntries()) { stringEntries.add(entry.toString()); } m.put("entries", stringEntries); FsPermission perm = status.getPermission(); if (perm != null) { m.put("permission", toString(perm)); if (perm.getAclBit()) { m.put("aclBit", true); } if (perm.getEncryptedBit()) { m.put("encBit", true); } } final Map<String, Map<String, Object>> finalMap = new TreeMap<String, Map<String, Object>>(); finalMap.put(AclStatus.class.getSimpleName(), m); Gson gson = new Gson(); return gson.toJson(finalMap); }
From source file:org.apache.sentry.tests.e2e.hdfs.TestDbHdfsBase.java
License:Apache License
/** * Verify extended acl entries are correctly synced up * @param expectedAcls//from w w w .j ava 2 s . co m * @param pathLoc * @param recursive * @throws Exception */ private void verifyAclsHelper(List<AclEntry> expectedAcls, String pathLoc, boolean recursive) throws Exception { int retry = 0; Path path = new Path(pathLoc); LOGGER.info("expectedAcls of [" + pathLoc + "] = " + expectedAcls.toString()); // Syncing up acls takes some time so make validation in a loop while (retry < NUM_RETRIES_FOR_ACLS) { AclStatus aclStatus = fileSystem.getAclStatus(path); List<AclEntry> actualAcls = new ArrayList<>(aclStatus.getEntries()); LOGGER.info("[" + retry + "] actualAcls of [" + pathLoc + "] = " + actualAcls.toString()); retry += 1; if (!actualAcls.isEmpty() && !actualAcls.contains(expectedAcls.get(expectedAcls.size() - 1))) { Thread.sleep(WAIT_SECS_FOR_ACLS); } else { for (AclEntry expected : expectedAcls) { assertTrue("Fail to find aclEntry: " + expected.toString(), actualAcls.contains(expected)); } break; } } assertThat(retry, lessThan(NUM_RETRIES_FOR_ACLS)); if (recursive && fileSystem.getFileStatus(path).isDirectory()) { FileStatus[] children = fileSystem.listStatus(path); for (FileStatus fs : children) { verifyAclsRecursive(expectedAcls, fs.getPath().toString(), recursive); } } }
From source file:org.apache.sentry.tests.e2e.hdfs.TestDbHdfsBase.java
License:Apache License
/** * Verify there is no specified acls gotten synced up in the path status * @param noAcls//from ww w.ja v a 2s.co m * @param pathLoc * @param recursive * @throws Exception */ private void verifyNoAclHelper(List<AclEntry> noAcls, String pathLoc, boolean recursive) throws Exception { int retry = 0; // Retry a couple of times in case the incorrect acls take time to be synced up while (retry < NUM_RETRIES_FOR_ACLS) { Path path = new Path(pathLoc); AclStatus aclStatus = fileSystem.getAclStatus(path); List<AclEntry> actualAcls = new ArrayList<>(aclStatus.getEntries()); LOGGER.info("[" + retry + "] actualAcls of [" + pathLoc + "] = " + actualAcls.toString()); Thread.sleep(1000); // wait for syncup retry += 1; for (AclEntry acl : actualAcls) { if (noAcls.contains(acl)) { fail("Path [ " + pathLoc + " ] should not contain " + acl.toString()); } } } Path path = new Path(pathLoc); if (recursive && fileSystem.getFileStatus(path).isDirectory()) { FileStatus[] children = fileSystem.listStatus(path); for (FileStatus fs : children) { verifyNoAclRecursive(noAcls, fs.getPath().toString(), recursive); } } }
From source file:org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegration.java
License:Apache License
@Test public void testEnd2End() throws Throwable { tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { "db1" }; roles = new String[] { "admin_role", "db_role", "tab_role", "p1_admin" }; admin = "hive"; Connection conn;//from w ww . j a va 2 s.c o m Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant role admin_role to group hive"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("create table p1 (s string) partitioned by (month int, day int)"); stmt.execute("alter table p1 add partition (month=1, day=1)"); stmt.execute("alter table p1 add partition (month=1, day=2)"); stmt.execute("alter table p1 add partition (month=2, day=1)"); stmt.execute("alter table p1 add partition (month=2, day=2)"); // db privileges stmt.execute("create database db5"); stmt.execute("create role db_role"); stmt.execute("create role tab_role"); stmt.execute("grant role db_role to group hbase"); stmt.execute("grant role tab_role to group flume"); stmt.execute("create table db5.p2(id int)"); stmt.execute("create role p1_admin"); stmt.execute("grant role p1_admin to group hbase"); // Verify default db is inaccessible initially verifyOnAllSubDirs("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); stmt.execute("grant all on database db5 to role db_role"); stmt.execute("use db5"); stmt.execute("grant all on table p2 to role tab_role"); stmt.execute("use default"); verifyOnAllSubDirs("/user/hive/warehouse/db5.db", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "flume", true); verifyOnPath("/user/hive/warehouse/db5.db", FsAction.ALL, "flume", false); loadData(stmt); verifyHDFSandMR(stmt); // Verify default db is STILL inaccessible after grants but tables are fine verifyOnPath("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); adminUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { // Simulate hdfs dfs -setfacl -m <aclantry> <path> AclStatus existing = miniDFS.getFileSystem().getAclStatus(new Path("/user/hive/warehouse/p1")); ArrayList<AclEntry> newEntries = new ArrayList<AclEntry>(existing.getEntries()); newEntries.add(AclEntry.parseAclEntry("user::---", true)); newEntries.add(AclEntry.parseAclEntry("group:bla:rwx", true)); newEntries.add(AclEntry.parseAclEntry("other::---", true)); miniDFS.getFileSystem().setAcl(new Path("/user/hive/warehouse/p1"), newEntries); return null; } }); stmt.execute("revoke select on table p1 from role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); // Verify default db grants work stmt.execute("grant select on database default to role p1_admin"); verifyOnPath("/user/hive/warehouse", FsAction.READ_EXECUTE, "hbase", true); // Verify default db grants are propagated to the tables verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); // Verify default db revokes work stmt.execute("revoke select on database default from role p1_admin"); verifyOnPath("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); stmt.execute("grant all on table p1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.ALL, "hbase", true); stmt.execute("revoke select on table p1 from role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify table rename works stmt.execute("alter table p1 rename to p3"); verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); stmt.execute("alter table p3 partition (month=1, day=1) rename to partition (month=1, day=3)"); verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/p3/month=1/day=3", FsAction.WRITE_EXECUTE, "hbase", true); // Test DB case insensitivity stmt.execute("create database extdb"); stmt.execute("grant all on database ExtDb to role p1_admin"); writeToPath("/tmp/external/ext100", 5, "foo", "bar"); writeToPath("/tmp/external/ext101", 5, "foo", "bar"); stmt.execute("use extdb"); stmt.execute("create table ext100 (s string) location \'/tmp/external/ext100\'"); verifyQuery(stmt, "ext100", 5); verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "hbase", true); stmt.execute("use default"); stmt.execute("use EXTDB"); stmt.execute("create table ext101 (s string) location \'/tmp/external/ext101\'"); verifyQuery(stmt, "ext101", 5); verifyOnAllSubDirs("/tmp/external/ext101", FsAction.ALL, "hbase", true); // Test table case insensitivity stmt.execute("grant all on table exT100 to role tab_role"); verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "flume", true); stmt.execute("use default"); //TODO: SENTRY-795: HDFS permissions do not sync when Sentry restarts in HA mode. if (!testSentryHA) { long beforeStop = System.currentTimeMillis(); sentryServer.stopAll(); long timeTakenForStopMs = System.currentTimeMillis() - beforeStop; LOGGER.info("Time taken for Sentry server stop: " + timeTakenForStopMs); // Verify that Sentry permission are still enforced for the "stale" period only if stop did not take too long if (timeTakenForStopMs < STALE_THRESHOLD) { verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); Thread.sleep((STALE_THRESHOLD - timeTakenForStopMs)); } else { LOGGER.warn("Sentry server stop took too long"); } // Verify that Sentry permission are NOT enforced AFTER "stale" period verifyOnAllSubDirs("/user/hive/warehouse/p3", null, "hbase", false); sentryServer.startAll(); } // Verify that After Sentry restart permissions are re-enforced verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); // Create new table and verify everything is fine after restart... stmt.execute("create table p2 (s string) partitioned by (month int, day int)"); stmt.execute("alter table p2 add partition (month=1, day=1)"); stmt.execute("alter table p2 add partition (month=1, day=2)"); stmt.execute("alter table p2 add partition (month=2, day=1)"); stmt.execute("alter table p2 add partition (month=2, day=2)"); verifyOnAllSubDirs("/user/hive/warehouse/p2", null, "hbase", false); stmt.execute("grant select on table p2 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); stmt.execute("grant select on table p2 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); // Create external table writeToPath("/tmp/external/ext1", 5, "foo", "bar"); stmt.execute("create table ext1 (s string) location \'/tmp/external/ext1\'"); verifyQuery(stmt, "ext1", 5); // Ensure existing group permissions are never returned.. verifyOnAllSubDirs("/tmp/external/ext1", null, "bar", false); verifyOnAllSubDirs("/tmp/external/ext1", null, "hbase", false); stmt.execute("grant all on table ext1 to role p1_admin"); verifyOnAllSubDirs("/tmp/external/ext1", FsAction.ALL, "hbase", true); stmt.execute("revoke select on table ext1 from role p1_admin"); verifyOnAllSubDirs("/tmp/external/ext1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify database operations works correctly stmt.execute("create database db1"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false); stmt.execute("create table db1.tbl1 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", null, "hbase", false); stmt.execute("create table db1.tbl2 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false); // Verify default db grants do not affect other dbs stmt.execute("grant all on database default to role p1_admin"); verifyOnPath("/user/hive/warehouse", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false); // Verify table rename works stmt.execute("create table q1 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/q1", FsAction.ALL, "hbase", true); stmt.execute("alter table q1 rename to q2"); verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); // Verify table GRANTS do not trump db GRANTS stmt.execute("grant select on table q2 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); stmt.execute("create table q3 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/q3", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); // Verify db privileges are propagated to tables stmt.execute("grant select on database db1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); // Verify default db revokes do not affect other dbs stmt.execute("revoke all on database default from role p1_admin"); verifyOnPath("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); stmt.execute("use db1"); stmt.execute("grant all on table tbl1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); // Verify recursive revoke stmt.execute("revoke select on database db1 from role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.WRITE_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false); // Verify cleanup.. stmt.execute("drop table tbl1"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl1"))); stmt.execute("drop table tbl2"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl2"))); stmt.execute("use default"); stmt.execute("drop database db1"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db"))); // START : Verify external table set location.. writeToPath("/tmp/external/tables/ext2_before/i=1", 5, "foo", "bar"); writeToPath("/tmp/external/tables/ext2_before/i=2", 5, "foo", "bar"); stmt.execute( "create external table ext2 (s string) partitioned by (i int) location \'/tmp/external/tables/ext2_before\'"); stmt.execute("alter table ext2 add partition (i=1)"); stmt.execute("alter table ext2 add partition (i=2)"); verifyQuery(stmt, "ext2", 10); verifyOnAllSubDirs("/tmp/external/tables/ext2_before", null, "hbase", false); stmt.execute("grant all on table ext2 to role p1_admin"); verifyOnPath("/tmp/external/tables/ext2_before", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true); writeToPath("/tmp/external/tables/ext2_after/i=1", 6, "foo", "bar"); writeToPath("/tmp/external/tables/ext2_after/i=2", 6, "foo", "bar"); stmt.execute("alter table ext2 set location \'hdfs:///tmp/external/tables/ext2_after\'"); // Even though table location is altered, partition location is still old (still 10 rows) verifyQuery(stmt, "ext2", 10); // You have to explicitly alter partition location.. verifyOnPath("/tmp/external/tables/ext2_before", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true); stmt.execute( "alter table ext2 partition (i=1) set location \'hdfs:///tmp/external/tables/ext2_after/i=1\'"); stmt.execute( "alter table ext2 partition (i=2) set location \'hdfs:///tmp/external/tables/ext2_after/i=2\'"); // Now that partition location is altered, it picks up new data (12 rows instead of 10) verifyQuery(stmt, "ext2", 12); verifyOnPath("/tmp/external/tables/ext2_before", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=1", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=2", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=2", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true); // END : Verify external table set location.. // Restart HDFS to verify if things are fine after re-start.. // TODO : this is currently commented out since miniDFS.restartNameNode() does // not work corectly on the version of hadoop sentry depends on // This has been verified to work on a real cluster. // Once miniDFS is fixed, this should be uncommented.. // miniDFS.shutdown(); // miniDFS.restartNameNode(true); // miniDFS.waitActive(); // verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); // verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); stmt.close(); conn.close(); }
From source file:org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegration.java
License:Apache License
private Map<String, FsAction> getAcls(Path path) throws Exception { AclStatus aclStatus = miniDFS.getFileSystem().getAclStatus(path); Map<String, FsAction> acls = new HashMap<String, FsAction>(); for (AclEntry ent : aclStatus.getEntries()) { if (ent.getType().equals(AclEntryType.GROUP)) { // In case of duplicate acl exist, exception should be thrown. if (acls.containsKey(ent.getName())) { throw new SentryAlreadyExistsException("The acl " + ent.getName()); } else { acls.put(ent.getName(), ent.getPermission()); }/*from w w w . j a v a2 s.c o m*/ } } return acls; }
From source file:org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase.java
License:Apache License
protected Map<String, FsAction> getAcls(AclEntryType type, Path path) throws Exception { AclStatus aclStatus = miniDFS.getFileSystem().getAclStatus(path); Map<String, FsAction> acls = new HashMap<String, FsAction>(); for (AclEntry ent : aclStatus.getEntries()) { if (ent.getType().equals(type)) { // In case of duplicate acl exist, exception should be thrown. if (acls.containsKey(ent.getName())) { throw new SentryAlreadyExistsException("The acl " + ent.getName() + " already exists.\n"); } else { acls.put(ent.getName(), ent.getPermission()); }// w ww. j a v a 2 s. c o m } } return acls; }
From source file:org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationEnd2End.java
License:Apache License
@Test public void testEnd2End() throws Throwable { tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { "db1" }; roles = new String[] { "admin_role", "db_role", "tab_role", "p1_admin" }; admin = "hive"; Connection conn;/*from w w w .jav a 2 s . co m*/ Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant role admin_role to group hive"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("create table p1 (s string) partitioned by (month int, day " + "int)"); stmt.execute("alter table p1 add partition (month=1, day=1)"); stmt.execute("alter table p1 add partition (month=1, day=2)"); stmt.execute("alter table p1 add partition (month=2, day=1)"); stmt.execute("alter table p1 add partition (month=2, day=2)"); // db privileges stmt.execute("create database db5"); stmt.execute("create role db_role"); stmt.execute("create role tab_role"); stmt.execute("grant role db_role to group hbase"); stmt.execute("grant role tab_role to group flume"); stmt.execute("create table db5.p2(id int)"); stmt.execute("create role p1_admin"); stmt.execute("grant role p1_admin to group hbase"); // Verify default db is inaccessible initially verifyGroupPermOnAllSubDirs("/user/hive/warehouse", null, "hbase", false); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); stmt.execute("grant all on database db5 to role db_role"); stmt.execute("use db5"); stmt.execute("grant all on table p2 to role tab_role"); stmt.execute("use default"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db5.db", FsAction.ALL, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "flume", true); verifyGroupPermOnPath("/user/hive/warehouse/db5.db", FsAction.ALL, "flume", false); loadData(stmt); verifyHDFSandMR(stmt); // Verify default db is STILL inaccessible after grants but tables are fine verifyGroupPermOnPath("/user/hive/warehouse", null, "hbase", false); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); adminUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { // Simulate hdfs dfs -setfacl -m <aclantry> <path> AclStatus existing = miniDFS.getFileSystem().getAclStatus(new Path("/user/hive/warehouse/p1")); ArrayList<AclEntry> newEntries = new ArrayList<AclEntry>(existing.getEntries()); newEntries.add(AclEntry.parseAclEntry("user::---", true)); newEntries.add(AclEntry.parseAclEntry("group:bla:rwx", true)); newEntries.add(AclEntry.parseAclEntry("other::---", true)); miniDFS.getFileSystem().setAcl(new Path("/user/hive/warehouse/p1"), newEntries); return null; } }); stmt.execute("revoke select on table p1 from role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); // Verify default db grants work stmt.execute("grant select on database default to role p1_admin"); verifyGroupPermOnPath("/user/hive/warehouse", FsAction.READ_EXECUTE, "hbase", true); // Verify default db grants are propagated to the tables verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); // Verify default db revokes work stmt.execute("revoke select on database default from role p1_admin"); verifyGroupPermOnPath("/user/hive/warehouse", null, "hbase", false); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); stmt.execute("grant all on table p1 to role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", FsAction.ALL, "hbase", true); stmt.execute("revoke select on table p1 from role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify table rename works when locations are also changed stmt.execute("alter table p1 rename to p3"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); //This is true as parent hive object's (p3) ACLS are used. verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3/month=1/day=1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify when oldName == newName and oldPath != newPath stmt.execute("alter table p3 partition (month=1, day=1) rename to partition (month=1, day=3)"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3/month=1/day=3", FsAction.WRITE_EXECUTE, "hbase", true); // Test DB case insensitivity stmt.execute("create database extdb"); stmt.execute("grant all on database ExtDb to role p1_admin"); writeToPath("/tmp/external/ext100", 5, "foo", "bar"); writeToPath("/tmp/external/ext101", 5, "foo", "bar"); stmt.execute("use extdb"); stmt.execute("create table ext100 (s string) location \'/tmp/external/ext100\'"); verifyQuery(stmt, "ext100", 5); verifyGroupPermOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "hbase", true); stmt.execute("use default"); stmt.execute("use EXTDB"); stmt.execute("create table ext101 (s string) location \'/tmp/external/ext101\'"); verifyQuery(stmt, "ext101", 5); verifyGroupPermOnAllSubDirs("/tmp/external/ext101", FsAction.ALL, "hbase", true); // Test table case insensitivity stmt.execute("grant all on table exT100 to role tab_role"); verifyGroupPermOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "flume", true); stmt.execute("drop table ext100"); stmt.execute("drop table ext101"); stmt.execute("use default"); stmt.execute("drop database extdb"); if (!testSentryHA) { long beforeStop = System.currentTimeMillis(); sentryServer.stopAll(); long timeTakenForStopMs = System.currentTimeMillis() - beforeStop; LOGGER.info("Time taken for Sentry server stop: " + timeTakenForStopMs); // Verify that Sentry permission are still enforced for the "stale" period only if stop did not take too long if (timeTakenForStopMs < STALE_THRESHOLD) { verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); Thread.sleep((STALE_THRESHOLD - timeTakenForStopMs)); } else { LOGGER.warn("Sentry server stop took too long"); } // Verify that Sentry permission are NOT enforced AFTER "stale" period verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3", null, "hbase", false); sentryServer.startAll(); } // Verify that After Sentry restart permissions are re-enforced verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); // Create new table and verify everything is fine after restart... stmt.execute("create table p2 (s string) partitioned by (month int, day int)"); stmt.execute("alter table p2 add partition (month=1, day=1)"); stmt.execute("alter table p2 add partition (month=1, day=2)"); stmt.execute("alter table p2 add partition (month=2, day=1)"); stmt.execute("alter table p2 add partition (month=2, day=2)"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p2", null, "hbase", false); stmt.execute("grant select on table p2 to role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); stmt.execute("grant select on table p2 to role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); // Create external table writeToPath("/tmp/external/ext1", 5, "foo", "bar"); stmt.execute("create table ext1 (s string) location \'/tmp/external/ext1\'"); verifyQuery(stmt, "ext1", 5); // Ensure existing group permissions are never returned.. verifyGroupPermOnAllSubDirs("/tmp/external/ext1", null, "bar", false); verifyGroupPermOnAllSubDirs("/tmp/external/ext1", null, "hbase", false); stmt.execute("grant all on table ext1 to role p1_admin"); verifyGroupPermOnAllSubDirs("/tmp/external/ext1", FsAction.ALL, "hbase", true); stmt.execute("revoke select on table ext1 from role p1_admin"); verifyGroupPermOnAllSubDirs("/tmp/external/ext1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify database operations works correctly stmt.execute("create database db1"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false); stmt.execute("create table db1.tbl1 (s string)"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", null, "hbase", false); stmt.execute("create table db1.tbl2 (s string)"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false); // Verify default db grants do not affect other dbs stmt.execute("grant all on database default to role p1_admin"); verifyGroupPermOnPath("/user/hive/warehouse", FsAction.ALL, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false); // Verify table rename works stmt.execute("create table q1 (s string)"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/q1", FsAction.ALL, "hbase", true); stmt.execute("alter table q1 rename to q2"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); // Verify table GRANTS do not trump db GRANTS stmt.execute("grant select on table q2 to role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); stmt.execute("create table q3 (s string)"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/q3", FsAction.ALL, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); // Verify db privileges are propagated to tables stmt.execute("grant select on database db1 to role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); // Verify default db revokes do not affect other dbs stmt.execute("revoke all on database default from role p1_admin"); verifyGroupPermOnPath("/user/hive/warehouse", null, "hbase", false); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); stmt.execute("use db1"); stmt.execute("grant all on table tbl1 to role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.ALL, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); // Verify recursive revoke stmt.execute("revoke select on database db1 from role p1_admin"); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.WRITE_EXECUTE, "hbase", true); verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false); // Verify cleanup.. stmt.execute("drop table tbl1"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl1"))); stmt.execute("drop table tbl2"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl2"))); stmt.execute("use default"); stmt.execute("drop database db1"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db"))); // START : Verify external table set location.. writeToPath("/tmp/external/tables/ext2_before/i=1", 5, "foo", "bar"); writeToPath("/tmp/external/tables/ext2_before/i=2", 5, "foo", "bar"); stmt.execute( "create external table ext2 (s string) partitioned by (i int) location \'/tmp/external/tables/ext2_before\'"); stmt.execute("alter table ext2 add partition (i=1)"); stmt.execute("alter table ext2 add partition (i=2)"); verifyQuery(stmt, "ext2", 10); verifyGroupPermOnAllSubDirs("/tmp/external/tables/ext2_before", null, "hbase", false); stmt.execute("grant all on table ext2 to role p1_admin"); verifyGroupPermOnPath("/tmp/external/tables/ext2_before", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true); writeToPath("/tmp/external/tables/ext2_after/i=1", 6, "foo", "bar"); writeToPath("/tmp/external/tables/ext2_after/i=2", 6, "foo", "bar"); stmt.execute("alter table ext2 set location \'hdfs:///tmp/external/tables/ext2_after\'"); // Even though table location is altered, partition location is still old (still 10 rows) verifyQuery(stmt, "ext2", 10); // You have to explicitly alter partition location.. verifyGroupPermOnPath("/tmp/external/tables/ext2_before", null, "hbase", false); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true); stmt.execute( "alter table ext2 partition (i=1) set location \'hdfs:///tmp/external/tables/ext2_after/i=1\'"); stmt.execute( "alter table ext2 partition (i=2) set location \'hdfs:///tmp/external/tables/ext2_after/i=2\'"); // Now that partition location is altered, it picks up new data (12 rows instead of 10) verifyQuery(stmt, "ext2", 12); verifyGroupPermOnPath("/tmp/external/tables/ext2_before", null, "hbase", false); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=1", null, "hbase", false); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=2", null, "hbase", false); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", null, "hbase", false); verifyGroupPermOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", null, "hbase", false); verifyGroupPermOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=2", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true); // END : Verify external table set location.. //Create a new table partition on the existing partition stmt.execute("create table tmp (s string) partitioned by (i int)"); stmt.execute("alter table tmp add partition (i=1)"); stmt.execute("alter table tmp partition (i=1) set location \'hdfs:///tmp/external/tables/ext2_after/i=1\'"); stmt.execute("grant all on table tmp to role tab_role"); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "flume", true); //Alter table rename of external table => oldName != newName, oldPath == newPath stmt.execute("alter table ext2 rename to ext3"); //Verify all original paths still have the privileges verifyGroupPermOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "flume", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=2", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyGroupPermOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true); // Restart HDFS to verify if things are fine after re-start.. // TODO : this is currently commented out since miniDFS.restartNameNode() does // not work corectly on the version of hadoop sentry depends on // This has been verified to work on a real cluster. // Once miniDFS is fixed, this should be uncommented.. // miniDFS.shutdown(); // miniDFS.restartNameNode(true); // miniDFS.waitActive(); // verifyGroupPermOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); // verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); stmt.close(); conn.close(); }
From source file:org.trustedanalytics.auth.gateway.hdfs.integration.HdfsGatewayIntegrationTest.java
License:Apache License
private void checkIfDirectoryExistsWithACL(Path path, String owner, String[] privilegedUsers, String[] privilegedGroups) throws IOException { AclStatus s = fileSystem.getAclStatus(path); assertThat(fileSystem.exists(path), equalTo(true)); assertThat(s.getOwner(), equalTo(owner)); List<String> usersWithAcl = s.getEntries().stream() .filter(entry -> entry.getType().equals(AclEntryType.USER)) .filter(entry -> entry.getScope().equals(AclEntryScope.ACCESS)).map(AclEntry::getName) .collect(toList());//from www . j av a 2 s.c om List<String> groupsWithAcl = s.getEntries().stream() .filter(entry -> entry.getType().equals(AclEntryType.GROUP)) .filter(entry -> entry.getScope().equals(AclEntryScope.ACCESS)).map(AclEntry::getName) .collect(toList()); List<String> usersWithDefaultAcl = s.getEntries().stream() .filter(entry -> entry.getType().equals(AclEntryType.USER)) .filter(entry -> entry.getScope().equals(AclEntryScope.ACCESS)).map(AclEntry::getName) .collect(toList()); List<String> groupsWithDefaultAcl = s.getEntries().stream() .filter(entry -> entry.getType().equals(AclEntryType.GROUP)) .filter(entry -> entry.getScope().equals(AclEntryScope.ACCESS)).map(AclEntry::getName) .collect(toList()); assertThat(usersWithAcl.size(), equalTo(privilegedUsers.length)); assertThat(usersWithAcl, containsInAnyOrder(privilegedUsers)); assertThat(groupsWithAcl.size(), equalTo(privilegedGroups.length)); assertThat(groupsWithAcl, containsInAnyOrder(privilegedGroups)); assertThat(usersWithDefaultAcl.size(), equalTo(privilegedUsers.length)); assertThat(usersWithDefaultAcl, containsInAnyOrder(privilegedUsers)); assertThat(groupsWithDefaultAcl.size(), equalTo(privilegedGroups.length)); assertThat(groupsWithDefaultAcl, containsInAnyOrder(privilegedGroups)); }