Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.sentry.tests.e2e.hdfs; import static org.apache.sentry.hdfs.ServiceConstants.ServerConfig.SENTRY_HDFS_INTEGRATION_PATH_PREFIXES; import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.ServerSocket; import java.net.URL; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.StringTokenizer; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.base.Preconditions; import org.apache.sentry.core.common.exception.SentryAlreadyExistsException; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.*; import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.security.UserGroupInformation; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.binding.hive.v2.SentryHiveAuthorizationTaskFactoryImplV2; import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.SentryINodeAttributesProvider; import org.apache.sentry.provider.db.SimpleDBProviderBackend; import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider; import org.apache.sentry.provider.file.PolicyFile; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.apache.sentry.tests.e2e.hive.StaticUserGroup; import org.apache.sentry.tests.e2e.hive.fs.MiniDFS; import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; import org.apache.sentry.tests.e2e.hive.hiveserver.InternalHiveServer; import org.apache.sentry.tests.e2e.hive.hiveserver.InternalMetastoreServer; import org.apache.sentry.tests.e2e.minisentry.SentrySrv; import org.apache.sentry.tests.e2e.minisentry.SentrySrvFactory; import org.fest.reflect.core.Reflection; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Maps; import com.google.common.io.Files; import com.google.common.io.Resources; import org.apache.hadoop.hive.metastore.api.Table; public class TestHDFSIntegration { private static final Logger LOGGER = LoggerFactory.getLogger(TestHDFSIntegration.class); public static class WordCountMapper extends MapReduceBase implements Mapper<LongWritable, Text, String, Long> { public void map(LongWritable key, Text value, OutputCollector<String, Long> output, Reporter reporter) throws IOException { StringTokenizer st = new StringTokenizer(value.toString()); while (st.hasMoreTokens()) { output.collect(st.nextToken(), 1L); } } } public static class SumReducer extends MapReduceBase implements Reducer<Text, Long, Text, Long> { public void reduce(Text key, Iterator<Long> values, OutputCollector<Text, Long> output, Reporter reporter) throws IOException { long sum = 0; while (values.hasNext()) { sum += values.next(); } output.collect(key, sum); } } private static final String MANAGED_PREFIXES = "/user/hive/warehouse,/tmp/external"; private static final int NUM_RETRIES = 10; private static final int RETRY_WAIT = 1000; private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry"; private static MiniDFSCluster miniDFS; private static InternalHiveServer hiveServer2; private static InternalMetastoreServer metastore; private static HiveMetaStoreClient hmsClient; private static int sentryPort = -1; protected static SentrySrv sentryServer; protected static boolean testSentryHA = false; private static final long STALE_THRESHOLD = 5000; // It is the interval in milliseconds that hdfs uses to get acl from sentry. Default is 500, but // we want it to be low in our tests so that changes reflect soon private static final long CACHE_REFRESH = 100; // It is Used to wait before verifying result in test. // We want to make sure the cache is updated in our tests so that changes reflect soon. The unit is milliseconds // It takes at most (ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT + ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT) // for sentry to get the path configured in Hive, adding ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT to be safe. // And then it takes at most CACHE_REFRESH for HDFS to get this from sentry, adding CACHE_REFRESH to be sure protected static final long WAIT_BEFORE_TESTVERIFY = ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT + ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT * 2 + CACHE_REFRESH * 2; // Time to wait before running next tests. The unit is milliseconds. // Deleting HDFS may finish, but HDFS may not be ready for creating the same file again. // We need to to make sure that creating the same file in the next test will succeed // If we don't wait, next test may get exception similar to // "org.apache.hadoop.security.AccessControlException Permission denied: user=hive, access=EXECUTE, // inode="/tmp/external/p1":hdfs:hdfs:drwxrwx---" protected static final long WAIT_BEFORE_NEXTTEST = 50; private static String fsURI; private static int hmsPort; private static File baseDir; private static File policyFileLocation; private static UserGroupInformation adminUgi; private static UserGroupInformation hiveUgi; // Variables which are used for cleanup after test // Please set these values in each test private Path tmpHDFSDir; private String[] dbNames; private String[] roles; private String admin; private static Configuration hadoopConf; protected static File assertCreateDir(File dir) { if (!dir.isDirectory()) { Assert.assertTrue("Failed creating " + dir, dir.mkdirs()); } return dir; } private static int findPort() throws IOException { ServerSocket socket = new ServerSocket(0); int port = socket.getLocalPort(); socket.close(); return port; } @BeforeClass public static void setup() throws Exception { Class.forName("org.apache.hive.jdbc.HiveDriver"); baseDir = Files.createTempDir(); policyFileLocation = new File(baseDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME); PolicyFile policyFile = PolicyFile.setAdminOnServer1("hive") .setUserGroupMapping(StaticUserGroup.getStaticMapping()); policyFile.write(policyFileLocation); adminUgi = UserGroupInformation.createUserForTesting(System.getProperty("user.name"), new String[] { "supergroup" }); hiveUgi = UserGroupInformation.createUserForTesting("hive", new String[] { "hive" }); // Create SentryService and its internal objects. // Set Sentry port createSentry(); // Create hive-site.xml that contains the metastore uri // it is used by HMSFollower configureHiveAndMetastoreForSentry(); // Start SentryService after Hive configuration hive-site.xml is available // So HMSFollower can contact metastore using its URI startSentry(); // Start HDFS and MR with Sentry Port. Set fsURI startDFSandYARN(); // Configure Hive and Metastore with Sentry Port and fsURI // Read src/test/resources/sentry-site.xml. // Create hive-site.xml and sentry-site.xml used by Hive. HiveConf hiveConf = configureHiveAndMetastore(); // Start Hive and Metastore after SentryService is started startHiveAndMetastore(hiveConf); } private static void startHiveAndMetastore(final HiveConf hiveConf) throws IOException, InterruptedException { startHiveAndMetastore(hiveConf, NUM_RETRIES); } private static void startHiveAndMetastore(final HiveConf hiveConf, final int retries) throws IOException, InterruptedException { hiveUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { metastore = new InternalMetastoreServer(hiveConf); new Thread() { @Override public void run() { try { metastore.start(); while (true) { Thread.sleep(1000L); } } catch (Exception e) { LOGGER.info("Could not start Hive Server"); } } }.start(); hmsClient = new HiveMetaStoreClient(hiveConf); startHiveServer2(retries, hiveConf); return null; } }); } private static void startHiveServer2(final int retries, HiveConf hiveConf) throws IOException, InterruptedException, SQLException { Connection conn = null; Thread th = null; final AtomicBoolean keepRunning = new AtomicBoolean(true); try { hiveServer2 = new InternalHiveServer(hiveConf); th = new Thread() { @Override public void run() { try { hiveServer2.start(); while (keepRunning.get()) { Thread.sleep(1000L); } } catch (Exception e) { LOGGER.info("Could not start Hive Server"); } } }; th.start(); Thread.sleep(RETRY_WAIT * 5); conn = hiveServer2.createConnection("hive", "hive"); } catch (Exception ex) { if (retries > 0) { try { keepRunning.set(false); hiveServer2.shutdown(); } catch (Exception e) { // Ignore } LOGGER.info("Re-starting Hive Server2 !!"); startHiveServer2(retries - 1, hiveConf); } } if (conn != null) { conn.close(); } } private static void startDFSandYARN() throws IOException, InterruptedException { adminUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, "target/test/data"); hadoopConf = new HdfsConfiguration(); hadoopConf.set(DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY, SentryINodeAttributesProvider.class.getName()); hadoopConf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); hadoopConf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); File dfsDir = assertCreateDir(new File(baseDir, "dfs")); hadoopConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dfsDir.getPath()); hadoopConf.set("hadoop.security.group.mapping", MiniDFS.PseudoGroupMappingService.class.getName()); Configuration.addDefaultResource("test.xml"); hadoopConf.set("sentry.authorization-provider.hdfs-path-prefixes", MANAGED_PREFIXES); hadoopConf.set("sentry.authorization-provider.cache-refresh-retry-wait.ms", "5000"); hadoopConf.set("sentry.authorization-provider.cache-refresh-interval.ms", String.valueOf(CACHE_REFRESH)); hadoopConf.set("sentry.authorization-provider.cache-stale-threshold.ms", String.valueOf(STALE_THRESHOLD)); hadoopConf.set("sentry.hdfs.service.security.mode", "none"); hadoopConf.set("sentry.hdfs.service.client.server.rpc-addresses", "localhost"); hadoopConf.set("sentry.hdfs.service.client.server.rpc-port", String.valueOf(sentryPort)); EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); miniDFS = new MiniDFSCluster.Builder(hadoopConf).build(); Path tmpPath = new Path("/tmp"); Path hivePath = new Path("/user/hive"); Path warehousePath = new Path(hivePath, "warehouse"); miniDFS.getFileSystem().mkdirs(warehousePath); boolean directory = miniDFS.getFileSystem().isDirectory(warehousePath); LOGGER.info("\n\n Is dir :" + directory + "\n\n"); LOGGER.info("\n\n DefaultFS :" + miniDFS.getFileSystem().getUri() + "\n\n"); fsURI = miniDFS.getFileSystem().getUri().toString(); hadoopConf.set("fs.defaultFS", fsURI); // Create Yarn cluster // miniMR = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); miniDFS.getFileSystem().mkdirs(tmpPath); miniDFS.getFileSystem().setPermission(tmpPath, FsPermission.valueOf("drwxrwxrwx")); miniDFS.getFileSystem().setOwner(hivePath, "hive", "hive"); miniDFS.getFileSystem().setOwner(warehousePath, "hive", "hive"); LOGGER.info("\n\n Owner :" + miniDFS.getFileSystem().getFileStatus(warehousePath).getOwner() + ", " + miniDFS.getFileSystem().getFileStatus(warehousePath).getGroup() + "\n\n"); LOGGER.info("\n\n Owner tmp :" + miniDFS.getFileSystem().getFileStatus(tmpPath).getOwner() + ", " + miniDFS.getFileSystem().getFileStatus(tmpPath).getGroup() + ", " + miniDFS.getFileSystem().getFileStatus(tmpPath).getPermission() + ", " + "\n\n"); int dfsSafeCheckRetry = 30; boolean hasStarted = false; for (int i = dfsSafeCheckRetry; i > 0; i--) { if (!miniDFS.getFileSystem().isInSafeMode()) { hasStarted = true; LOGGER.info("HDFS safemode check num times : " + (31 - i)); break; } } if (!hasStarted) { throw new RuntimeException("HDFS hasnt exited safe mode yet.."); } return null; } }); } private static void createSentry() throws Exception { try { hiveUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { Configuration sentryConf = new Configuration(false); sentryConf.set(SENTRY_HDFS_INTEGRATION_PATH_PREFIXES, MANAGED_PREFIXES); Map<String, String> properties = Maps.newHashMap(); properties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND, SimpleDBProviderBackend.class.getName()); properties.put(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname, SentryHiveAuthorizationTaskFactoryImplV2.class.getName()); properties.put(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS.varname, "2"); properties.put("hive.exec.local.scratchdir", Files.createTempDir().getAbsolutePath()); properties.put(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE); // properties.put("sentry.service.server.compact.transport", "true"); properties.put("sentry.hive.testing.mode", "true"); properties.put("sentry.service.reporting", "JMX"); properties.put(ServerConfig.ADMIN_GROUPS, "hive,admin"); properties.put(ServerConfig.RPC_ADDRESS, "localhost"); properties.put(ServerConfig.RPC_PORT, String.valueOf(sentryPort > 0 ? sentryPort : 0)); properties.put(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false"); properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING); properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath()); properties.put(ServerConfig.SENTRY_STORE_JDBC_URL, "jdbc:derby:;databaseName=" + baseDir.getPath() + "/sentrystore_db;create=true"); properties.put(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy"); properties.put("sentry.service.processor.factories", "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); properties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin"); properties.put(ServerConfig.RPC_MIN_THREADS, "3"); for (Map.Entry<String, String> entry : properties.entrySet()) { sentryConf.set(entry.getKey(), entry.getValue()); } sentryServer = SentrySrvFactory.create(SentrySrvFactory.SentrySrvType.INTERNAL_SERVER, sentryConf, testSentryHA ? 2 : 1); sentryPort = sentryServer.get(0).getAddress().getPort(); sentryServer.startAll(); LOGGER.info("\n\n Sentry service started \n\n"); return null; } }); } catch (Exception e) { //An exception happening in above block will result in a wrapped UndeclaredThrowableException. throw new Exception(e.getCause()); } } private static void startSentry() throws Exception { try { hiveUgi.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { sentryServer.startAll(); LOGGER.info("\n\n Sentry service started \n\n"); return null; } }); } catch (Exception ex) { //An exception happening in above block will result in a wrapped UndeclaredThrowableException. throw new Exception(ex.getCause()); } } private static HiveConf configureHiveAndMetastoreForSentry() throws IOException, InterruptedException { final HiveConf hiveConfiguration = new HiveConf(); hiveUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { HiveConf hiveConf = hiveConfiguration; hmsPort = findPort(); LOGGER.info("\n\n HMS port : " + hmsPort + "\n\n"); // Sets hive.metastore.authorization.storage.checks to true, so that // disallow the operations such as drop-partition if the user in question // doesn't have permissions to delete the corresponding directory // on the storage. hiveConf.set("hive.metastore.authorization.storage.checks", "true"); hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort); hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user (beeline) skip meta store check File confDir = assertCreateDir(new File(baseDir, "etc")); File hiveSite = new File(confDir, "hive-site.xml"); hiveConf.set("hive.server2.enable.doAs", "false"); OutputStream out = new FileOutputStream(hiveSite); hiveConf.writeXml(out); out.close(); Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class) .set(hiveSite.toURI().toURL()); return null; } }); return hiveConfiguration; } private static HiveConf configureHiveAndMetastore() throws IOException, InterruptedException { final HiveConf hiveConfiguration = new HiveConf(); hiveUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { HiveConf hiveConf = hiveConfiguration; hiveConf.set("sentry.metastore.plugins", "org.apache.sentry.hdfs.MetastorePlugin"); hiveConf.set("sentry.service.client.server.rpc-addresses", "localhost"); hiveConf.set("sentry.hdfs.service.client.server.rpc-addresses", "localhost"); hiveConf.set("sentry.hdfs.service.client.server.rpc-port", String.valueOf(sentryPort)); hiveConf.set("sentry.service.client.server.rpc-port", String.valueOf(sentryPort)); // hiveConf.set("sentry.service.server.compact.transport", "true"); // hiveConf.set("sentry.service.client.compact.transport", "true"); hiveConf.set("sentry.service.security.mode", "none"); hiveConf.set("sentry.hdfs.service.security.mode", "none"); hiveConf.set("sentry.hdfs.init.update.retry.delay.ms", "500"); hiveConf.set("sentry.hive.provider.backend", "org.apache.sentry.provider.db.SimpleDBProviderBackend"); hiveConf.set("sentry.provider", LocalGroupResourceAuthorizationProvider.class.getName()); hiveConf.set("sentry.hive.provider", LocalGroupResourceAuthorizationProvider.class.getName()); hiveConf.set("sentry.hive.provider.resource", policyFileLocation.getPath()); hiveConf.set("sentry.hive.testing.mode", "true"); hiveConf.set("sentry.hive.server", "server1"); hiveConf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING); hiveConf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath()); hiveConf.set("fs.defaultFS", fsURI); hiveConf.set("fs.default.name", fsURI); hiveConf.set("hive.metastore.execute.setugi", "true"); hiveConf.set("hive.metastore.warehouse.dir", "hdfs:///user/hive/warehouse"); hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + "/metastore_db;create=true"); hiveConf.set("javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver"); hiveConf.set("javax.jdo.option.ConnectionUserName", "hive"); hiveConf.set("javax.jdo.option.ConnectionPassword", "hive"); hiveConf.set("datanucleus.autoCreateSchema", "true"); hiveConf.set("datanucleus.fixedDatastore", "false"); hiveConf.set("datanucleus.autoStartMechanism", "SchemaTable"); hmsPort = findPort(); LOGGER.info("\n\n HMS port : " + hmsPort + "\n\n"); // Sets hive.metastore.authorization.storage.checks to true, so that // disallow the operations such as drop-partition if the user in question // doesn't have permissions to delete the corresponding directory // on the storage. hiveConf.set("hive.metastore.authorization.storage.checks", "true"); hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort); hiveConf.set("hive.metastore.pre.event.listeners", "org.apache.sentry.binding.metastore.MetastoreAuthzBinding"); hiveConf.set("hive.metastore.event.listeners", "org.apache.sentry.binding.metastore.SentryMetastorePostEventListenerNotificationLog"); hiveConf.set("hcatalog.message.factory.impl.json", "org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory"); hiveConf.set("hive.security.authorization.task.factory", "org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl"); hiveConf.set("hive.server2.session.hook", "org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook"); hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user (beeline) skip meta store check HiveAuthzConf authzConf = new HiveAuthzConf(Resources.getResource("sentry-site.xml")); authzConf.addResource(hiveConf); File confDir = assertCreateDir(new File(baseDir, "etc")); File accessSite = new File(confDir, HiveAuthzConf.AUTHZ_SITE_FILE); OutputStream out = new FileOutputStream(accessSite); authzConf.set("fs.defaultFS", fsURI); authzConf.writeXml(out); out.close(); hiveConf.set("hive.sentry.conf.url", accessSite.getPath()); LOGGER.info("Sentry client file : " + accessSite.getPath()); File hiveSite = new File(confDir, "hive-site.xml"); hiveConf.set("hive.server2.enable.doAs", "false"); hiveConf.set(HiveAuthzConf.HIVE_SENTRY_CONF_URL, accessSite.toURI().toURL().toExternalForm()); out = new FileOutputStream(hiveSite); hiveConf.writeXml(out); out.close(); Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class) .set(hiveSite.toURI().toURL()); return null; } }); return hiveConfiguration; } @After public void cleanAfterTest() throws Exception { //Clean up database Connection conn; Statement stmt; Preconditions.checkArgument(admin != null && dbNames != null && roles != null && tmpHDFSDir != null, "Test case did not set some of these values required for clean up: admin, dbNames, roles, tmpHDFSDir"); conn = hiveServer2.createConnection(admin, admin); stmt = conn.createStatement(); for (String dbName : dbNames) { stmt.execute("drop database if exists " + dbName + " cascade"); } stmt.close(); conn.close(); //Clean up roles conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); for (String role : roles) { stmt.execute("drop role " + role); } stmt.close(); conn.close(); //Clean up hdfs directories miniDFS.getFileSystem().delete(tmpHDFSDir, true); tmpHDFSDir = null; dbNames = null; roles = null; admin = null; Thread.sleep(WAIT_BEFORE_NEXTTEST); // make sure the clean up is done before next test starts. otherwise, the next test may fail } @AfterClass public static void cleanUp() throws Exception { try { if (miniDFS != null) { miniDFS.shutdown(); } } finally { try { if (hiveServer2 != null) { hiveServer2.shutdown(); } } finally { try { if (metastore != null) { metastore.shutdown(); } } finally { sentryServer.close(); } } } } @Test public void testEnd2End() throws Throwable { tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { "db1" }; roles = new String[] { "admin_role", "db_role", "tab_role", "p1_admin" }; admin = "hive"; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant role admin_role to group hive"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("create table p1 (s string) partitioned by (month int, day int)"); stmt.execute("alter table p1 add partition (month=1, day=1)"); stmt.execute("alter table p1 add partition (month=1, day=2)"); stmt.execute("alter table p1 add partition (month=2, day=1)"); stmt.execute("alter table p1 add partition (month=2, day=2)"); // db privileges stmt.execute("create database db5"); stmt.execute("create role db_role"); stmt.execute("create role tab_role"); stmt.execute("grant role db_role to group hbase"); stmt.execute("grant role tab_role to group flume"); stmt.execute("create table db5.p2(id int)"); stmt.execute("create role p1_admin"); stmt.execute("grant role p1_admin to group hbase"); // Verify default db is inaccessible initially verifyOnAllSubDirs("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); stmt.execute("grant all on database db5 to role db_role"); stmt.execute("use db5"); stmt.execute("grant all on table p2 to role tab_role"); stmt.execute("use default"); verifyOnAllSubDirs("/user/hive/warehouse/db5.db", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "flume", true); verifyOnPath("/user/hive/warehouse/db5.db", FsAction.ALL, "flume", false); loadData(stmt); verifyHDFSandMR(stmt); // Verify default db is STILL inaccessible after grants but tables are fine verifyOnPath("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); adminUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { // Simulate hdfs dfs -setfacl -m <aclantry> <path> AclStatus existing = miniDFS.getFileSystem().getAclStatus(new Path("/user/hive/warehouse/p1")); ArrayList<AclEntry> newEntries = new ArrayList<AclEntry>(existing.getEntries()); newEntries.add(AclEntry.parseAclEntry("user::---", true)); newEntries.add(AclEntry.parseAclEntry("group:bla:rwx", true)); newEntries.add(AclEntry.parseAclEntry("other::---", true)); miniDFS.getFileSystem().setAcl(new Path("/user/hive/warehouse/p1"), newEntries); return null; } }); stmt.execute("revoke select on table p1 from role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); // Verify default db grants work stmt.execute("grant select on database default to role p1_admin"); verifyOnPath("/user/hive/warehouse", FsAction.READ_EXECUTE, "hbase", true); // Verify default db grants are propagated to the tables verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); // Verify default db revokes work stmt.execute("revoke select on database default from role p1_admin"); verifyOnPath("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); stmt.execute("grant all on table p1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.ALL, "hbase", true); stmt.execute("revoke select on table p1 from role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify table rename works stmt.execute("alter table p1 rename to p3"); verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); stmt.execute("alter table p3 partition (month=1, day=1) rename to partition (month=1, day=3)"); verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/p3/month=1/day=3", FsAction.WRITE_EXECUTE, "hbase", true); // Test DB case insensitivity stmt.execute("create database extdb"); stmt.execute("grant all on database ExtDb to role p1_admin"); writeToPath("/tmp/external/ext100", 5, "foo", "bar"); writeToPath("/tmp/external/ext101", 5, "foo", "bar"); stmt.execute("use extdb"); stmt.execute("create table ext100 (s string) location \'/tmp/external/ext100\'"); verifyQuery(stmt, "ext100", 5); verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "hbase", true); stmt.execute("use default"); stmt.execute("use EXTDB"); stmt.execute("create table ext101 (s string) location \'/tmp/external/ext101\'"); verifyQuery(stmt, "ext101", 5); verifyOnAllSubDirs("/tmp/external/ext101", FsAction.ALL, "hbase", true); // Test table case insensitivity stmt.execute("grant all on table exT100 to role tab_role"); verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "flume", true); stmt.execute("use default"); //TODO: SENTRY-795: HDFS permissions do not sync when Sentry restarts in HA mode. if (!testSentryHA) { long beforeStop = System.currentTimeMillis(); sentryServer.stopAll(); long timeTakenForStopMs = System.currentTimeMillis() - beforeStop; LOGGER.info("Time taken for Sentry server stop: " + timeTakenForStopMs); // Verify that Sentry permission are still enforced for the "stale" period only if stop did not take too long if (timeTakenForStopMs < STALE_THRESHOLD) { verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); Thread.sleep((STALE_THRESHOLD - timeTakenForStopMs)); } else { LOGGER.warn("Sentry server stop took too long"); } // Verify that Sentry permission are NOT enforced AFTER "stale" period verifyOnAllSubDirs("/user/hive/warehouse/p3", null, "hbase", false); sentryServer.startAll(); } // Verify that After Sentry restart permissions are re-enforced verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true); // Create new table and verify everything is fine after restart... stmt.execute("create table p2 (s string) partitioned by (month int, day int)"); stmt.execute("alter table p2 add partition (month=1, day=1)"); stmt.execute("alter table p2 add partition (month=1, day=2)"); stmt.execute("alter table p2 add partition (month=2, day=1)"); stmt.execute("alter table p2 add partition (month=2, day=2)"); verifyOnAllSubDirs("/user/hive/warehouse/p2", null, "hbase", false); stmt.execute("grant select on table p2 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); stmt.execute("grant select on table p2 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); // Create external table writeToPath("/tmp/external/ext1", 5, "foo", "bar"); stmt.execute("create table ext1 (s string) location \'/tmp/external/ext1\'"); verifyQuery(stmt, "ext1", 5); // Ensure existing group permissions are never returned.. verifyOnAllSubDirs("/tmp/external/ext1", null, "bar", false); verifyOnAllSubDirs("/tmp/external/ext1", null, "hbase", false); stmt.execute("grant all on table ext1 to role p1_admin"); verifyOnAllSubDirs("/tmp/external/ext1", FsAction.ALL, "hbase", true); stmt.execute("revoke select on table ext1 from role p1_admin"); verifyOnAllSubDirs("/tmp/external/ext1", FsAction.WRITE_EXECUTE, "hbase", true); // Verify database operations works correctly stmt.execute("create database db1"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false); stmt.execute("create table db1.tbl1 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", null, "hbase", false); stmt.execute("create table db1.tbl2 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false); // Verify default db grants do not affect other dbs stmt.execute("grant all on database default to role p1_admin"); verifyOnPath("/user/hive/warehouse", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false); // Verify table rename works stmt.execute("create table q1 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/q1", FsAction.ALL, "hbase", true); stmt.execute("alter table q1 rename to q2"); verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); // Verify table GRANTS do not trump db GRANTS stmt.execute("grant select on table q2 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); stmt.execute("create table q3 (s string)"); verifyOnAllSubDirs("/user/hive/warehouse/q3", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true); // Verify db privileges are propagated to tables stmt.execute("grant select on database db1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); // Verify default db revokes do not affect other dbs stmt.execute("revoke all on database default from role p1_admin"); verifyOnPath("/user/hive/warehouse", null, "hbase", false); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); stmt.execute("use db1"); stmt.execute("grant all on table tbl1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.ALL, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true); // Verify recursive revoke stmt.execute("revoke select on database db1 from role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.WRITE_EXECUTE, "hbase", true); verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false); // Verify cleanup.. stmt.execute("drop table tbl1"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl1"))); stmt.execute("drop table tbl2"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl2"))); stmt.execute("use default"); stmt.execute("drop database db1"); Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db"))); // START : Verify external table set location.. writeToPath("/tmp/external/tables/ext2_before/i=1", 5, "foo", "bar"); writeToPath("/tmp/external/tables/ext2_before/i=2", 5, "foo", "bar"); stmt.execute( "create external table ext2 (s string) partitioned by (i int) location \'/tmp/external/tables/ext2_before\'"); stmt.execute("alter table ext2 add partition (i=1)"); stmt.execute("alter table ext2 add partition (i=2)"); verifyQuery(stmt, "ext2", 10); verifyOnAllSubDirs("/tmp/external/tables/ext2_before", null, "hbase", false); stmt.execute("grant all on table ext2 to role p1_admin"); verifyOnPath("/tmp/external/tables/ext2_before", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true); writeToPath("/tmp/external/tables/ext2_after/i=1", 6, "foo", "bar"); writeToPath("/tmp/external/tables/ext2_after/i=2", 6, "foo", "bar"); stmt.execute("alter table ext2 set location \'hdfs:///tmp/external/tables/ext2_after\'"); // Even though table location is altered, partition location is still old (still 10 rows) verifyQuery(stmt, "ext2", 10); // You have to explicitly alter partition location.. verifyOnPath("/tmp/external/tables/ext2_before", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true); stmt.execute( "alter table ext2 partition (i=1) set location \'hdfs:///tmp/external/tables/ext2_after/i=1\'"); stmt.execute( "alter table ext2 partition (i=2) set location \'hdfs:///tmp/external/tables/ext2_after/i=2\'"); // Now that partition location is altered, it picks up new data (12 rows instead of 10) verifyQuery(stmt, "ext2", 12); verifyOnPath("/tmp/external/tables/ext2_before", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=1", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=2", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", null, "hbase", false); verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=2", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=1/stuff.txt", FsAction.ALL, "hbase", true); verifyOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true); // END : Verify external table set location.. // Restart HDFS to verify if things are fine after re-start.. // TODO : this is currently commented out since miniDFS.restartNameNode() does // not work corectly on the version of hadoop sentry depends on // This has been verified to work on a real cluster. // Once miniDFS is fixed, this should be uncommented.. // miniDFS.shutdown(); // miniDFS.restartNameNode(true); // miniDFS.waitActive(); // verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true); // verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true); stmt.close(); conn.close(); } /** * Make sure non HDFS paths are not added to the object - location map. * @throws Throwable */ @Test public void testNonHDFSLocations() throws Throwable { String dbName = "db2"; tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "user_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant all on uri 'file:///tmp/external' to role admin_role"); stmt.execute("grant all on uri 'hdfs:///tmp/external' to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); conn = hiveServer2.createConnection(admin, admin); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.close(); conn.close(); conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role user_role"); stmt.execute("grant all on database " + dbName + " to role user_role"); stmt.execute("grant role user_role to group " + StaticUserGroup.USERGROUP1); stmt.close(); conn.close(); conn = hiveServer2.createConnection(admin, admin); stmt = conn.createStatement(); miniDFS.getFileSystem().mkdirs(tmpHDFSDir); miniDFS.getFileSystem().setOwner(tmpHDFSDir, "hive", "hive"); //External table on local file system miniDFS.getFileSystem().mkdirs(new Path("/tmp/external/tab1_loc")); stmt.execute("use " + dbName); stmt.execute("create external table tab1(a int) location 'file:///tmp/external/tab1_loc'"); verifyOnAllSubDirs("/tmp/external/tab1_loc", null, StaticUserGroup.USERGROUP1, false); //External partitioned table on local file system miniDFS.getFileSystem().mkdirs(new Path("/tmp/external/tab2_loc/i=1")); stmt.execute( "create external table tab2 (s string) partitioned by (i int) location 'file:///tmp/external/tab2_loc'"); verifyOnAllSubDirs("/tmp/external/tab2_loc", null, StaticUserGroup.USERGROUP1, false); //Partition on local file system stmt.execute("alter table tab2 add partition (i=1)"); stmt.execute("alter table tab2 partition (i=1) set location 'file:///tmp/external/tab2_loc/i=1'"); verifyOnAllSubDirs("/tmp/external/tab2_loc/i=1", null, StaticUserGroup.USERGROUP1, false); //HDFS to local file system, also make sure does not specifying scheme still works stmt.execute("create external table tab3(a int) location '/tmp/external/tab3_loc'"); // SENTRY-546 // verifyOnAllSubDirs("/tmp/external/tab3_loc", FsAction.ALL, StaticUserGroup.USERGROUP1, true); verifyOnAllSubDirs("/tmp/external/tab3_loc", null, StaticUserGroup.USERGROUP1, true); stmt.execute("alter table tab3 set location 'file:///tmp/external/tab3_loc'"); verifyOnAllSubDirs("/tmp/external/tab3_loc", null, StaticUserGroup.USERGROUP1, false); //Local file system to HDFS stmt.execute("create table tab4(a int) location 'file:///tmp/external/tab4_loc'"); stmt.execute("alter table tab4 set location 'hdfs:///tmp/external/tab4_loc'"); miniDFS.getFileSystem().mkdirs(new Path("/tmp/external/tab4_loc")); // SENTRY-546 // verifyOnAllSubDirs("/tmp/external/tab4_loc", FsAction.ALL, StaticUserGroup.USERGROUP1, true); verifyOnAllSubDirs("/tmp/external/tab4_loc", null, StaticUserGroup.USERGROUP1, true); stmt.close(); conn.close(); } @Ignore("SENTRY-546") @Test public void testExternalTable() throws Throwable { String dbName = "db2"; tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant all on uri 'hdfs:///tmp/external' to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("create external table tab1(a int) location '/tmp/external/tab1_loc'"); verifyOnAllSubDirs("/tmp/external/tab1_loc", FsAction.ALL, StaticUserGroup.ADMINGROUP, true); stmt.close(); conn.close(); } /** * Make sure when events such as table creation fail, the path should not be sync to NameNode plugin. */ @Test public void testTableCreationFailure() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); if (!miniDFS.getFileSystem().exists(tmpHDFSDir)) { miniDFS.getFileSystem().mkdirs(tmpHDFSDir); } miniDFS.getFileSystem().setPermission(tmpHDFSDir, FsPermission.valueOf("drwxrwx---")); Path partitionDir = new Path("/tmp/external/p1"); if (!miniDFS.getFileSystem().exists(partitionDir)) { miniDFS.getFileSystem().mkdirs(partitionDir); } dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant all on uri 'hdfs:///tmp/external' to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); stmt.execute("grant role admin_role to group " + StaticUserGroup.HIVE); stmt.close(); conn.close(); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); // Expect table creation to fail because hive:hive does not have // permission to write at parent directory. try { stmt.execute("create external table tab1(a int) location 'hdfs:///tmp/external/p1'"); Assert.fail("Expect table creation to fail"); } catch (Exception ex) { LOGGER.error("Exception when creating table: " + ex.getMessage()); } // When the table creation failed, the path will not be managed by sentry. And the // permission of the path will not be hive:hive. verifyOnAllSubDirs("/tmp/external/p1", null, StaticUserGroup.HIVE, true); stmt.close(); conn.close(); } /** * Make sure when events such as add partition fail, the path should not be sync to NameNode plugin. */ @Test public void testAddPartitionFailure() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); if (!miniDFS.getFileSystem().exists(tmpHDFSDir)) { miniDFS.getFileSystem().mkdirs(tmpHDFSDir); } miniDFS.getFileSystem().setPermission(tmpHDFSDir, FsPermission.valueOf("drwxrwx--x")); Path partitionDir = new Path("/tmp/external/p1"); if (!miniDFS.getFileSystem().exists(partitionDir)) { miniDFS.getFileSystem().mkdirs(partitionDir); } dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); stmt.close(); conn.close(); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("create external table tab2 (s string) partitioned by (month int)"); // Expect adding partition to fail because hive:hive does not have // permission to write at parent directory. try { stmt.execute("alter table tab2 add partition (month = 1) location '/tmp/external/p1'"); Assert.fail("Expect adding partition to fail"); } catch (Exception ex) { LOGGER.error("Exception when adding partition: " + ex.getMessage()); } // When the table creation failed, the path will not be managed by sentry. And the // permission of the path will not be hive:hive. verifyOnAllSubDirs("/tmp/external/p1", null, StaticUserGroup.HIVE, true); stmt.close(); conn.close(); } /** * Make sure when events such as drop table fail, the path should not be sync to NameNode plugin. */ @Test public void testDropTableFailure() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); if (!miniDFS.getFileSystem().exists(tmpHDFSDir)) { miniDFS.getFileSystem().mkdirs(tmpHDFSDir); } miniDFS.getFileSystem().setPermission(tmpHDFSDir, FsPermission.valueOf("drwxrwxrwx")); Path partitionDir = new Path("/tmp/external/p1"); if (!miniDFS.getFileSystem().exists(partitionDir)) { miniDFS.getFileSystem().mkdirs(partitionDir); } dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); stmt.close(); conn.close(); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("create external table tab1(a int) location 'hdfs:///tmp/external/p1'"); miniDFS.getFileSystem().setPermission(tmpHDFSDir, FsPermission.valueOf("drwxrwx---")); // Expect dropping table to fail because hive:hive does not have // permission to write at parent directory when // hive.metastore.authorization.storage.checks property is true. try { stmt.execute("drop table tab1"); Assert.fail("Expect dropping table to fail"); } catch (Exception ex) { LOGGER.error("Exception when creating table: " + ex.getMessage()); } // When the table dropping failed, the path will still be managed by sentry. And the // permission of the path still should be hive:hive. verifyOnAllSubDirs("/tmp/external/p1", FsAction.ALL, StaticUserGroup.HIVE, true); stmt.close(); conn.close(); } /** * Make sure when events such as drop table fail, the path should not be sync to NameNode plugin. */ @Test public void testDropPartitionFailure() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); if (!miniDFS.getFileSystem().exists(tmpHDFSDir)) { miniDFS.getFileSystem().mkdirs(tmpHDFSDir); } miniDFS.getFileSystem().setPermission(tmpHDFSDir, FsPermission.valueOf("drwxrwxrwx")); Path partitionDir = new Path("/tmp/external/p1"); if (!miniDFS.getFileSystem().exists(partitionDir)) { miniDFS.getFileSystem().mkdirs(partitionDir); } dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); stmt.close(); conn.close(); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("create table tab3 (s string) partitioned by (month int)"); stmt.execute("alter table tab3 add partition (month = 1) location '/tmp/external/p1'"); miniDFS.getFileSystem().setPermission(tmpHDFSDir, FsPermission.valueOf("drwxrwx---")); // Expect dropping partition to fail because because hive:hive does not have // permission to write at parent directory. try { stmt.execute("ALTER TABLE tab3 DROP PARTITION (month = 1)"); Assert.fail("Expect dropping partition to fail"); } catch (Exception ex) { LOGGER.error("Exception when dropping partition: " + ex.getMessage()); } // When the partition dropping failed, the path for the partition will still // be managed by sentry. And the permission of the path still should be hive:hive. verifyOnAllSubDirs("/tmp/external/p1", FsAction.ALL, StaticUserGroup.HIVE, true); stmt.close(); conn.close(); } @Test public void testColumnPrivileges() throws Throwable { String dbName = "db2"; tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "tab_role", "db_role", "col_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role with grant option"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("use " + dbName); stmt.execute("create table p1 (s string) partitioned by (month int, day int)"); stmt.execute("alter table p1 add partition (month=1, day=1)"); stmt.execute("alter table p1 add partition (month=1, day=2)"); stmt.execute("alter table p1 add partition (month=2, day=1)"); stmt.execute("alter table p1 add partition (month=2, day=2)"); loadData(stmt); stmt.execute("create role db_role"); stmt.execute("grant select on database " + dbName + " to role db_role"); stmt.execute("create role tab_role"); stmt.execute("grant select on p1 to role tab_role"); stmt.execute("create role col_role"); stmt.execute("grant select(s) on p1 to role col_role"); stmt.execute("grant role col_role to group " + StaticUserGroup.USERGROUP1); stmt.execute("grant role tab_role to group " + StaticUserGroup.USERGROUP2); stmt.execute("grant role col_role to group " + StaticUserGroup.USERGROUP2); stmt.execute("grant role db_role to group " + StaticUserGroup.USERGROUP3); stmt.execute("grant role col_role to group " + StaticUserGroup.USERGROUP3); stmt.execute("grant role col_role to group " + StaticUserGroup.ADMINGROUP); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode //User with just column level privileges cannot read HDFS verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false); //User with permissions on table and column can read HDFS file verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); //User with permissions on db and column can read HDFS file verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP3, true); //User with permissions on server and column cannot read HDFS file //TODO:SENTRY-751 verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.ADMINGROUP, false); stmt.close(); conn.close(); } /* TODO:SENTRY-819 */ @Test public void testAllColumn() throws Throwable { String dbName = "db2"; tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "col_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role with grant option"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("use " + dbName); stmt.execute("create table p1 (c1 string, c2 string) partitioned by (month int, day int)"); stmt.execute("alter table p1 add partition (month=1, day=1)"); loadDataTwoCols(stmt); stmt.execute("create role col_role"); stmt.execute("grant select(c1,c2) on p1 to role col_role"); stmt.execute("grant role col_role to group " + StaticUserGroup.USERGROUP1); Thread.sleep(100); //User with privileges on all columns of the data cannot still read the HDFS files verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false); stmt.close(); conn.close(); } //SENTRY-780 @Test public void testViews() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); try { stmt.execute("create database " + dbName); stmt.execute("create table test(a string)"); stmt.execute("create view testView as select * from test"); stmt.execute("create or replace view testView as select * from test"); stmt.execute("drop view testView"); } catch (Exception s) { throw s; } stmt.close(); conn.close(); } //SENTRY-884 @Test public void testAccessToTableDirectory() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "table_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("use " + dbName); stmt.execute("create table tb1(a string)"); stmt.execute("create role table_role"); stmt.execute("grant all on table tb1 to role table_role"); stmt.execute("grant role table_role to group " + StaticUserGroup.USERGROUP1); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode //Verify user1 is able to access table directory verifyAccessToPath(StaticUserGroup.USER1_1, StaticUserGroup.USERGROUP1, "/user/hive/warehouse/db1.db/tb1", true); stmt.close(); conn.close(); } /* SENTRY-953 */ @Test public void testAuthzObjOnPartitionMultipleTables() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external"); miniDFS.getFileSystem().mkdirs(tmpHDFSDir); Path partitionDir = new Path("/tmp/external/p1"); miniDFS.getFileSystem().mkdirs(partitionDir); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "tab1_role", "tab2_role", "tab3_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); // Create external table tab1 on location '/tmp/external/p1'. // Create tab1_role, and grant it with insert permission on table tab1 to user_group1. conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("use " + dbName); stmt.execute( "create external table tab1 (s string) partitioned by (month int) location '/tmp/external/p1'"); stmt.execute("create role tab1_role"); stmt.execute("grant insert on table tab1 to role tab1_role"); stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'. verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); // Create external table tab2 and partition on location '/tmp/external'. // Create tab2_role, and grant it with select permission on table tab2 to user_group2. stmt.execute("create external table tab2 (s string) partitioned by (month int)"); stmt.execute("alter table tab2 add partition (month = 1) location '/tmp/external'"); stmt.execute("create role tab2_role"); stmt.execute("grant select on table tab2 to role tab2_role"); stmt.execute("grant role tab2_role to group " + StaticUserGroup.USERGROUP2); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode // Verify that user_group2 have select(read_execute) permission on both paths. verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab2", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); // Create table tab3 and partition on the same location '/tmp/external' as tab2. // Create tab3_role, and grant it with insert permission on table tab3 to user_group3. stmt.execute("create table tab3 (s string) partitioned by (month int)"); stmt.execute("alter table tab3 add partition (month = 1) location '/tmp/external'"); stmt.execute("create role tab3_role"); stmt.execute("grant insert on table tab3 to role tab3_role"); stmt.execute("grant role tab3_role to group " + StaticUserGroup.USERGROUP3); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode // When two partitions of different tables pointing to the same location with different grants, // ACLs should have union (no duplicates) of both rules. verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); // When alter the table name (tab2 to be tabx), ACLs should remain the same. stmt.execute("alter table tab2 rename to tabx"); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); // When drop a partition that shares the same location with other partition belonging to // other table, should still have the other table permissions. stmt.execute("ALTER TABLE tabx DROP PARTITION (month = 1)"); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); // When drop a table that has a partition shares the same location with other partition // belonging to other table, should still have the other table permissions. stmt.execute("DROP TABLE IF EXISTS tabx"); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); stmt.close(); conn.close(); miniDFS.getFileSystem().delete(partitionDir, true); } /* SENTRY-953 */ @Test public void testAuthzObjOnPartitionSameTable() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external/p1"); miniDFS.getFileSystem().mkdirs(tmpHDFSDir); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "tab1_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); // Create table tab1 and partition on the same location '/tmp/external/p1'. // Create tab1_role, and grant it with insert permission on table tab1 to user_group1. conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("use " + dbName); stmt.execute("create table tab1 (s string) partitioned by (month int)"); stmt.execute("alter table tab1 add partition (month = 1) location '/tmp/external/p1'"); stmt.execute("create role tab1_role"); stmt.execute("grant insert on table tab1 to role tab1_role"); stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'. verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); // When two partitions of the same table pointing to the same location, // ACLS should not be repeated. Exception will be thrown if there are duplicates. stmt.execute("alter table tab1 add partition (month = 2) location '/tmp/external/p1'"); verifyOnPath("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); stmt.close(); conn.close(); } /* SENTRY-953 */ @Test public void testAuthzObjOnMultipleTables() throws Throwable { String dbName = "db1"; tmpHDFSDir = new Path("/tmp/external/p1"); miniDFS.getFileSystem().mkdirs(tmpHDFSDir); dbNames = new String[] { dbName }; roles = new String[] { "admin_role", "tab1_role", "tab2_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); // Create external table tab1 on location '/tmp/external/p1'. // Create tab1_role, and grant it with insert permission on table tab1 to user_group1. conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("use " + dbName); stmt.execute( "create external table tab1 (s string) partitioned by (month int) location '/tmp/external/p1'"); stmt.execute("create role tab1_role"); stmt.execute("grant insert on table tab1 to role tab1_role"); stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'. verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); // Create table tab2 on the same location '/tmp/external/p1' as table tab1. // Create tab2_role, and grant it with select permission on table tab2 to user_group1. stmt.execute("create table tab2 (s string) partitioned by (month int) location '/tmp/external/p1'"); stmt.execute("create role tab2_role"); stmt.execute("grant select on table tab2 to role tab2_role"); stmt.execute("grant role tab2_role to group " + StaticUserGroup.USERGROUP1); // When two tables pointing to the same location, ACLS should have union (no duplicates) // of both rules. verifyOnPath("/tmp/external/p1", FsAction.ALL, StaticUserGroup.USERGROUP1, true); // When drop table tab1, ACLs of tab2 still remain. stmt.execute("DROP TABLE IF EXISTS tab1"); Thread.sleep(WAIT_BEFORE_TESTVERIFY);//Wait till sentry cache is updated in Namenode verifyOnPath("/tmp/external/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP1, true); stmt.close(); conn.close(); } private void verifyAccessToPath(String user, String group, String path, boolean hasPermission) throws Exception { Path p = new Path(path); UserGroupInformation hadoopUser = UserGroupInformation.createUserForTesting(user, new String[] { group }); FileSystem fs = DFSTestUtil.getFileSystemAs(hadoopUser, hadoopConf); try { fs.listFiles(p, true); if (!hasPermission) { Assert.assertFalse("Expected listing files to fail", false); } } catch (Exception e) { if (hasPermission) { throw e; } } } private void verifyQuery(Statement stmt, String table, int n) throws Throwable { verifyQuery(stmt, table, n, NUM_RETRIES); } private void verifyQuery(Statement stmt, String table, int n, int retry) throws Throwable { ResultSet rs = null; try { rs = stmt.executeQuery("select * from " + table); int numRows = 0; while (rs.next()) { numRows++; } Assert.assertEquals(n, numRows); } catch (Throwable th) { if (retry > 0) { Thread.sleep(RETRY_WAIT); verifyQuery(stmt, table, n, retry - 1); } else { throw th; } } } /** * SENTRY-1002: * Ensure the paths with no scheme will not cause NPE during paths update. */ @Test public void testMissingScheme() throws Throwable { // In the local test environment, EXTERNAL_SENTRY_SERVICE is false, // set the default URI scheme to be hdfs. boolean testConfOff = Boolean.valueOf(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false")); if (!testConfOff) { PathsUpdate.setDefaultScheme("hdfs"); } tmpHDFSDir = new Path("/tmp/external"); if (!miniDFS.getFileSystem().exists(tmpHDFSDir)) { miniDFS.getFileSystem().mkdirs(tmpHDFSDir); } Path partitionDir = new Path("/tmp/external/p1"); if (!miniDFS.getFileSystem().exists(partitionDir)) { miniDFS.getFileSystem().mkdirs(partitionDir); } String dbName = "db1"; String tblName = "tab1"; dbNames = new String[] { dbName }; roles = new String[] { "admin_role" }; admin = StaticUserGroup.ADMIN1; Connection conn; Statement stmt; conn = hiveServer2.createConnection("hive", "hive"); stmt = conn.createStatement(); stmt.execute("create role admin_role"); stmt.execute("grant all on server server1 to role admin_role"); stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); stmt.close(); conn.close(); conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); stmt = conn.createStatement(); stmt.execute("create database " + dbName); stmt.execute("create external table " + dbName + "." + tblName + "(s string) location '/tmp/external/p1'"); // Deep copy of table tab1 Table tbCopy = hmsClient.getTable(dbName, tblName); // Change the location of the table to strip the scheme. StorageDescriptor sd = hmsClient.getTable(dbName, tblName).getSd(); sd.setLocation("/tmp/external"); tbCopy.setSd(sd); // Alter table tab1 to be tbCopy which is at scheme-less location. // And the corresponding path will be updated to sentry server. hmsClient.alter_table(dbName, "tab1", tbCopy); Assert.assertEquals(hmsClient.getTable(dbName, tblName).getSd().getLocation(), "/tmp/external"); verifyOnPath("/tmp/external", FsAction.ALL, StaticUserGroup.HIVE, true); stmt.close(); conn.close(); } private void loadData(Statement stmt) throws IOException, SQLException { FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f1.txt")); f1.writeChars("m1d1_t1\n"); f1.writeChars("m1d1_t2\n"); f1.writeChars("m1d1_t3\n"); f1.flush(); f1.close(); stmt.execute("load data inpath \'/tmp/f1.txt\' overwrite into table p1 partition (month=1, day=1)"); FSDataOutputStream f2 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt")); f2.writeChars("m2d2_t4\n"); f2.writeChars("m2d2_t5\n"); f2.writeChars("m2d2_t6\n"); f2.flush(); f2.close(); stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=2, day=2)"); ResultSet rs = stmt.executeQuery("select * from p1"); List<String> vals = new ArrayList<String>(); while (rs.next()) { vals.add(rs.getString(1)); } Assert.assertEquals(6, vals.size()); rs.close(); } private void loadDataTwoCols(Statement stmt) throws IOException, SQLException { FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt")); f1.writeChars("m1d1_t1, m1d1_t2\n"); f1.writeChars("m1d1_t2, m1d1_t2\n"); f1.writeChars("m1d1_t3, m1d1_t2\n"); f1.flush(); f1.close(); stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=1, day=1)"); ResultSet rs = stmt.executeQuery("select * from p1"); List<String> vals = new ArrayList<String>(); while (rs.next()) { vals.add(rs.getString(1)); } Assert.assertEquals(3, vals.size()); rs.close(); } private void writeToPath(String path, int numRows, String user, String group) throws IOException { Path p = new Path(path); miniDFS.getFileSystem().mkdirs(p); miniDFS.getFileSystem().setOwner(p, user, group); // miniDFS.getFileSystem().setPermission(p, FsPermission.valueOf("-rwxrwx---")); FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path(path + "/stuff.txt")); for (int i = 0; i < numRows; i++) { f1.writeChars("random" + i + "\n"); } f1.flush(); f1.close(); miniDFS.getFileSystem().setOwner(new Path(path + "/stuff.txt"), "asuresh", "supergroup"); miniDFS.getFileSystem().setPermission(new Path(path + "/stuff.txt"), FsPermission.valueOf("-rwxrwx---")); } private void verifyHDFSandMR(Statement stmt) throws Throwable { // hbase user should not be allowed to read... UserGroupInformation hbaseUgi = UserGroupInformation.createUserForTesting("hbase", new String[] { "hbase" }); hbaseUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { try { miniDFS.getFileSystem().open(new Path("/user/hive/warehouse/p1/month=1/day=1/f1.txt")); Assert.fail("Should not be allowed !!"); } catch (Exception e) { Assert.assertEquals("Wrong Error : " + e.getMessage(), true, e.getMessage().contains("Permission denied: user=hbase")); } return null; } }); // WordCount should fail.. // runWordCount(new JobConf(miniMR.getConfig()), "/user/hive/warehouse/p1/month=1/day=1", "/tmp/wc_out"); stmt.execute("grant select on table p1 to role p1_admin"); verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true); // hbase user should now be allowed to read... hbaseUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { Path p = new Path("/user/hive/warehouse/p1/month=2/day=2/f2.txt"); BufferedReader in = new BufferedReader(new InputStreamReader(miniDFS.getFileSystem().open(p))); String line = null; List<String> lines = new ArrayList<String>(); do { line = in.readLine(); if (line != null) { lines.add(line); } } while (line != null); Assert.assertEquals(3, lines.size()); in.close(); return null; } }); } private void verifyOnAllSubDirs(String path, FsAction fsAction, String group, boolean groupShouldExist) throws Throwable { verifyOnAllSubDirs(path, fsAction, group, groupShouldExist, true); } private void verifyOnPath(String path, FsAction fsAction, String group, boolean groupShouldExist) throws Throwable { verifyOnAllSubDirs(path, fsAction, group, groupShouldExist, false); } private void verifyOnAllSubDirs(String path, FsAction fsAction, String group, boolean groupShouldExist, boolean recurse) throws Throwable { verifyOnAllSubDirs(new Path(path), fsAction, group, groupShouldExist, recurse, NUM_RETRIES); } private void verifyOnAllSubDirs(Path p, FsAction fsAction, String group, boolean groupShouldExist, boolean recurse, int retry) throws Throwable { FileStatus fStatus = null; try { fStatus = miniDFS.getFileSystem().getFileStatus(p); if (groupShouldExist) { Assert.assertEquals("Error at verifying Path action : " + p + " ;", fsAction, getAcls(p).get(group)); } else { Assert.assertFalse("Error at verifying Path : " + p + " ," + " group : " + group + " ;", getAcls(p).containsKey(group)); } } catch (Throwable th) { if (retry > 0) { Thread.sleep(RETRY_WAIT); verifyOnAllSubDirs(p, fsAction, group, groupShouldExist, recurse, retry - 1); } else { throw th; } } if (recurse && fStatus.isDirectory()) { FileStatus[] children = miniDFS.getFileSystem().listStatus(p); for (FileStatus fs : children) { verifyOnAllSubDirs(fs.getPath(), fsAction, group, groupShouldExist, recurse, NUM_RETRIES); } } } private Map<String, FsAction> getAcls(Path path) throws Exception { AclStatus aclStatus = miniDFS.getFileSystem().getAclStatus(path); Map<String, FsAction> acls = new HashMap<String, FsAction>(); for (AclEntry ent : aclStatus.getEntries()) { if (ent.getType().equals(AclEntryType.GROUP)) { // In case of duplicate acl exist, exception should be thrown. if (acls.containsKey(ent.getName())) { throw new SentryAlreadyExistsException("The acl " + ent.getName()); } else { acls.put(ent.getName(), ent.getPermission()); } } } return acls; } /* private void runWordCount(JobConf job, String inPath, String outPath) throws Exception { Path in = new Path(inPath); Path out = new Path(outPath); miniDFS.getFileSystem().delete(out, true); job.setJobName("TestWC"); JobClient jobClient = new JobClient(job); RunningJob submittedJob = null; FileInputFormat.setInputPaths(job, in); FileOutputFormat.setOutputPath(job, out); job.set("mapreduce.output.textoutputformat.separator", " "); job.setInputFormat(TextInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setMapperClass(WordCountMapper.class); job.setReducerClass(SumReducer.class); job.setOutputFormat(TextOutputFormat.class); job.setNumReduceTasks(1); job.setInt("mapreduce.map.maxattempts", 1); job.setInt("mapreduce.reduce.maxattempts", 1); submittedJob = jobClient.submitJob(job); if (!jobClient.monitorAndPrintJob(job, submittedJob)) { throw new IOException("job Failed !!"); } } */ }