| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
| * implied. |
| * See the License for the specific language governing permissions |
| * and |
| * limitations under the License. |
| */ |
| |
| package org.apache.sentry.tests.e2e.hdfs; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.permission.FsAction; |
| import org.apache.sentry.hdfs.SentryAuthorizationConstants; |
| import org.apache.sentry.service.common.ServiceConstants; |
| import org.junit.BeforeClass; |
| import org.junit.Test; |
| |
| import java.sql.Connection; |
| import java.sql.Statement; |
| |
| /** |
| * this test class includes tests to verify the behaviour of sentry server |
| * when the HDFS sync feature is toggled on/off |
| */ |
| public class TestHDFSIntegrationTogglingConf extends TestHDFSIntegrationBase { |
| |
| private static long getSleepTimeAfterFollowerRestart(Configuration conf) { |
| long followerInitDelay = conf.getLong(ServiceConstants.ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS, |
| ServiceConstants.ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT); |
| long followerInterval = conf.getLong(ServiceConstants.ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, |
| ServiceConstants.ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT); |
| long refreshIntervalMillisec = conf.getInt( |
| SentryAuthorizationConstants.CACHE_REFRESH_INTERVAL_KEY, |
| SentryAuthorizationConstants.CACHE_REFRESH_INTERVAL_DEFAULT); |
| |
| return (followerInitDelay + followerInterval + refreshIntervalMillisec) * 2; |
| } |
| |
| private static void enableHdfsSync(int serverIndex) throws Exception { |
| Configuration newConfig = new Configuration(sentryConf); |
| newConfig.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, |
| "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); |
| newConfig.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, |
| "org.apache.sentry.hdfs.SentryPlugin"); |
| newConfig.set(ServiceConstants.ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS, |
| "1000"); |
| sentryServer.restartHMSFollower(newConfig, serverIndex, |
| getSleepTimeAfterFollowerRestart(newConfig)); |
| } |
| |
| private static void disableHdfsSync(int serverIndex) throws Exception { |
| Configuration newConfig = new Configuration(sentryConf); |
| newConfig.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, ""); |
| newConfig.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, ""); |
| newConfig.set(ServiceConstants.ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS, |
| "1000"); |
| |
| sentryServer.restartHMSFollower(newConfig, serverIndex, |
| getSleepTimeAfterFollowerRestart(newConfig)); |
| } |
| |
| @BeforeClass |
| public static void setup() throws Exception { |
| hdfsSyncEnabled = true; |
| TestHDFSIntegrationBase.setup(); |
| } |
| |
| /** |
| * Test makes sure that the namenode is not synced with the new change to HMS when |
| * processor and sentry_plugin for HDFS sync are not configured. |
| * |
| * @throws Throwable |
| */ |
| @Test |
| public void testDisablingHDFSSync() throws Throwable { |
| disableHdfsSync(0); |
| dbNames = new String[]{"db1"}; |
| roles = new String[]{"admin_role", "tab_role"}; |
| admin = "hive"; |
| |
| Connection conn; |
| Statement stmt; |
| conn = hiveServer2.createConnection("hive", "hive"); |
| stmt = conn.createStatement(); |
| stmt.execute("create role admin_role"); |
| stmt.execute("grant role admin_role to group hive"); |
| stmt.execute("grant all on server server1 to role admin_role"); |
| |
| // db privileges |
| stmt.execute("create database db1"); |
| stmt.execute("create role tab_role"); |
| stmt.execute("grant role tab_role to group flume"); |
| stmt.execute("create table db1.p2(id int)"); |
| |
| stmt.execute("use db1"); |
| stmt.execute("grant all on table p2 to role tab_role"); |
| stmt.execute("use default"); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db", FsAction.ALL, "hbase", false); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/p2", FsAction.ALL, "flume", false); |
| verifyGroupPermOnPath("/user/hive/warehouse/db1.db", FsAction.ALL, "flume", false); |
| |
| //Enabling HDFS sync back in sentry server |
| enableHdfsSync(0); |
| } |
| |
| /** |
| * Test makes sure that HDFS sync configurations in sentryserver are toggled multiple times. |
| * <ul> |
| * <li>When processor and sentry_plugin for HDFS sync are configured, |
| * Namenode should have all the HMS path and permission updates.</li> |
| * <li>When processor and sentry_plugin for HDFS sync are configured, |
| * Namenode should not have the HMS path updates.</li> |
| * <li>When processor and sentry_plugin for HDFS sync are configured again, |
| * Namenode should not have the HMS path updates by getting HMS full snapshot |
| * from sentry server.</li> |
| * </ul> |
| * |
| * @throws Throwable |
| */ |
| @Test |
| public void testEnablingDisablingHDFSSync() throws Throwable { |
| dbNames = new String[]{"db1", "db6"}; |
| roles = new String[]{"admin_role", "db_role", "tab_role", "p1_admin"}; |
| admin = "hive"; |
| |
| Connection conn; |
| Statement stmt; |
| conn = hiveServer2.createConnection("hive", "hive"); |
| stmt = conn.createStatement(); |
| stmt.execute("create role admin_role"); |
| stmt.execute("grant role admin_role to group hive"); |
| stmt.execute("grant all on server server1 to role admin_role"); |
| stmt.execute("create table p1 (s string) partitioned by (month int, day " + |
| "int)"); |
| stmt.execute("alter table p1 add partition (month=1, day=1)"); |
| |
| // db privileges |
| stmt.execute("create database db1"); |
| stmt.execute("create role db_role"); |
| stmt.execute("create role tab_role"); |
| stmt.execute("grant role db_role to group hbase"); |
| stmt.execute("grant role tab_role to group flume"); |
| stmt.execute("create table db1.p2(id int)"); |
| |
| stmt.execute("create role p1_admin"); |
| stmt.execute("grant role p1_admin to group hbase"); |
| |
| // Verify default db is inaccessible initially |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse", null, "hbase", false); |
| |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false); |
| |
| stmt.execute("grant all on database db1 to role db_role"); |
| stmt.execute("use db1"); |
| stmt.execute("grant all on table p2 to role tab_role"); |
| stmt.execute("use default"); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db", FsAction.ALL, "hbase", true); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/p2", FsAction.ALL, "hbase", true); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db/p2", FsAction.ALL, "flume", true); |
| verifyGroupPermOnPath("/user/hive/warehouse/db1.db", FsAction.ALL, "flume", false); |
| |
| loadData(stmt); |
| |
| verifyHDFSandMR(stmt); |
| |
| //Disabling HDFS sync in sentry server |
| disableHdfsSync(0); |
| |
| stmt.execute("revoke all on database db1 from role db_role"); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db1.db", FsAction.ALL, "hbase", false); |
| |
| // create a table and grant all to db_role |
| stmt.execute("create database db6"); |
| stmt.execute("grant all on database db6 to role db_role"); |
| |
| // verify that db_role does not have required ACL's as HDFS sync is disabled in sentry server. |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db6.db", FsAction.ALL, "hbase", false); |
| |
| //Create table in db6 and grant all privileges to tab role |
| stmt.execute("use db6"); |
| stmt.execute("create table db6.p1(id int)"); |
| stmt.execute("grant all on table db6.p1 to role tab_role"); |
| |
| // verify that tab_role does not have required permissions |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db6.db/p1", FsAction.ALL, "flume", false); |
| |
| //Enabling HDFS sync in sentry server |
| enableHdfsSync(0); |
| |
| // As HDFS sync is re-enabled, sentry should take full snapshot and send it NN. |
| // db_role and tab_role should have required privileges. |
| // Checks below will make sure that sentry/NN have the updates that happened |
| // to HMS objects when HDFS was disabled. |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db6.db", FsAction.ALL, "hbase", true); |
| verifyGroupPermOnAllSubDirs("/user/hive/warehouse/db6.db/p1", FsAction.ALL, "flume", true); |
| |
| stmt.close(); |
| conn.close(); |
| } |
| } |