| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Hive Docker helpers and declarative plugin scenarios (beeline, policies, ``HivePluginTest``).""" | |
| import socket | |
| import time | |
| import pytest | |
| from ranger_test_utils.plugin_test_users import PLUGIN_TEST_USER | |
| from ranger_test_utils.plugin_enforcement_utils import ( | |
| logger as plugin_enforcement_logger, | |
| normalize_audit_log_filter, | |
| ) | |
| from ranger_test_utils.utils import ( | |
| DOCKER_PLUGIN_TEST_USER_PASSWORD, | |
| POLICY_URL, | |
| RANGER_ADMIN_BASE_URL, | |
| assert_http_ok, | |
| get_test_logger, | |
| log_testcase_begin, | |
| unique_suffix, | |
| ) | |
| logger = get_test_logger(__name__) | |
| BEELINE_LOG_OUTPUT_MAX_CHARS = 12000 | |
| BEELINE_COMPLETE_LOG_MAX_CHARS = 100000 | |
| HIVE_SERVICE_NAME = "dev_hive" | |
| HIVE_CONTAINER = "ranger-hive" | |
| HADOOP_CONTAINER = "ranger-hadoop" | |
| KDC_CONTAINER = "ranger-kdc" | |
| ADMIN_CONTAINER = "ranger" | |
| KERBEROS_REALM = "EXAMPLE.COM" | |
| HIVE_HS2_PRINCIPAL = "hive/ranger-hive.rangernw@" + KERBEROS_REALM | |
| HIVE_SERVICE_KEYTAB = "/etc/keytabs/hive.keytab" | |
| HIVE_JDBC_BEELINE = ( | |
| "jdbc:hive2://localhost:10000/default;principal=" + HIVE_HS2_PRINCIPAL | |
| ) | |
| HIVE_TEST_USER = PLUGIN_TEST_USER | |
| HIVE_TEST_USER_PASSWORD = DOCKER_PLUGIN_TEST_USER_PASSWORD | |
| HIVE_AUTHZ_DENIED_SUBSTR = "Permission denied" | |
| HIVE_AUDIT_POLICY_SERVICE_USERS = ["hive"] | |
| HIVE_AUDIT_POLICY_ACCESS_TYPES = [ | |
| "select", | |
| "update", | |
| "create", | |
| "drop", | |
| "alter", | |
| "index", | |
| "lock", | |
| "all", | |
| ] | |
| POLICY_PROPAGATION_SLEEP_SEC = 15 | |
| _docker_client = None | |
| def get_docker_client(): | |
| global _docker_client | |
| if _docker_client is None: | |
| import docker | |
| _docker_client = docker.from_env() | |
| return _docker_client | |
| def _container_running(name): | |
| client = get_docker_client() | |
| try: | |
| container = client.containers.get(name) | |
| except Exception: | |
| return False | |
| container.reload() | |
| return container.status == "running" | |
| def _hs2_port_open(host="127.0.0.1", port=10000, timeout=2.0): | |
| try: | |
| with socket.create_connection((host, port), timeout=timeout): | |
| return True | |
| except OSError: | |
| return False | |
| def preflight_hive_stack(): | |
| """Skip tests when required Docker services or HS2 are unavailable.""" | |
| required = [ADMIN_CONTAINER, KDC_CONTAINER, HADOOP_CONTAINER, HIVE_CONTAINER] | |
| missing = [name for name in required if not _container_running(name)] | |
| if missing: | |
| pytest.skip( | |
| "Hive functional-tests need running containers: " | |
| + ", ".join(missing) | |
| + ". Run ./run-tests.sh postgres hive from functional-tests/." | |
| ) | |
| if not _hs2_port_open(): | |
| pytest.skip( | |
| "HiveServer2 is not accepting connections on localhost:10000 " | |
| "(is ranger-hive healthy?)" | |
| ) | |
| def _user_principal(username): | |
| return username + "@" + KERBEROS_REALM | |
| def _user_keytab_in_hive_container(username): | |
| return "/etc/keytabs/" + username + ".keytab" | |
| def _log_multiline_block(header, text, max_chars=BEELINE_LOG_OUTPUT_MAX_CHARS): | |
| """Write multi-line command output to pytest.log (one line per source line).""" | |
| body = text if text else "(empty)" | |
| if len(body) > max_chars: | |
| body = ( | |
| body[:max_chars] | |
| + "\n... [truncated, total " | |
| + str(len(text)) | |
| + " chars]" | |
| ) | |
| logger.info("%s", header) | |
| for line in body.splitlines(): | |
| logger.info(" %s", line) | |
| def _log_text_block(title, text, max_chars=BEELINE_COMPLETE_LOG_MAX_CHARS): | |
| """Log a full stdout/stderr block.""" | |
| logger.info("%s", title) | |
| body = text if text and text.strip() else "(empty)" | |
| if len(body) > max_chars: | |
| body = ( | |
| body[:max_chars] | |
| + "\n... [truncated, total " | |
| + str(len(text)) | |
| + " chars]" | |
| ) | |
| for line in body.splitlines(): | |
| logger.info("%s", line) | |
| def _log_beeline_run_detail(exit_code, stdout, stderr, combined, sql, beeline_cmd, expect_substring=None): | |
| """Log beeline query, command, exit code, stdout, and stderr.""" | |
| logger.info("Exit Code: %s", exit_code) | |
| _log_text_block("Output of beeline command is:", stdout) | |
| _log_text_block("Error of beeline command is:", stderr) | |
| if "Error:" in combined or "FAILED:" in combined: | |
| sql_result = "FAILED (server/beeline error in output)" | |
| elif exit_code != 0: | |
| sql_result = "FAILED (non-zero exit)" | |
| elif "INFO : OK" in combined or "No rows affected" in combined: | |
| sql_result = "SUCCESS" | |
| else: | |
| sql_result = "UNKNOWN" | |
| logger.info( | |
| "Beeline summary: exit_code=%s sql_result=%s user_query=%s", | |
| exit_code, | |
| sql_result, | |
| sql, | |
| ) | |
| logger.info("Beeline command: %s", beeline_cmd) | |
| if expect_substring: | |
| if expect_substring in combined: | |
| logger.info( | |
| 'Found expected string "%s" in beeline output', | |
| expect_substring, | |
| ) | |
| else: | |
| logger.info( | |
| 'Expected string "%s" NOT found in beeline output', | |
| expect_substring, | |
| ) | |
| def ensure_kerberos_user(username): | |
| """Create a user Kerberos principal and keytab on KDC, visible in ranger-hive.""" | |
| from ranger_test_utils.plugin_test_users import ( | |
| PLUGIN_KERBEROS_STACK_HIVE, | |
| ensure_kerberos_keytab_for_stack, | |
| ) | |
| ensure_kerberos_keytab_for_stack(username, PLUGIN_KERBEROS_STACK_HIVE) | |
| def verify_kerberos_keytab_in_hive(username): | |
| """kinit in ranger-hive using the user's keytab (fail fast if keytab is bad).""" | |
| hive = get_docker_client().containers.get(HIVE_CONTAINER) | |
| principal = _user_principal(username) | |
| keytab = _user_keytab_in_hive_container(username) | |
| shell = ( | |
| "kdestroy -A 2>/dev/null || true; " | |
| "kinit -kt " | |
| + keytab | |
| + " " | |
| + principal | |
| + "; klist" | |
| ) | |
| exit_code, output = hive.exec_run(["bash", "-c", shell], user="hive", demux=True) | |
| combined = ((output[0] or b"") + (output[1] or b"")).decode() | |
| if exit_code != 0: | |
| raise RuntimeError( | |
| "Kerberos kinit failed in " | |
| + HIVE_CONTAINER | |
| + " for " | |
| + principal | |
| + ": " | |
| + combined | |
| ) | |
| logger.info( | |
| "Verified Kerberos keytab for %s in %s: %s", | |
| principal, | |
| HIVE_CONTAINER, | |
| combined.strip().splitlines()[-1] if combined.strip() else "(ok)", | |
| ) | |
| def prepare_hive_test_namespace(database, table): | |
| """Create DB and ensure test table absent — runs as hive service user.""" | |
| run_beeline( | |
| None, | |
| "create database if not exists " + database + ";", | |
| service_user=True, | |
| ) | |
| run_beeline( | |
| None, | |
| "drop table if exists " + database + "." + table + ";", | |
| service_user=True, | |
| ) | |
| def cleanup_hive_test_namespace(database, table): | |
| """Drop test table and database — runs as hive service user.""" | |
| run_beeline( | |
| None, | |
| "drop table if exists " + database + "." + table + ";", | |
| service_user=True, | |
| ) | |
| run_beeline( | |
| None, | |
| "drop database if exists " + database + " cascade;", | |
| service_user=True, | |
| ) | |
| def run_beeline(username, sql, service_user=False, expect_substring=None): | |
| """ | |
| Run a beeline statement inside ranger-hive (-e). | |
| Returns (exit_code, combined_output). | |
| """ | |
| hive = get_docker_client().containers.get(HIVE_CONTAINER) | |
| if service_user: | |
| principal = HIVE_HS2_PRINCIPAL | |
| keytab = HIVE_SERVICE_KEYTAB | |
| ranger_user_label = "hive (service)" | |
| else: | |
| principal = _user_principal(username) | |
| keytab = _user_keytab_in_hive_container(username) | |
| ranger_user_label = username | |
| sql_stripped = sql.strip() | |
| logger.info("Going to run query is: %s", sql_stripped) | |
| logger.info( | |
| "Beeline context: container=%s docker_exec_user=hive ranger_user=%s", | |
| HIVE_CONTAINER, | |
| ranger_user_label, | |
| ) | |
| logger.info( | |
| "Kerberos: kdestroy -A; kinit -kt %s %s", | |
| keytab, | |
| principal, | |
| ) | |
| escaped_sql = sql_stripped.replace("\\", "\\\\").replace('"', '\\"') | |
| beeline_cmd = ( | |
| '/opt/hive/bin/beeline -u "' | |
| + HIVE_JDBC_BEELINE | |
| + '" --outputformat=tsv2 -e "' | |
| + escaped_sql | |
| + '"' | |
| ) | |
| logger.info("RUNNING: %s", beeline_cmd) | |
| # Clear shared hive-user credential cache so service-user and test-user runs | |
| # do not reuse each other's tickets (klist -s alone would skip kinit incorrectly). | |
| shell = ( | |
| "kdestroy -A 2>/dev/null || true; " | |
| "kinit -kt " | |
| + keytab | |
| + " " | |
| + principal | |
| + "; " | |
| + "kinit_rc=$?; " | |
| + 'if [ "$kinit_rc" -ne 0 ]; then exit "$kinit_rc"; fi; ' | |
| + beeline_cmd | |
| ) | |
| exit_code, output = hive.exec_run(["bash", "-c", shell], user="hive", demux=True) | |
| stdout = (output[0] or b"").decode() | |
| stderr = (output[1] or b"").decode() | |
| combined = stdout + stderr | |
| _log_beeline_run_detail( | |
| exit_code, | |
| stdout, | |
| stderr, | |
| combined, | |
| sql_stripped, | |
| beeline_cmd, | |
| expect_substring=expect_substring, | |
| ) | |
| return exit_code, combined | |
| def list_policies_for_service(session, service_name=HIVE_SERVICE_NAME): | |
| url = RANGER_ADMIN_BASE_URL + "/plugins/policies/service/name/" + service_name | |
| response = session.get(url) | |
| assert_http_ok(response, 200, "List policies for " + service_name, url) | |
| return response.json().get("policies") or [] | |
| def set_all_policies_enabled(session, enabled, service_name=HIVE_SERVICE_NAME): | |
| policies = list_policies_for_service(session, service_name) | |
| for policy in policies: | |
| policy_id = policy.get("id") | |
| if policy_id is None: | |
| continue | |
| policy["isEnabled"] = enabled | |
| put_url = RANGER_ADMIN_BASE_URL + "/plugins/policies/" + str(policy_id) | |
| response = session.put(put_url, json=policy) | |
| assert_http_ok( | |
| response, | |
| 200, | |
| "Set isEnabled=" + str(enabled) + " on policy " + str(policy_id), | |
| put_url, | |
| ) | |
| def build_audit_only_policy_payload(service_name, suffix): | |
| accesses = [ | |
| {"type": perm, "isAllowed": True} | |
| for perm in HIVE_AUDIT_POLICY_ACCESS_TYPES | |
| ] | |
| return { | |
| "isEnabled": True, | |
| "service": service_name, | |
| "name": "audit policy for hive - " + suffix, | |
| "policyType": 0, | |
| "policyPriority": 0, | |
| "description": "Hive audit policy for functional-tests", | |
| "isAuditEnabled": True, | |
| "resources": { | |
| "database": { | |
| "values": ["*"], | |
| "isExcludes": False, | |
| "isRecursive": False, | |
| }, | |
| "table": { | |
| "values": ["*"], | |
| "isExcludes": False, | |
| "isRecursive": False, | |
| }, | |
| "column": { | |
| "values": ["*"], | |
| "isExcludes": False, | |
| "isRecursive": False, | |
| }, | |
| }, | |
| "policyItems": [ | |
| { | |
| "accesses": accesses, | |
| "users": list(HIVE_AUDIT_POLICY_SERVICE_USERS), | |
| "groups": [], | |
| "conditions": [], | |
| "delegateAdmin": False, | |
| } | |
| ], | |
| "denyPolicyItems": [], | |
| "allowExceptions": [], | |
| "denyExceptions": [], | |
| "dataMaskPolicyItems": [], | |
| "rowFilterPolicyItems": [], | |
| "serviceType": "hive", | |
| "options": {}, | |
| "validitySchedules": [], | |
| "policyLabels": [""], | |
| "zoneName": "", | |
| } | |
| def create_audit_only_policy(session, suffix=None, service_name=HIVE_SERVICE_NAME): | |
| if suffix is None: | |
| suffix = unique_suffix() | |
| payload = build_audit_only_policy_payload(service_name, suffix) | |
| response = session.post(POLICY_URL, json=payload) | |
| assert_http_ok(response, (200, 201), "Create audit-only hive policy", POLICY_URL) | |
| body = response.json() | |
| policy_id = body.get("id") | |
| assert policy_id is not None, "Create policy response missing id: " + response.text | |
| return policy_id, suffix | |
| def delete_policies_by_id(session, policy_ids): | |
| for policy_id in policy_ids: | |
| url = POLICY_URL + str(policy_id) | |
| response = session.delete(url) | |
| assert_http_ok(response, (200, 204), "Delete policy " + str(policy_id), url) | |
| def wait_for_policy_propagation(seconds=POLICY_PROPAGATION_SLEEP_SEC): | |
| logger.info("Waiting %ss for Hive plugin policy propagation", seconds) | |
| time.sleep(seconds) | |
| def execute_hive_test( | |
| sql, | |
| username=HIVE_TEST_USER, | |
| *, | |
| admin_session=None, | |
| verify_output_contains=None, | |
| verify_output_not_contains=None, | |
| audit_filter=None, | |
| expect_audit_event=True, | |
| repo_name=HIVE_SERVICE_NAME, | |
| ): | |
| """ | |
| Run beeline as ``username``, assert on CLI output, optionally validate access audits. | |
| Uses :func:`ranger_test_utils.access_audit_utils.begin_access_audit_capture` | |
| for Solr/OpenSearch-backed audit checks (shared with other plugin suites). | |
| """ | |
| from ranger_test_utils.access_audit_utils import begin_access_audit_capture | |
| audit_capture = None | |
| if audit_filter is not None and admin_session is not None: | |
| audit_capture = begin_access_audit_capture( | |
| admin_session, | |
| repo_name, | |
| username, | |
| ) | |
| exit_code, output = run_beeline( | |
| username, | |
| sql, | |
| expect_substring=verify_output_contains, | |
| ) | |
| if verify_output_contains is not None: | |
| assert verify_output_contains in output, ( | |
| "Expected text not found in beeline output; exit_code=" | |
| + str(exit_code) | |
| + " output=" | |
| + output | |
| ) | |
| if verify_output_not_contains is not None: | |
| assert verify_output_not_contains not in output, ( | |
| "Unexpected text in beeline output; exit_code=" | |
| + str(exit_code) | |
| + " output=" | |
| + output | |
| ) | |
| if audit_capture is not None: | |
| audit_capture.verify( | |
| audit_filter, | |
| expect_audit_event=expect_audit_event, | |
| ) | |
| return exit_code, output | |
| class HivePluginTestContext: | |
| """ | |
| Per-module context (assign ``admin_session`` in ``setup_module``). | |
| Class attributes mirror common ranger-qe ``XAHiveTestCase`` fields used in ports. | |
| """ | |
| USER_USER1 = HIVE_TEST_USER | |
| USER_USER2 = "hrt_22" | |
| PASS_USER1 = None | |
| PASS_USER2 = None | |
| result_denied = "0" | |
| result_allowed = "1" | |
| access_type = { | |
| "CREATE": "CREATE", | |
| "USE": "USE", | |
| "DROP": "DROP", | |
| "SELECT": "SELECT", | |
| "UPDATE": "UPDATE", | |
| "ALTER": "ALTER", | |
| "INSERT": "INSERT", | |
| "DELETE": "DELETE", | |
| "GRANT": "GRANT", | |
| "REVOKE": "REVOKE", | |
| } | |
| def __init__(self): | |
| self.admin_session = None | |
| self.repo_name = HIVE_SERVICE_NAME | |
| class HivePluginTest: | |
| """One Hive enforcement scenario: log id, beeline SQL, verify output, optional audit.""" | |
| STDERR_CONTAINS = "stderr_contains" | |
| STDOUT_CONTAINS = "stdout_contains" | |
| STDERR_NOT_CONTAINS = "stderr_not_contains" | |
| STDOUT_NOT_CONTAINS = "stdout_not_contains" | |
| def __init__( | |
| self, | |
| test_id, | |
| test_case, | |
| sql, | |
| req_user, | |
| req_user_pass=None, | |
| *, | |
| verify_criteria=STDERR_CONTAINS, | |
| verify_str=None, | |
| new_audit_log_must_exist=True, | |
| audit_log_filter=None, | |
| validate_impala=False, | |
| verifyCriteria=None, | |
| verifyStr=None, | |
| newAuditLogMustExist=None, | |
| auditLogFilter=None, | |
| validateImpala=None, | |
| hSql=None, | |
| ): | |
| self.test_id = test_id | |
| self.test_case = test_case | |
| self.sql = hSql if hSql is not None else sql | |
| self.req_user = req_user | |
| self.req_user_pass = req_user_pass | |
| self.verify_criteria = ( | |
| verifyCriteria if verifyCriteria is not None else verify_criteria | |
| ) | |
| self.verify_str = ( | |
| verifyStr | |
| if verifyStr is not None | |
| else (verify_str if verify_str is not None else HIVE_AUTHZ_DENIED_SUBSTR) | |
| ) | |
| self.new_audit_log_must_exist = ( | |
| newAuditLogMustExist | |
| if newAuditLogMustExist is not None | |
| else new_audit_log_must_exist | |
| ) | |
| self.audit_log_filter = ( | |
| auditLogFilter if auditLogFilter is not None else audit_log_filter | |
| ) | |
| self.validate_impala = ( | |
| validateImpala if validateImpala is not None else validate_impala | |
| ) | |
| def execute(self): | |
| if self.validate_impala: | |
| pytest.skip("Impala validation is not enabled in Docker functional-tests") | |
| log_testcase_begin(plugin_enforcement_logger, self.test_id) | |
| verify_contains = None | |
| verify_not_contains = None | |
| if self.verify_criteria in ( | |
| self.STDERR_CONTAINS, | |
| self.STDOUT_CONTAINS, | |
| ): | |
| verify_contains = self.verify_str | |
| elif self.verify_criteria in ( | |
| self.STDERR_NOT_CONTAINS, | |
| self.STDOUT_NOT_CONTAINS, | |
| ): | |
| verify_not_contains = self.verify_str | |
| else: | |
| raise ValueError("Unsupported verify_criteria: " + str(self.verify_criteria)) | |
| admin_session = getattr(self.test_case, "admin_session", None) | |
| repo_name = getattr(self.test_case, "repo_name", HIVE_SERVICE_NAME) | |
| audit_filter = normalize_audit_log_filter(self.audit_log_filter) | |
| execute_hive_test( | |
| self.sql, | |
| username=self.req_user, | |
| admin_session=admin_session, | |
| verify_output_contains=verify_contains, | |
| verify_output_not_contains=verify_not_contains, | |
| audit_filter=audit_filter, | |
| expect_audit_event=self.new_audit_log_must_exist, | |
| repo_name=repo_name, | |
| ) |