blob: ba3fbf91c6a8bd517edc9c0b31db17334e913fca [file]
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""HBase Docker helpers and declarative plugin scenarios (shell, policies, ``HBasePluginTest``)."""
import base64
import socket
import time
import pytest
from ranger_test_utils.plugin_enforcement_utils import (
logger as plugin_enforcement_logger,
normalize_audit_log_filter,
)
from ranger_test_utils.utils import (
POLICY_URL,
assert_http_ok,
get_test_logger,
log_testcase_begin,
unique_suffix,
)
from ranger_test_utils.hive_utils import (
KDC_CONTAINER,
POLICY_PROPAGATION_SLEEP_SEC,
get_docker_client,
)
from ranger_test_utils.plugin_test_users import PLUGIN_TEST_USER
logger = get_test_logger(__name__)
HBASE_SHELL_LOG_OUTPUT_MAX_CHARS = 12000
HBASE_SERVICE_NAME = "dev_hbase"
HBASE_CONTAINER = "ranger-hbase"
HADOOP_CONTAINER = "ranger-hadoop"
ADMIN_CONTAINER = "ranger"
KERBEROS_REALM = "EXAMPLE.COM"
HBASE_SERVICE_PRINCIPAL = "hbase/ranger-hbase.rangernw@" + KERBEROS_REALM
HBASE_SERVICE_KEYTAB = "/etc/keytabs/hbase.keytab"
HBASE_TEST_USER = PLUGIN_TEST_USER
HBASE_AUTHZ_DENIED_SUBSTR = "AccessDeniedException"
HBASE_AUDIT_POLICY_SERVICE_USERS = ["hbase"]
HBASE_AUDIT_POLICY_ACCESS_TYPES = ["create", "read", "write", "admin"]
def _container_running(name):
client = get_docker_client()
try:
container = client.containers.get(name)
except Exception:
return False
container.reload()
return container.status == "running"
def _master_ui_port_open(host="127.0.0.1", port=16010, timeout=2.0):
try:
with socket.create_connection((host, port), timeout=timeout):
return True
except OSError:
return False
def preflight_hbase_stack():
"""Skip tests when required Docker services or HBase master UI are unavailable."""
required = [
ADMIN_CONTAINER,
KDC_CONTAINER,
HADOOP_CONTAINER,
HBASE_CONTAINER,
]
missing = [name for name in required if not _container_running(name)]
if missing:
pytest.skip(
"HBase functional-tests need running containers: "
+ ", ".join(missing)
+ ". Run ./run-tests.sh postgres hbase from functional-tests/."
)
if not _master_ui_port_open():
pytest.skip(
"HBase master UI is not accepting connections on localhost:16010 "
"(is ranger-hbase healthy?)"
)
def _user_principal(username):
return username + "@" + KERBEROS_REALM
def _user_keytab_in_hbase_container(username):
return "/etc/keytabs/" + username + ".keytab"
def _log_multiline_block(header, text, max_chars=HBASE_SHELL_LOG_OUTPUT_MAX_CHARS):
body = text if text else "(empty)"
if len(body) > max_chars:
body = (
body[:max_chars]
+ "\n... [truncated, total "
+ str(len(text))
+ " chars]"
)
logger.info("%s", header)
for line in body.splitlines():
logger.info(" %s", line)
def ensure_kerberos_user(username):
"""Create a user Kerberos principal and keytab on KDC, visible in ranger-hbase."""
from ranger_test_utils.plugin_test_users import (
PLUGIN_KERBEROS_STACK_HBASE,
ensure_kerberos_keytab_for_stack,
)
ensure_kerberos_keytab_for_stack(username, PLUGIN_KERBEROS_STACK_HBASE)
def verify_kerberos_keytab_in_hbase(username):
"""kinit in ranger-hbase using the user's keytab (fail fast if keytab is bad)."""
hbase = get_docker_client().containers.get(HBASE_CONTAINER)
principal = _user_principal(username)
keytab = _user_keytab_in_hbase_container(username)
shell = (
"kdestroy -A 2>/dev/null || true; "
"kinit -kt "
+ keytab
+ " "
+ principal
+ "; klist"
)
exit_code, output = hbase.exec_run(["bash", "-c", shell], user="hbase", demux=True)
combined = ((output[0] or b"") + (output[1] or b"")).decode()
if exit_code != 0:
raise RuntimeError(
"Kerberos kinit failed in "
+ HBASE_CONTAINER
+ " for "
+ principal
+ ": "
+ combined
)
logger.info(
"Verified Kerberos keytab for %s in %s",
principal,
HBASE_CONTAINER,
)
def run_hbase_shell(username, shell_lines, service_user=False, expect_substring=None):
"""
Run HBase shell commands inside ranger-hbase (stdin).
Returns (exit_code, combined_output).
"""
hbase = get_docker_client().containers.get(HBASE_CONTAINER)
if service_user:
principal = HBASE_SERVICE_PRINCIPAL
keytab = HBASE_SERVICE_KEYTAB
ranger_user_label = "hbase (service)"
else:
principal = _user_principal(username)
keytab = _user_keytab_in_hbase_container(username)
ranger_user_label = username
script_body = "\n".join(shell_lines) + "\n"
logger.info("Going to run HBase shell script:\n%s", script_body)
logger.info(
"HBase shell context: container=%s docker_exec_user=hbase ranger_user=%s",
HBASE_CONTAINER,
ranger_user_label,
)
encoded = base64.b64encode(script_body.encode("utf-8")).decode("ascii")
hbase_cmd = "echo " + encoded + " | base64 -d | /opt/hbase/bin/hbase shell"
shell = (
"kdestroy -A 2>/dev/null || true; "
"kinit -kt "
+ keytab
+ " "
+ principal
+ "; "
+ "kinit_rc=$?; "
+ 'if [ "$kinit_rc" -ne 0 ]; then exit "$kinit_rc"; fi; '
+ hbase_cmd
)
exit_code, output = hbase.exec_run(["bash", "-c", shell], user="hbase", demux=True)
stdout = (output[0] or b"").decode()
stderr = (output[1] or b"").decode()
combined = stdout + stderr
_log_multiline_block("HBase shell exit_code=%s stdout:" % exit_code, stdout)
_log_multiline_block("HBase shell stderr:", stderr)
if expect_substring:
if expect_substring in combined:
logger.info(
'Found expected string "%s" in HBase shell output',
expect_substring,
)
else:
logger.info(
'Expected string "%s" NOT found in HBase shell output',
expect_substring,
)
return exit_code, combined
def prepare_hbase_test_table(table_name):
"""Ensure test table is absent — runs as HBase service user."""
run_hbase_shell(
None,
[
"if exists '" + table_name + "'",
" disable '" + table_name + "'",
" drop '" + table_name + "'",
"end",
],
service_user=True,
)
def cleanup_hbase_test_table(table_name):
"""Drop test table if present — runs as HBase service user."""
run_hbase_shell(
None,
[
"if exists '" + table_name + "'",
" disable '" + table_name + "'",
" drop '" + table_name + "'",
"end",
],
service_user=True,
)
def build_audit_only_policy_payload(service_name, suffix):
accesses = [
{"type": perm, "isAllowed": True}
for perm in HBASE_AUDIT_POLICY_ACCESS_TYPES
]
return {
"isEnabled": True,
"service": service_name,
"name": "audit policy for hbase - " + suffix,
"policyType": 0,
"policyPriority": 0,
"description": "HBase audit policy for functional-tests",
"isAuditEnabled": True,
"resources": {
"table": {
"values": ["*"],
"isExcludes": False,
"isRecursive": False,
},
"column-family": {
"values": ["*"],
"isExcludes": False,
"isRecursive": False,
},
"column": {
"values": ["*"],
"isExcludes": False,
"isRecursive": False,
},
},
"policyItems": [
{
"accesses": accesses,
"users": list(HBASE_AUDIT_POLICY_SERVICE_USERS),
"groups": [],
"conditions": [],
"delegateAdmin": False,
}
],
"denyPolicyItems": [],
"allowExceptions": [],
"denyExceptions": [],
"dataMaskPolicyItems": [],
"rowFilterPolicyItems": [],
"serviceType": "hbase",
"options": {},
"validitySchedules": [],
"policyLabels": [""],
"zoneName": "",
}
def create_audit_only_policy(session, suffix=None, service_name=HBASE_SERVICE_NAME):
if suffix is None:
suffix = unique_suffix()
payload = build_audit_only_policy_payload(service_name, suffix)
response = session.post(POLICY_URL, json=payload)
assert_http_ok(response, (200, 201), "Create audit-only hbase policy", POLICY_URL)
body = response.json()
policy_id = body.get("id")
assert policy_id is not None, "Create policy response missing id: " + response.text
return policy_id, suffix
def wait_for_hbase_policy_propagation(seconds=POLICY_PROPAGATION_SLEEP_SEC):
logger.info("Waiting %ss for HBase plugin policy propagation", seconds)
time.sleep(seconds)
def execute_hbase_test(
shell_lines,
username=HBASE_TEST_USER,
*,
admin_session=None,
verify_output_contains=None,
verify_output_not_contains=None,
audit_filter=None,
expect_audit_event=True,
repo_name=HBASE_SERVICE_NAME,
service_user=False,
):
"""
Run HBase shell as ``username``, assert on output, optionally validate access audits.
Uses :func:`ranger_test_utils.access_audit_utils.begin_access_audit_capture`
(Solr/OpenSearch via Admin REST when the audit pipeline is enabled).
"""
from ranger_test_utils.access_audit_utils import begin_access_audit_capture
if isinstance(shell_lines, str):
shell_lines = [shell_lines]
audit_capture = None
if (
audit_filter is not None
and admin_session is not None
and not service_user
):
audit_capture = begin_access_audit_capture(
admin_session,
repo_name,
username,
)
exit_code, output = run_hbase_shell(
username,
shell_lines,
service_user=service_user,
expect_substring=verify_output_contains,
)
if verify_output_contains is not None:
assert verify_output_contains in output, (
"Expected text not found in HBase shell output; exit_code="
+ str(exit_code)
+ " output="
+ output
)
if verify_output_not_contains is not None:
assert verify_output_not_contains not in output, (
"Unexpected text in HBase shell output; exit_code="
+ str(exit_code)
+ " output="
+ output
)
if audit_capture is not None:
audit_capture.verify(
audit_filter,
expect_audit_event=expect_audit_event,
)
return exit_code, output
class HBasePluginTestContext:
"""
Per-module context (assign ``admin_session`` in ``setup_module``).
Class attributes mirror common ranger-qe ``XAHBaseTestCase`` fields used in ports.
"""
USER_USER1 = HBASE_TEST_USER
USER_USER2 = "hrt_22"
result_denied = "0"
result_allowed = "1"
access_type = {
"CREATE_TABLE": "createTable",
"PUT": "put",
"SCAN": "scannerOpen",
"GET": "get",
"DISABLE_TABLE": "disableTable",
"DELETE_TABLE": "deleteTable",
"GRANT": "grant",
"REVOKE": "revoke",
}
def __init__(self):
self.admin_session = None
self.repo_name = HBASE_SERVICE_NAME
class HBasePluginTest:
"""One HBase enforcement scenario: log id, shell cmds, verify output, optional audit."""
STDERR_CONTAINS = "stderr_contains"
STDOUT_CONTAINS = "stdout_contains"
STDERR_NOT_CONTAINS = "stderr_not_contains"
STDOUT_NOT_CONTAINS = "stdout_not_contains"
def __init__(
self,
test_id,
test_case,
shell_cmds=None,
req_user=None,
*,
verify_criteria=STDOUT_CONTAINS,
verify_str=None,
new_audit_log_must_exist=True,
audit_log_filter=None,
logoutput=True,
verifyCriteria=None,
verifyStr=None,
newAuditLogMustExist=None,
auditLogFilter=None,
hBaseCmds=None,
reqUser=None,
checkAuditLog=None,
):
self.test_id = test_id
self.test_case = test_case
if hBaseCmds is not None:
self.shell_cmds = hBaseCmds
elif shell_cmds is not None:
self.shell_cmds = shell_cmds
else:
raise ValueError("Provide shell_cmds or hBaseCmds")
self.req_user = reqUser if reqUser is not None else req_user
if self.req_user is None:
raise ValueError("Provide req_user or reqUser")
self.logoutput = logoutput
self.verify_criteria = (
verifyCriteria if verifyCriteria is not None else verify_criteria
)
self.verify_str = (
verifyStr
if verifyStr is not None
else (verify_str if verify_str is not None else HBASE_AUTHZ_DENIED_SUBSTR)
)
self.new_audit_log_must_exist = (
newAuditLogMustExist
if newAuditLogMustExist is not None
else new_audit_log_must_exist
)
self.audit_log_filter = (
auditLogFilter if auditLogFilter is not None else audit_log_filter
)
_ = checkAuditLog
def execute(self):
log_testcase_begin(plugin_enforcement_logger, self.test_id)
verify_contains = None
verify_not_contains = None
if self.verify_criteria in (
self.STDERR_CONTAINS,
self.STDOUT_CONTAINS,
):
verify_contains = self.verify_str
elif self.verify_criteria in (
self.STDERR_NOT_CONTAINS,
self.STDOUT_NOT_CONTAINS,
):
verify_not_contains = self.verify_str
else:
raise ValueError("Unsupported verify_criteria: " + str(self.verify_criteria))
admin_session = getattr(self.test_case, "admin_session", None)
repo_name = getattr(self.test_case, "repo_name", HBASE_SERVICE_NAME)
audit_filter = normalize_audit_log_filter(self.audit_log_filter)
execute_hbase_test(
self.shell_cmds,
username=self.req_user,
admin_session=admin_session,
verify_output_contains=verify_contains,
verify_output_not_contains=verify_not_contains,
audit_filter=audit_filter,
expect_audit_event=self.new_audit_log_must_exist,
repo_name=repo_name,
)