blob: bffe31dcefe6cac66abaf484270e31ff7a4e30c4 [file]
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
import pytest
import docker
import requests
from hdfs.test_config import (KMS_CONTAINER, HADOOP_NAMENODE_LOG_PATH, KMS_LOG_PATH)
from kms.utils import (
BASE_URL,
PARAMS,
ensure_ticket,
get_kms_container,
krb_requests,
)
from hdfs.test_config import HDFS_USER
import subprocess
import tempfile
import os
# Setup Docker Client
client = docker.from_env()
RANGER_AUTH = ("keyadmin", "rangerR0cks!")
RANGER_ADMIN_AUTH = ("admin", "rangerR0cks!")
RANGER_POLICY_URL = "http://localhost:6080/service/public/v2/api/policy"
RANGER_SERVICE_POLICY_URL = "http://localhost:6080/service/public/v2/api/service/dev_kms/policy"
RANGER_USERS_URL = "http://localhost:6080/service/xusers/secure/users"
RANGER_USER_BY_NAME_URL = "http://localhost:6080/service/xusers/users/userName"
DEFAULT_KMS_POLICY_NAME = "all - keyname"
HDFS_KMS_USERS = ["keyadmin", "nn", "hdfs", "hive", "hbase", "dn"]
HDFS_KMS_ACCESSES = [
"CREATE", "DELETE", "ROLLOVER", "GET", "GETKEYS", "GETMETADATA",
"GENERATEEEK", "DECRYPTEEK", "SETKEYMATERIAL",
]
KMS_HOME = "/opt/ranger/kms"
KMS_SITE_PATH = f"{KMS_HOME}/ews/webapp/WEB-INF/classes/conf/kms-site.xml"
HADOOP_CONTAINER = "ranger-hadoop"
HIVE_PRINCIPAL = "hive/ranger-hadoop.rangernw@EXAMPLE.COM"
HIVE_KEYTAB = "/etc/keytabs/hive.keytab"
KDC_CONTAINER = "ranger-kdc"
HDFS_PRINCIPAL = "hdfs/ranger-hadoop.rangernw@EXAMPLE.COM"
HDFS_KEYTAB = "/etc/keytabs/hdfs.keytab"
HBASE_PRINCIPAL = "hbase/ranger-hadoop.rangernw@EXAMPLE.COM"
HBASE_KEYTAB = "/etc/keytabs/hbase.keytab"
# Map each OS user to their Kerberos principal and keytab.
# Defined here (near the constants) so run_command can reference it clearly.
_KERBEROS_CREDENTIALS = {
"hdfs": (HDFS_PRINCIPAL, HDFS_KEYTAB),
"hive": (HIVE_PRINCIPAL, HIVE_KEYTAB),
"hbase": (HBASE_PRINCIPAL, HBASE_KEYTAB),
}
#to run all HDFS commands
def run_command(container, cmd, user, fail_on_error=True, return_exit_code=False):
# For string commands on kerberos-capable users, wrap with kinit automatically.
# klist -s checks for an existing valid ticket first — skips kinit when one is present,
# avoiding redundant re-authentication on every command call.
# List commands (e.g. from run_kerberos_command) are passed through unchanged.
if isinstance(cmd, str) and user in _KERBEROS_CREDENTIALS:
principal, keytab = _KERBEROS_CREDENTIALS[user]
actual_cmd = ["bash", "-c",
f"klist -s 2>/dev/null || kinit -kt {keytab} {principal} 2>/dev/null; {cmd}"]
else:
actual_cmd = cmd
# demux=True separates stdout and stderr — prevents hadoop usage text (stderr)
# from leaking into the captured output returned to callers and printed in tests.
exit_code, (stdout, stderr) = container.exec_run(actual_cmd, user=user, demux=True)
stdout_text = (stdout or b"").decode()
stderr_text = (stderr or b"").decode()
# Negative/failure-detection tests (return_exit_code=True) need stderr to validate
# the error type (e.g. Permission denied vs auth failure). Positive tests get stdout
# only, which keeps hadoop usage dumps from polluting the console.
output_response = stdout_text if not return_exit_code else stdout_text + stderr_text
if exit_code != 0 and fail_on_error:
kms_container = client.containers.get(KMS_CONTAINER)
hadoop_logs, kms_logs = get_error_logs(container, kms_container)
pytest.fail(f"""
Command failed: {cmd}
Exit Code: {exit_code}
Stdout:
{stdout_text}
Stderr:
{stderr_text}
Hadoop Container Logs:
{hadoop_logs}
KMS Container Logs:
{kms_logs}
""")
if return_exit_code:
return output_response, exit_code
return output_response
#fetch logs from hadoop and KMS file
def get_error_logs(hadoop_container, kms_container):
# Get Hadoop NameNode logs
hadoop_log_cmd = f"tail -n 50 {HADOOP_NAMENODE_LOG_PATH}"
_, hadoop_logs = hadoop_container.exec_run(hadoop_log_cmd, user='hdfs')
hadoop_logs_decoded = hadoop_logs.decode()
hadoop_error_lines = [line for line in hadoop_logs_decoded.split("\n") if "ERROR" in line or "Exception" in line or "WARN" in line]
hadoop_error_text = "\n".join(hadoop_error_lines) if hadoop_error_lines else "No recent errors in Hadoop Namenode logs."
# Get KMS logs
kms_log_cmd = f"tail -n 50 {KMS_LOG_PATH}"
_, kms_logs = kms_container.exec_run(kms_log_cmd, user='root')
kms_logs_decoded = kms_logs.decode()
kms_error_lines = [line for line in kms_logs_decoded.split("\n") if "ERROR" in line or "Exception" in line or "WARN" in line]
kms_error_text = "\n".join(kms_error_lines) if kms_error_lines else "No recent errors in KMS logs."
return hadoop_error_text, kms_error_text
def _ensure_kdc_service_principal(username: str, service: str = "ranger-hadoop") -> str:
# Create service principal in KDC and copy keytab to ranger-hadoop.
principal = f"{username}/{service}.rangernw@EXAMPLE.COM"
keytab_name = f"{username}.keytab"
dest_path = f"/etc/keytabs/{keytab_name}"
kdc_tmp = f"/tmp/{keytab_name}"
kdc = client.containers.get(KDC_CONTAINER)
hadoop = client.containers.get(HADOOP_CONTAINER)
exit_code, output = kdc.exec_run(f'kadmin.local -q "getprinc {principal}"', user="root")
if exit_code != 0 or b"does not exist" in output:
ec, out = kdc.exec_run(f'kadmin.local -q "addprinc -randkey {principal}"', user="root")
if ec != 0:
raise RuntimeError(f"addprinc failed for {principal}: {out.decode()}")
exit_code, output = kdc.exec_run(
f'kadmin.local -q "xst -k {kdc_tmp} {principal}"', user="root"
)
if exit_code != 0:
raise RuntimeError(f"keytab export failed for {principal}: {output.decode()}")
local = os.path.join(tempfile.gettempdir(), keytab_name)
subprocess.check_call(f"docker cp {KDC_CONTAINER}:{kdc_tmp} {local}", shell=True)
subprocess.check_call(f"docker cp {local} {HADOOP_CONTAINER}:{dest_path}", shell=True)
hadoop.exec_run(f"chmod 444 {dest_path}", user="root")
hadoop.exec_run(f"chown {username}:{username} {dest_path}", user="root")
os.remove(local)
return principal
def ensure_hadoop_user_keytab(username: str) -> None:
_ensure_kdc_service_principal(username)
def run_kerberos_command(container, cmd, user, principal, keytab, **kwargs):
# Use klist -s to skip kinit when a valid ticket already exists for this user.
shell = (
f"klist -s 2>/dev/null || kinit -kt {keytab} {principal} 2>/dev/null; "
f'HADOOP_OPTS="-Dhadoop.security.authentication=kerberos" {cmd}'
)
return run_command(container, ["bash", "-c", shell], user, **kwargs)
def _ranger_policies_from_response(data):
if isinstance(data, list):
return data
return data.get("policies", [])
def ensure_ranger_user(username: str) -> None:
# Create a Ranger xuser if missing (required before adding to policies).
resp = requests.get(
f"{RANGER_USER_BY_NAME_URL}/{username}",
auth=RANGER_ADMIN_AUTH,
timeout=30,
)
if resp.status_code == 200:
return
payload = {
"name": username,
"firstName": username,
"lastName": "User",
"password": "Password123!",
"description": "hdfs encryption test user",
"status": 1,
"isVisible": 1,
"userSource": 0,
"userRoleList": ["ROLE_USER"],
}
resp = requests.post(RANGER_USERS_URL, auth=RANGER_ADMIN_AUTH, json=payload, timeout=30)
if resp.status_code not in (200, 201):
print(f"WARN: create Ranger user {username}: {resp.status_code} {resp.text}")
def _wait_for_kms_up(kms_container, timeout=180) -> None:
# Block until the KMS port accepts connections inside the container.
deadline = time.time() + timeout
while time.time() < deadline:
kms_container.reload()
if kms_container.status == "running":
ec, _ = kms_container.exec_run(
["bash", "-c", "exec 3<>/dev/tcp/localhost/9292"], user="root"
)
if ec == 0:
return
time.sleep(3)
pytest.fail(f"ranger-kms did not come up within {timeout}s")
def ensure_kms_kerberos_rules(kms_container) -> bool:
# Patch KMS kerberos name rules at runtime. Returns True if KMS was restarted.
kms_container.reload()
if kms_container.status != "running":
kms_container.start()
_wait_for_kms_up(kms_container)
exit_code, _ = kms_container.exec_run(
f"grep -q 'hive/ranger-hadoop' {KMS_SITE_PATH}", user="root"
)
if exit_code == 0:
return False
rules = [
# [ndj]n already covers nn, dn, jn — no separate dn rule needed
"RULE:[2:$1/$2@$0]([ndj]n/ranger-hadoop\\.rangernw@EXAMPLE\\.COM)s/.*/keyadmin/",
"RULE:[2:$1/$2@$0](hive/ranger-hadoop\\.rangernw@EXAMPLE\\.COM)s/.*/hive/",
]
for rule in rules:
kms_container.exec_run(f"sed -i '/DEFAULT/i {rule}' {KMS_SITE_PATH}", user="root")
kms_container.restart()
_wait_for_kms_up(kms_container)
print("Patched KMS kerberos rules, restarted ranger-kms container.")
return True
def ensure_kms_hdfs_policy() -> None:
# Grant HDFS principals KMS access by updating the default wildcard policy.
# Ensure every user in HDFS_KMS_USERS exists in Ranger first — service accounts
# like 'nn' (NameNode) and 'dn' (DataNode) are not auto-created in Ranger admin
# and will cause a 400 if added to a policy without being registered first.
for username in HDFS_KMS_USERS:
ensure_ranger_user(username)
resp = requests.get(RANGER_SERVICE_POLICY_URL, auth=RANGER_AUTH, timeout=30)
if resp.status_code != 200:
print(f"WARN: could not fetch KMS policies: {resp.status_code} {resp.text}")
return
policies = _ranger_policies_from_response(resp.json())
target = None
for policy in policies:
name = policy.get("name") or policy.get("policyName", "")
keyname_values = policy.get("resources", {}).get("keyname", {}).get("values", [])
if name == DEFAULT_KMS_POLICY_NAME or keyname_values == ["*"]:
target = policy
break
if target is None:
print("WARN: default KMS wildcard policy not found, skipping update.")
return
policy_items = target.get("policyItems") or [{"accesses": [], "users": []}]
if not policy_items:
policy_items = [{"accesses": [], "users": []}]
existing_users = set(policy_items[0].get("users") or [])
needed_users = set(HDFS_KMS_USERS)
# Ranger returns access types in lowercase; compare case-insensitively
existing_access = {a["type"].lower() for a in policy_items[0].get("accesses", [])}
needed_access = {a.lower() for a in HDFS_KMS_ACCESSES}
if needed_users.issubset(existing_users) and needed_access.issubset(existing_access):
print("KMS policy already grants required HDFS users and accesses.")
return
policy_items[0]["users"] = sorted(existing_users | needed_users)
for access_type in HDFS_KMS_ACCESSES:
if access_type.lower() not in existing_access:
policy_items[0].setdefault("accesses", []).append(
{"type": access_type.lower(), "isAllowed": True}
)
target["policyItems"] = policy_items
policy_id = target["id"]
resp = requests.put(
f"{RANGER_POLICY_URL}/{policy_id}",
auth=RANGER_AUTH,
json=target,
timeout=30,
)
if resp.status_code not in (200, 201):
print(f"WARN: KMS policy update failed: {resp.status_code} {resp.text}")
return
print(f"Updated KMS policy {target.get('name', policy_id)}, waiting for sync...")
time.sleep(30)
def _kms_key_missing(resp) -> bool:
return "does not exist" in resp.text.lower()
def delete_kms_key(key_name: str) -> None:
# Delete KMS key if present. Ignores missing-key responses.
# Only refresh the keyadmin ticket when no valid ticket exists — avoids a full
# kdestroy+kinit on every key deletion (e.g. 6 calls in cleanup_test_artifacts).
ec, _ = get_kms_container().exec_run("klist -s 2>/dev/null", user="root")
if ec != 0:
ensure_ticket()
resp = krb_requests.delete(f"{BASE_URL}/key/{key_name}", params=PARAMS)
if resp.status_code in (200, 404):
return
if resp.status_code in (403, 500) and _kms_key_missing(resp):
return
if resp.status_code == 403:
print(f"WARN: delete key {key_name}: {resp.status_code} {resp.text}")
return
def ensure_kms_key(key_name: str, cipher="AES/CTR/NoPadding", length=128) -> None:
# Idempotent: remove stale key, then create fresh.
delete_kms_key(key_name)
resp = krb_requests.post(
f"{BASE_URL}/keys",
json={"name": key_name, "cipher": cipher, "length": length},
params=PARAMS,
)
assert resp.status_code == 201, f"Key creation failed: {resp.text}"
def delete_hdfs_path(container, path: str) -> None:
# Remove HDFS path if present. Uses klist -s check before kinit.
# demux=True so errors appear in stderr (not mixed into stdout).
shell = (
f"klist -s 2>/dev/null || kinit -kt {HDFS_KEYTAB} {HDFS_PRINCIPAL} 2>/dev/null; "
f"hdfs dfs -rm -R -f {path}"
)
exit_code, (stdout, stderr) = container.exec_run(
["bash", "-c", shell], user=HDFS_USER, demux=True
)
if exit_code != 0:
stderr_text = (stderr or b"").decode()
stdout_text = (stdout or b"").decode()
print(f"WARN: delete_hdfs_path {path} failed (exit {exit_code}): {stderr_text or stdout_text}")
def cleanup_encryption_zone(container, ez_path: str) -> None:
# Remove EZ directory (Must run before deleting bound KMS key)
delete_hdfs_path(container, ez_path)
def cleanup_test_artifacts(container, keys: list, ez_paths: list) -> None:
#Full cleanup: EZ paths first, then KMS keys.
for path in ez_paths:
cleanup_encryption_zone(container, path)
for key in keys:
delete_kms_key(key)
def create_encryption_zone(container, ez_name: str, key_name: str) -> None:
#Idempotent EZ setup: clean path, mkdir, createZone.
ez_path = f"/{ez_name}"
cleanup_encryption_zone(container, ez_path)
run_command(container, f"hdfs dfs -mkdir {ez_path}", HDFS_USER)
run_command(container, f"hdfs crypto -createZone -keyName {key_name} -path {ez_path}", HDFS_USER)