blob: f72b448f431b6328f67e92a4b37c17422f60052d [file]
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import docker
import pytest
import time
from hdfs.test_config import (
CORE_SITE_XML_PATH,
HADOOP_CONTAINER,
HDFS_USER,
KMS_CONTAINER,
KMS_PROPERTY,
TEST_EZ_PATHS,
TEST_KMS_KEYS,
)
# Setup Docker Client
client = docker.from_env()
def _session_needs_encryption_setup(session):
"""Encryption/KMS session setup — skip for hdfs_policy-only runs."""
if not session.items:
return False
for item in session.items:
if item.get_closest_marker("hdfs_policy") is None:
return True
return False
@pytest.fixture(scope="session")
def hadoop_container():
container = client.containers.get(HADOOP_CONTAINER)
return container
def ensure_key_provider_and_simple_auth(container) -> bool:
"""
Ensures:
1) KMS provider property exists
2) hadoop.security.authentication = simple
Returns True if the file was modified.
"""
changed = False
# 1) Ensure KMS provider property exists
exit_code, _ = container.exec_run(
f"grep -q 'hadoop.security.key.provider.path' {CORE_SITE_XML_PATH}",
user="root",
)
if exit_code != 0:
container.exec_run(
f"sed -i '/<\\/configuration>/i {KMS_PROPERTY}' {CORE_SITE_XML_PATH}",
user="root",
)
changed = True
# 2) Fix stale host.docker.internal from previous runs
exit_code, _ = container.exec_run(
f"grep -q 'host.docker.internal' {CORE_SITE_XML_PATH}",
user="root",
)
if exit_code == 0:
container.exec_run(
f"sed -i 's|host.docker.internal|ranger-kms.rangernw|g' {CORE_SITE_XML_PATH}",
user="root",
)
changed = True
return changed
def ensure_user_exists(container, username: str) -> None:
exit_code, _ = container.exec_run(f"id -u {username}", user="root")
if exit_code == 0:
return
container.exec_run(f"useradd -m -s /bin/bash {username}", user="root")
container.exec_run(f"usermod -aG hadoop {username}", user="root")
def ensure_kms_provider_configured(container) -> bool:
"""
Ensures KMS provider property exists in core-site.xml.
Returns True if the file was modified (restart needed).
"""
changed = False
# 1) Ensure KMS provider property exists
exit_code, _ = container.exec_run(
f"grep -q 'hadoop.security.key.provider.path' {CORE_SITE_XML_PATH}",
user="root",
)
if exit_code != 0:
container.exec_run(
f"sed -i '/<\\/configuration>/i {KMS_PROPERTY}' {CORE_SITE_XML_PATH}",
user="root",
)
changed = True
# 2) Fix stale host.docker.internal from previous runs
exit_code, _ = container.exec_run(
f"grep -q 'host.docker.internal' {CORE_SITE_XML_PATH}",
user="root",
)
if exit_code == 0:
container.exec_run(
f"sed -i 's|host.docker.internal|ranger-kms.rangernw|g' {CORE_SITE_XML_PATH}",
user="root",
)
changed = True
return changed
@pytest.fixture(scope="session", autouse=True)
def setup_encryption_environment(request):
if not _session_needs_encryption_setup(request.session):
yield
return
hadoop_container = client.containers.get(HADOOP_CONTAINER)
from hdfs.utils import (
cleanup_test_artifacts,
ensure_hadoop_user_keytab,
ensure_kms_hdfs_policy,
ensure_kms_kerberos_rules,
)
from kms.utils import ensure_keyadmin_keytab, ensure_ticket
changed = ensure_kms_provider_configured(hadoop_container)
if changed:
hadoop_container.restart()
time.sleep(30)
ensure_user_exists(hadoop_container, "hive")
ensure_user_exists(hadoop_container, "hbase")
ensure_hadoop_user_keytab("hive")
ensure_hadoop_user_keytab("hbase")
# safemode leave with kerberos (hdfs principal already has keytab in container)
hadoop_container.exec_run(
[
"bash",
"-c",
"kinit -kt /etc/keytabs/hdfs.keytab hdfs/ranger-hadoop.rangernw@EXAMPLE.COM 2>/dev/null;"
"hdfs dfsadmin -safemode leave",
],
user=HDFS_USER,
)
kms_container = client.containers.get(KMS_CONTAINER)
ensure_kms_kerberos_rules(kms_container)
ensure_keyadmin_keytab()
ensure_ticket()
ensure_kms_hdfs_policy()
cleanup_test_artifacts(hadoop_container, TEST_KMS_KEYS, TEST_EZ_PATHS)
yield
cleanup_test_artifacts(hadoop_container, TEST_KMS_KEYS, TEST_EZ_PATHS)