| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| import docker | |
| import pytest | |
| import time | |
| from hdfs.test_config import ( | |
| CORE_SITE_XML_PATH, | |
| HADOOP_CONTAINER, | |
| HDFS_USER, | |
| KMS_CONTAINER, | |
| KMS_PROPERTY, | |
| TEST_EZ_PATHS, | |
| TEST_KMS_KEYS, | |
| ) | |
| # Setup Docker Client | |
| client = docker.from_env() | |
| def _session_needs_encryption_setup(session): | |
| """Encryption/KMS session setup — skip for hdfs_policy-only runs.""" | |
| if not session.items: | |
| return False | |
| for item in session.items: | |
| if item.get_closest_marker("hdfs_policy") is None: | |
| return True | |
| return False | |
| @pytest.fixture(scope="session") | |
| def hadoop_container(): | |
| container = client.containers.get(HADOOP_CONTAINER) | |
| return container | |
| def ensure_key_provider_and_simple_auth(container) -> bool: | |
| """ | |
| Ensures: | |
| 1) KMS provider property exists | |
| 2) hadoop.security.authentication = simple | |
| Returns True if the file was modified. | |
| """ | |
| changed = False | |
| # 1) Ensure KMS provider property exists | |
| exit_code, _ = container.exec_run( | |
| f"grep -q 'hadoop.security.key.provider.path' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| if exit_code != 0: | |
| container.exec_run( | |
| f"sed -i '/<\\/configuration>/i {KMS_PROPERTY}' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| changed = True | |
| # 2) Fix stale host.docker.internal from previous runs | |
| exit_code, _ = container.exec_run( | |
| f"grep -q 'host.docker.internal' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| if exit_code == 0: | |
| container.exec_run( | |
| f"sed -i 's|host.docker.internal|ranger-kms.rangernw|g' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| changed = True | |
| return changed | |
| def ensure_user_exists(container, username: str) -> None: | |
| exit_code, _ = container.exec_run(f"id -u {username}", user="root") | |
| if exit_code == 0: | |
| return | |
| container.exec_run(f"useradd -m -s /bin/bash {username}", user="root") | |
| container.exec_run(f"usermod -aG hadoop {username}", user="root") | |
| def ensure_kms_provider_configured(container) -> bool: | |
| """ | |
| Ensures KMS provider property exists in core-site.xml. | |
| Returns True if the file was modified (restart needed). | |
| """ | |
| changed = False | |
| # 1) Ensure KMS provider property exists | |
| exit_code, _ = container.exec_run( | |
| f"grep -q 'hadoop.security.key.provider.path' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| if exit_code != 0: | |
| container.exec_run( | |
| f"sed -i '/<\\/configuration>/i {KMS_PROPERTY}' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| changed = True | |
| # 2) Fix stale host.docker.internal from previous runs | |
| exit_code, _ = container.exec_run( | |
| f"grep -q 'host.docker.internal' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| if exit_code == 0: | |
| container.exec_run( | |
| f"sed -i 's|host.docker.internal|ranger-kms.rangernw|g' {CORE_SITE_XML_PATH}", | |
| user="root", | |
| ) | |
| changed = True | |
| return changed | |
| @pytest.fixture(scope="session", autouse=True) | |
| def setup_encryption_environment(request): | |
| if not _session_needs_encryption_setup(request.session): | |
| yield | |
| return | |
| hadoop_container = client.containers.get(HADOOP_CONTAINER) | |
| from hdfs.utils import ( | |
| cleanup_test_artifacts, | |
| ensure_hadoop_user_keytab, | |
| ensure_kms_hdfs_policy, | |
| ensure_kms_kerberos_rules, | |
| ) | |
| from kms.utils import ensure_keyadmin_keytab, ensure_ticket | |
| changed = ensure_kms_provider_configured(hadoop_container) | |
| if changed: | |
| hadoop_container.restart() | |
| time.sleep(30) | |
| ensure_user_exists(hadoop_container, "hive") | |
| ensure_user_exists(hadoop_container, "hbase") | |
| ensure_hadoop_user_keytab("hive") | |
| ensure_hadoop_user_keytab("hbase") | |
| # safemode leave with kerberos (hdfs principal already has keytab in container) | |
| hadoop_container.exec_run( | |
| [ | |
| "bash", | |
| "-c", | |
| "kinit -kt /etc/keytabs/hdfs.keytab hdfs/ranger-hadoop.rangernw@EXAMPLE.COM 2>/dev/null;" | |
| "hdfs dfsadmin -safemode leave", | |
| ], | |
| user=HDFS_USER, | |
| ) | |
| kms_container = client.containers.get(KMS_CONTAINER) | |
| ensure_kms_kerberos_rules(kms_container) | |
| ensure_keyadmin_keytab() | |
| ensure_ticket() | |
| ensure_kms_hdfs_policy() | |
| cleanup_test_artifacts(hadoop_container, TEST_KMS_KEYS, TEST_EZ_PATHS) | |
| yield | |
| cleanup_test_artifacts(hadoop_container, TEST_KMS_KEYS, TEST_EZ_PATHS) |