blob: a791aca8bd44fe892ff576aa1cc6593f0ee7383a [file]
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Shared helpers for Apache Ranger functional-tests (Docker pytest).
"""
import logging
import random
import string
import uuid
from datetime import datetime, timezone
from pathlib import Path
import requests
# --- Ranger Admin defaults (local Docker) ---
RANGER_ADMIN_BASE_URL = "http://localhost:6080/service"
PUBLIC_V2_BASE = RANGER_ADMIN_BASE_URL + "/public/v2/api"
SERVICE_URL = PUBLIC_V2_BASE + "/service/"
POLICY_URL = PUBLIC_V2_BASE + "/policy/"
XUSERS_BASE = RANGER_ADMIN_BASE_URL + "/xusers"
DEFAULT_ADMIN_AUTH = ("admin", "rangerR0cks!")
# Avoid indefinite hangs when Ranger Admin or Docker is not up (plugin session setup).
RANGER_ADMIN_REQUEST_TIMEOUT_SEC = 30
DEFAULT_JSON_HEADERS = {
"Accept": "application/json",
"Content-Type": "application/json",
}
# Users/groups used by migrated Argus public-API policy tests (created via xusers if missing).
ARGUS_POLICY_USERS = ["hrt_21", "hrt_22"]
ARGUS_POLICY_GROUPS = ["finance", "audit"]
HIVE_JDBC_URL = "jdbc:hive2://127.0.0.1:10000/default"
HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"
# --- Logging ---
# functional-tests/ (parent of ranger_test_utils/)
FUNCTIONAL_TESTS_DIR = Path(__file__).resolve().parent.parent
LOGS_DIR = FUNCTIONAL_TESTS_DIR / "logs"
DEFAULT_PYTEST_LOG_FILE = LOGS_DIR / "pytest.log"
def ensure_logs_directory():
"""Create functional-tests/logs if missing (pytest log_file target)."""
LOGS_DIR.mkdir(parents=True, exist_ok=True)
def configure_test_logging(level=logging.INFO, module_name=None):
"""
Call once from setup_module.
Ensures logs/ exists for pytest.ini log_file=logs/pytest.log and sets logger levels.
"""
ensure_logs_directory()
logging.getLogger().setLevel(level)
if module_name:
logging.getLogger(module_name).setLevel(level)
get_test_logger(module_name).info(
"Test logging configured; log file: %s",
DEFAULT_PYTEST_LOG_FILE,
)
def get_test_logger(module_name):
"""
Module logger for tests.
"""
log = logging.getLogger(module_name)
log.setLevel(logging.INFO)
return log
def log_http_response(logger, step, response, max_body_length=400):
"""Log HTTP status and a short response body for each REST call."""
logger.info("%s: HTTP %s", step, response.status_code)
if not response.text:
return
body = response.text
if len(body) > max_body_length:
body = body[:max_body_length] + "..."
logger.info("%s response body: %s", step, body)
# --- HTTP assertions ---
def assert_http_ok(response, ok_codes, step, url=""):
"""
Check response.status_code. On failure, raise with step, URL, and body text.
ok_codes can be 200 or (200, 201).
"""
if isinstance(ok_codes, int):
expected = (ok_codes,)
else:
expected = ok_codes
if response.status_code in expected:
return
message = step + " failed: HTTP " + str(response.status_code)
message += " (expected " + str(expected) + ")"
if url:
message += "\nURL: " + url
message += "\nResponse body:\n" + response.text
raise AssertionError(message)
class _RangerAdminSession(requests.Session):
"""requests.Session with a default timeout on every call."""
def request(self, method, url, **kwargs):
kwargs.setdefault("timeout", RANGER_ADMIN_REQUEST_TIMEOUT_SEC)
return super().request(method, url, **kwargs)
def create_ranger_admin_session(auth=None, headers=None):
"""
Build a requests.Session for Ranger Admin REST calls (Docker defaults).
Sets HTTP basic auth and JSON Accept/Content-Type headers on the session.
"""
if auth is None:
auth = DEFAULT_ADMIN_AUTH
if headers is None:
headers = DEFAULT_JSON_HEADERS
session = _RangerAdminSession()
session.auth = auth
session.headers.update(headers)
return session
def preflight_ranger_admin_http():
"""Skip plugin tests when Ranger Admin is not reachable on Docker defaults."""
import pytest
url = RANGER_ADMIN_BASE_URL + "/plugins/definitions?page=0&pageSize=1"
try:
response = requests.get(
url,
auth=DEFAULT_ADMIN_AUTH,
headers=DEFAULT_JSON_HEADERS,
timeout=RANGER_ADMIN_REQUEST_TIMEOUT_SEC,
)
except requests.RequestException as exc:
pytest.skip(
"Ranger Admin is not reachable at "
+ RANGER_ADMIN_BASE_URL
+ " ("
+ str(exc)
+ "). Start the stack: cd functional-tests && ./run-tests.sh postgres hive"
)
if response.status_code not in (200, 401, 403):
pytest.skip(
"Ranger Admin returned HTTP "
+ str(response.status_code)
+ " at "
+ url
+ ". Wait for containers to finish starting, then retry."
)
# --- Test run IDs ---
def unique_suffix(length=None):
"""
Unique string for service/policy names so parallel runs do not clash.
length is None (default): UTC timestamp + 6 hex characters (existing callers).
length is an int: random lowercase letters and digits.
"""
if length is not None:
if length < 1:
raise ValueError("unique_suffix length must be >= 1")
alphabet = string.ascii_lowercase + string.digits
return "".join(random.choice(alphabet) for _ in range(length))
time_part = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
random_part = uuid.uuid4().hex[:6]
return time_part + random_part
def qe_test_id_from_pytest_name(function_name, suite="HIVE"):
"""
test_01_01_table_create_employee_no_policy_as_user1_deny ->
HIVE.test_01_01_TableCreateEmployeeNoPolicyAsUser1Deny
"""
if not function_name.startswith("test_"):
return suite + "." + function_name
body = function_name[len("test_") :]
parts = body.split("_")
if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
section = parts[0] + "_" + parts[1]
title = "".join(word.capitalize() for word in parts[2:])
return suite + ".test_" + section + "_" + title
title = "".join(word.capitalize() for word in parts)
return suite + ".test_" + title
def log_testcase_begin(logger, test_id, component="ranger.functional-tests"):
"""
Log at the start of a test.
"""
logger.info("%s", component)
logger.info("TESTCASE BEGIN: testId=%s", test_id)
def log_testcase_begin_for_pytest(
logger, pytest_function_name, suite="HIVE", component="ranger.functional-tests"
):
"""TESTCASE BEGIN from pytest test function name (request.node.name)."""
test_id = qe_test_id_from_pytest_name(pytest_function_name, suite=suite)
log_testcase_begin(logger, test_id, component=component)
# --- xusers: ensure principals exist ---
# Password for Docker plugin suites
DOCKER_PLUGIN_TEST_USER_PASSWORD = "Password@123"
def ensure_user_exists(
session,
username,
password="Test@123",
last_name="api_tests",
):
"""Create a Ranger user via xusers REST if it is not already present."""
lookup_url = XUSERS_BASE + "/users/userName/" + username
lookup = session.get(lookup_url)
if lookup.status_code == 200:
return
payload = {
"name": username,
"firstName": username,
"lastName": last_name,
"emailAddress": username + "@example.com",
"password": password,
"status": 1,
"isVisible": 1,
"userSource": 1,
"userRoleList": ["ROLE_USER"],
"groupIdList": [],
"groupNameList": [],
}
create_url = XUSERS_BASE + "/secure/users"
create = session.post(create_url, json=payload)
assert_http_ok(create, 200, "Create Ranger user " + username, create_url)
def ensure_group_exists(session, group_name):
"""Create a Ranger group via xusers REST if it is not already present."""
lookup_url = XUSERS_BASE + "/groups/groupName/" + group_name
lookup = session.get(lookup_url)
if lookup.status_code == 200:
return
payload = {"name": group_name, "groupSource": 0}
create_url = XUSERS_BASE + "/secure/groups"
create = session.post(create_url, json=payload)
assert_http_ok(create, 200, "Create Ranger group " + group_name, create_url)
def ensure_argus_policy_principals(session):
"""Create users and groups referenced by migrated Argus policy tests."""
for username in ARGUS_POLICY_USERS:
ensure_user_exists(session, username)
for group_name in ARGUS_POLICY_GROUPS:
ensure_group_exists(session, group_name)
def ensure_user_assigned_to_groups(session, username, group_names):
"""Assign Ranger xuser group membership (idempotent for functional-tests)."""
if not group_names:
return
url = XUSERS_BASE + "/users/userinfo"
payload = {
"xuserInfo": {"name": username},
"xgroupInfo": [{"name": group_name} for group_name in group_names],
}
response = session.post(url, json=payload)
assert_http_ok(
response,
200,
"Assign groups "
+ ",".join(group_names)
+ " to user "
+ username,
url,
)
# --- Hive public API v2 JSON builders ---
def split_csv(value):
"""Turn 'a, b ,c' into ['a', 'b', 'c']."""
return [part.strip() for part in value.split(",") if part.strip()]
def build_hive_service_payload(
name,
description,
username="policymgr",
password="policymgr",
jdbc_url=HIVE_JDBC_URL,
is_enabled=True,
):
"""JSON body for POST/PUT on /public/v2/api/service/ (Hive type)."""
return {
"name": name,
"type": "hive",
"description": description,
"isEnabled": is_enabled,
"configs": {
"username": username,
"password": password,
"jdbc.driverClassName": HIVE_DRIVER,
"jdbc.url": jdbc_url,
"commonNameForCertificate": "",
},
}
def resource_block(values):
"""Standard Ranger policy resource block (database/table/column)."""
return {
"values": values,
"isExcludes": False,
"isRecursive": False,
}
def perm_map_list_to_policy_items(perm_map_list):
"""
Convert legacy test shape to Public API v2 policyItems.
Each entry: {"userList": [...], "groupList": [...], "permList": ["select", ...]}.
"""
items = []
for entry in perm_map_list:
accesses = [
{"type": perm, "isAllowed": True}
for perm in entry.get("permList", [])
]
items.append(
{
"users": entry.get("userList", []),
"groups": entry.get("groupList", []),
"accesses": accesses,
"delegateAdmin": False,
}
)
return items
def build_hive_policy_payload(
service_name,
policy_name,
database_list,
table_list,
column_list,
perm_map_list,
description="",
is_enabled=True,
is_audit_enabled=True,
policy_id=None,
):
"""JSON body for POST/PUT on /public/v2/api/policy/."""
payload = {
"service": service_name,
"name": policy_name,
"description": description,
"isEnabled": is_enabled,
"isAuditEnabled": is_audit_enabled,
"resources": {
"database": resource_block(split_csv(database_list)),
"table": resource_block(split_csv(table_list)),
"column": resource_block(split_csv(column_list)),
},
"policyItems": perm_map_list_to_policy_items(perm_map_list),
}
if policy_id is not None:
payload["id"] = policy_id
return payload
# --- Compare expected vs actual service/policy JSON ---
def service_matches(expected, actual, attributes=None):
"""True if selected top-level service fields match."""
if attributes is None:
attributes = ["name", "description", "type"]
return all(actual.get(key) == expected.get(key) for key in attributes)
def get_allowed_access_types(policy_item):
"""Sorted list of permission types marked isAllowed on a policy item."""
types = [
access.get("type")
for access in policy_item.get("accesses", [])
if access.get("isAllowed", True)
]
return sorted(types)
def single_policy_item_matches(expected_item, actual_item):
"""True if users, groups, and allowed access types match one policy item."""
if sorted(expected_item.get("users", [])) != sorted(actual_item.get("users", [])):
return False
if sorted(expected_item.get("groups", [])) != sorted(
actual_item.get("groups", [])
):
return False
return get_allowed_access_types(expected_item) == get_allowed_access_types(
actual_item
)
def policy_items_match(expected_items, actual_items):
"""True if every expected policy item has a matching actual item (order ignored)."""
if len(expected_items) != len(actual_items):
return False
for expected_item in expected_items:
if not any(
single_policy_item_matches(expected_item, actual_item)
for actual_item in actual_items
):
return False
return True
def resource_values_match(expected_resources, actual_resources, resource_name):
"""True if database/table/column value lists match (order ignored)."""
exp_vals = sorted(
expected_resources.get(resource_name, {}).get("values", [])
)
act_vals = sorted(actual_resources.get(resource_name, {}).get("values", []))
return exp_vals == act_vals
def policy_matches(expected, actual):
"""True if name, service, hive resources, and policy items match."""
if expected.get("name") != actual.get("name"):
return False
if expected.get("service") != actual.get("service"):
return False
exp_res = expected.get("resources", {})
act_res = actual.get("resources", {})
for resource_name in ("database", "table", "column"):
if not resource_values_match(exp_res, act_res, resource_name):
return False
return policy_items_match(
expected.get("policyItems", []),
actual.get("policyItems", []),
)
def find_service_by_id(services, service_id):
"""Return the service dict with the given id, or None."""
for service in services:
if service.get("id") == service_id:
return service
return None
def find_policy_by_id(policies, policy_id):
"""Return the policy dict with the given id, or None."""
for policy in policies:
if policy.get("id") == policy_id:
return policy
return None