This is an automated email from the ASF dual-hosted git repository.
pradeepagrawal8184 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new f56788a1b RANGER-5831: Add Docker system tests to verify Hive denies
operations for users without an explicit allow policy (#1300)
f56788a1b is described below
commit f56788a1bdb84b9297f17563813bb7c301e04ea9
Author: Vishal Bhavsar <[email protected]>
AuthorDate: Fri Oct 9 14:34:43 2026 +0530
RANGER-5831: Add Docker system tests to verify Hive denies operations for
users without an explicit allow policy (#1300)
---
functional-tests/.gitignore | 3 +
functional-tests/hive/test_hive_01.py | 171 +++++++
functional-tests/pytest.ini | 1 +
.../ranger_test_utils/access_audit_utils.py | 126 ++++++
functional-tests/ranger_test_utils/hive_utils.py | 493 +++++++++++++++++++++
functional-tests/ranger_test_utils/utils.py | 66 ++-
functional-tests/readme.md | 14 +-
functional-tests/run-tests.sh | 14 +-
8 files changed, 882 insertions(+), 6 deletions(-)
diff --git a/functional-tests/.gitignore b/functional-tests/.gitignore
index 606df5c16..1734553a9 100644
--- a/functional-tests/.gitignore
+++ b/functional-tests/.gitignore
@@ -6,3 +6,6 @@ myenv/
# Pytest cache
.pytest_cache/
+
+# Pytest file log (pytest.ini log_file=logs/pytest.log)
+logs/
diff --git a/functional-tests/hive/test_hive_01.py
b/functional-tests/hive/test_hive_01.py
new file mode 100644
index 000000000..e44ac1f31
--- /dev/null
+++ b/functional-tests/hive/test_hive_01.py
@@ -0,0 +1,171 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Authorization is asserted via beeline output only. Access-audit asserts are
deferred until
+Hive plugin audits are reliably indexed in Docker — use
ranger_test_utils.access_audit_utils
+when ready.
+
+AUDIT_FILTERS (future):
+ test_01_01: accessType=CREATE, accessResult=0, aclEnforcer=ranger-acl,
+ resourcePath=<db>/<table>, repoName=dev_hive, requestUser=hrt_21
+ test_01_02: accessType=USE, accessResult=0, aclEnforcer=ranger-acl,
+ resourcePath=<db>, repoName=dev_hive, requestUser=hrt_21
+"""
+
+import pytest
+
+from ranger_test_utils.hive_utils import (
+ HIVE_AUTHZ_DENIED_SUBSTR,
+ HIVE_SERVICE_NAME,
+ HIVE_TEST_USER,
+ cleanup_hive_test_namespace,
+ create_audit_only_policy,
+ delete_policies_by_id,
+ ensure_hive_test_user_in_ranger,
+ ensure_kerberos_user,
+ preflight_hive_stack,
+ prepare_hive_test_namespace,
+ run_beeline,
+ set_all_policies_enabled,
+ wait_for_policy_propagation,
+)
+from ranger_test_utils.utils import (
+ configure_test_logging,
+ create_ranger_admin_session,
+ get_test_logger,
+ log_testcase_begin_for_pytest,
+ unique_suffix,
+)
+
+pytestmark = pytest.mark.hive
+
+logger = get_test_logger(__name__)
+
+_admin_session = None
+_audit_policy_id = None
+_audit_suffix = None
+_run_suffix = None
+_test_database = None
+_test_table = None
+
+
+def setup_module():
+ configure_test_logging(module_name=__name__)
+ logger.info("=== setup_module: start ===")
+ preflight_hive_stack()
+
+ global _admin_session, _audit_policy_id, _audit_suffix
+ global _run_suffix, _test_database, _test_table
+ _admin_session = create_ranger_admin_session()
+ _run_suffix = unique_suffix(4)
+ _test_database = "db_" + _run_suffix
+ _test_table = "employee_" + _run_suffix
+ logger.info(
+ "Test resources: database=%s table=%s",
+ _test_database,
+ _test_table,
+ )
+
+ logger.info("Ensuring Ranger user and Kerberos principal for %s",
HIVE_TEST_USER)
+ ensure_hive_test_user_in_ranger(_admin_session)
+ ensure_kerberos_user(HIVE_TEST_USER)
+
+ logger.info(
+ "Hive service setup: create database %s and drop table %s.%s if
present",
+ _test_database,
+ _test_database,
+ _test_table,
+ )
+ prepare_hive_test_namespace(_test_database, _test_table)
+
+ logger.info("Disabling all policies on %s and creating audit-only policy",
HIVE_SERVICE_NAME)
+ set_all_policies_enabled(_admin_session, False, HIVE_SERVICE_NAME)
+ _audit_policy_id, _audit_suffix = create_audit_only_policy(_admin_session)
+ logger.info("Created audit policy id=%s suffix=%s", _audit_policy_id,
_audit_suffix)
+ wait_for_policy_propagation()
+ logger.info("=== setup_module: done ===")
+
+
+def teardown_module():
+ if _admin_session is None:
+ return
+
+ logger.info("=== teardown_module: start ===")
+ if _test_database is not None and _test_table is not None:
+ logger.info(
+ "Hive service cleanup: drop %s.%s and database %s",
+ _test_database,
+ _test_table,
+ _test_database,
+ )
+ cleanup_hive_test_namespace(_test_database, _test_table)
+
+ if _audit_policy_id is not None:
+ logger.info("Deleting audit policy id=%s", _audit_policy_id)
+ delete_policies_by_id(_admin_session, [_audit_policy_id])
+
+ logger.info("Re-enabling all policies on %s", HIVE_SERVICE_NAME)
+ set_all_policies_enabled(_admin_session, True, HIVE_SERVICE_NAME)
+ wait_for_policy_propagation()
+ logger.info("=== teardown_module: done ===")
+
+
+def test_01_01_table_create_employee_no_policy_as_user1_deny(request):
+ log_testcase_begin_for_pytest(logger, request.node.name)
+ sql = (
+ "create table "
+ + _test_database
+ + "."
+ + _test_table
+ + "(name String);"
+ )
+ logger.info(
+ "Running beeline CREATE TABLE as %s (expect deny): %s",
+ HIVE_TEST_USER,
+ sql,
+ )
+ exit_code, output = run_beeline(
+ HIVE_TEST_USER,
+ sql,
+ expect_substring=HIVE_AUTHZ_DENIED_SUBSTR,
+ )
+ assert HIVE_AUTHZ_DENIED_SUBSTR in output, (
+ "Expected authorization denial in beeline output; exit_code="
+ + str(exit_code)
+ + " output="
+ + output
+ )
+
+
+def test_01_02_database_use_default_no_policy_as_user1_deny(request):
+ log_testcase_begin_for_pytest(logger, request.node.name)
+ sql = "use " + _test_database + ";"
+ logger.info(
+ "Running beeline USE database as %s (expect deny): %s",
+ HIVE_TEST_USER,
+ sql,
+ )
+ exit_code, output = run_beeline(
+ HIVE_TEST_USER,
+ sql,
+ expect_substring=HIVE_AUTHZ_DENIED_SUBSTR,
+ )
+ assert HIVE_AUTHZ_DENIED_SUBSTR in output, (
+ "Expected authorization denial in beeline output; exit_code="
+ + str(exit_code)
+ + " output="
+ + output
+ )
diff --git a/functional-tests/pytest.ini b/functional-tests/pytest.ini
index 87e6e65d6..45db54be6 100644
--- a/functional-tests/pytest.ini
+++ b/functional-tests/pytest.ini
@@ -27,6 +27,7 @@ markers =
secure_endpoint: secure endpoint test cases
rolerest: rolerest test cases
api: public API v2 test cases
+ hive: Hive plugin enforcement (Docker HS2)
pythonpath = .
# No console logging during pytest; logger.info -> logs/pytest.log (tail -f
logs/pytest.log).
log_cli = false
diff --git a/functional-tests/ranger_test_utils/access_audit_utils.py
b/functional-tests/ranger_test_utils/access_audit_utils.py
new file mode 100644
index 000000000..c37d3f765
--- /dev/null
+++ b/functional-tests/ranger_test_utils/access_audit_utils.py
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Ranger Admin access-audit query helpers (Docker functional-tests).
+
+Hive plugin tests do not call these yet — Hive access audits are not reliably
+indexed in the default Docker stack. Wire into hive tests once audit pipeline
works.
+"""
+
+import logging
+import time
+
+from ranger_test_utils.utils import RANGER_ADMIN_BASE_URL
+
+logger = logging.getLogger(__name__)
+
+ACCESS_AUDIT_PATH = RANGER_ADMIN_BASE_URL + "/assets/accessAudit"
+
+
+def fetch_last_access_audit_event_time(session):
+ """
+ Return the newest access audit eventTime, or None if none / request failed.
+ """
+ params = {
+ "pageSize": 25,
+ "startIndex": 0,
+ "sortBy": "eventTime",
+ "sortType": "desc",
+ }
+ response = session.get(ACCESS_AUDIT_PATH, params=params)
+ if response.status_code != 200:
+ return None
+ audits = response.json().get("vXAccessAudits") or []
+ if not audits:
+ return None
+ return audits[0].get("eventTime")
+
+
+def wait_for_access_audits(
+ session,
+ *,
+ after_event_time=None,
+ request_user=None,
+ repo_name=None,
+ access_type=None,
+ resource_path=None,
+ access_result=None,
+ acl_enforcer=None,
+ resource_type=None,
+ security_zone=None,
+ max_iterations=12,
+ sleep_time=5,
+):
+ """
+ Poll Ranger Admin access audits matching query filters.
+
+ When after_event_time is set, only audits with eventTime greater than that
+ value are returned.
+ """
+ query_params = {
+ "pageSize": 25,
+ "startIndex": 0,
+ "sortBy": "eventTime",
+ "sortType": "desc",
+ }
+ if request_user is not None:
+ query_params["requestUser"] = request_user
+ if repo_name is not None:
+ query_params["repoName"] = repo_name
+ if resource_type is not None:
+ query_params["resourceType"] = resource_type
+ if access_type is not None:
+ query_params["accessType"] = access_type
+ if resource_path is not None:
+ query_params["resourcePath"] = resource_path
+ if access_result is not None:
+ query_params["accessResult"] = access_result
+ if security_zone is not None:
+ query_params["zoneName"] = security_zone
+ if acl_enforcer is not None:
+ query_params["aclEnforcer"] = acl_enforcer
+
+ for _ in range(max_iterations):
+ response = session.get(ACCESS_AUDIT_PATH, params=query_params)
+ if response.status_code != 200:
+ time.sleep(sleep_time)
+ continue
+
+ audits = response.json().get("vXAccessAudits") or []
+ if not audits:
+ time.sleep(sleep_time)
+ continue
+
+ if after_event_time is None:
+ return audits
+
+ audits.sort(key=lambda row: row.get("eventTime", 0), reverse=True)
+ newer = [
+ row
+ for row in audits
+ if row.get("eventTime", 0) > after_event_time
+ ]
+ if newer:
+ return newer
+ time.sleep(sleep_time)
+
+ return None
+
+
+def assert_access_audits_found(session, **kwargs):
+ """Convenience assert for future hive audit tests."""
+ audits = wait_for_access_audits(session, **kwargs)
+ assert audits, "Expected access audit events matching filters: " +
str(kwargs)
diff --git a/functional-tests/ranger_test_utils/hive_utils.py
b/functional-tests/ranger_test_utils/hive_utils.py
new file mode 100644
index 000000000..a1361e080
--- /dev/null
+++ b/functional-tests/ranger_test_utils/hive_utils.py
@@ -0,0 +1,493 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Hive Docker helpers for Ranger functional-tests (HS2 + plugin
enforcement)."""
+
+import socket
+import time
+
+import pytest
+
+from ranger_test_utils.utils import (
+ POLICY_URL,
+ RANGER_ADMIN_BASE_URL,
+ XUSERS_BASE,
+ assert_http_ok,
+ get_test_logger,
+ unique_suffix,
+)
+
+logger = get_test_logger(__name__)
+
+BEELINE_LOG_OUTPUT_MAX_CHARS = 12000
+BEELINE_COMPLETE_LOG_MAX_CHARS = 100000
+
+HIVE_SERVICE_NAME = "dev_hive"
+HIVE_CONTAINER = "ranger-hive"
+HADOOP_CONTAINER = "ranger-hadoop"
+KDC_CONTAINER = "ranger-kdc"
+ADMIN_CONTAINER = "ranger"
+
+KERBEROS_REALM = "EXAMPLE.COM"
+HIVE_HS2_PRINCIPAL = "hive/ranger-hive.rangernw@" + KERBEROS_REALM
+HIVE_SERVICE_KEYTAB = "/etc/keytabs/hive.keytab"
+HIVE_JDBC_BEELINE = (
+ "jdbc:hive2://localhost:10000/default;principal=" + HIVE_HS2_PRINCIPAL
+)
+
+HIVE_TEST_USER = "hrt_21"
+HIVE_TEST_USER_PASSWORD = "Password@123"
+
+HIVE_AUTHZ_DENIED_SUBSTR = "Permission denied"
+
+HIVE_AUDIT_POLICY_SERVICE_USERS = ["hive"]
+HIVE_AUDIT_POLICY_ACCESS_TYPES = [
+ "select",
+ "update",
+ "create",
+ "drop",
+ "alter",
+ "index",
+ "lock",
+ "all",
+]
+
+POLICY_PROPAGATION_SLEEP_SEC = 15
+
+_docker_client = None
+
+
+def get_docker_client():
+ global _docker_client
+ if _docker_client is None:
+ import docker
+
+ _docker_client = docker.from_env()
+ return _docker_client
+
+
+def _container_running(name):
+ client = get_docker_client()
+ try:
+ container = client.containers.get(name)
+ except Exception:
+ return False
+ container.reload()
+ return container.status == "running"
+
+
+def _hs2_port_open(host="127.0.0.1", port=10000, timeout=2.0):
+ try:
+ with socket.create_connection((host, port), timeout=timeout):
+ return True
+ except OSError:
+ return False
+
+
+def preflight_hive_stack():
+ """Skip tests when required Docker services or HS2 are unavailable."""
+ required = [ADMIN_CONTAINER, KDC_CONTAINER, HADOOP_CONTAINER,
HIVE_CONTAINER]
+ missing = [name for name in required if not _container_running(name)]
+ if missing:
+ pytest.skip(
+ "Hive functional-tests need running containers: "
+ + ", ".join(missing)
+ + ". Run ./run-tests.sh postgres hive from functional-tests/."
+ )
+
+ if not _hs2_port_open():
+ pytest.skip(
+ "HiveServer2 is not accepting connections on localhost:10000 "
+ "(is ranger-hive healthy?)"
+ )
+
+
+def _user_principal(username):
+ return username + "@" + KERBEROS_REALM
+
+
+def _user_keytab_in_hive_container(username):
+ return "/etc/keytabs/" + username + ".keytab"
+
+
+def _log_multiline_block(header, text, max_chars=BEELINE_LOG_OUTPUT_MAX_CHARS):
+ """Write multi-line command output to pytest.log (one line per source
line)."""
+ body = text if text else "(empty)"
+ if len(body) > max_chars:
+ body = (
+ body[:max_chars]
+ + "\n... [truncated, total "
+ + str(len(text))
+ + " chars]"
+ )
+ logger.info("%s", header)
+ for line in body.splitlines():
+ logger.info(" %s", line)
+
+
+def _log_text_block(title, text, max_chars=BEELINE_COMPLETE_LOG_MAX_CHARS):
+ """Log a full stdout/stderr block."""
+ logger.info("%s", title)
+ body = text if text and text.strip() else "(empty)"
+ if len(body) > max_chars:
+ body = (
+ body[:max_chars]
+ + "\n... [truncated, total "
+ + str(len(text))
+ + " chars]"
+ )
+ for line in body.splitlines():
+ logger.info("%s", line)
+
+
+def _log_beeline_qe_style(exit_code, stdout, stderr, combined, sql,
beeline_cmd, expect_substring=None):
+ """beeline logging: query, command, exit code, stdout, stderr."""
+ logger.info("Exit Code: %s", exit_code)
+ _log_text_block("Output of beeline command is:", stdout)
+ _log_text_block("Error of beeline command is:", stderr)
+
+ if "Error:" in combined or "FAILED:" in combined:
+ sql_result = "FAILED (server/beeline error in output)"
+ elif exit_code != 0:
+ sql_result = "FAILED (non-zero exit)"
+ elif "INFO : OK" in combined or "No rows affected" in combined:
+ sql_result = "SUCCESS"
+ else:
+ sql_result = "UNKNOWN"
+
+ logger.info(
+ "Beeline summary: exit_code=%s sql_result=%s user_query=%s",
+ exit_code,
+ sql_result,
+ sql,
+ )
+ logger.info("Beeline command: %s", beeline_cmd)
+
+ if expect_substring:
+ if expect_substring in combined:
+ logger.info(
+ 'Found expected string "%s" in beeline output',
+ expect_substring,
+ )
+ else:
+ logger.info(
+ 'Expected string "%s" NOT found in beeline output',
+ expect_substring,
+ )
+
+
+def ensure_kerberos_user(username):
+ """Create a user Kerberos principal and keytab on KDC, visible in
ranger-hive."""
+ kdc = get_docker_client().containers.get(KDC_CONTAINER)
+ principal = _user_principal(username)
+ keytab_in_kdc = "/etc/keytabs/ranger-hive/" + username + ".keytab"
+ keytab_in_hive = _user_keytab_in_hive_container(username)
+
+ logger.info(
+ "Kerberos setup on %s: addprinc/ktadd for %s -> %s (visible in hive as
%s)",
+ KDC_CONTAINER,
+ principal,
+ keytab_in_kdc,
+ keytab_in_hive,
+ )
+
+ add_cmd = 'kadmin.local -q "addprinc -randkey ' + principal + '"'
+ exit_add, out_add = kdc.exec_run(add_cmd, user="root", demux=True)
+ _log_multiline_block(
+ "kadmin addprinc exit_code=%s" % exit_add,
+ ((out_add[0] or b"") + (out_add[1] or b"")).decode(),
+ max_chars=2000,
+ )
+
+ kdc.exec_run("rm -f " + keytab_in_kdc, user="root")
+ ktadd_cmd = (
+ 'kadmin.local -q "ktadd -k '
+ + keytab_in_kdc
+ + " "
+ + principal
+ + '"'
+ )
+ exit_kt, out_kt = kdc.exec_run(ktadd_cmd, user="root", demux=True)
+ kt_out = ((out_kt[0] or b"") + (out_kt[1] or b"")).decode()
+ _log_multiline_block(
+ "kadmin ktadd exit_code=%s" % exit_kt,
+ kt_out,
+ max_chars=2000,
+ )
+ if exit_kt != 0:
+ raise RuntimeError(
+ "ktadd failed for "
+ + principal
+ + " (keytab "
+ + keytab_in_hive
+ + "): "
+ + kt_out
+ )
+ kdc.exec_run("chmod 444 " + keytab_in_kdc, user="root")
+ verify_kerberos_keytab_in_hive(username)
+
+
+def verify_kerberos_keytab_in_hive(username):
+ """kinit in ranger-hive using the user's keytab (fail fast if keytab is
bad)."""
+ hive = get_docker_client().containers.get(HIVE_CONTAINER)
+ principal = _user_principal(username)
+ keytab = _user_keytab_in_hive_container(username)
+ shell = (
+ "kdestroy -A 2>/dev/null || true; "
+ "kinit -kt "
+ + keytab
+ + " "
+ + principal
+ + "; klist"
+ )
+ exit_code, output = hive.exec_run(["bash", "-c", shell], user="hive",
demux=True)
+ combined = ((output[0] or b"") + (output[1] or b"")).decode()
+ if exit_code != 0:
+ raise RuntimeError(
+ "Kerberos kinit failed in "
+ + HIVE_CONTAINER
+ + " for "
+ + principal
+ + ": "
+ + combined
+ )
+ logger.info(
+ "Verified Kerberos keytab for %s in %s: %s",
+ principal,
+ HIVE_CONTAINER,
+ combined.strip().splitlines()[-1] if combined.strip() else "(ok)",
+ )
+
+
+def prepare_hive_test_namespace(database, table):
+ """Create DB and ensure test table absent — runs as hive service user."""
+ run_beeline(
+ None,
+ "create database if not exists " + database + ";",
+ service_user=True,
+ )
+ run_beeline(
+ None,
+ "drop table if exists " + database + "." + table + ";",
+ service_user=True,
+ )
+
+
+def cleanup_hive_test_namespace(database, table):
+ """Drop test table and database — runs as hive service user."""
+ run_beeline(
+ None,
+ "drop table if exists " + database + "." + table + ";",
+ service_user=True,
+ )
+ run_beeline(
+ None,
+ "drop database if exists " + database + " cascade;",
+ service_user=True,
+ )
+
+
+def ensure_hive_test_user_in_ranger(session, username=HIVE_TEST_USER):
+ """Ensure hrt_21 exists in Ranger xusers (QE password)."""
+ lookup_url = XUSERS_BASE + "/users/userName/" + username
+ lookup = session.get(lookup_url)
+ if lookup.status_code == 200:
+ return
+ payload = {
+ "name": username,
+ "firstName": username,
+ "lastName": "hive_test",
+ "emailAddress": username + "@example.com",
+ "password": HIVE_TEST_USER_PASSWORD,
+ "status": 1,
+ "isVisible": 1,
+ "userSource": 1,
+ "userRoleList": ["ROLE_USER"],
+ "groupIdList": [],
+ "groupNameList": [],
+ }
+ create_url = XUSERS_BASE + "/secure/users"
+ create = session.post(create_url, json=payload)
+ assert_http_ok(create, 200, "Create Ranger user " + username, create_url)
+
+
+def run_beeline(username, sql, service_user=False, expect_substring=None):
+ """
+ Run a beeline statement inside ranger-hive (-e, QE-style logging).
+
+ Returns (exit_code, combined_output).
+ """
+ hive = get_docker_client().containers.get(HIVE_CONTAINER)
+ if service_user:
+ principal = HIVE_HS2_PRINCIPAL
+ keytab = HIVE_SERVICE_KEYTAB
+ ranger_user_label = "hive (service)"
+ else:
+ principal = _user_principal(username)
+ keytab = _user_keytab_in_hive_container(username)
+ ranger_user_label = username
+
+ sql_stripped = sql.strip()
+ logger.info("Going to run query is: %s", sql_stripped)
+ logger.info(
+ "Beeline context: container=%s docker_exec_user=hive ranger_user=%s",
+ HIVE_CONTAINER,
+ ranger_user_label,
+ )
+ logger.info(
+ "Kerberos: kdestroy -A; kinit -kt %s %s",
+ keytab,
+ principal,
+ )
+
+ escaped_sql = sql_stripped.replace("\\", "\\\\").replace('"', '\\"')
+ beeline_cmd = (
+ '/opt/hive/bin/beeline -u "'
+ + HIVE_JDBC_BEELINE
+ + '" --outputformat=tsv2 -e "'
+ + escaped_sql
+ + '"'
+ )
+ logger.info("RUNNING: %s", beeline_cmd)
+
+ # Clear shared hive-user credential cache so service-user and test-user
runs
+ # do not reuse each other's tickets (klist -s alone would skip kinit
incorrectly).
+ shell = (
+ "kdestroy -A 2>/dev/null || true; "
+ "kinit -kt "
+ + keytab
+ + " "
+ + principal
+ + "; "
+ + "kinit_rc=$?; "
+ + 'if [ "$kinit_rc" -ne 0 ]; then exit "$kinit_rc"; fi; '
+ + beeline_cmd
+ )
+ exit_code, output = hive.exec_run(["bash", "-c", shell], user="hive",
demux=True)
+ stdout = (output[0] or b"").decode()
+ stderr = (output[1] or b"").decode()
+ combined = stdout + stderr
+
+ _log_beeline_qe_style(
+ exit_code,
+ stdout,
+ stderr,
+ combined,
+ sql_stripped,
+ beeline_cmd,
+ expect_substring=expect_substring,
+ )
+
+ return exit_code, combined
+
+
+def list_policies_for_service(session, service_name=HIVE_SERVICE_NAME):
+ url = RANGER_ADMIN_BASE_URL + "/plugins/policies/service/name/" +
service_name
+ response = session.get(url)
+ assert_http_ok(response, 200, "List policies for " + service_name, url)
+ return response.json().get("policies") or []
+
+
+def set_all_policies_enabled(session, enabled, service_name=HIVE_SERVICE_NAME):
+ policies = list_policies_for_service(session, service_name)
+ for policy in policies:
+ policy_id = policy.get("id")
+ if policy_id is None:
+ continue
+ policy["isEnabled"] = enabled
+ put_url = RANGER_ADMIN_BASE_URL + "/plugins/policies/" + str(policy_id)
+ response = session.put(put_url, json=policy)
+ assert_http_ok(
+ response,
+ 200,
+ "Set isEnabled=" + str(enabled) + " on policy " + str(policy_id),
+ put_url,
+ )
+
+
+def build_audit_only_policy_payload(service_name, suffix):
+ accesses = [
+ {"type": perm, "isAllowed": True}
+ for perm in HIVE_AUDIT_POLICY_ACCESS_TYPES
+ ]
+ return {
+ "isEnabled": True,
+ "service": service_name,
+ "name": "audit policy for hive - " + suffix,
+ "policyType": 0,
+ "policyPriority": 0,
+ "description": "Hive audit policy for functional-tests",
+ "isAuditEnabled": True,
+ "resources": {
+ "database": {
+ "values": ["*"],
+ "isExcludes": False,
+ "isRecursive": False,
+ },
+ "table": {
+ "values": ["*"],
+ "isExcludes": False,
+ "isRecursive": False,
+ },
+ "column": {
+ "values": ["*"],
+ "isExcludes": False,
+ "isRecursive": False,
+ },
+ },
+ "policyItems": [
+ {
+ "accesses": accesses,
+ "users": list(HIVE_AUDIT_POLICY_SERVICE_USERS),
+ "groups": [],
+ "conditions": [],
+ "delegateAdmin": False,
+ }
+ ],
+ "denyPolicyItems": [],
+ "allowExceptions": [],
+ "denyExceptions": [],
+ "dataMaskPolicyItems": [],
+ "rowFilterPolicyItems": [],
+ "serviceType": "hive",
+ "options": {},
+ "validitySchedules": [],
+ "policyLabels": [""],
+ "zoneName": "",
+ }
+
+
+def create_audit_only_policy(session, suffix=None,
service_name=HIVE_SERVICE_NAME):
+ if suffix is None:
+ suffix = unique_suffix()
+ payload = build_audit_only_policy_payload(service_name, suffix)
+ response = session.post(POLICY_URL, json=payload)
+ assert_http_ok(response, (200, 201), "Create audit-only hive policy",
POLICY_URL)
+ body = response.json()
+ policy_id = body.get("id")
+ assert policy_id is not None, "Create policy response missing id: " +
response.text
+ return policy_id, suffix
+
+
+def delete_policies_by_id(session, policy_ids):
+ for policy_id in policy_ids:
+ url = POLICY_URL + str(policy_id)
+ response = session.delete(url)
+ assert_http_ok(response, (200, 204), "Delete policy " +
str(policy_id), url)
+
+
+def wait_for_policy_propagation(seconds=POLICY_PROPAGATION_SLEEP_SEC):
+ logger.info("Waiting %ss for Hive plugin policy propagation", seconds)
+ time.sleep(seconds)
diff --git a/functional-tests/ranger_test_utils/utils.py
b/functional-tests/ranger_test_utils/utils.py
index 8c1b6fd75..4bcb1e03d 100644
--- a/functional-tests/ranger_test_utils/utils.py
+++ b/functional-tests/ranger_test_utils/utils.py
@@ -19,8 +19,11 @@
"""
import logging
+import random
+import string
import uuid
from datetime import datetime, timezone
+from pathlib import Path
import requests
@@ -49,15 +52,31 @@
# --- Logging ---
+# functional-tests/ (parent of ranger_test_utils/)
+FUNCTIONAL_TESTS_DIR = Path(__file__).resolve().parent.parent
+LOGS_DIR = FUNCTIONAL_TESTS_DIR / "logs"
+DEFAULT_PYTEST_LOG_FILE = LOGS_DIR / "pytest.log"
+
+
+def ensure_logs_directory():
+ """Create functional-tests/logs if missing (pytest log_file target)."""
+ LOGS_DIR.mkdir(parents=True, exist_ok=True)
+
def configure_test_logging(level=logging.INFO, module_name=None):
"""
Call once from setup_module.
+ Ensures logs/ exists for pytest.ini log_file=logs/pytest.log and sets
logger levels.
"""
+ ensure_logs_directory()
logging.getLogger().setLevel(level)
if module_name:
logging.getLogger(module_name).setLevel(level)
+ get_test_logger(module_name).info(
+ "Test logging configured; log file: %s",
+ DEFAULT_PYTEST_LOG_FILE,
+ )
def get_test_logger(module_name):
@@ -121,13 +140,56 @@ def create_ranger_admin_session(auth=None, headers=None):
# --- Test run IDs ---
-def unique_suffix():
- """Unique string for service/policy names so parallel runs do not clash."""
+def unique_suffix(length=None):
+ """
+ Unique string for service/policy names so parallel runs do not clash.
+
+ length is None (default): UTC timestamp + 6 hex characters (existing
callers).
+ length is an int: random lowercase letters and digits.
+ """
+ if length is not None:
+ if length < 1:
+ raise ValueError("unique_suffix length must be >= 1")
+ alphabet = string.ascii_lowercase + string.digits
+ return "".join(random.choice(alphabet) for _ in range(length))
time_part = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
random_part = uuid.uuid4().hex[:6]
return time_part + random_part
+def qe_test_id_from_pytest_name(function_name, suite="HIVE"):
+ """
+ test_01_01_table_create_employee_no_policy_as_user1_deny ->
+ HIVE.test_01_01_TableCreateEmployeeNoPolicyAsUser1Deny
+ """
+ if not function_name.startswith("test_"):
+ return suite + "." + function_name
+ body = function_name[len("test_") :]
+ parts = body.split("_")
+ if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
+ section = parts[0] + "_" + parts[1]
+ title = "".join(word.capitalize() for word in parts[2:])
+ return suite + ".test_" + section + "_" + title
+ title = "".join(word.capitalize() for word in parts)
+ return suite + ".test_" + title
+
+
+def log_testcase_begin(logger, test_id, component="ranger.functional-tests"):
+ """
+ Log at the start of a test.
+ """
+ logger.info("%s", component)
+ logger.info("TESTCASE BEGIN: testId=%s", test_id)
+
+
+def log_testcase_begin_for_pytest(
+ logger, pytest_function_name, suite="HIVE",
component="ranger.functional-tests"
+):
+ """TESTCASE BEGIN from pytest test function name (request.node.name)."""
+ test_id = qe_test_id_from_pytest_name(pytest_function_name, suite=suite)
+ log_testcase_begin(logger, test_id, component=component)
+
+
# --- xusers: ensure principals exist ---
diff --git a/functional-tests/readme.md b/functional-tests/readme.md
index 2104d57f9..a7c6a317a 100644
--- a/functional-tests/readme.md
+++ b/functional-tests/readme.md
@@ -31,6 +31,7 @@ This test suite validates REST API endpoints for Apache
Ranger services — Admi
| **servicerest** | Test cases for Ranger Admin Service REST APIs |
| **tagrest** | Test cases for Ranger Admin Tag REST APIs |
| **api** | Public API v2 service and policy CRUD (admin Docker only) |
+| **hive** | Hive Ranger plugin enforcement via HS2/beeline (requires
`ranger-hadoop` + `ranger-hive`) |
---
@@ -46,6 +47,9 @@ functional-tests/
├── servicerest/ # Tests on Ranger Service REST APIs
├── tagrest/ # Tests on Ranger Tag REST APIs
├── api/ # Public API v2 (service / policy) tests
+├── hive/ # Hive plugin enforcement tests
+├── ranger_test_utils/hive_utils.py
+├── ranger_test_utils/access_audit_utils.py # For future access-audit asserts
│
├── pytest.ini # Registers custom pytest markers
├── run-tests.sh # Script to automate setup and test execution
@@ -158,7 +162,7 @@ Pass `db-type` and `test-suites` directly to skip prompts.
Use env vars for the
```
- `db-type` — First argument. Valid values: `postgres`, `mysql`, `oracle`.
-- `test-suites` — Space-separated list: `hdfs`, `kms`, `rolerest`,
`xuserrest`, `servicerest`, `tagrest`.
+- `test-suites` — Space-separated list: `hdfs`, `hive`, `kms`, `rolerest`,
`xuserrest`, `servicerest`, `tagrest`, `api`.
Examples:
@@ -204,10 +208,18 @@ Regardless of which test suites you choose, these
containers always start:
| Suite | Extra Container |
|---|---|
| `hdfs` | `ranger-hadoop` |
+| `hive` | `ranger-hadoop`, `ranger-hive` |
| `kms` | _(already in base)_ |
| `rolerest`, `xuserrest`, `servicerest`, `tagrest`| _(none — use Ranger Admin
API only)_ |
---
+
+### Hive suite notes
+
+- Service repo name in Docker is `dev_hive` (not QE `cm_hive`).
+- Tests currently assert **beeline authorization output only**. Hive access
audits helpers live in `ranger_test_utils/access_audit_utils.py` for a
follow-up once auditing works.
+- Example: `./run-tests.sh postgres hive`
+
## Test Reports
After execution, an HTML report is automatically generated for each suite in
the `functional-tests/` directory:
diff --git a/functional-tests/run-tests.sh b/functional-tests/run-tests.sh
index 8b8f2ebbd..efa7dae95 100755
--- a/functional-tests/run-tests.sh
+++ b/functional-tests/run-tests.sh
@@ -17,10 +17,10 @@
# All available test suites (pytest folders)
-ALL_TEST_SUITES=(rolerest xuserrest servicerest tagrest hdfs kms api)
+ALL_TEST_SUITES=(rolerest xuserrest servicerest tagrest hdfs kms api hive)
# Suites that have actual docker-compose services
-DOCKER_SERVICES=(hdfs kms)
+DOCKER_SERVICES=(hdfs kms hive)
# Test-only suites (no docker-compose file needed)
TEST_ONLY_SUITES=(rolerest xuserrest servicerest tagrest api)
@@ -106,7 +106,7 @@ for service in "${EXTRA_SERVICES[@]}"; do
for ds in "${DOCKER_SERVICES[@]}"; do
if [[ "$service" == "$ds" ]]; then
case "$service" in
- hdfs)
+ hdfs|hive)
# 'hive' arg downloads hadoop + tez (both needed by
Dockerfile.ranger-hadoop)
# 'hadoop' arg alone does NOT download tez
DOCKER_BACKED+=("hive")
@@ -193,6 +193,10 @@ for service in "${EXTRA_SERVICES[@]}"; do
hdfs)
DOCKER_FILES+=("-f" "docker-compose.ranger-hadoop.yml")
;;
+ hive)
+ DOCKER_FILES+=("-f" "docker-compose.ranger-hadoop.yml")
+ DOCKER_FILES+=("-f" "docker-compose.ranger-hive.yml")
+ ;;
kms)
# already included in base
;;
@@ -218,6 +222,10 @@ for service in "${EXTRA_SERVICES[@]}"; do
hdfs)
ALL_SERVICES+=("ranger-hadoop")
;;
+ hive)
+ ALL_SERVICES+=("ranger-hadoop")
+ ALL_SERVICES+=("ranger-hive")
+ ;;
kms)
: # already in BASE_SERVICES
;;
