This is an automated email from the ASF dual-hosted git repository.
ramackri pushed a commit to branch ranger-2.9
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.9 by this push:
new 695bfcc53 RANGER-5637: Fix Ozone plugin docker CI smoke-test failures
695bfcc53 is described below
commit 695bfcc53009fb12ced2d6c75c1081599cbd3435
Author: ramk <[email protected]>
AuthorDate: Wed Jul 29 09:05:36 2026 +0530
RANGER-5637: Fix Ozone plugin docker CI smoke-test failures
Backport docker/CI fixes for plugins-docker-build on ranger-2.9:
SCM wait in ranger-ozone-setup.sh, UseContainerSupport in docker-config,
OM depends_on scm/datanode, extractOzoneIfNeeded cache, and drop ozone rm
in CI.
---
.github/workflows/ci.yml | 3 +-
.../ranger-docker/docker-compose.ranger-ozone.yml | 9 +--
dev-support/ranger-docker/download-archives.sh | 73 ++++++++++++++++++----
.../ranger-docker/scripts/ozone/docker-config | 4 ++
.../scripts/ozone/ranger-ozone-setup.sh | 27 +++++++-
5 files changed, 98 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c7bdf6b2e..61defe70e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -205,10 +205,11 @@ jobs:
restore-keys: |
${{ runner.os }}-ranger-downloads-
+ # Populates downloads/ for plugins-docker-build; Ozone tarball + extract
dir are
+ # cached together (extractOzoneIfNeeded in download-archives.sh).
- name: Run download-archives.sh
run: |
cd dev-support/ranger-docker
- rm -rf downloads/ozone-*
./download-archives.sh hadoop hive hbase kafka knox ozone
- name: Build all ranger-plugin images
diff --git a/dev-support/ranger-docker/docker-compose.ranger-ozone.yml
b/dev-support/ranger-docker/docker-compose.ranger-ozone.yml
index 99b2b6dfa..9ed275643 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-ozone.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-ozone.yml
@@ -12,8 +12,6 @@ services:
command: bash -c "sudo mkdir -p /data/hdds /data/metadata && sudo chown -R
hadoop:hadoop /data && exec /opt/hadoop/bin/ozone datanode"
env_file:
- ./scripts/ozone/docker-config
- environment:
- OZONE_OPTS:
om:
build:
context: .
@@ -35,12 +33,16 @@ services:
- 9874:9874
- 9862:9862
depends_on:
+ scm:
+ condition: service_started
+ datanode:
+ condition: service_started
ranger:
condition: service_started
ranger-solr:
condition: service_started
environment:
- OZONE_OPTS: -XX:-UseContainerSupport
-Dcom.sun.net.ssl.checkRevocation=false
+ OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false
OZONE_HOME: /opt/hadoop
OZONE_PLUGIN_VERSION: ${OZONE_PLUGIN_VERSION}
OZONE_VERSION: ${OZONE_VERSION}
@@ -63,7 +65,6 @@ services:
- ./scripts/ozone/docker-config
environment:
OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- OZONE_OPTS:
command: bash -c "(test -f /data/metadata/scm/current/VERSION ||
/opt/hadoop/bin/ozone scm --init) && exec /opt/hadoop/bin/ozone scm"
networks:
diff --git a/dev-support/ranger-docker/download-archives.sh
b/dev-support/ranger-docker/download-archives.sh
index 0bfca41ad..efb303b8f 100755
--- a/dev-support/ranger-docker/download-archives.sh
+++ b/dev-support/ranger-docker/download-archives.sh
@@ -35,12 +35,71 @@ downloadIfNotPresent() {
then
echo "downloading ${urlBase}/${fileName}.."
- curl -L ${urlBase}/${fileName} --output downloads/${fileName}
+ curl -fL --retry 3 --retry-all-errors --retry-delay 10 --connect-timeout
30 \
+ ${urlBase}/${fileName} --output downloads/${fileName}
else
echo "file already in cache: ${fileName}"
fi
}
+# Ozone is special among plugin archives: docker-compose.ranger-ozone.yml
bind-mounts the
+# *extracted* tree (downloads/ozone-${OZONE_VERSION}/), not the .tar.gz. On a
warm CI
+# cache hit, downloadIfNotPresent skips network I/O but we still need a valid
extract dir.
+extractOzoneIfNeeded() {
+ local tarball="downloads/ozone-${OZONE_VERSION}.tar.gz"
+ local extractDir="downloads/ozone-${OZONE_VERSION}"
+
+ downloadIfNotPresent ozone-${OZONE_VERSION}.tar.gz
https://archive.apache.org/dist/ozone/${OZONE_VERSION}
+
+ if [ ! -f "${tarball}" ]; then
+ echo "ERROR: missing ${tarball}" >&2
+ exit 1
+ fi
+
+ local stampFile="${extractDir}/.ozone-extract.stamp"
+ local tarballId
+ if stat --version >/dev/null 2>&1; then
+ tarballId=$(stat -c '%Y:%s' "${tarball}")
+ else
+ tarballId=$(stat -f '%m:%z' "${tarball}")
+ fi
+
+ local needExtract=false
+ if [ ! -d "${extractDir}" ]; then
+ needExtract=true
+ echo "ozone extract dir missing: ${extractDir}"
+ elif [ ! -f "${extractDir}/bin/ozone" ]; then
+ needExtract=true
+ echo "ozone extract dir incomplete, re-extracting: ${extractDir}"
+ rm -rf "${extractDir}"
+ elif [ ! -f "${stampFile}" ] || [ "$(cat "${stampFile}")" != "${tarballId}"
]; then
+ needExtract=true
+ echo "ozone tarball changed or stamp missing, re-extracting"
+ rm -rf "${extractDir}"
+ else
+ echo "ozone extract dir up to date: ${extractDir}"
+ fi
+
+ if [ "${needExtract}" = true ]; then
+ tar xzf "${tarball}" --directory=downloads/
+ echo "${tarballId}" > "${stampFile}"
+ fi
+
+ local stale
+ for stale in downloads/ozone-*.tar.gz; do
+ [ -e "${stale}" ] || continue
+ [ "${stale}" = "${tarball}" ] && continue
+ echo "removing stale ozone tarball: ${stale}"
+ rm -f "${stale}"
+ done
+ for stale in downloads/ozone-*/; do
+ [ -d "${stale}" ] || continue
+ [ "${stale}" = "${extractDir}/" ] && continue
+ echo "removing stale ozone extract dir: ${stale}"
+ rm -rf "${stale}"
+ done
+}
+
downloadIfNotPresent postgresql-42.2.16.jre7.jar
"https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7";
downloadIfNotPresent mysql-connector-java-8.0.28.jar
"https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28";
downloadIfNotPresent ojdbc8.jar
https://download.oracle.com/otn-pub/otn_software/jdbc/236
@@ -55,11 +114,7 @@ then
downloadIfNotPresent hadoop-${HIVE_HADOOP_VERSION}.tar.gz
https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION}
downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz
https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz
https://archive.apache.org/dist/knox/${KNOX_VERSION}
- downloadIfNotPresent ozone-${OZONE_VERSION}.tar.gz
https://archive.apache.org/dist/ozone/${OZONE_VERSION}
- if [ ! -d downloads/ozone-${OZONE_VERSION} ]
- then
- tar xvfz downloads/ozone-${OZONE_VERSION}.tar.gz --directory=downloads/
- fi
+ extractOzoneIfNeeded
else
for arg in "$@"; do
if [[ $arg == 'hadoop' ]]
@@ -80,11 +135,7 @@ else
downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz
https://archive.apache.org/dist/knox/${KNOX_VERSION}
elif [[ $arg == 'ozone' ]]
then
- downloadIfNotPresent ozone-${OZONE_VERSION}.tar.gz
https://archive.apache.org/dist/ozone/${OZONE_VERSION}
- if [ ! -d downloads/ozone-${OZONE_VERSION} ]
- then
- tar xvfz downloads/ozone-${OZONE_VERSION}.tar.gz --directory=downloads/
- fi
+ extractOzoneIfNeeded
else
echo "Passed argument $arg is invalid!"
fi
diff --git a/dev-support/ranger-docker/scripts/ozone/docker-config
b/dev-support/ranger-docker/scripts/ozone/docker-config
index ae06dde87..e1716c02d 100644
--- a/dev-support/ranger-docker/scripts/ozone/docker-config
+++ b/dev-support/ranger-docker/scripts/ozone/docker-config
@@ -58,6 +58,10 @@ OZONE-SITE.XML_hdds.container.ratis.datastream.enabled=true
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
+# GitHub Actions ubuntu-24.04 (kernel 6.12+) cgroup layout breaks JDK container
+# metrics and can NPE during Ratis JMX init; disable container support for CI
smoke tests.
+OZONE_OPTS=-XX:-UseContainerSupport
+
no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
# Explicitly enable filesystem snapshot feature for this Docker compose cluster
diff --git a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh
b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh
old mode 100644
new mode 100755
index 8c52474fd..72bd1f32f
--- a/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh
+++ b/dev-support/ranger-docker/scripts/ozone/ranger-ozone-setup.sh
@@ -18,6 +18,28 @@
cd "${OZONE_HOME}"/ranger-ozone-plugin || exit
+# SCM Ratis leader election can lag compose depends_on; avoid OM failing with
+# Connection refused / ServerNotLeaderException on first SCM RPC.
+wait_for_scm() {
+ local scm_host="${OZONE_SCM_HOST:-scm}"
+ local scm_port="${OZONE_SCM_CLIENT_PORT:-9863}"
+ local max_wait_sec="${OZONE_SCM_WAIT_SEC:-120}"
+ local deadline=$((SECONDS + max_wait_sec))
+
+ echo "Waiting for SCM at ${scm_host}:${scm_port} (up to ${max_wait_sec}s)..."
+ while (( SECONDS < deadline )); do
+ if (echo > "/dev/tcp/${scm_host}/${scm_port}") 2>/dev/null; then
+ sleep 5
+ echo "SCM port open; proceeding with Ozone plugin enable"
+ return 0
+ fi
+ sleep 2
+ done
+ echo "ERROR: timed out waiting for SCM at ${scm_host}:${scm_port}" >&2
+ exit 1
+}
+wait_for_scm
+
if [[ ! -f "${OZONE_HOME}"/.setupDone ]];
then
if [ ! -d conf ]; then
@@ -26,8 +48,9 @@ then
else
echo "conf directory exists already!"
fi
- echo "export JAVA_HOME=${JAVA_HOME}" >> conf/ozone-env.sh
- sudo JAVA_HOME=/usr/lib/jvm/jre/ ./enable-ozone-plugin.sh
+ enable_java_home="${JAVA_HOME:-/usr/lib/jvm/jre/}"
+ echo "export JAVA_HOME=${enable_java_home}" >> conf/ozone-env.sh
+ sudo JAVA_HOME="${enable_java_home}" ./enable-ozone-plugin.sh
touch "${OZONE_HOME}"/.setupDone
else
echo "Ranger Ozone Plugin Installation is already complete!"
