This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 390391d1ad Issue #8923 : Run MongoDB without AVX and keep JCraft JSch
off the classpath (#8924)
390391d1ad is described below
commit 390391d1ad25fdd720d71a88ac89d35ac03532c3
Author: Matt Casters <[email protected]>
AuthorDate: Wed Oct 14 18:45:40 2026 +0200
Issue #8923 : Run MongoDB without AVX and keep JCraft JSch off the
classpath (#8924)
Jenkins agent builds44 has no AVX, so mongo:8 and Atlas Local die with
SIGILL
before the mongo tests start. The images now run mongod natively when AVX is
present and under qemu-user otherwise.
hadoop-common's jsch 0.1.55 was unpacked into lib/core next to mwiede jsch.
Filesystem order sometimes loaded 0.1.55, which OpenSSH 9 rejects for public
keys, so the SSH private-key test failed with "Auth fail".
---
.../integration-tests/integration-tests-mongo.yaml | 28 +++++++++++----
docker/integration-tests/mongo-atlas.Dockerfile | 41 +++++++++++++++++++++
docker/integration-tests/mongo-noavx-wrapper.sh | 34 ++++++++++++++++++
docker/integration-tests/mongo.Dockerfile | 42 ++++++++++++++++++++++
integration-tests/scripts/run-tests-docker.sh | 5 +++
plugins/engines/beam/src/assembly/assembly.xml | 5 +++
6 files changed, 149 insertions(+), 6 deletions(-)
diff --git a/docker/integration-tests/integration-tests-mongo.yaml
b/docker/integration-tests/integration-tests-mongo.yaml
index fb3d47df63..c9aec57f77 100644
--- a/docker/integration-tests/integration-tests-mongo.yaml
+++ b/docker/integration-tests/integration-tests-mongo.yaml
@@ -30,13 +30,19 @@ services:
- mongo-atlas
mongo-atlas:
- image: mongodb/mongodb-atlas-local
+ # Built image: native mongod when the CPU has AVX, qemu-user otherwise.
+ # See mongo-atlas.Dockerfile. HOP_MONGO_FORCE_QEMU=1 forces emulation.
+ build:
+ context: .
+ dockerfile: mongo-atlas.Dockerfile
+ image: hop-it-mongo-atlas:local
# The replica set member host is this hostname. It must be the Compose DNS
name clients use.
# "mongo" is the other service on this network, and a member named mongo
talks to mongo:8.
hostname: mongo-atlas
environment:
- MONGODB_INITDB_ROOT_USERNAME=root
- MONGODB_INITDB_ROOT_PASSWORD=example
+ - HOP_MONGO_FORCE_QEMU=${HOP_MONGO_FORCE_QEMU:-}
ports:
- 27017:27017
# The image's own check waits until mongod has finished its init restart.
A plain ping
@@ -46,20 +52,30 @@ services:
test: ["CMD", "runner", "healthcheck"]
interval: 10s
timeout: 10s
- retries: 12
- start_period: 40s
+ retries: 30
+ # start_period does not delay an already-healthy native start. qemu-user
+ # brings mongod up slowly on a CPU without AVX.
+ start_period: 180s
mongo:
- image: mongo:8
+ # Built image: native mongod when the CPU has AVX, qemu-user otherwise.
+ # See mongo.Dockerfile. HOP_MONGO_FORCE_QEMU=1 forces emulation.
+ build:
+ context: .
+ dockerfile: mongo.Dockerfile
+ image: hop-it-mongo:8
hostname: mongo8
environment:
- MONGO_INITDB_ROOT_USERNAME=root
- MONGO_INITDB_ROOT_PASSWORD=example
+ - HOP_MONGO_FORCE_QEMU=${HOP_MONGO_FORCE_QEMU:-}
ports:
- 27018:27017
healthcheck:
test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
interval: 10s
timeout: 5s
- retries: 5
- start_period: 30s
+ retries: 12
+ # start_period does not delay an already-healthy native start. qemu-user
+ # init (mongod --fork) on a CPU without AVX needs the extra grace.
+ start_period: 180s
diff --git a/docker/integration-tests/mongo-atlas.Dockerfile
b/docker/integration-tests/mongo-atlas.Dockerfile
new file mode 100644
index 0000000000..103705bcb6
--- /dev/null
+++ b/docker/integration-tests/mongo-atlas.Dockerfile
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Same AVX workaround as mongo.Dockerfile. The Atlas Local image runs
+# MongoDB 8, which dies with SIGILL (exit 132) on a CPU without AVX.
+# mongot is a Java process with a scalar fallback, so only mongod is wrapped.
+# The image's runner looks mongod up on PATH and execs /usr/bin/mongod.
+
+FROM debian:stable-slim AS qemu
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends qemu-user-static \
+ && rm -rf /var/lib/apt/lists/*
+
+FROM mongodb/mongodb-atlas-local
+
+USER root
+
+COPY --from=qemu /usr/bin/qemu-x86_64-static /usr/bin/qemu-x86_64-static
+COPY mongo-noavx-wrapper.sh /usr/lib/hop-it/mongo-noavx-wrapper.sh
+
+RUN mv /usr/bin/mongod /usr/lib/hop-it/mongod \
+ && install -m 755 /usr/lib/hop-it/mongo-noavx-wrapper.sh /usr/bin/mongod \
+ && chmod 755 /usr/bin/qemu-x86_64-static
+
+USER mongod
diff --git a/docker/integration-tests/mongo-noavx-wrapper.sh
b/docker/integration-tests/mongo-noavx-wrapper.sh
new file mode 100644
index 0000000000..83aba7ecbf
--- /dev/null
+++ b/docker/integration-tests/mongo-noavx-wrapper.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Installed in place of a MongoDB binary. The real binary is
+# /usr/lib/hop-it/<basename>. Hosts with AVX run it directly. Hosts without
+# AVX (and HOP_MONGO_FORCE_QEMU=1) run it under qemu-user, which emulates the
+# AVX instructions MongoDB 5+ is built with. Without this, mongod dies with
+# SIGILL (exit 132) on those agents.
+
+name=$(basename "$0")
+real="/usr/lib/hop-it/${name}"
+
+if [ "${HOP_MONGO_FORCE_QEMU:-}" != "1" ] && grep -qE '^flags.* avx( .*|$)'
/proc/cpuinfo; then
+ exec "$real" "$@"
+fi
+
+echo "hop-it: starting ${name} under qemu-x86_64 because AVX is not available"
>&2
+exec /usr/bin/qemu-x86_64-static -cpu max "$real" "$@"
diff --git a/docker/integration-tests/mongo.Dockerfile
b/docker/integration-tests/mongo.Dockerfile
new file mode 100644
index 0000000000..0325a044e0
--- /dev/null
+++ b/docker/integration-tests/mongo.Dockerfile
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# MongoDB 5+ is built with AVX. On a CPU that does not advertise that flag,
+# the official image's mongod dies with SIGILL (exit 132) and the mongo
+# integration tests never start. qemu-user emulates those instructions when
+# AVX is missing. Hosts that have AVX keep the native binary.
+
+FROM debian:stable-slim AS qemu
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends qemu-user-static \
+ && rm -rf /var/lib/apt/lists/*
+
+FROM mongo:8
+
+USER root
+
+COPY --from=qemu /usr/bin/qemu-x86_64-static /usr/bin/qemu-x86_64-static
+COPY mongo-noavx-wrapper.sh /usr/lib/hop-it/mongo-noavx-wrapper.sh
+
+# The official entrypoint starts "mongod" by PATH. /usr/local/bin is before
+# /usr/bin, but replacing the binary itself also covers an absolute exec.
+# mongosh is a Node.js shell and does not need this wrapper.
+RUN mv /usr/bin/mongod /usr/lib/hop-it/mongod \
+ && install -m 755 /usr/lib/hop-it/mongo-noavx-wrapper.sh /usr/bin/mongod \
+ && chmod 755 /usr/bin/qemu-x86_64-static
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index adbc1c5cf1..9c127544ec 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -491,6 +491,11 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER}
SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build
--abort-on-container-exit \
|| COMPOSE_EXIT=$?
+ elif [ "${PROJECT_NAME}" = "mongo" ]; then
+ echo "Rebuilding mongo images so a CPU without AVX can start mongod"
+ PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER}
SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
+ docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build
--abort-on-container-exit \
+ || COMPOSE_EXIT=$?
else
PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER}
SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit \
diff --git a/plugins/engines/beam/src/assembly/assembly.xml
b/plugins/engines/beam/src/assembly/assembly.xml
index df1edc03c7..3c97fe308f 100644
--- a/plugins/engines/beam/src/assembly/assembly.xml
+++ b/plugins/engines/beam/src/assembly/assembly.xml
@@ -100,6 +100,11 @@
pulls org.postgresql:postgresql back into the runtime
tree. -->
<exclude>io.debezium:debezium-connector-postgres</exclude>
<exclude>org.postgresql:postgresql</exclude>
+ <!-- hadoop-common depends on com.jcraft:jsch 0.1.55. A second
jsch jar
+ in lib/core is loaded in filesystem order. 0.1.55 fails
the SSH
+ private-key test with "Auth fail" against OpenSSH 9.x.
mwiede
+ jsch, packaged by hop-core, stays. -->
+ <exclude>com.jcraft:jsch</exclude>
</excludes>
<outputDirectory>lib/core</outputDirectory>
</dependencySet>
