This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 2763aca4af Add guardrails for plugin packaging regressions. fixes
#8041 (#8170)
2763aca4af is described below
commit 2763aca4afd90de873c612bf751a9a5e6258ccfb
Author: Bart Maertens <[email protected]>
AuthorDate: Mon Aug 31 09:29:46 2026 +0200
Add guardrails for plugin packaging regressions. fixes #8041 (#8170)
---
.github/workflows/marketplace_classpath.yml | 179 ++++++++++
tools/check-plugin-classpath.sh | 491 ++++++++++++++++++++++++++++
tools/fetch-apache-marketplace.sh | 202 ++++++++++++
tools/marketplace-it-aliases.txt | 31 ++
tools/marketplace-it-suites.sh | 60 ++++
tools/plugin-classpath-allowlist.txt | 29 ++
6 files changed, 992 insertions(+)
diff --git a/.github/workflows/marketplace_classpath.yml
b/.github/workflows/marketplace_classpath.yml
new file mode 100644
index 0000000000..e11d8acca1
--- /dev/null
+++ b/.github/workflows/marketplace_classpath.yml
@@ -0,0 +1,179 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+
+# Check that every marketplace plugin still works when a user installs it.
+#
+# Two failure modes, neither of which any other job here can see:
+#
+# * a plugin zip that ships its dependencies but no plugin jar — an assembly
+# <include> that matches no artifact produces exactly this, silently;
+# * a plugin whose own classes reference a package that is on no classpath
once
+# installed, because a dependency was mis-scoped or dropped by a wildcard
+# <exclusion> and so landed neither in the plugin's lib/ nor in lib/core.
+#
+# Both only fail in a user's install. Issue #8036 (hop-tech-parquet) was the
second
+# kind: it worked only because another plugin happened to put the same Hadoop
jars
+# in lib/core, and broke when that plugin left the default client.
+#
+# This job builds nothing. It downloads the published client and plugin zips
and
+# checks those, which is why it costs about three minutes instead of a full
+# assembly build. The consequence is that it runs after a merge, not on a PR: a
+# pull request does not produce plugin zips (pr_build_code.yml passes
+# -Dassemblies=false -Dmodule.zips=false), so there is nothing for it to look
at
+# until the nightly deploy publishes new snapshots.
+#
+# The client zip is the baseline, deliberately, and the container image is
not. The
+# image is built from a client that has already had install-wave1-plugins.sh
run
+# over it, so every plugin is present and each one can borrow the others' jars
—
+# which is precisely how this class of bug hides.
+name: Marketplace plugin classpath
+
+on:
+ schedule:
+ # Snapshots are published by the "Hop Orchestration Platform" job, which
deploys
+ # on merge rather than on a schedule — three to five times on a normal
weekday.
+ # (The daily IT job runs `clean install` and publishes nothing, so its
schedule
+ # is not what this job should track.) The fetch always resolves the newest
+ # snapshot from maven-metadata.xml, so a run is never stale relative to
main; it
+ # is only ever as old as the last merge. Two slots keep the worst-case
delay at
+ # about twelve hours: midday catches the European morning's merges, late
evening
+ # catches the rest of the day.
+ - cron: '0 12,22 * * *'
+ workflow_dispatch:
+ inputs:
+ version:
+ description: 'Version to check (default: this branch''s project
version, e.g. 2.20.0-SNAPSHOT). A release like 2.19.0 checks what users
actually downloaded.'
+ required: false
+ default: ''
+ type: string
+
+concurrency:
+ group: marketplace-classpath-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ check:
+ name: Check marketplace plugin zips
+ runs-on: ubuntu-latest
+ env:
+ # The nightly integration-test run's test report. "lastCompletedBuild"
keeps the
+ # links current without plumbing build numbers between two CI systems.
The API
+ # hangs off this root, while the human-readable per-suite pages sit one
level
+ # down under "(root)" — that segment is required for the pages and must
not be
+ # used for the API.
+ IT_REPORT_ROOT:
'https://ci-builds.apache.org/job/Hop/job/Hop-integration-tests/lastCompletedBuild/testReport'
+
+ steps:
+ - uses: actions/checkout@v4
+
+ # jdeps is a JDK tool: a JRE does not have it.
+ - name: Set up JDK 21
+ uses: actions/setup-java@v4
+ with:
+ java-version: '21'
+ distribution: 'temurin'
+
+ - name: Resolve version
+ id: version
+ run: |
+ set -euo pipefail
+ version='${{ github.event.inputs.version }}'
+ if [ -z "$version" ]; then
+ # -N keeps this to the root pom rather than loading the whole
reactor.
+ version=$(mvn -N -q -DforceStdout help:evaluate
-Dexpression=project.version)
+ fi
+ [ -n "$version" ] || { echo "could not determine the project
version"; exit 1; }
+ echo "version=$version" >> "$GITHUB_OUTPUT"
+ echo "checking $version"
+
+ # Releases come from the ASF dist archive, snapshots from the snapshot
+ # repository. The plugin list is read from the client's own
+ # full-client-env.yaml, so the list and the baseline are always the same
+ # build.
+ - name: Download client and marketplace plugin zips
+ run: |
+ ./tools/fetch-apache-marketplace.sh \
+ --version '${{ steps.version.outputs.version }}' \
+ --dest "$RUNNER_TEMP/marketplace"
+
+ # A renamed integration-test project silently turns a report link into a
404,
+ # which reads as "covered" when it is not. Warn rather than fail:
Jenkins being
+ # unreachable says nothing about the plugin zips this job is here to
check.
+ # Derived from optional-plugins.yaml and the integration-tests/
directories, so
+ # a new plugin whose IT project follows the naming convention is picked
up
+ # without editing anything.
+ - name: Pair plugins with their integration-test projects
+ run: |
+ ./tools/marketplace-it-suites.sh | tee "$RUNNER_TEMP/it-suites.txt"
+
+ - name: Verify the integration-test suite names still exist
+ continue-on-error: true
+ run: |
+ set -uo pipefail
+ suites=$(curl -fsSL --max-time 60 \
+ "$IT_REPORT_ROOT/api/json?tree=suites\[name\]" | tr ',' '\n' |
+ sed -n 's/.*"name":"\([^"]*\)".*/\1/p' | sort -u)
+ if [ -z "$suites" ]; then
+ echo "::warning::could not read the Jenkins test report; skipping
the link check"
+ exit 0
+ fi
+ missing=0
+ while read -r plugin suite; do
+ case "$plugin" in ''|\#*) continue ;; esac
+ if ! printf '%s\n' "$suites" | grep -qxF "$suite"; then
+ echo "::warning::$plugin maps to integration-test suite
'$suite', which is not in the latest run"
+ missing=$((missing + 1))
+ fi
+ done < "$RUNNER_TEMP/it-suites.txt"
+ echo "$missing stale mapping(s)"
+
+ - name: Check
+ run: |
+ ./tools/check-plugin-classpath.sh \
+ --client "$RUNNER_TEMP/marketplace/client.zip" \
+ --plugins "$RUNNER_TEMP/marketplace/plugins.txt" \
+ --label '${{ steps.version.outputs.version }}' \
+ --junit reports/marketplace-classpath.xml \
+ --report reports/summary.md \
+ --it-suites "$RUNNER_TEMP/it-suites.txt" \
+ --it-base "$IT_REPORT_ROOT/(root)"
+
+ # Rendered on the run's summary page, so the per-plugin status is the
first
+ # thing a reader sees rather than something buried in the log.
+ - name: Publish summary
+ if: always()
+ run: cat reports/summary.md >> "$GITHUB_STEP_SUMMARY" || true
+
+ # Proves the check can still fail. Without this a packaging change that
makes
+ # every plugin unanalysable would show up as a green run.
+ - name: Self test
+ run: |
+ ./tools/check-plugin-classpath.sh \
+ --client "$RUNNER_TEMP/marketplace/client.zip" \
+ --plugins "$RUNNER_TEMP/marketplace/plugins.txt" \
+ --label '${{ steps.version.outputs.version }}' \
+ --self-test
+
+ - name: Upload report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: marketplace-classpath-report
+ path: reports/
+ if-no-files-found: ignore
diff --git a/tools/check-plugin-classpath.sh b/tools/check-plugin-classpath.sh
new file mode 100755
index 0000000000..308ec06b42
--- /dev/null
+++ b/tools/check-plugin-classpath.sh
@@ -0,0 +1,491 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Fail when a marketplace plugin zip cannot work once it is installed.
+#
+# Two failure modes, both silent in every other part of the build:
+#
+# 1. The zip ships no plugin jar at all. Its dependencies are there, its
+# version.xml is there, and installing it adds nothing that Hop can load.
+# An assembly <include> that matches no artifact produces exactly this and
+# maven-assembly-plugin does not warn.
+#
+# 2. The plugin's own classes reference a package that is on no classpath
once
+# the plugin is installed, because a dependency was mis-scoped or dropped
by
+# a wildcard <exclusion> and so landed neither in the plugin's lib/ nor
in the
+# client's lib/core. The plugin then dies with NoClassDefFoundError on
first
+# use, in the user's install, not in CI.
+#
+# Each plugin is checked against a baseline client that represents what its
users
+# actually have. That baseline matters: with every plugin present, one plugin's
+# lib/core contribution covers another's gap, which is how these bugs survive.
+# For a marketplace plugin the baseline is the plain client. For a plugin that
+# ships inside a client, install its companions with --install first.
+#
+# jdeps is run over the plugin's own jars only, never over the third-party
jars it
+# bundles: those are full of optional dependencies that are on no classpath by
+# design, and several are modular, which makes jdeps abort with a module
resolution
+# error rather than report. Restricting the input to first-party classes keeps
the
+# check silent when healthy and loud when a plugin is genuinely broken.
+#
+# Usage:
+# tools/check-plugin-classpath.sh --client <zip|dir> --plugins <list>
[options]
+#
+# --client <zip|dir> Baseline client to install onto. Required.
+# --plugins <file> Plugin list, one per line: "<artifactId>
<path-to-zip>".
+# A '|' separator is accepted too. '#' comments and
blank
+# lines are ignored. Required.
+# --install <zip> Install this zip into the baseline before checking
+# anything. Repeatable. Use for the plugins that ship
+# inside the client the marketplace plugins install
onto.
+# --allowlist <file> Default: tools/plugin-classpath-allowlist.txt next to
this
+# script.
+# --label <name> Name for the run in output and JUnit. Default
"plugins".
+# --junit <file> Also write a JUnit report, one testcase per plugin.
+# --report <file> Also write a Markdown status table, one row per
plugin.
+# --it-suites <file> "<artifactId> <suite>" pairs naming the
integration-test
+# project that covers a plugin needing a live service.
Those
+# plugins report as needs-service rather than as a gap.
+# --it-base <url> Base URL for the test report, to turn the suite name
into a
+# link. Without it the suite is named but not linked.
+# --plugin <id> Check only this artifactId.
+# --allow-missing Do not fail on plugins whose zip was not built.
+# --self-test Prove the check can still fail, then exit.
+#
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+CLIENT=""
+PLUGIN_LIST=""
+ALLOWLIST="${HERE}/plugin-classpath-allowlist.txt"
+LABEL="plugins"
+JUNIT=""
+REPORT=""
+IT_SUITES=""
+IT_BASE=""
+ONLY_PLUGIN=""
+ALLOW_MISSING=false
+SELF_TEST=false
+INSTALL_ZIPS=()
+
+die() {
+ echo "ERROR: $*" >&2
+ exit 2
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --client) shift; CLIENT="${1:-}"; [[ -n "${CLIENT}" ]] || die "--client
needs a value" ;;
+ --plugins) shift; PLUGIN_LIST="${1:-}"; [[ -n "${PLUGIN_LIST}" ]] || die
"--plugins needs a value" ;;
+ --install) shift; [[ -n "${1:-}" ]] || die "--install needs a value";
INSTALL_ZIPS+=("$1") ;;
+ --allowlist) shift; ALLOWLIST="${1:-}"; [[ -n "${ALLOWLIST}" ]] || die
"--allowlist needs a value" ;;
+ --label) shift; LABEL="${1:-}"; [[ -n "${LABEL}" ]] || die "--label needs a
value" ;;
+ --junit) shift; JUNIT="${1:-}"; [[ -n "${JUNIT}" ]] || die "--junit needs a
value" ;;
+ --report) shift; REPORT="${1:-}"; [[ -n "${REPORT}" ]] || die "--report
needs a value" ;;
+ --it-suites) shift; IT_SUITES="${1:-}"; [[ -n "${IT_SUITES}" ]] || die
"--it-suites needs a value" ;;
+ --it-base) shift; IT_BASE="${1:-}"; [[ -n "${IT_BASE}" ]] || die "--it-base
needs a value" ;;
+ --plugin) shift; ONLY_PLUGIN="${1:-}"; [[ -n "${ONLY_PLUGIN}" ]] || die
"--plugin needs an artifactId" ;;
+ --allow-missing) ALLOW_MISSING=true ;;
+ --self-test) SELF_TEST=true ;;
+ -h | --help) sed -n '17,65p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//';
exit 0 ;;
+ *) die "unknown argument: $1" ;;
+ esac
+ shift
+done
+
+[[ -n "${CLIENT}" ]] || die "--client is required"
+[[ -n "${PLUGIN_LIST}" ]] || die "--plugins is required"
+[[ -f "${PLUGIN_LIST}" ]] || die "plugin list not found: ${PLUGIN_LIST}"
+command -v jdeps >/dev/null 2>&1 || die "jdeps not on PATH — this needs a JDK,
not a JRE"
+command -v unzip >/dev/null 2>&1 || die "unzip not on PATH"
+
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/plugin-classpath.XXXXXX")"
+trap 'rm -rf "${WORK}"' EXIT
+
+# ---------------------------------------------------------------- baseline
+
+BASE="${WORK}/baseline"
+if [[ -d "${CLIENT}" ]]; then
+ mkdir -p "${BASE}"
+ # Copied rather than used in place: --install writes into the baseline, and a
+ # check must never mutate the caller's client.
+ cp -R "${CLIENT}/." "${BASE}/"
+elif [[ -f "${CLIENT}" ]]; then
+ echo "==> Unpacking $(basename "${CLIENT}")"
+ unzip -q -o "${CLIENT}" -d "${BASE}"
+else
+ die "--client is neither a file nor a directory: ${CLIENT}"
+fi
+
+# Client zips unpack to a single top directory; extracted container roots do
not.
+if [[ ! -d "${BASE}/lib" ]]; then
+ inner="$(find "${BASE}" -maxdepth 2 -type d -name lib -print -quit
2>/dev/null || true)"
+ [[ -n "${inner}" ]] || die "no lib/ directory found under ${CLIENT}"
+ BASE="$(dirname "${inner}")"
+fi
+
+for zip in ${INSTALL_ZIPS[@]+"${INSTALL_ZIPS[@]}"}; do
+ [[ -f "${zip}" ]] || die "--install zip not found: ${zip}"
+ echo " installing into baseline: $(basename "${zip}")"
+ unzip -q -o "${zip}" -d "${BASE}"
+done
+
+BASE_CP=""
+while IFS= read -r jar; do
+ BASE_CP="${BASE_CP}${jar}:"
+done < <(find "${BASE}/lib" "${BASE}/plugins" -name '*.jar' 2>/dev/null | sort)
+[[ -n "${BASE_CP}" ]] || die "no jars found in the baseline client at ${BASE}"
+
+# ---------------------------------------------------------------- helpers
+
+# Jars that are the plugin's own code: directly under
plugins/<category>/<name>/,
+# as opposed to its bundled dependencies, which the assembly puts in that
+# directory's lib/.
+own_jars() { find "$1/plugins" -name '*.jar' 2>/dev/null | grep -v '/lib/' |
sort || true; }
+
+# What a zip is, before deciding whether it is broken:
+# plugin — ships first-party jars, check them
+# nojar — ships dependencies under plugins/**/lib but no plugin jar.
Broken:
+# installing it adds libraries and nothing that Hop can load.
+# library — ships no plugins/ tree at all, only lib/ jars. A shared library
+# published as a zip, with nothing of its own to check.
+# empty — no jars anywhere
+classify() {
+ local dir="$1"
+ if [[ -n "$(own_jars "${dir}")" ]]; then echo plugin; return; fi
+ if [[ -n "$(find "${dir}/plugins" -name '*.jar' 2>/dev/null | head -1)" ]];
then echo nojar; return; fi
+ if [[ -n "$(find "${dir}" -name '*.jar' 2>/dev/null | head -1)" ]]; then
echo library; return; fi
+ echo empty
+}
+
+# $2, when set, is a jar basename to withhold from the classpath (--self-test).
+plugin_cp() {
+ local dir="$1" withhold="${2:-}" cp="" jar
+ while IFS= read -r jar; do
+ [[ -n "${withhold}" && "$(basename "${jar}")" == "${withhold}" ]] &&
continue
+ cp="${cp}${jar}:"
+ done < <(find "${dir}" -name '*.jar' 2>/dev/null | sort)
+ if [[ -z "${withhold}" ]]; then
+ cp="${cp}${BASE_CP}"
+ else
+ while IFS= read -r jar; do
+ [[ "$(basename "${jar}")" == "${withhold}" ]] && continue
+ cp="${cp}${jar}:"
+ done < <(find "${BASE}/lib" "${BASE}/plugins" -name '*.jar' 2>/dev/null |
sort)
+ fi
+ printf '%s\n' "${cp}"
+}
+
+run_jdeps() {
+ local dir="$1" withhold="${2:-}" cp jars
+ cp="$(plugin_cp "${dir}" "${withhold}")"
+ jars="$(own_jars "${dir}")"
+ [[ -n "${jars}" ]] || return 3
+ # shellcheck disable=SC2086
+ jdeps -q --multi-release 21 -cp "${cp}" ${jars} 2>&1 || true
+}
+
+# "sourcePackage missingPackage" per unresolved reference. jdeps also emits a
bare
+# "<jar> -> not found" summary; only the indented package lines name the source
+# package, which is what makes a finding actionable.
+parse_missing() { awk '/ not found$/ && /^[[:space:]]/ { print $1, $3 }' |
sort -u; }
+
+allowed() {
+ local id="$1" pkg="$2"
+ [[ -f "${ALLOWLIST}" ]] || return 1
+ awk -v id="${id}" -v pkg="${pkg}" '
+ /^[[:space:]]*#/ { next }
+ NF < 2 { next }
+ $1 == id && (pkg == $2 || index(pkg, $2 ".") == 1) { found = 1; exit }
+ END { exit found ? 0 : 1 }
+ ' "${ALLOWLIST}"
+}
+
+# The integration-test project covering this plugin, if it needs a live
service.
+it_suite() {
+ [[ -n "${IT_SUITES}" && -f "${IT_SUITES}" ]] || return 1
+ awk -v id="$1" '/^[[:space:]]*#/ { next } NF < 2 { next } $1 == id { print
$2; found = 1; exit }
+ END { exit found ? 0 : 1 }' "${IT_SUITES}"
+}
+
+xml_escape() { sed -e 's/&/\&/g' -e 's/</\</g' -e 's/>/\>/g'; }
+
+# ---------------------------------------------------------------- plugin list
+
+IDS=()
+ZIPS=()
+while IFS= read -r line || [[ -n "${line}" ]]; do
+ line="${line%%#*}"
+ line="$(printf '%s' "${line}" | tr '|' ' ')"
+ # shellcheck disable=SC2086
+ set -- ${line}
+ [[ $# -ge 2 ]] || continue
+ [[ -z "${ONLY_PLUGIN}" || "$1" == "${ONLY_PLUGIN}" ]] || continue
+ IDS+=("$1")
+ ZIPS+=("$2")
+done <"${PLUGIN_LIST}"
+
+[[ ${#IDS[@]} -gt 0 ]] || die "no plugins to check from ${PLUGIN_LIST}"
+
+# ---------------------------------------------------------------- self test
+
+# Picks a plugin whose own classes resolve against a jar the plugin itself
ships,
+# withholds that jar, and expects the missing package to be reported. Derived
from
+# the build rather than hard-coded, so it keeps working as plugins come and go.
+if [[ "${SELF_TEST}" == true ]]; then
+ echo "==> Self test (${LABEL})"
+ i=0
+ while [[ ${i} -lt ${#IDS[@]} ]]; do
+ id="${IDS[$i]}"; zip="${ZIPS[$i]}"; i=$((i + 1))
+ [[ -f "${zip}" ]] || continue
+ dir="${WORK}/st"; rm -rf "${dir}"; mkdir -p "${dir}"
+ unzip -q -o "${zip}" -d "${dir}"
+ [[ "$(classify "${dir}")" == plugin ]] || continue
+ out="$(run_jdeps "${dir}")" || continue
+ echo "${out}" | grep -q '^Exception in thread' && continue
+ [[ -n "$(echo "${out}" | parse_missing)" ]] && continue
+
+ # Dependency jars only. The plugin's own jars are jdeps *input*, so
withholding
+ # one from the classpath changes nothing and would fail the self test
spuriously.
+ own_jars "${dir}" | xargs -n1 basename 2>/dev/null | sort -u
>"${WORK}/own.txt"
+ find "${dir}" -name '*.jar' -exec basename {} \; | sort -u |
+ grep -Fxv -f "${WORK}/own.txt" >"${WORK}/shipped.txt" || true
+ # jdeps names JDK modules in the same column ("java.base"), so keep real
jars only.
+ canary="$(echo "${out}" |
+ awk 'NF == 4 && $2 == "->" && $4 ~ /\.jar$/ { print $4 }' |
+ sort -u | grep -Fx -f "${WORK}/shipped.txt" | head -1 || true)"
+ [[ -n "${canary}" ]] || continue
+
+ echo " canary plugin: ${id}"
+ echo " withholding: ${canary}"
+ findings="$(run_jdeps "${dir}" "${canary}" | parse_missing || true)"
+ if [[ -z "${findings}" ]]; then
+ echo
+ echo "SELF TEST FAILED: withholding ${canary} produced no finding."
+ exit 1
+ fi
+ echo " reported:"
+ echo "${findings}" | sed 's/^/ /'
+ echo
+ echo "Self test passed."
+ exit 0
+ done
+ die "no plugin with a usable canary jar; build the plugin zips first"
+fi
+
+# ---------------------------------------------------------------- main
+
+echo "==> Checking ${#IDS[@]} ${LABEL} plugin(s)"
+echo " baseline: ${CLIENT}"
+echo
+
+ok=0; allowedcount=0; failed=0; missing=0; errored=0; libs=0
+CASES="${WORK}/cases"
+: >"${CASES}"
+
+# One line per plugin. Detail can be multi-line, which would otherwise turn
every
+# continuation line into its own testcase, so newlines are folded onto a record
+# separator here and unfolded when the JUnit report is written.
+# Fields are separated by a unit separator, not a tab: tab is an IFS whitespace
+# character, so `read` collapses runs of it and an empty detail field would
silently
+# shift every column after it. Embedded newlines fold onto a record separator
so one
+# plugin stays one line.
+record() {
+ printf '%s\037%s\037%s\037%s\n' \
+ "$1" "$2" "$(printf '%s' "$3" | tr '\n' '\036')" "${4:-0}" >>"${CASES}"
+}
+
+# Sample pipelines and workflows the zip ships. Plugins put these under
+# config/projects/samples via src/main/samples, and the client pre-registers
that
+# project, so they are what a per-plugin smoke test would run.
+sample_count() {
+ find "$1" \( -name '*.hpl' -o -name '*.hwf' \) 2>/dev/null | wc -l | tr -d '
'
+}
+
+i=0
+while [[ ${i} -lt ${#IDS[@]} ]]; do
+ id="${IDS[$i]}"; zip="${ZIPS[$i]}"; i=$((i + 1))
+
+ if [[ ! -f "${zip}" ]]; then
+ echo " MISSING ${id} — ${zip}"
+ missing=$((missing + 1))
+ record "${id}" skipped "zip not built: ${zip}"
+ continue
+ fi
+
+ dir="${WORK}/stage"; rm -rf "${dir}"; mkdir -p "${dir}"
+ unzip -q -o "${zip}" -d "${dir}"
+ samples="$(sample_count "${dir}")"
+
+ case "$(classify "${dir}")" in
+ library)
+ echo " LIBRARY ${id} (no plugin tree, nothing to check)"
+ libs=$((libs + 1))
+ record "${id}" skipped "shared library zip" "${samples}"
+ continue
+ ;;
+ nojar)
+ echo " FAIL ${id}"
+ echo " the zip ships dependencies but no plugin jar"
+ echo " installing it adds nothing Hop can load"
+ echo " check the assembly <include> matches the module's
groupId:artifactId"
+ failed=$((failed + 1))
+ record "${id}" failure "zip ships dependencies under plugins/**/lib but no
plugin jar" "${samples}"
+ continue
+ ;;
+ empty)
+ echo " FAIL ${id} — the zip contains no jars at all"
+ failed=$((failed + 1))
+ record "${id}" failure "zip contains no jars" "${samples}"
+ continue
+ ;;
+ esac
+
+ out="$(run_jdeps "${dir}")"
+
+ # jdeps refusing to run must never read as a pass: that is how a plugin
quietly
+ # stops being checked.
+ if echo "${out}" | grep -q '^Exception in thread'; then
+ detail="$(echo "${out}" | grep '^Exception in thread' | head -1)"
+ echo " ERROR ${id} — jdeps could not analyse the plugin:"
+ echo " ${detail}"
+ errored=$((errored + 1))
+ record "${id}" error "${detail}" "${samples}"
+ continue
+ fi
+
+ hits="$(echo "${out}" | parse_missing)"
+ if [[ -z "${hits}" ]]; then
+ echo " OK ${id}"
+ ok=$((ok + 1))
+ record "${id}" pass "" "${samples}"
+ continue
+ fi
+
+ plugin_failed=false
+ shown=""
+ detail=""
+ while read -r src pkg; do
+ [[ -n "${src}" ]] || continue
+ if allowed "${id}" "${pkg}"; then
+ shown="${shown} allowed: ${pkg}"$'\n'
+ else
+ shown="${shown} ${src} -> ${pkg}"$'\n'
+ detail="${detail}${src} -> ${pkg}"$'\n'
+ plugin_failed=true
+ fi
+ done <<<"${hits}"
+
+ if [[ "${plugin_failed}" == true ]]; then
+ echo " FAIL ${id}"
+ printf '%s' "${shown}"
+ failed=$((failed + 1))
+ record "${id}" failure "${detail}" "${samples}"
+ else
+ echo " ALLOWED ${id}"
+ printf '%s' "${shown}"
+ allowedcount=$((allowedcount + 1))
+ record "${id}" pass "" "${samples}"
+ fi
+done
+
+echo
+echo "Summary (${LABEL}): ok=${ok} allowed=${allowedcount} library=${libs}
failed=${failed} errored=${errored} missing=${missing}"
+
+# ---------------------------------------------------------------- junit
+
+if [[ -n "${JUNIT}" ]]; then
+ mkdir -p "$(dirname "${JUNIT}")"
+ total=$(wc -l <"${CASES}" | tr -d ' ')
+ {
+ echo '<?xml version="1.0" encoding="UTF-8"?>'
+ printf '<testsuite name="plugin-classpath-%s" tests="%s" failures="%s"
errors="%s" skipped="%s">\n' \
+ "${LABEL}" "${total}" "${failed}" "${errored}" "$((missing + libs))"
+ while IFS=$'\037' read -r id status detail _samples; do
+ printf ' <testcase classname="plugin-classpath.%s" name="%s">'
"${LABEL}" "${id}"
+ case "${status}" in
+ pass) ;;
+ failure) printf '<failure message="unresolved classpath
references"><![CDATA[%s]]></failure>' "$(printf '%s' "${detail}" | tr '\036'
'\n')" ;;
+ error) printf '<error message="jdeps failed"><![CDATA[%s]]></error>'
"$(printf '%s' "${detail}" | tr '\036' '\n')" ;;
+ skipped) printf '<skipped message="%s"/>' "$(printf '%s' "${detail}" |
tr '\036' ' ' | xml_escape)" ;;
+ esac
+ printf '</testcase>\n'
+ done <"${CASES}"
+ echo '</testsuite>'
+ } >"${JUNIT}"
+ echo "JUnit report: ${JUNIT}"
+fi
+
+# ---------------------------------------------------------------- report
+
+if [[ -n "${REPORT}" ]]; then
+ mkdir -p "$(dirname "${REPORT}")"
+ {
+ printf '### Marketplace plugins — %s\n\n' "${LABEL}"
+ printf '| Plugin | Classpath | Samples |\n|---|---|---|\n'
+ while IFS=$'\037' read -r id status detail samples; do
+ case "${status}" in
+ pass) cp_cell="pass" ;;
+ failure) cp_cell="**fail**" ;;
+ error) cp_cell="**error**" ;;
+ skipped) cp_cell="not checked — $(printf '%s' "${detail}" | tr '\036' '
')" ;;
+ *) cp_cell="${status}" ;;
+ esac
+
+ # A plugin an integration test already covers points at that test rather
than
+ # reading as untested — most of them need a live service and cannot run
here at
+ # all. It is evidence, not a pass: those tests run against a full
distribution,
+ # where a plugin can borrow another plugin's jars — the very thing the
classpath
+ # column exists to rule out.
+ if suite="$(it_suite "${id}")"; then
+ if [[ -n "${IT_BASE}" ]]; then
+ samples_cell="covered by IT — [\`${suite}\`
↗](${IT_BASE%/}/${suite}/)"
+ else
+ samples_cell="covered by IT — ${suite}"
+ fi
+ elif [[ "${samples:-0}" -gt 0 ]]; then
+ samples_cell="not run (${samples})"
+ else
+ samples_cell="no sample"
+ fi
+ printf '| %s | %s | %s |\n' "${id}" "${cp_cell}" "${samples_cell}"
+ done <"${CASES}"
+
+ printf '\nClasspath: ok=%s allowed=%s library=%s failed=%s errored=%s
missing=%s\n' \
+ "${ok}" "${allowedcount}" "${libs}" "${failed}" "${errored}" "${missing}"
+ printf '\n%s\n' '- **no sample** — ships no pipeline under
`config/projects/samples`, so nothing can smoke-test it.'
+ printf '%s\n' '- **not run** — ships samples; running them per plugin is
not wired up yet.'
+ printf '%s\n' '- **covered by IT** — not smoke-tested here (these mostly
need a live service), but exercised by the nightly integration tests.
Supporting evidence only: those run against a full distribution, where a plugin
can borrow another plugin'"'"'s jars.'
+ } >"${REPORT}"
+ echo "Report: ${REPORT}"
+fi
+
+status=0
+if [[ ${failed} -gt 0 || ${errored} -gt 0 ]]; then
+ status=1
+fi
+if [[ ${missing} -gt 0 ]]; then
+ if [[ "${ALLOW_MISSING}" == true ]]; then
+ echo "Ignoring ${missing} plugin zip(s) that were not built
(--allow-missing)."
+ else
+ echo "${missing} plugin zip(s) were not built, so they were not checked."
+ status=1
+ fi
+fi
+
+[[ ${status} -eq 0 ]] && echo "Plugin classpath check passed (${LABEL})."
+exit ${status}
diff --git a/tools/fetch-apache-marketplace.sh
b/tools/fetch-apache-marketplace.sh
new file mode 100755
index 0000000000..4fd0bee5b8
--- /dev/null
+++ b/tools/fetch-apache-marketplace.sh
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Download an Apache Hop client and the marketplace plugin zips that go with
it,
+# so tools/check-plugin-classpath.sh can check them the way a user's install
would
+# see them.
+#
+# The client zip is the right baseline and the container image is not: the
image is
+# built from a client that already had every marketplace plugin installed into
it
+# (Apache Hop's Jenkinsfile unpacks the client, runs install-wave1-plugins.sh,
then
+# builds the image from that directory). Checking against the image would let
each
+# plugin borrow the jars of all the others, which is exactly the bug being
hunted.
+# The published client zip is what a user downloads, and it is plugin-free.
+#
+# The plugin list is read from the client's own full-client-env.yaml rather
than
+# from Apache Hop's optional-plugins.yaml over the network, so the list and the
+# baseline are always the same version.
+#
+# Releases and snapshots are both supported. A release client comes from the
ASF
+# dist archive; a -SNAPSHOT client comes from the snapshot repository, where
the
+# assemblies are published under the artifactId hop-client (the
deploy-snapshots
+# excludes drop hop-assemblies*, which that name does not match). Snapshot
files are
+# timestamped, so their real names have to be resolved per artifact from
+# maven-metadata.xml — the plain <artifactId>-<version>-SNAPSHOT.zip name is a
404.
+#
+# Usage:
+# tools/fetch-apache-marketplace.sh --version 2.19.0 --dest <dir> [--only
<id>]...
+# tools/fetch-apache-marketplace.sh --version 2.20.0-SNAPSHOT --dest <dir>
+# tools/fetch-apache-marketplace.sh --version 2.19.0 --dest <dir>
--client-only
+#
+# Writes <dir>/client.zip, <dir>/zips/*.zip and <dir>/plugins.txt (the
+# "<artifactId> <path>" list that check-plugin-classpath.sh --plugins expects).
+#
+# --client-only stops after the client, for callers that just need a baseline
to
+# install something else onto.
+#
+set -euo pipefail
+
+VERSION=""
+DEST=""
+CLIENT_ONLY=false
+ONLY=()
+
+die() { echo "ERROR: $*" >&2; exit 2; }
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --version) shift; VERSION="${1:-}"; [[ -n "${VERSION}" ]] || die "--version
needs a value" ;;
+ --dest) shift; DEST="${1:-}"; [[ -n "${DEST}" ]] || die "--dest needs a
value" ;;
+ --only) shift; [[ -n "${1:-}" ]] || die "--only needs an artifactId";
ONLY+=("$1") ;;
+ --client-only) CLIENT_ONLY=true ;;
+ -h | --help) sed -n '17,50p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//';
exit 0 ;;
+ *) die "unknown argument: $1" ;;
+ esac
+ shift
+done
+
+[[ -n "${VERSION}" ]] || die "--version is required"
+[[ -n "${DEST}" ]] || die "--dest is required"
+command -v curl >/dev/null 2>&1 || die "curl not on PATH"
+command -v unzip >/dev/null 2>&1 || die "unzip not on PATH"
+
+# archive.apache.org keeps every release; downloads.apache.org only the
current one,
+# so a pinned older version resolves on the archive and not on the mirror.
+DIST_URLS=(
+
"https://archive.apache.org/dist/hop/${VERSION}/apache-hop-client-${VERSION}.zip";
+
"https://downloads.apache.org/hop/${VERSION}/apache-hop-client-${VERSION}.zip";
+)
+# The marketplace itself resolves plugins from the ASF group first and Central
as a
+# fallback (see MarketplaceConfig in Apache Hop); mirror that order here so
this
+# check fails on the same artifacts a user's install would get.
+REPO_URLS=(
+ "https://repository.apache.org/content/groups/public";
+ "https://repo1.maven.org/maven2";
+)
+
+case "${VERSION}" in
+*-SNAPSHOT) IS_SNAPSHOT=true ;;
+*) IS_SNAPSHOT=false ;;
+esac
+
+mkdir -p "${DEST}/zips"
+CLIENT_ZIP="${DEST}/client.zip"
+
+fetch() {
+ local out="$1" url
+ shift
+ for url in "$@"; do
+ if curl -fsSL --retry 3 --retry-delay 2 -o "${out}.part" "${url}"; then
+ mv "${out}.part" "${out}"
+ printf '%s\n' "${url}"
+ return 0
+ fi
+ done
+ rm -f "${out}.part"
+ return 1
+}
+
+# A snapshot version directory holds timestamped files only; maven-metadata.xml
+# names the current one. Each artifact is resolved separately: they are
usually from
+# the same deploy run, but nothing guarantees it, and reusing one artifact's
+# timestamp for another silently 404s.
+snapshot_name() {
+ local repo="$1" art="$2" value
+ value="$(curl -fsSL --retry 2 --max-time 60 \
+ "${repo}/org/apache/hop/${art}/${VERSION}/maven-metadata.xml" 2>/dev/null |
+ tr '<' '\n' | sed -n 's:^value>::p' | tail -1 || true)"
+ [[ -n "${value}" ]] || return 1
+ printf '%s-%s.zip\n' "${art}" "${value}"
+}
+
+# Candidate URLs for one artifact's zip, across both repositories.
+zip_urls() {
+ local art="$1" repo name
+ for repo in "${REPO_URLS[@]}"; do
+ if [[ "${IS_SNAPSHOT}" == true ]]; then
+ name="$(snapshot_name "${repo}" "${art}")" || continue
+ else
+ name="${art}-${VERSION}.zip"
+ fi
+ printf '%s\n' "${repo}/org/apache/hop/${art}/${VERSION}/${name}"
+ done
+}
+
+if [[ -f "${CLIENT_ZIP}" ]]; then
+ echo "==> Client already downloaded: ${CLIENT_ZIP}"
+elif [[ "${IS_SNAPSHOT}" == true ]]; then
+ echo "==> Downloading Apache Hop ${VERSION} client from the snapshot
repository"
+ urls=()
+ while IFS= read -r u; do urls+=("$u"); done < <(zip_urls hop-client)
+ [[ ${#urls[@]} -gt 0 ]] || die "no snapshot client published for ${VERSION}"
+ from="$(fetch "${CLIENT_ZIP}" "${urls[@]}")" ||
+ die "could not download the snapshot client for ${VERSION}"
+ echo " from ${from}"
+else
+ echo "==> Downloading Apache Hop ${VERSION} client"
+ from="$(fetch "${CLIENT_ZIP}" "${DIST_URLS[@]}")" ||
+ die "could not download the client zip for ${VERSION} (tried
${DIST_URLS[*]})"
+ echo " from ${from}"
+fi
+
+if [[ "${CLIENT_ONLY}" == true ]]; then
+ echo " client: ${CLIENT_ZIP}"
+ exit 0
+fi
+
+# full-client-env.yaml is generated from optional-plugins.yaml and ships in the
+# client, so the list matches the baseline by construction.
+ENV_FILE="$(unzip -Z1 "${CLIENT_ZIP}" '*full-client-env.yaml' 2>/dev/null |
head -1 || true)"
+[[ -n "${ENV_FILE}" ]] ||
+ die "full-client-env.yaml not found in the client zip; this Hop version
predates the marketplace registry"
+unzip -p "${CLIENT_ZIP}" "${ENV_FILE}" >"${DEST}/full-client-env.yaml"
+
+ARTIFACTS="$(awk '/^[[:space:]]*-[[:space:]]*artifactId:/ { print $3 }'
"${DEST}/full-client-env.yaml")"
+[[ -n "${ARTIFACTS}" ]] || die "no artifactIds in full-client-env.yaml"
+
+wanted() {
+ [[ ${#ONLY[@]} -eq 0 ]] && return 0
+ local a
+ for a in "${ONLY[@]}"; do [[ "$a" == "$1" ]] && return 0; done
+ return 1
+}
+
+: >"${DEST}/plugins.txt"
+count=0
+failed=0
+for art in ${ARTIFACTS}; do
+ wanted "${art}" || continue
+ zip="${DEST}/zips/${art}-${VERSION}.zip"
+ if [[ ! -f "${zip}" ]]; then
+ urls=()
+ while IFS= read -r u; do urls+=("$u"); done < <(zip_urls "${art}")
+ if [[ ${#urls[@]} -eq 0 ]] || ! fetch "${zip}" "${urls[@]}" >/dev/null;
then
+ # Not fatal on its own: the check reports it as a missing zip, which is a
+ # more useful failure than aborting the whole fetch here.
+ echo " MISSING ${art}"
+ failed=$((failed + 1))
+ continue
+ fi
+ fi
+ printf '%s %s\n' "${art}" "${zip}" >>"${DEST}/plugins.txt"
+ count=$((count + 1))
+done
+
+echo "==> ${count} plugin zip(s) ready in ${DEST}/zips"
+[[ ${failed} -gt 0 ]] && echo " ${failed} could not be downloaded"
+echo " client: ${CLIENT_ZIP}"
+echo " plugins: ${DEST}/plugins.txt"
+exit 0
diff --git a/tools/marketplace-it-aliases.txt b/tools/marketplace-it-aliases.txt
new file mode 100644
index 0000000000..487eb1e3b3
--- /dev/null
+++ b/tools/marketplace-it-aliases.txt
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Exceptions for tools/marketplace-it-suites.sh.
+#
+# That script pairs a marketplace plugin with the integration-test project
covering
+# it by taking the last segment of the plugin's modulePath and looking for a
+# directory of that name under integration-tests/. Twelve of the sixteen pairs
fall
+# out of that rule on their own; these four are the ones where the two trees
chose
+# different names.
+#
+# Format: <last segment of modulePath> <integration-tests project>
+# Add a line only when the names genuinely differ — a new plugin whose IT
project
+# matches its module name needs nothing here.
+
+beam beam_directrunner
+google gcp
+mongodb mongo
+script scripting
diff --git a/tools/marketplace-it-suites.sh b/tools/marketplace-it-suites.sh
new file mode 100755
index 0000000000..5b4400f367
--- /dev/null
+++ b/tools/marketplace-it-suites.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Pair each marketplace plugin with the integration-test project that covers
it, for
+# tools/check-plugin-classpath.sh --it-suites.
+#
+# Most marketplace plugins cannot be smoke-tested from a GitHub runner: they
need a
+# database, a cloud account or a cluster. The integration tests cover those,
so the
+# report links to the matching suite instead of leaving the row looking
untested.
+#
+# Both sides of the pairing are already in the repository, so this is derived
rather
+# than maintained by hand: the plugin list and its modulePath come from
+# optional-plugins.yaml (via list-marketplace-plugins.sh) and the projects are
the
+# directories under integration-tests/. A plugin is paired when the last
segment of
+# its modulePath matches a project name. Only the handful of cases where the
two
+# trees picked different names need an entry in marketplace-it-aliases.txt, so
a new
+# plugin that follows the convention is picked up with no edit here.
+#
+# The project directory name is also the suite name Jenkins publishes, which
is what
+# makes the report link work.
+#
+# Usage:
+# tools/marketplace-it-suites.sh # "<artifactId> <suite>" per line
+#
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+LIST="${ROOT}/tools/list-marketplace-plugins.sh"
+ALIASES="${ROOT}/tools/marketplace-it-aliases.txt"
+IT_DIR="${ROOT}/integration-tests"
+
+[[ -x "${LIST}" ]] || chmod +x "${LIST}" 2>/dev/null || true
+
+alias_for() {
+ [[ -f "${ALIASES}" ]] || return 1
+ awk -v n="$1" '/^[[:space:]]*#/ { next } NF < 2 { next } $1 == n { print $2;
found = 1; exit }
+ END { exit found ? 0 : 1 }' "${ALIASES}"
+}
+
+while IFS='|' read -r artifact module; do
+ [[ -n "${artifact}" && -n "${module}" ]] || continue
+ name="${module##*/}"
+ suite="$(alias_for "${name}" || true)"
+ [[ -n "${suite}" ]] || suite="${name}"
+ [[ -d "${IT_DIR}/${suite}" ]] || continue
+ printf '%s %s\n' "${artifact}" "${suite}"
+done < <("${LIST}")
diff --git a/tools/plugin-classpath-allowlist.txt
b/tools/plugin-classpath-allowlist.txt
new file mode 100644
index 0000000000..7a2ef09ab3
--- /dev/null
+++ b/tools/plugin-classpath-allowlist.txt
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Packages that tools/check-plugin-classpath.sh is allowed to leave unresolved.
+#
+# Format: <artifactId> <package prefix>
+# A prefix matches the package itself and everything below it, so
+# "edu.stanford.nlp" also covers "edu.stanford.nlp.simple".
+#
+# Every entry needs a comment saying why the jar legitimately does not ship.
+# The default answer to a new unresolved package is to fix the packaging, not
+# to add a line here.
+
+# stanford-corenlp is GPLv3 (ASF category X) and is deliberately kept out of
the
+# plugin zip; users download it themselves. See the hop.pluginlib.exclude*
+# properties and the comment above them in
plugins/transforms/stanfordnlp/pom.xml.
+hop-transform-stanfordnlp edu.stanford.nlp
