This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 08448bf72b Issue #8722 : Docker options to install marketplace plugins
(#8723)
08448bf72b is described below
commit 08448bf72bd5b46c7dc5e34b3bccb69ecba0758b
Author: Matt Casters <[email protected]>
AuthorDate: Tue Oct 6 15:58:34 2026 +0200
Issue #8722 : Docker options to install marketplace plugins (#8723)
* Issue #8722 : Docker options to install marketplace plugins
* Issue #8722 : Skip repeat marketplace installs and stop putting repo
passwords on the command line.
* Issue #8722 : Install marketplace plugins from the web image at startup.
* Issue #8722 : Document the stock apache/hop container
Describe running the published image with drivers and marketplace
plugins installed at start, and the project read through a Git VFS
connection.
---
assemblies/static/src/main/resources/hop | 26 ++
assemblies/static/src/main/resources/hop-run.sh | 26 ++
assemblies/static/src/main/resources/hop-server.sh | 26 ++
docker/resources/load-and-execute.sh | 88 +++++
docker/resources/run-web.sh | 92 +++++
docker/unified.Dockerfile | 19 +-
docker/web.Dockerfile | 3 +-
docs/hop-user-manual/modules/ROOT/pages/cloud.adoc | 7 +-
.../modules/ROOT/pages/docker-container.adoc | 70 ++++
.../ROOT/pages/getting-started/hop-next-steps.adoc | 2 +-
.../modules/ROOT/pages/hop-server/deploying.adoc | 154 +++++++-
.../hop/marketplace/catalog/PluginDiscovery.java | 13 +-
.../marketplace/command/MarketplaceCommand.java | 164 +++++++-
.../marketplace/config/MarketplaceRepository.java | 32 +-
.../apache/hop/marketplace/install/HopHome.java | 56 ++-
.../hop/marketplace/install/InstallReceipt.java | 4 +
.../hop/marketplace/install/PluginInstaller.java | 100 ++++-
.../catalog/PluginDiscoveryResolveInstallTest.java | 3 +
.../command/InstallCommandParsingTest.java | 79 ++++
.../MarketplaceRepositoryBrowserTypeTest.java | 31 ++
.../hop/marketplace/install/HopHomeTest.java | 19 +
.../marketplace/install/PluginInstallerTest.java | 425 +++++++++++++++++++++
22 files changed, 1404 insertions(+), 35 deletions(-)
diff --git a/assemblies/static/src/main/resources/hop
b/assemblies/static/src/main/resources/hop
index c1d81fc8c6..fb7b48035f 100755
--- a/assemblies/static/src/main/resources/hop
+++ b/assemblies/static/src/main/resources/hop
@@ -127,6 +127,32 @@ if [ -d "plugins/engines/beam" ]; then
CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
fi
+# Shared jars installed beside an extra plugins folder
(HOP_PLUGIN_BASE_FOLDERS) are not under
+# this install's lib/core. The folder name must be "plugins"; its parent holds
lib/core.
+if [ -n "${HOP_PLUGIN_BASE_FOLDERS:-}" ]; then
+ _hop_plugin_rest="${HOP_PLUGIN_BASE_FOLDERS}"
+ while [ -n "${_hop_plugin_rest}" ]; do
+ _hop_plugin_folder="${_hop_plugin_rest%%,*}"
+ case "${_hop_plugin_rest}" in
+ *,*) _hop_plugin_rest="${_hop_plugin_rest#*,}" ;;
+ *) _hop_plugin_rest="" ;;
+ esac
+
_hop_plugin_folder="${_hop_plugin_folder#"${_hop_plugin_folder%%[![:space:]]*}"}"
+
_hop_plugin_folder="${_hop_plugin_folder%"${_hop_plugin_folder##*[![:space:]]}"}"
+ [ -z "${_hop_plugin_folder}" ] && continue
+ if [ "$(basename "${_hop_plugin_folder}")" = "plugins" ]; then
+ _hop_plugin_home="$(dirname "${_hop_plugin_folder}")"
+ if [ -d "${_hop_plugin_home}/lib/core" ]; then
+ case ":${CLASSPATH}:" in
+ *":${_hop_plugin_home}/lib/core/*:"*) ;;
+ *) CLASSPATH="${_hop_plugin_home}/lib/core/*:${CLASSPATH}" ;;
+ esac
+ fi
+ fi
+ done
+ unset _hop_plugin_rest _hop_plugin_folder _hop_plugin_home
+fi
+
# The JVM runs from the Hop installation, so user.dir is never where the user
typed the command.
# Pass the directory they were in, for subcommands which take a path from them.
"${_HOP_JAVA}" ${HOP_OPTIONS} -Dhop.origin.dir="${ORIGINDIR}"
-Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}"
org.apache.hop.hop.Hop "$@"
diff --git a/assemblies/static/src/main/resources/hop-run.sh
b/assemblies/static/src/main/resources/hop-run.sh
index c6836cbb5d..38f353a421 100755
--- a/assemblies/static/src/main/resources/hop-run.sh
+++ b/assemblies/static/src/main/resources/hop-run.sh
@@ -111,6 +111,32 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
+# Shared jars installed beside an extra plugins folder
(HOP_PLUGIN_BASE_FOLDERS) are not under
+# this install's lib/core. The folder name must be "plugins"; its parent holds
lib/core.
+if [ -n "${HOP_PLUGIN_BASE_FOLDERS:-}" ]; then
+ _hop_plugin_rest="${HOP_PLUGIN_BASE_FOLDERS}"
+ while [ -n "${_hop_plugin_rest}" ]; do
+ _hop_plugin_folder="${_hop_plugin_rest%%,*}"
+ case "${_hop_plugin_rest}" in
+ *,*) _hop_plugin_rest="${_hop_plugin_rest#*,}" ;;
+ *) _hop_plugin_rest="" ;;
+ esac
+
_hop_plugin_folder="${_hop_plugin_folder#"${_hop_plugin_folder%%[![:space:]]*}"}"
+
_hop_plugin_folder="${_hop_plugin_folder%"${_hop_plugin_folder##*[![:space:]]}"}"
+ [ -z "${_hop_plugin_folder}" ] && continue
+ if [ "$(basename "${_hop_plugin_folder}")" = "plugins" ]; then
+ _hop_plugin_home="$(dirname "${_hop_plugin_folder}")"
+ if [ -d "${_hop_plugin_home}/lib/core" ]; then
+ case ":${CLASSPATH}:" in
+ *":${_hop_plugin_home}/lib/core/*:"*) ;;
+ *) CLASSPATH="${_hop_plugin_home}/lib/core/*:${CLASSPATH}" ;;
+ esac
+ fi
+ fi
+ done
+ unset _hop_plugin_rest _hop_plugin_folder _hop_plugin_home
+fi
+
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.run.HopRun "$@"
EXITCODE=$?
diff --git a/assemblies/static/src/main/resources/hop-server.sh
b/assemblies/static/src/main/resources/hop-server.sh
index 08adcdb47d..d363b2fb43 100755
--- a/assemblies/static/src/main/resources/hop-server.sh
+++ b/assemblies/static/src/main/resources/hop-server.sh
@@ -116,6 +116,32 @@ if [ -n "${HOP_SPARK_CLIENT_VERSION:-}" ] && [ -d
"lib/spark-clients/${HOP_SPARK
CLASSPATH="${CLASSPATH}:lib/spark-clients/${HOP_SPARK_CLIENT_VERSION}/*"
fi
+# Shared jars installed beside an extra plugins folder
(HOP_PLUGIN_BASE_FOLDERS) are not under
+# this install's lib/core. The folder name must be "plugins"; its parent holds
lib/core.
+if [ -n "${HOP_PLUGIN_BASE_FOLDERS:-}" ]; then
+ _hop_plugin_rest="${HOP_PLUGIN_BASE_FOLDERS}"
+ while [ -n "${_hop_plugin_rest}" ]; do
+ _hop_plugin_folder="${_hop_plugin_rest%%,*}"
+ case "${_hop_plugin_rest}" in
+ *,*) _hop_plugin_rest="${_hop_plugin_rest#*,}" ;;
+ *) _hop_plugin_rest="" ;;
+ esac
+
_hop_plugin_folder="${_hop_plugin_folder#"${_hop_plugin_folder%%[![:space:]]*}"}"
+
_hop_plugin_folder="${_hop_plugin_folder%"${_hop_plugin_folder##*[![:space:]]}"}"
+ [ -z "${_hop_plugin_folder}" ] && continue
+ if [ "$(basename "${_hop_plugin_folder}")" = "plugins" ]; then
+ _hop_plugin_home="$(dirname "${_hop_plugin_folder}")"
+ if [ -d "${_hop_plugin_home}/lib/core" ]; then
+ case ":${CLASSPATH}:" in
+ *":${_hop_plugin_home}/lib/core/*:"*) ;;
+ *) CLASSPATH="${_hop_plugin_home}/lib/core/*:${CLASSPATH}" ;;
+ esac
+ fi
+ fi
+ done
+ unset _hop_plugin_rest _hop_plugin_folder _hop_plugin_home
+fi
+
"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath
"${CLASSPATH}" org.apache.hop.www.HopServer "$@"
EXITCODE=$?
diff --git a/docker/resources/load-and-execute.sh
b/docker/resources/load-and-execute.sh
index b579b3e5b7..60931c0ee7 100755
--- a/docker/resources/load-and-execute.sh
+++ b/docker/resources/load-and-execute.sh
@@ -75,6 +75,91 @@ install_jdbc_drivers() {
done
}
+# Download Marketplace plugins on container start, before Hop is launched.
+# Driven by environment variables:
+# HOP_PLUGINS_DOWNLOAD comma-separated plugin coordinates, installed
in one hop process:
+# short names, artifactId, artifactId:version,
or groupId:artifactId:version
+# e.g.
"org.hopper:hopper-edw:0.10.0,org.apache.hop:hop-tech-parquet:2.20.0"
+# HOP_PLUGINS_MAVEN_REPO optional Maven/Artifactory/Nexus base URL,
tried first. Other
+# configured repositories are still used when a
plugin is not there.
+# HOP_PLUGINS_REPO_ID repository id for that URL (default:
corporate-repo). Credential
+# variables are HOP_MARKETPLACE_<ID>_USERNAME,
_PASSWORD and _TOKEN.
+# HOP_PLUGINS_REPO_USERNAME optional username, exported as
HOP_MARKETPLACE_<ID>_USERNAME
+# HOP_PLUGINS_REPO_PASSWORD optional password or token. Exported as
_PASSWORD, or _TOKEN when
+# the auth type is token. Never passed as
--password.
+# HOP_PLUGINS_REPO_AUTH_TYPE optional authentication type: auto (default),
none, basic, token
+# HOP_PLUGINS_REPO_TYPE optional browse API: auto (default), nexus,
forgejo, jfrog, or maven
+# HOP_PLUGINS_ENV_FILE optional install spec file or URL
(hop-env.yaml / hop-marketplace-repo.yaml)
+# HOP_PLUGIN_BASE_FOLDERS optional writable plugins directory. The first
one that is not the
+# image's own plugins folder receives the
install.
+install_marketplace_plugins() {
+ local env_file="${HOP_PLUGINS_ENV_FILE:-}"
+ if [ -n "${env_file}" ]; then
+ log "Applying Hop marketplace environment file: ${env_file}"
+ if ! "${DEPLOYMENT_PATH}"/hop marketplace apply -f "${env_file}"; then
+ log "Error: failed to apply marketplace environment file '${env_file}'"
+ exitWithCode 8
+ fi
+ fi
+
+ local plugins="${HOP_PLUGINS_DOWNLOAD:-}"
+ if [ -z "${plugins}" ]; then
+ return 0
+ fi
+
+ local repo_id="${HOP_PLUGINS_REPO_ID:-corporate-repo}"
+ local repo_prefix
+ repo_prefix="$(printf '%s' "${repo_id}" | tr '[:lower:]' '[:upper:]' | sed
's/[^A-Z0-9]/_/g')"
+
+ local install_args=()
+ local repo_url="${HOP_PLUGINS_MAVEN_REPO:-}"
+ if [ -n "${repo_url}" ]; then
+ install_args+=("--repo-url=${repo_url}" "--repo-id=${repo_id}")
+ fi
+
+ local repo_type="${HOP_PLUGINS_REPO_TYPE:-}"
+ if [ -n "${repo_type}" ]; then
+ install_args+=("--repo-type=${repo_type}")
+ fi
+
+ local auth_type="${HOP_PLUGINS_REPO_AUTH_TYPE:-}"
+ if [ -n "${auth_type}" ]; then
+ install_args+=("--auth-type=${auth_type}")
+ fi
+
+ # MarketplaceRepository reads these. They must not be placed on the hop
command line.
+ if [ -n "${HOP_PLUGINS_REPO_USERNAME:-}" ]; then
+ export
"HOP_MARKETPLACE_${repo_prefix}_USERNAME=${HOP_PLUGINS_REPO_USERNAME}"
+ fi
+ if [ -n "${HOP_PLUGINS_REPO_PASSWORD:-}" ]; then
+ case "${auth_type}" in
+ token | TOKEN | Token)
+ export
"HOP_MARKETPLACE_${repo_prefix}_TOKEN=${HOP_PLUGINS_REPO_PASSWORD}"
+ ;;
+ *)
+ export
"HOP_MARKETPLACE_${repo_prefix}_PASSWORD=${HOP_PLUGINS_REPO_PASSWORD}"
+ ;;
+ esac
+ fi
+
+ local specs=()
+ local spec
+ for spec in ${plugins//,/ }; do
+ spec="$(echo "${spec}" | tr -d '[:space:]')"
+ [ -z "${spec}" ] && continue
+ specs+=("${spec}")
+ done
+ if [ ${#specs[@]} -eq 0 ]; then
+ return 0
+ fi
+
+ log "Installing marketplace plugins: ${plugins}"
+ if ! "${DEPLOYMENT_PATH}"/hop marketplace install "${specs[@]}"
"${install_args[@]}"; then
+ log "Error: failed to install marketplace plugins: ${plugins}"
+ exitWithCode 8
+ fi
+}
+
# write the hop-server config to a configuration file
# to avoid the password of the server being shown in ps
#
@@ -158,6 +243,9 @@ fi
# Download requested JDBC drivers (HOP_DRIVERS_DOWNLOAD) before Hop starts.
install_jdbc_drivers
+# Download requested Marketplace plugins (HOP_PLUGINS_DOWNLOAD) before Hop
starts.
+install_marketplace_plugins
+
# Set empty defaults on the Hop command options.
#
HOP_COMMAND="${HOP_COMMAND:-}"
diff --git a/docker/resources/run-web.sh b/docker/resources/run-web.sh
index 089c0c7e49..f6fea4c941 100755
--- a/docker/resources/run-web.sh
+++ b/docker/resources/run-web.sh
@@ -75,6 +75,95 @@ install_jdbc_drivers() {
done
}
+# Download Marketplace plugins on container start, before Tomcat is launched.
+# Driven by environment variables:
+# HOP_PLUGINS_DOWNLOAD comma-separated plugin coordinates, installed
in one hop process:
+# short names, artifactId, artifactId:version,
or groupId:artifactId:version
+# e.g.
"org.hopper:hopper-edw:0.10.0,org.apache.hop:hop-tech-parquet:2.20.0"
+# HOP_PLUGINS_MAVEN_REPO optional Maven/Artifactory/Nexus base URL,
tried first. Other
+# configured repositories are still used when a
plugin is not there.
+# HOP_PLUGINS_REPO_ID repository id for that URL (default:
corporate-repo). Credential
+# variables are HOP_MARKETPLACE_<ID>_USERNAME,
_PASSWORD and _TOKEN.
+# HOP_PLUGINS_REPO_USERNAME optional username, exported as
HOP_MARKETPLACE_<ID>_USERNAME
+# HOP_PLUGINS_REPO_PASSWORD optional password or token. Exported as
_PASSWORD, or _TOKEN when
+# the auth type is token. Never passed as
--password.
+# HOP_PLUGINS_REPO_AUTH_TYPE optional authentication type: auto (default),
none, basic, token
+# HOP_PLUGINS_REPO_TYPE optional browse API: auto (default), nexus,
forgejo, jfrog, or maven
+# HOP_PLUGINS_ENV_FILE optional install spec file or URL
(hop-env.yaml / hop-marketplace-repo.yaml)
+# HOP_PLUGIN_BASE_FOLDERS optional writable plugins directory. The first
one that is not the
+# image's own plugins folder receives the
install.
+install_marketplace_plugins() {
+ local env_file="${HOP_PLUGINS_ENV_FILE:-}"
+ local plugins="${HOP_PLUGINS_DOWNLOAD:-}"
+ if [ -z "${env_file}" ] && [ -z "${plugins}" ]; then
+ return 0
+ fi
+
+ if [ -n "${env_file}" ]; then
+ log "Applying Hop marketplace environment file: ${env_file}"
+ if ! "${DEPLOYMENT_PATH}"/hop marketplace apply -f "${env_file}"; then
+ log "Error: failed to apply marketplace environment file '${env_file}'"
+ exitWithCode 8
+ fi
+ fi
+
+ if [ -z "${plugins}" ]; then
+ return 0
+ fi
+
+ local repo_id="${HOP_PLUGINS_REPO_ID:-corporate-repo}"
+ local repo_prefix
+ repo_prefix="$(printf '%s' "${repo_id}" | tr '[:lower:]' '[:upper:]' | sed
's/[^A-Z0-9]/_/g')"
+
+ local install_args=()
+ local repo_url="${HOP_PLUGINS_MAVEN_REPO:-}"
+ if [ -n "${repo_url}" ]; then
+ install_args+=("--repo-url=${repo_url}" "--repo-id=${repo_id}")
+ fi
+
+ local repo_type="${HOP_PLUGINS_REPO_TYPE:-}"
+ if [ -n "${repo_type}" ]; then
+ install_args+=("--repo-type=${repo_type}")
+ fi
+
+ local auth_type="${HOP_PLUGINS_REPO_AUTH_TYPE:-}"
+ if [ -n "${auth_type}" ]; then
+ install_args+=("--auth-type=${auth_type}")
+ fi
+
+ # MarketplaceRepository reads these. They must not be placed on the hop
command line.
+ if [ -n "${HOP_PLUGINS_REPO_USERNAME:-}" ]; then
+ export
"HOP_MARKETPLACE_${repo_prefix}_USERNAME=${HOP_PLUGINS_REPO_USERNAME}"
+ fi
+ if [ -n "${HOP_PLUGINS_REPO_PASSWORD:-}" ]; then
+ case "${auth_type}" in
+ token | TOKEN | Token)
+ export
"HOP_MARKETPLACE_${repo_prefix}_TOKEN=${HOP_PLUGINS_REPO_PASSWORD}"
+ ;;
+ *)
+ export
"HOP_MARKETPLACE_${repo_prefix}_PASSWORD=${HOP_PLUGINS_REPO_PASSWORD}"
+ ;;
+ esac
+ fi
+
+ local specs=()
+ local spec
+ for spec in ${plugins//,/ }; do
+ spec="$(echo "${spec}" | tr -d '[:space:]')"
+ [ -z "${spec}" ] && continue
+ specs+=("${spec}")
+ done
+ if [ ${#specs[@]} -eq 0 ]; then
+ return 0
+ fi
+
+ log "Installing marketplace plugins: ${plugins}"
+ if ! "${DEPLOYMENT_PATH}"/hop marketplace install "${specs[@]}"
"${install_args[@]}"; then
+ log "Error: failed to install marketplace plugins: ${plugins}"
+ exitWithCode 8
+ fi
+}
+
# Ensure HOP_AUDIT_FOLDER exists and is writable by the hop process (per-user
data under
# users/<username>/). Prefer the configured path; fall back to
/tmp/hop-web-audit under
# java.io.tmpdir when a bind-mount is not writable.
@@ -223,6 +312,9 @@ fi
# Download requested JDBC drivers (HOP_DRIVERS_DOWNLOAD) before Tomcat starts.
install_jdbc_drivers
+# Download requested Marketplace plugins (HOP_PLUGINS_DOWNLOAD) before Tomcat
starts.
+install_marketplace_plugins
+
# if we have a /config/tomcat-users.xml file, copy it to the conf folder.
if [ -f "/config/tomcat-users.xml" ]; then
log "copying users file to /usr/local/tomcat/conf/"
diff --git a/docker/unified.Dockerfile b/docker/unified.Dockerfile
index dbad5cd03d..7c4f3b3d50 100644
--- a/docker/unified.Dockerfile
+++ b/docker/unified.Dockerfile
@@ -256,7 +256,8 @@ RUN chmod +x /build/hop-web-prepared/webapps/ROOT/*.sh
# Fix hop-config.json
RUN sed -i 's/config\/projects/${HOP_CONFIG_FOLDER}\/projects/g'
/build/hop-web-prepared/webapps/ROOT/config/hop-config.json
-# Set the correct classpath for hop scripts
+# Set the correct classpath for hop scripts. hop (no suffix) is the
marketplace entrypoint.
+RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
/build/hop-web-prepared/webapps/ROOT/hop
RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
/build/hop-web-prepared/webapps/ROOT/hop-run.sh
RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
/build/hop-web-prepared/webapps/ROOT/hop-conf.sh
RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
/build/hop-web-prepared/webapps/ROOT/hop-search.sh
@@ -293,6 +294,14 @@ ENV HOP_SHARED_JDBC_FOLDERS=
ENV HOP_DRIVERS_DOWNLOAD=
ENV HOP_DRIVERS_ACCEPT_LICENSE=
ENV HOP_DRIVERS_MAVEN_REPO=
+ENV HOP_PLUGINS_DOWNLOAD=
+ENV HOP_PLUGINS_MAVEN_REPO=
+ENV HOP_PLUGINS_REPO_ID=
+ENV HOP_PLUGINS_REPO_USERNAME=
+ENV HOP_PLUGINS_REPO_PASSWORD=
+ENV HOP_PLUGINS_REPO_AUTH_TYPE=
+ENV HOP_PLUGINS_REPO_TYPE=
+ENV HOP_PLUGINS_ENV_FILE=
ENV HOP_PROJECT_NAME=
ENV HOP_PROJECT_DIRECTORY=
ENV HOP_PROJECT_FOLDER=
@@ -372,6 +381,14 @@ ENV HOP_SHARED_JDBC_FOLDERS="${CATALINA_HOME}/jdbc-drivers"
ENV HOP_DRIVERS_DOWNLOAD=
ENV HOP_DRIVERS_ACCEPT_LICENSE=
ENV HOP_DRIVERS_MAVEN_REPO=
+ENV HOP_PLUGINS_DOWNLOAD=
+ENV HOP_PLUGINS_MAVEN_REPO=
+ENV HOP_PLUGINS_REPO_ID=
+ENV HOP_PLUGINS_REPO_USERNAME=
+ENV HOP_PLUGINS_REPO_PASSWORD=
+ENV HOP_PLUGINS_REPO_AUTH_TYPE=
+ENV HOP_PLUGINS_REPO_TYPE=
+ENV HOP_PLUGINS_ENV_FILE=
ENV HOP_GUI_ZOOM_FACTOR=1.0
ENV HOP_PROJECT_FOLDER=
ENV HOP_PROJECT_CONFIG_FILE_NAME=project-config.json
diff --git a/docker/web.Dockerfile b/docker/web.Dockerfile
index 4e7697d88f..96685cf0b7 100644
--- a/docker/web.Dockerfile
+++ b/docker/web.Dockerfile
@@ -104,7 +104,8 @@ RUN sed -i
's/config\/projects/${HOP_CONFIG_FOLDER}\/projects/g' "${CATALINA_HOM
RUN mkdir -p "$CATALINA_HOME"/lib/swt/linux/x86_64
-# set the correct classpath for hop-conf and hop-run
+# set the correct classpath for hop, hop-conf and hop-run
+RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
${CATALINA_HOME}/webapps/ROOT/hop
RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
${CATALINA_HOME}/webapps/ROOT/hop-run.sh
RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
${CATALINA_HOME}/webapps/ROOT/hop-conf.sh
RUN sed -i 's&lib/core/*&../../lib/*:WEB-INF/lib/*:lib/core/*&g'
${CATALINA_HOME}/webapps/ROOT/hop-search.sh
diff --git a/docs/hop-user-manual/modules/ROOT/pages/cloud.adoc
b/docs/hop-user-manual/modules/ROOT/pages/cloud.adoc
index d65a4d0be3..a1a9d00db5 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/cloud.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/cloud.adoc
@@ -64,7 +64,7 @@ A Hop project is a folder: `project-config.json`, a
`metadata/` folder, pipeline
See xref:projects/index.adoc[Projects and environments].
Something has to put that folder inside the runtime.
-The three patterns on xref:hop-server/deploying.adoc[Deploying Hop Server] are
not Hop Server specific — they are the three ways a project reaches *any*
runtime, server or not:
+The patterns on xref:hop-server/deploying.adoc[Deploying Hop Server] are not
Hop Server specific — they are the ways a project reaches *any* runtime, server
or not:
* *Baked into an image.* CI builds a container image that contains the
project, and the platform pulls that image.
The build is the deployment; a run is immutable and reproducible.
@@ -72,10 +72,11 @@ See xref:hop-server/deploy-project-image.adoc[Project in a
Docker image].
* *Fetched at start-up.* The image is the stock `apache/hop` image and the
project is pulled in when the container starts — from git, from object storage,
or from a mounted volume.
`HOP_CUSTOM_ENTRYPOINT_EXTENSION_SHELL_FILE_PATH` exists for exactly this: a
script that runs before Hop starts.
See xref:hop-server/deploy-git-checkout.adoc[Git checkout on the server].
+The same stock image can install JDBC drivers and marketplace plugins as it
starts, and open the pipelines and workflows through a
xref:hop-server/deploying.adoc#StandardApacheHopContainer[Git VFS connection],
with no clone script and no derived image.
* *Sent by a client.* A
xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[remote
run configuration] with *Export linked resources to server* ships a ZIP to a
server that owns no files.
See xref:hop-server/deploy-export-resources.adoc[Remote run with export
resources].
-Baking the project into an image is the optimal choice on a container
platform, because it is the only one of the three where the running unit is
fully described by an image tag.
+Baking the project into an image is the optimal choice on a container
platform, because it is the only one of these where the running unit is fully
described by an image tag.
Fetching at start-up is attractive when many small projects share one image,
at the cost of a network dependency on every start.
Data files are a separate question from project files.
@@ -254,7 +255,7 @@ If your provider is not in these lists, the question to ask
is which standard it
== See also
* xref:docker-container.adoc[Hop in Docker] — the image, its environment
variables and its two run modes
-* xref:hop-server/deploying.adoc[Deploying Hop Server] — the three ways a
project reaches a server, compared
+* xref:hop-server/deploying.adoc[Deploying Hop Server] — the ways a project
reaches a server, compared
* xref:projects/index.adoc[Projects and environments] — what belongs in the
project and what belongs in the environment
* xref:metadata-types/execution-information-location.adoc[Execution
Information Location] — where run history is kept
* xref:how-to-guides/scheduling-workflows-and-pipelines.adoc[Scheduling
workflows and pipelines] — cron, systemd, Jenkins, Kubernetes CronJob, Airflow
diff --git a/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
b/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
index 25ca98a996..4e45eb12ed 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/docker-container.adoc
@@ -448,6 +448,76 @@ The download is supported by the client/server image
(running pipelines, workflo
NOTE: `HOP_DRIVERS_DOWNLOAD` is available from Hop version 2.19.
+[[DownloadingMarketplacePlugins]]
+== Downloading Marketplace plugins
+
+The Docker container can install optional or third-party plugins from the Hop
marketplace or a corporate Maven repository (e.g. Sonatype Nexus, JFrog
Artifactory) on startup *before* Hop starts. The stock image does this at
runtime, so a custom image is not required. A repeated start skips a plugin
when its receipt version matches and its files are still present. Every
coordinate is installed by one `hop marketplace install` invocation.
+
+Set `HOP_PLUGINS_DOWNLOAD` to a comma-separated list of plugin coordinates.
Each entry can be:
+
+* An artifact id or short name (e.g. `hop-tech-parquet`, `datavault`), when a
browsable repository or the Apache catalog can resolve it.
+* An artifact id with version (e.g. `hopper-edw:0.10.0`), with the same
limitation.
+* Full Maven coordinates (e.g. `org.hopper:hopper-edw:0.10.0`).
+
+`HOP_PLUGINS_MAVEN_REPO` is the repository to try first. The configured Apache
and Maven Central repositories are still used when a plugin is not in that
repository, so a private plugin and an Apache optional plugin can be requested
together. Pass a full `groupId:artifactId:version` for the private one when
that repository cannot be browsed.
+
+A repository that cannot be browsed (a plain Maven repository, or
`HOP_PLUGINS_REPO_TYPE=maven`) does not accept a short name. The install fails
and asks for `groupId:artifactId:version` instead of guessing `org.apache.hop`.
`HOP_PLUGINS_REPO_TYPE` selects the browse API: `auto` (default), `nexus`,
`forgejo`, `jfrog`, or `maven`. `auto` infers Nexus, Artifactory, or Forgejo
from the URL.
+
+`HOP_PLUGINS_REPO_ID` is the id of that repository (default `corporate-repo`).
The entrypoint does not put the password on the `hop` command line. It exports
`HOP_MARKETPLACE_<ID>_USERNAME` and either `HOP_MARKETPLACE_<ID>_PASSWORD` or,
when the auth type is `token`, `HOP_MARKETPLACE_<ID>_TOKEN`. For the default id
those names are `HOP_MARKETPLACE_CORPORATE_REPO_USERNAME`,
`HOP_MARKETPLACE_CORPORATE_REPO_PASSWORD` and
`HOP_MARKETPLACE_CORPORATE_REPO_TOKEN`. Any other id is uppercased, an [...]
+
+* `HOP_PLUGINS_REPO_USERNAME`: Basic auth username.
+* `HOP_PLUGINS_REPO_PASSWORD`: Basic auth password or bearer token.
+* `HOP_PLUGINS_REPO_AUTH_TYPE`: `auto` (default), `none`, `basic`, or `token`.
+
+The global `HOP_MARKETPLACE_USERNAME` and `HOP_MARKETPLACE_PASSWORD` variables
are also picked up.
+
+.Install a private plugin and an Apache plugin, keeping them on a volume
+[source,shell]
+----
+docker run -it --rm \
+ --env HOP_PLUGINS_DOWNLOAD="org.hopper:hopper-edw:0.10.0,hop-tech-parquet" \
+ --env
HOP_PLUGINS_MAVEN_REPO="https://repository.data-hopper.com/repository/hop-community-plugins/";
\
+ --env HOP_PLUGINS_REPO_ID=data-hopper \
+ --env HOP_PLUGIN_BASE_FOLDERS=/opt/hop/plugins,/files/plugins \
+ --env HOP_PROJECT_FOLDER=/project \
+ --env HOP_PROJECT_NAME=my-project \
+ --env HOP_FILE_PATH='${PROJECT_HOME}/main.hpl' \
+ --env HOP_RUN_CONFIG=local \
+ --mount type=bind,source=/path/to/project,target=/project \
+ --mount type=volume,source=hop-plugins,target=/files/plugins \
+ apache/hop
+----
+
+`HOP_PLUGIN_BASE_FOLDERS` replaces the default `plugins` folder. Include the
image plugins path and the mounted path, as in the example. The install is
written to the first writable directory in that list that is not the image
install itself. Shared `lib/core` jars are written next to that plugins folder
and added to the classpath of `hop`, `hop-run` and `hop-server`.
+
+Plugins written only into the container filesystem are not kept when the
container is removed. They are kept when that install directory is a mounted
volume, or when the install happened while building an image.
+
+The Hop web image installs marketplace plugins at startup as well. The image
rewrites the `hop` classpath the same way as `hop-run.sh`, so it includes
`WEB-INF/lib` (where the Hop jars live) as well as `lib/core`. Shared jars from
a plugin are installed into `WEB-INF/lib` so Tomcat and `hop` both load them.
Plugin folders still follow `HOP_PLUGIN_BASE_FOLDERS`.
+
+Building a derived image is optional. It avoids the download on every start
when a custom image is acceptable:
+
+[source,dockerfile]
+----
+FROM apache/hop:2.20.0
+RUN /opt/hop/hop marketplace install \
+ org.hopper:hopper-edw:0.10.0 \
+ org.apache.hop:hop-tech-parquet:2.20.0 \
+ --repo-url
https://repository.data-hopper.com/repository/hop-community-plugins/ \
+ --repo-id data-hopper
+----
+
+If you maintain a declarative Hop environment install spec (`hop-env.yaml`) or
repository definition (`hop-marketplace-repo.yaml`), you can point to it
directly using `HOP_PLUGINS_ENV_FILE`:
+
+[source,shell]
+----
+docker run -it --rm \
+ --env
HOP_PLUGINS_ENV_FILE=https://raw.githubusercontent.com/ProjectDataHopper/hopper-edw/main/hop-marketplace-repo.yaml
\
+ ... \
+ apache/hop
+----
+
+NOTE: `HOP_PLUGINS_DOWNLOAD` is available from Hop version 2.20.
+
== Custom Entrypoint Extension Shell Script
To make the Hop Docker image even more flexible, we added a
```HOP_CUSTOM_ENTRYPOINT_EXTENSION_SHELL_FILE_PATH``` variable that accepts a
path to a custom shell script (that you provide).This shell script will run
when you start the container before your Hop project is registered with the
container's Hop config and before your Hop workflow or pipeline gets kicked off.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-next-steps.adoc
b/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-next-steps.adoc
index 6bb033009f..86b1bd3489 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-next-steps.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/getting-started/hop-next-steps.adoc
@@ -50,7 +50,7 @@ Where you go next depends on what you're trying to do.
== Run it somewhere else
* xref:projects/index.adoc[Projects] - the full reference for projects and
environments
-* xref:hop-server/deploying.adoc[Deploying Hop Server] - three ways to get a
project onto a running server
+* xref:hop-server/deploying.adoc[Deploying Hop Server] - how a project gets
onto a running server, including a stock `apache/hop` container
* xref:docker-container.adoc[Hop in Docker] - the `apache/hop` image
*
xref:pipeline/pipeline-run-configurations/pipeline-run-configurations.adoc[Pipeline
run configurations] - run the same pipeline on Apache Spark, Apache Flink or
Google Dataflow without changing it
* xref:logging/logging-basics.adoc[Logging] - log levels and where to send the
output
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-server/deploying.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-server/deploying.adoc
index cc8751be03..a0299da637 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-server/deploying.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-server/deploying.adoc
@@ -16,7 +16,7 @@ under the License.
////
[[DeployingHopServer]]
:imagesdir: ../../assets/images
-:description: How to get a Hop project onto a Hop Server: bake it into a
Docker image, send it from a client with export resources, or check it out with
git on the host.
+:description: How to get a Hop project onto a Hop Server: bake it into a
Docker image, send it from a client with export resources, check it out with
git on the host, or run the stock apache/hop image and read the project through
a Git VFS connection.
= Deploying Hop Server
@@ -26,15 +26,17 @@ See
xref:how-to-guides/scheduling-workflows-and-pipelines.adoc[Scheduling workfl
This page is the deployment story: after you develop a project, how does a
running Hop Server get the files and the configuration it needs?
-The same three patterns are how a project reaches any runtime, not only a
server.
+The same four patterns are how a project reaches any runtime, not only a
server.
For the wider picture -- containers, schedulers, secrets, object storage and
run history on a cloud platform -- see xref:cloud.adoc[Running Apache Hop in
the cloud].
-There are three patterns that cover almost every production setup:
+There are four patterns that cover almost every production setup:
* xref:hop-server/deploy-project-image.adoc[Project in a Docker image] -- CI
builds an image that contains the project.
You can roll that image as a long-lived Hop Server, but you often do not need
to: a scheduler can start a *short-lived* copy, run one pipeline or workflow,
and exit.
* xref:hop-server/deploy-export-resources.adoc[Remote run with export
resources] -- the client owns the project and sends a ZIP to a "dumb" server.
* xref:hop-server/deploy-git-checkout.adoc[Git checkout on the server] -- the
host has a clone; updating is `git pull`.
+* <<StandardApacheHopContainer,Standard apache/hop container>> -- the
published image, unchanged.
+JDBC drivers and marketplace plugins are installed when the container starts,
and the pipelines and workflows are read from git through a named VFS
connection.
== What you deploy
@@ -53,6 +55,7 @@ The official Docker image does this from `HOP_PROJECT_*` /
`HOP_ENVIRONMENT_*` v
On a host you register the project and environment once with
xref:hop-tools/hop-conf/hop-conf.adoc[hop-conf], then start the server with
`-e`.
The environment already references its project, so you do not pass `-j` as
well.
This is how xref:hop-server/deploy-project-image.adoc[the image] and
xref:hop-server/deploy-git-checkout.adoc[the git checkout] work.
+The <<StandardApacheHopContainer,standard container>> is the same shape with a
smaller folder: the enabled project holds the Git connection and the metadata
the run needs, and `HOP_FILE_PATH` points at a file inside the repository.
Environment configuration files must be present *on the server*.
* *As a payload from the client*.
@@ -66,46 +69,184 @@ When both a project and `<metadata_folder>` are set, both
providers sit on the m
== Choose a pattern
-[cols="1,2,2,2",options="header"]
+[cols="1,2,2,2,2",options="header"]
|===
-| |Project in a Docker image |Export resources from a client |Git checkout on
the server
+| |Project in a Docker image |Export resources from a client |Git checkout on
the server |Standard `apache/hop` container
|Who owns the project files
|The image (immutable per build)
|The client (Airflow, hop-run, Hop Gui)
|The server host (working tree)
+|The git remote.
+The container holds a small project (the connection and the metadata) and
checks the repository out when a file is opened
|How you update
|Rebuild and roll the container
|Change the client project; the next run sends a new ZIP
|`git pull` (restart only for environment or server config)
+|Change the revision on the connection.
+A moving branch is fetched again when the connection says so
|Environment configuration
|Mounted or injected on the *server*
|Applied on the *client*
|Files on the *server*, usually outside the clone
+|Mounted or injected on the *server*
|Extra files (JDBC, plugins, `.properties`, JSON schemas)
|`COPY` into the image or a volume
|*Not sent.* Must already exist on the server, or be rewritten with the
named-resource folder mapping
|Present in the clone or installed on the host
+|Downloaded at start, or mounted.
+See <<StandardApacheHopContainer>>
|File paths
|`+${PROJECT_HOME}+` is real on the server
|Paths are rewritten; see the
xref:hop-server/deploy-export-resources.adoc[export resources caveats]
|`+${PROJECT_HOME}+` is the clone directory
+|The file you run is a Git VFS URL such as `ops:///workflows/daily.hwf`.
+`+${PROJECT_HOME}+` is the small local project, not the repository
|Best for
|Enterprise CI/CD, Kubernetes, OpenShift; the same image as a short-lived job
*or* a long-lived server
|Orchestrators that already have the project; a server that should not own
files
|Smaller or ops-friendly hosts with git access
+|A platform where building and rolling a custom image is the expensive step
|Poor fit
-|You cannot rebuild images
+|You cannot rebuild images.
+Use <<StandardApacheHopContainer>> instead
|Heavy file I/O, extra config files, large data files
|You cannot (or must not) run `git pull` on production hosts
+|Projects that open their files through `+${PROJECT_HOME}+`, or metadata that
has to be read from inside the repository
|===
+[[StandardApacheHopContainer]]
+== Standard apache/hop container
+
+The published `apache/hop` image can take the place of a derived image.
+A Jenkins, GitHub Actions or GitLab CI job usually builds that derived image
to add JDBC drivers, install a few marketplace plugins, and clone the project.
+Here the entrypoint installs the drivers and the plugins before Hop starts,
and a xref:metadata-types/git-vfs-connection.adoc[Git repository] connection
checks the project out when Hop opens a file.
+
+The checkout is read-only by default.
+A write changes the working copy on disk, and nothing is committed or pushed.
+
+This needs Hop 2.20, which is where `HOP_PLUGINS_DOWNLOAD` and the Git VFS
plugin arrive together.
+`HOP_DRIVERS_DOWNLOAD` is already in 2.19.
+
+=== Drivers and plugins
+
+Set `HOP_DRIVERS_DOWNLOAD` to a comma-separated list of driver ids, and
`HOP_DRIVERS_ACCEPT_LICENSE=true` when one of them is a restricted driver
(Oracle, MySQL, MariaDB, DB2, ...).
+`HOP_DRIVERS_MAVEN_REPO` points the download at an internal Nexus or
Artifactory.
+See xref:docker-container.adoc#DownloadingJDBCDrivers[Downloading JDBC
drivers].
+A jar that `hop driver list` does not know is still a file you mount.
+
+Set `HOP_PLUGINS_DOWNLOAD` to the marketplace plugins the run needs.
+The Git driver is one of them: `hop-tech-git-vfs`.
+The stock image does not contain it.
+A private plugin and that Apache plugin can be listed together;
`HOP_PLUGINS_MAVEN_REPO` is tried first and the other configured repositories
are still used.
+See xref:docker-container.adoc#DownloadingMarketplacePlugins[Downloading
Marketplace plugins].
+
+A later start skips a plugin whose receipt version is already on disk.
+Drivers are downloaded on every start.
+`lib/jdbc` and the plugins directory have to be writable.
+On a read-only root filesystem, point `HOP_SHARED_JDBC_FOLDERS` and
`HOP_PLUGIN_BASE_FOLDERS` at a volume.
+
+=== The project you mount
+
+Set `HOP_PROJECT_FOLDER` to a real directory.
+Hop resolves that home before it loads named VFS connections, and the Git
scheme exists only after the connection has been read from a folder Hop can
already open.
+
+Mount a small project that holds the connection and the metadata the run needs.
+The pipelines and workflows stay in the remote repository.
+The mount in the example is read-only, so `project-config.json` has to be in
it already.
+The entrypoint keeps that file (`--project-keep-config-file`).
+A missing file would be created with *Enforce executions in project home*
turned on, and hop-run would then refuse `ops:///workflows/daily.hwf` because
that path is outside `/bootstrap`.
+
+[source,json]
+----
+{
+ "metadataBaseFolder" : "${PROJECT_HOME}/metadata",
+ "parentProjectName" : "default",
+ "enforcingExecutionInHome" : false
+}
+----
+
+`parentProjectName` set to `default` picks up the `local` run configuration
shipped in the image.
+The `metadata/` directory inside the git repository is a different folder.
+Hop reads metadata from the bootstrap project (and from that `default`
parent), which is where database connections the pipeline uses belong.
+
+[source,bash]
+----
+docker run --rm \
+ -e HOP_DRIVERS_DOWNLOAD="oracle,mariadb:3.4.1" \
+ -e HOP_DRIVERS_ACCEPT_LICENSE=true \
+ -e HOP_PLUGINS_DOWNLOAD="hop-tech-git-vfs" \
+ -e HOP_PROJECT_FOLDER=/bootstrap \
+ -e HOP_PROJECT_NAME=bootstrap \
+ -e HOP_ENVIRONMENT_NAME=prod \
+ -e HOP_ENVIRONMENT_CONFIG_FILE_NAME_PATHS=/config/prod.json \
+ -e HOP_FILE_PATH='ops:///workflows/daily.hwf' \
+ -e HOP_RUN_CONFIG=local \
+ -v /etc/hop/bootstrap:/bootstrap:ro \
+ -v /etc/hop/environments/prod.json:/config/prod.json:ro \
+ -v /etc/hop/secrets/deploy-key:/run/secrets/deploy-key:ro \
+ -v /etc/hop/secrets/known_hosts:/run/secrets/known_hosts:ro \
+ apache/hop:<version>
+----
+
+`/bootstrap` also needs `metadata/git-vfs-connection/ops.json`.
+`HOP_RUN_CONFIG=local` resolves through the `default` parent, so you only add
a run configuration file when the run should use a different one.
+
+Omit `HOP_FILE_PATH` and `HOP_RUN_CONFIG` and the same container starts Hop
Server instead of hop-run.
+The server still needs the bootstrap project, the connection and the key.
+
+The connection for an SSH deploy key, GitHub or GitLab, looks like this.
+`authType` is the name of the enum value.
+`name` is the VFS scheme.
+
+[source,json]
+----
+{
+ "name" : "ops",
+ "repositoryUrl" : "[email protected]:data/your-project.git",
+ "revision" : "a1b2c3d4e5f6789012345678901234567890abcd",
+ "authType" : "DEPLOY_KEY",
+ "privateKeyFile" : "/run/secrets/deploy-key",
+ "sshUser" : "git",
+ "knownHostsFile" : "/run/secrets/known_hosts",
+ "readOnly" : true
+}
+----
+
+A GitLab deploy token is the same file with `authType` set to
`USERNAME_PASSWORD`, an `https://` repository URL, the token's user name, and
the token as the password.
+The token needs the `read_repository` scope.
+Leave the token out of the URL.
+
+Pin `revision` to a commit id.
+Leave the checkout folder empty on a short-lived container: the working copy
goes under the temporary directory and disappears with the container.
+A long-lived server should set a checkout folder and a maximum checkout age,
so a later commit on a branch is picked up without a restart.
+See xref:metadata-types/git-vfs-connection.adoc[Git repository].
+
+Mount a `known_hosts` file, as in the example.
+`acceptUnknownHosts` set to `true` is the alternative when the host key cannot
be supplied.
+A fresh container has no `~/.ssh/known_hosts`, and the connection refuses the
server until one of those two is set.
+
+=== What the paths mean
+
+`ops:///workflows/daily.hwf` is that file in the repository the connection
named `ops` checked out.
+
+`+${PROJECT_HOME}+` is `/bootstrap`, the folder you mounted.
+A pipeline stored as `+${PROJECT_HOME}+/pipelines/load.hpl` resolves against
that folder.
+
+A path relative to the file that is running stays on the git scheme.
+For a workflow opened as `ops:///workflows/daily.hwf`,
`+${Internal.Entry.Current.Folder}+` is the `workflows` directory inside the
checkout, so a workflow that refers to its neighbours that way keeps reading
the repository.
+
+A project that reaches its files through `+${PROJECT_HOME}+` still wants
xref:hop-server/deploy-project-image.adoc[an image] or
xref:hop-server/deploy-git-checkout.adoc[a checkout].
+
+The container needs the git host on every start.
+The connection names the revision, and the image tag names the Hop version.
+
== You often do not need a Hop Server
If the project is already in an image, a scheduler can start that image as a
*short-lived* container (locally, or on Docker, Kubernetes, OpenShift, ...),
run one pipeline or workflow, and exit.
@@ -114,6 +255,7 @@ There is no long-lived Hop Server to keep alive, upgrade or
authenticate.
The official `apache/hop` image does both jobs.
Set `HOP_FILE_PATH` and `HOP_RUN_CONFIG` and the entrypoint runs
xref:hop-run/index.adoc[hop-run] and exits.
Omit them and it starts Hop Server.
+On the <<StandardApacheHopContainer,stock image>>, `HOP_FILE_PATH` can be a
Git VFS URL.
Because the container disappears when the run finishes, send
xref:metadata-types/execution-information-location.adoc[execution information]
somewhere that outlives it: a mounted volume (file or caching-file location), a
relational database, OpenSearch, Neo4j or Elastic.
You can inspect those runs later from the
xref:hop-gui/perspective-execution-information.adoc[Execution Information
perspective] in Hop Gui.
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/PluginDiscovery.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/PluginDiscovery.java
index fa788babd6..da3139f461 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/PluginDiscovery.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/catalog/PluginDiscovery.java
@@ -45,8 +45,16 @@ public final class PluginDiscovery {
/**
* Resolved install target: Maven coordinates plus an optional preferred
repository id (from
* discovery {@code source}) tried first in the install fallback chain.
+ *
+ * @param discovered true when the coordinate came from a catalog or browse
hit, false when it is
+ * a literal parse (including the {@code org.apache.hop} fallback)
*/
- public record InstallTarget(MavenCoordinates coordinates, String
preferredRepoId) {}
+ public record InstallTarget(
+ MavenCoordinates coordinates, String preferredRepoId, boolean
discovered) {
+ public InstallTarget(MavenCoordinates coordinates, String preferredRepoId)
{
+ this(coordinates, preferredRepoId, false);
+ }
+ }
/**
* Full marketplace discovery: bundled Apache optional catalog plus every
enabled repository with
@@ -193,7 +201,8 @@ public final class PluginDiscovery {
preferredRepo = chosen.getSource();
}
- return new InstallTarget(new MavenCoordinates(groupId, artifactId,
version), preferredRepo);
+ return new InstallTarget(
+ new MavenCoordinates(groupId, artifactId, version), preferredRepo,
true);
}
/**
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
index b734abd648..49c64d625f 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/command/MarketplaceCommand.java
@@ -23,6 +23,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.Getter;
@@ -141,7 +142,7 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
+ " Every coordinate is resolved before the first download, so a
typo fails before"
+ " anything is fetched. A plugin that fails to install does not
stop the others;"
+ " the command exits non-zero when any of them failed.")
- static class InstallCommand extends MarketplaceSubCommand {
+ public static class InstallCommand extends MarketplaceSubCommand {
@Parameters(
index = "0",
arity = "1..*",
@@ -155,17 +156,79 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
@Option(
names = {"--repo"},
description =
- "Use only this repository id (skip fallback chain). Default:
prefer discovery source,"
- + " then primary and other enabled repos.")
+ "Use only this repository id (skip fallback chain) or base URL.
Default: prefer discovery"
+ + " source, then primary and other enabled repos.")
private String repoId;
+ @Option(
+ names = {"--repo-url"},
+ description =
+ "Maven repository base URL to try first (corporate Artifactory,
Nexus, or a plain"
+ + " Maven repository). Other configured repositories are still
used when the"
+ + " plugin is not there. Pass --repo <id> (not a URL) to use
only one repository.")
+ private String repoUrl;
+
+ @Option(
+ names = {"--repo-id"},
+ description =
+ "Repository id created for --repo-url. Defaults to the
HOP_PLUGINS_REPO_ID"
+ + " environment variable, or corporate-repo. Credentials are
read from"
+ + " HOP_MARKETPLACE_<ID>_USERNAME,
HOP_MARKETPLACE_<ID>_PASSWORD and"
+ + " HOP_MARKETPLACE_<ID>_TOKEN.")
+ private String adHocRepoId;
+
+ @Option(
+ names = {"--username"},
+ description = "Optional Basic auth username for the repository
specified in --repo-url")
+ private String username;
+
+ @Option(
+ names = {"--password"},
+ description =
+ "Optional Basic auth password or bearer token for the repository
specified in"
+ + " --repo-url. Prefer HOP_MARKETPLACE_<ID>_PASSWORD or _TOKEN
so the secret is"
+ + " not on the process command line.")
+ private String password;
+
+ @Option(
+ names = {"--auth-type"},
+ description = "Authentication type for --repo-url: auto (default),
none, basic or token")
+ private String authType;
+
+ @Option(
+ names = {"--repo-type"},
+ description =
+ "Browse API for --repo-url: auto (default), nexus, forgejo, jfrog,
or maven. maven is"
+ + " a plain repository that cannot be browsed; pass
groupId:artifactId:version.")
+ private String repoType;
+
@Override
public void run() {
try {
+ if (log == null) {
+ log = new LogChannel("Marketplace");
+ }
MarketplaceConfig config = MarketplaceConfig.load();
if (!config.isEnabled()) {
throw new HopException("Marketplace is disabled in hop-config.json");
}
+
+ // If repoId is an HTTP/HTTPS URL and repoUrl was omitted, treat it as
repoUrl
+ if (StringUtils.isBlank(repoUrl)
+ && StringUtils.isNotBlank(repoId)
+ && (repoId.startsWith("http://";) ||
repoId.startsWith("https://";))) {
+ repoUrl = repoId;
+ repoId = null;
+ }
+
+ MarketplaceRepository adHoc = registerAdHocRepository(config);
+ // --repo <id> stays exclusive. --repo-url is preferred and still
falls back, including
+ // when --repo was only the URL we just turned into the ad-hoc
repository.
+ String forceRepoId = null;
+ if (StringUtils.isNotBlank(repoId) && (adHoc == null ||
!repoId.equals(adHoc.getId()))) {
+ forceRepoId = repoId;
+ }
+
Path hopHome = HopHome.resolve();
PluginInstaller installer = new PluginInstaller(log, hopHome, config);
// Activate any previously staged plugins first
@@ -178,6 +241,9 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
PluginDiscovery.InstallTarget target =
PluginDiscovery.resolveInstall(
coordinate, config.getGroupId(),
resolveDefaultVersion(config), config, log);
+ if (adHoc != null && forceRepoId == null) {
+ rejectUnresolvedShortName(coordinate, target, adHoc);
+ }
printResolution(coordinate, target);
targets.add(target);
}
@@ -192,8 +258,12 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
// Prints "[2/5] hop-tech-parquet"; silent for a single install.
progress.item(gav.artifactId(), i, targets.size());
try {
+ String preferred = target.preferredRepoId();
+ if (adHoc != null && forceRepoId == null) {
+ preferred = adHoc.getId();
+ }
InstallReceipt receipt =
- installer.install(gav, true, repoId,
target.preferredRepoId(), progress);
+ installer.install(gav, true, forceRepoId, preferred,
progress);
installed++;
System.out.println(installedMessage(gav, receipt, hopHome,
targets.size() == 1));
} catch (Exception e) {
@@ -257,6 +327,9 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
private static String installedMessage(
MavenCoordinates gav, InstallReceipt receipt, Path hopHome, boolean
single) {
+ if (receipt.isAlreadyPresent()) {
+ return "Plugin " + gav.gav() + " is already installed under " +
hopHome + ".";
+ }
return "Plugin "
+ gav.gav()
+ " installed under "
@@ -267,6 +340,89 @@ public class MarketplaceCommand implements Runnable,
IHopCommand, IHasHopMetadat
// For a batch the restart hint belongs on the summary line, not on
every plugin.
+ (single ? ". Restart Hop to load it." : ".");
}
+
+ /**
+ * Default id is {@code corporate-repo}, matching {@code
HOP_PLUGINS_REPO_ID}. An id that is
+ * already configured is reused so credential variable names stay stable
across starts.
+ */
+ private MarketplaceRepository registerAdHocRepository(MarketplaceConfig
config) {
+ if (StringUtils.isBlank(repoUrl)) {
+ return null;
+ }
+ String id = resolveAdHocRepoId();
+ MarketplaceRepository adHoc = config.findRepository(id);
+ if (adHoc == null) {
+ adHoc = new MarketplaceRepository(id, "Ad-hoc Repository", repoUrl,
false);
+ if (config.getRepositories() == null) {
+ config.setRepositories(new ArrayList<>());
+ }
+ config.getRepositories().add(0, adHoc);
+ } else {
+ adHoc.setUrl(repoUrl);
+ adHoc.setEnabled(true);
+ }
+ if (StringUtils.isNotBlank(username)) {
+ adHoc.setUsername(username);
+ }
+ if (StringUtils.isNotBlank(password)) {
+ adHoc.setPassword(password);
+ }
+ if (StringUtils.isNotBlank(authType)) {
+ adHoc.setAuthType(authType);
+ }
+ applyRepoType(adHoc, repoType);
+ return adHoc;
+ }
+
+ private String resolveAdHocRepoId() {
+ if (StringUtils.isNotBlank(adHocRepoId)) {
+ return adHocRepoId.trim();
+ }
+ String fromEnv = System.getenv("HOP_PLUGINS_REPO_ID");
+ if (StringUtils.isNotBlank(fromEnv)) {
+ return fromEnv.trim();
+ }
+ return "corporate-repo";
+ }
+
+ static void applyRepoType(MarketplaceRepository repo, String repoType) {
+ String type = StringUtils.trimToEmpty(repoType);
+ if (type.isEmpty() ||
MarketplaceRepository.BROWSER_AUTO.equalsIgnoreCase(type)) {
+ repo.setBrowserType(MarketplaceRepository.BROWSER_AUTO);
+ repo.setBrowse(repo.supportsBrowseApi());
+ return;
+ }
+ if ("maven".equalsIgnoreCase(type) || "none".equalsIgnoreCase(type)) {
+ repo.setBrowserType(MarketplaceRepository.BROWSER_AUTO);
+ repo.setBrowse(false);
+ return;
+ }
+ repo.setBrowserType(type.toLowerCase(Locale.ROOT));
+ repo.setBrowse(true);
+ }
+
+ /**
+ * A short name against a repository that cannot be browsed must not be
rewritten to {@code
+ * org.apache.hop:artifact:version}. A full {@code
groupId:artifactId:version} is unchanged, and
+ * a name that discovery already resolved (the Apache catalog, for
example) is kept.
+ */
+ static void rejectUnresolvedShortName(
+ String coordinate, PluginDiscovery.InstallTarget target,
MarketplaceRepository adHoc)
+ throws HopException {
+ if (adHoc == null || target == null || target.discovered() ||
adHoc.canBrowse()) {
+ return;
+ }
+ String trimmed = coordinate == null ? "" : coordinate.trim();
+ if (trimmed.split(":", -1).length >= 3) {
+ return;
+ }
+ throw new HopException(
+ "Repository '"
+ + adHoc.getId()
+ + "' cannot be browsed, so '"
+ + coordinate
+ + "' cannot be resolved. Pass groupId:artifactId:version.");
+ }
}
@Command(
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepository.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepository.java
index 9eddec0fe9..99b567f78b 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepository.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/config/MarketplaceRepository.java
@@ -29,6 +29,7 @@ import java.util.function.UnaryOperator;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.marketplace.catalog.NexusRepositoryBrowser;
import org.apache.hop.marketplace.catalog.OptionalPluginInfo;
@Getter
@@ -277,12 +278,12 @@ public class MarketplaceRepository {
}
/**
- * Environment lookup, replaceable in tests. Package-private on purpose:
credential resolution is
- * otherwise untestable, and it is the part most likely to go subtly wrong.
+ * Environment lookup, replaceable in tests. Credential resolution is
otherwise untestable, and it
+ * is the part most likely to go subtly wrong.
*/
private static UnaryOperator<String> environment = System::getenv;
- static void setEnvironmentForTesting(UnaryOperator<String> lookup) {
+ public static void setEnvironmentForTesting(UnaryOperator<String> lookup) {
environment = lookup == null ? System::getenv : lookup;
}
@@ -363,6 +364,31 @@ public class MarketplaceRepository {
return StringUtils.isAllBlank(username, password) && hasCredentials();
}
+ /**
+ * True when this repository can list plugins (a catalog URL, or a browse
API the URL or {@link
+ * #browserType} actually supports). A plain Maven repository is not
browsable: callers must pass
+ * {@code groupId:artifactId:version} instead of a short name.
+ */
+ public boolean canBrowse() {
+ return isBrowse() && supportsBrowseApi();
+ }
+
+ /**
+ * Whether the resolved browse backend can list this repository, ignoring
the {@link #browse}
+ * flag. Explicit {@code jfrog} or {@code forgejo} counts; {@code nexus}
only when the URL is a
+ * Nexus {@code /repository/<name>/} base.
+ */
+ public boolean supportsBrowseApi() {
+ if (StringUtils.isNotBlank(catalogUrl)) {
+ return true;
+ }
+ return switch (effectiveBrowserType()) {
+ case BROWSER_FORGEJO, BROWSER_JFROG -> true;
+ case BROWSER_NEXUS -> NexusRepositoryBrowser.isNexusBrowseUrl(url);
+ default -> false;
+ };
+ }
+
/**
* Resolved browse backend. When {@link #browserType} is blank or {@code
auto}, the URL decides: a
* Forgejo / Gitea package registry ({@code .../api/packages/{owner}/maven})
selects {@code
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/HopHome.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/HopHome.java
index 2c3e08b99d..35ffa42ee5 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/HopHome.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/HopHome.java
@@ -21,6 +21,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@@ -42,9 +43,36 @@ public final class HopHome {
public static Path resolve() throws HopException {
List<String> tried = new ArrayList<>();
- for (Path candidate : candidates()) {
+ Path imageHome = null;
+ for (Path candidate : imageCandidates()) {
Path abs = candidate.toAbsolutePath().normalize();
tried.add(abs.toString());
+ if (isHopHome(abs)) {
+ imageHome = abs;
+ break;
+ }
+ }
+
+ // A writable plugins directory named in HOP_PLUGIN_BASE_FOLDERS (usually
a mounted volume) is
+ // the install target, so a read-only image root is not the only place
plugins can live. The
+ // launcher passes the variable as a system property; the environment is
only a fallback when
+ // the process was not started by that launcher.
+ for (Path home : pluginHomes(true)) {
+ Path abs = home.toAbsolutePath().normalize();
+ if (imageHome != null && imageHome.equals(abs)) {
+ continue;
+ }
+ if (isWritablePluginsHome(abs)) {
+ return abs;
+ }
+ }
+ if (imageHome != null) {
+ return imageHome;
+ }
+
+ for (Path home : pluginHomes(false)) {
+ Path abs = home.toAbsolutePath().normalize();
+ tried.add(abs.toString());
if (isHopHome(abs)) {
return abs;
}
@@ -56,7 +84,7 @@ public final class HopHome {
+ String.join(", ", tried));
}
- private static Set<Path> candidates() {
+ private static Set<Path> imageCandidates() {
Set<Path> paths = new LinkedHashSet<>();
Path cwd = Paths.get(System.getProperty("user.dir", "."));
paths.add(cwd);
@@ -65,16 +93,27 @@ public final class HopHome {
paths.add(parent);
}
- // Web deployments can keep the writable plugins and configuration outside
the binaries.
- // Resolve those configured locations without changing the launcher
behavior above.
+ // Web deployments can keep configuration outside the binaries.
addConfiguredParent(paths, System.getProperty(HOP_CONFIG_FOLDER),
"config");
addConfiguredParent(paths, System.getenv(HOP_CONFIG_FOLDER), "config");
- addConfiguredPluginParents(paths,
System.getProperty(HOP_PLUGIN_BASE_FOLDERS));
- addConfiguredPluginParents(paths, System.getenv(HOP_PLUGIN_BASE_FOLDERS));
return paths;
}
- private static void addConfiguredPluginParents(Set<Path> paths, String
configured) {
+ private static List<Path> pluginHomes(boolean propertyOnly) {
+ List<Path> homes = new ArrayList<>();
+ addConfiguredPluginParents(homes,
System.getProperty(HOP_PLUGIN_BASE_FOLDERS));
+ if (!propertyOnly) {
+ addConfiguredPluginParents(homes,
System.getenv(HOP_PLUGIN_BASE_FOLDERS));
+ }
+ return homes;
+ }
+
+ private static boolean isWritablePluginsHome(Path home) {
+ Path plugins = home.resolve("plugins");
+ return Files.isDirectory(plugins) && Files.isWritable(plugins);
+ }
+
+ private static void addConfiguredPluginParents(Collection<Path> paths,
String configured) {
if (configured == null || configured.isBlank()) {
return;
}
@@ -83,7 +122,8 @@ public final class HopHome {
}
}
- private static void addConfiguredParent(Set<Path> paths, String configured,
String childName) {
+ private static void addConfiguredParent(
+ Collection<Path> paths, String configured, String childName) {
if (configured == null || configured.isBlank()) {
return;
}
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/InstallReceipt.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/InstallReceipt.java
index 12edeb6063..9b4e6f0d64 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/InstallReceipt.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/InstallReceipt.java
@@ -17,6 +17,7 @@
package org.apache.hop.marketplace.install;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
@@ -39,6 +40,9 @@ public class InstallReceipt {
private List<String> paths = new ArrayList<>();
private boolean pendingActivation;
+ /** Set when {@code install} returned this receipt without downloading
again. Not persisted. */
+ @JsonIgnore private boolean alreadyPresent;
+
public InstallReceipt() {
// Jackson
}
diff --git
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginInstaller.java
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginInstaller.java
index 79afdcbb42..8096c5d026 100644
---
a/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginInstaller.java
+++
b/plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginInstaller.java
@@ -36,6 +36,7 @@ import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.json.HopJson;
import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.marketplace.config.MarketplaceConfig;
import org.apache.hop.marketplace.config.MarketplaceRepository;
import org.apache.hop.marketplace.resolve.MavenCoordinates;
@@ -57,15 +58,16 @@ public class PluginInstaller {
private final MavenRepositoryClient client;
public PluginInstaller(ILogChannel log, Path hopHome, MarketplaceConfig
config) {
- this.log = log;
- this.hopHome = hopHome;
- this.config = config;
- this.client = new MavenRepositoryClient(log);
+ this(
+ log != null ? log : new LogChannel("PluginInstaller"),
+ hopHome,
+ config,
+ new MavenRepositoryClient(log != null ? log : new
LogChannel("PluginInstaller")));
}
PluginInstaller(
ILogChannel log, Path hopHome, MarketplaceConfig config,
MavenRepositoryClient client) {
- this.log = log;
+ this.log = log != null ? log : new LogChannel("PluginInstaller");
this.hopHome = hopHome;
this.config = config;
this.client = client;
@@ -119,6 +121,17 @@ public class PluginInstaller {
IInstallListener listener)
throws HopException {
IInstallListener progress = listener == null ? IInstallListener.NONE :
listener;
+ InstallReceipt already = satisfiedInstall(coordinates);
+ if (already != null) {
+ already.setAlreadyPresent(true);
+ log.logBasic(
+ "Plugin "
+ + coordinates.gav()
+ + " is already installed (receipt "
+ + already.getVersion()
+ + "); skipping download.");
+ return already;
+ }
Path downloadDir = hopHome.resolve(STAGING_DIR).resolve(".download");
Path zipFile =
downloadDir.resolve(coordinates.artifactId() + "-" +
coordinates.version() + ".zip");
@@ -197,6 +210,49 @@ public class PluginInstaller {
}
}
+ /**
+ * A repeated install can skip the download when the receipt names this
version and every file it
+ * recorded is still on disk.
+ */
+ private InstallReceipt satisfiedInstall(MavenCoordinates coordinates) throws
HopException {
+ InstallReceipt receipt = readReceipt(hopHome, coordinates.artifactId());
+ if (receipt == null || receipt.isPendingActivation()) {
+ return null;
+ }
+ if (!coordinates.version().equals(receipt.getVersion())) {
+ return null;
+ }
+ if (StringUtils.isNotBlank(receipt.getGroupId())
+ && !receipt.getGroupId().equals(coordinates.groupId())) {
+ return null;
+ }
+ if (!receiptFilesPresent(receipt)) {
+ return null;
+ }
+ return receipt;
+ }
+
+ private boolean receiptFilesPresent(InstallReceipt receipt) {
+ List<String> paths = receipt.getPaths();
+ if (paths == null || paths.isEmpty()) {
+ return false;
+ }
+ boolean file = false;
+ for (String relative : paths) {
+ if (StringUtils.isBlank(relative)) {
+ continue;
+ }
+ Path path = activationTarget(relative);
+ if (!Files.exists(path)) {
+ return false;
+ }
+ if (Files.isRegularFile(path)) {
+ file = true;
+ }
+ }
+ return file;
+ }
+
private List<MarketplaceRepository> resolveRepositories(
String forceRepoId, String preferredRepoId) throws HopException {
if (StringUtils.isNotBlank(forceRepoId)) {
@@ -254,7 +310,7 @@ public class PluginInstaller {
try {
for (String relative : relativePaths) {
Path from = stageRoot.resolve(relative);
- Path to = hopHome.resolve(relative);
+ Path to = activationTarget(relative);
if (Files.isDirectory(from)) {
Files.createDirectories(to);
} else if (Files.isRegularFile(from)) {
@@ -324,6 +380,38 @@ public class PluginInstaller {
return normalized.equals("lib/core") || normalized.startsWith("lib/core/");
}
+ /** Web layout keeps shared jars in {@code WEB-INF/lib} instead of {@code
lib/core}. */
+ private Path activationTarget(String relative) {
+ Path standard = hopHome.resolve(relative);
+ String normalized = relative == null ? "" : relative.replace('\\', '/');
+ if (!normalized.startsWith("lib/core/") || normalized.endsWith("/")) {
+ return standard;
+ }
+ Path webLib = webInfLib();
+ if (webLib == null) {
+ return standard;
+ }
+ Path name = Path.of(normalized).getFileName();
+ return name == null ? standard : webLib.resolve(name);
+ }
+
+ private Path webInfLib() {
+ Path fromCwd =
+ Path.of(System.getProperty("user.dir", "."))
+ .resolve("WEB-INF")
+ .resolve("lib")
+ .toAbsolutePath()
+ .normalize();
+ if (Files.isDirectory(fromCwd)) {
+ return fromCwd;
+ }
+ Path fromHome =
hopHome.resolve("webapps").resolve("ROOT").resolve("WEB-INF").resolve("lib");
+ if (Files.isDirectory(fromHome)) {
+ return fromHome;
+ }
+ return null;
+ }
+
/**
* Whether activation should skip copying a staged lib/core file onto an
existing target.
*
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/PluginDiscoveryResolveInstallTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/PluginDiscoveryResolveInstallTest.java
index 1606d44774..28b9248dd5 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/PluginDiscoveryResolveInstallTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/catalog/PluginDiscoveryResolveInstallTest.java
@@ -18,6 +18,7 @@
package org.apache.hop.marketplace.catalog;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -61,6 +62,7 @@ class PluginDiscoveryResolveInstallTest {
"org.example:foo:9.9.9", "org.apache.hop", "2.19.0-SNAPSHOT",
null, null);
assertEquals("org.example:foo:9.9.9", target.coordinates().gav());
assertNull(target.preferredRepoId());
+ assertFalse(target.discovered());
}
@Test
@@ -70,6 +72,7 @@ class PluginDiscoveryResolveInstallTest {
"totally-unknown-plugin", "org.apache.hop", "2.19.0-SNAPSHOT",
null, null);
assertEquals(
"org.apache.hop:totally-unknown-plugin:2.19.0-SNAPSHOT",
target.coordinates().gav());
+ assertFalse(target.discovered());
}
private static OptionalPluginInfo plugin(String artifactId, String version,
String source) {
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/command/InstallCommandParsingTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/command/InstallCommandParsingTest.java
index ef106bafd6..6d2789cafa 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/command/InstallCommandParsingTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/command/InstallCommandParsingTest.java
@@ -19,8 +19,13 @@ package org.apache.hop.marketplace.command;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.marketplace.catalog.PluginDiscovery;
+import org.apache.hop.marketplace.config.MarketplaceRepository;
+import org.apache.hop.marketplace.resolve.MavenCoordinates;
import org.junit.jupiter.api.Test;
import picocli.CommandLine;
@@ -57,6 +62,80 @@ class InstallCommandParsingTest {
assertEquals("local-nexus", parsed.matchedOptionValue("--repo", null));
}
+ @Test
+ void repoUrlAndCredentialsOptionsAreParsed() {
+ CommandLine commandLine = new CommandLine(new
MarketplaceCommand.InstallCommand());
+ CommandLine.ParseResult parsed =
+ commandLine.parseArgs(
+ "--repo-url",
+
"https://repository.data-hopper.com/repository/hop-community-plugins/";,
+ "--username",
+ "testuser",
+ "--password",
+ "testpass",
+ "--auth-type",
+ "basic",
+ "org.hopper:hopper-edw:0.10.0");
+ assertEquals(List.of("org.hopper:hopper-edw:0.10.0"),
parsed.matchedPositional(0).getValue());
+ assertEquals(
+ "https://repository.data-hopper.com/repository/hop-community-plugins/";,
+ parsed.matchedOptionValue("--repo-url", null));
+ assertEquals("testuser", parsed.matchedOptionValue("--username", null));
+ assertEquals("testpass", parsed.matchedOptionValue("--password", null));
+ assertEquals("basic", parsed.matchedOptionValue("--auth-type", null));
+ }
+
+ @Test
+ void repoIdAndRepoTypeOptionsAreParsed() {
+ CommandLine commandLine = new CommandLine(new
MarketplaceCommand.InstallCommand());
+ CommandLine.ParseResult parsed =
+ commandLine.parseArgs(
+ "--repo-url",
+ "https://maven.example/releases/";,
+ "--repo-id",
+ "corporate-repo",
+ "--repo-type",
+ "maven",
+ "com.acme:acme-plugin:1.0.0");
+ assertEquals("corporate-repo", parsed.matchedOptionValue("--repo-id",
null));
+ assertEquals("maven", parsed.matchedOptionValue("--repo-type", null));
+ }
+
+ @Test
+ void shortNameOnUnbrowsableRepositoryAsksForFullCoordinates() {
+ MarketplaceRepository repo =
+ new MarketplaceRepository("corporate-repo",
"https://maven.example/releases/";);
+ repo.setBrowse(false);
+ PluginDiscovery.InstallTarget target =
+ new PluginDiscovery.InstallTarget(
+ new MavenCoordinates("org.apache.hop", "acme-plugin", "1.0.0"),
null);
+ HopException ex =
+ assertThrows(
+ HopException.class,
+ () ->
+ MarketplaceCommand.InstallCommand.rejectUnresolvedShortName(
+ "acme-plugin:1.0.0", target, repo));
+ assertTrue(ex.getMessage().contains("cannot be browsed"));
+ assertTrue(ex.getMessage().contains("groupId:artifactId:version"));
+ }
+
+ @Test
+ void
fullCoordinateAndDiscoveredNameAreAcceptedWhenRepositoryCannotBeBrowsed()
throws Exception {
+ MarketplaceRepository repo =
+ new MarketplaceRepository("corporate-repo",
"https://maven.example/releases/";);
+ repo.setBrowse(false);
+ MarketplaceCommand.InstallCommand.rejectUnresolvedShortName(
+ "com.acme:acme-plugin:1.0.0",
+ new PluginDiscovery.InstallTarget(
+ new MavenCoordinates("com.acme", "acme-plugin", "1.0.0"), null),
+ repo);
+ MarketplaceCommand.InstallCommand.rejectUnresolvedShortName(
+ "hop-tech-parquet",
+ new PluginDiscovery.InstallTarget(
+ new MavenCoordinates("org.apache.hop", "hop-tech-parquet",
"2.20.0"), null, true),
+ repo);
+ }
+
@Test
void atLeastOneCoordinateIsRequired() {
assertThrows(CommandLine.MissingParameterException.class,
this::parseNothing);
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryBrowserTypeTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryBrowserTypeTest.java
index 7713f3b1b8..5157f73622 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryBrowserTypeTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/config/MarketplaceRepositoryBrowserTypeTest.java
@@ -199,4 +199,35 @@ class MarketplaceRepositoryBrowserTypeTest {
"https://example.org/${artifactId}-${version}.zip";,
config.findRepository("acme").getUrlTemplate());
}
+
+ @Test
+ void plainMavenRepositoryCannotBeBrowsed() {
+ MarketplaceRepository repo =
+ new MarketplaceRepository("corporate-repo",
"https://maven.example/releases/";);
+ repo.setBrowse(true);
+ assertFalse(repo.supportsBrowseApi());
+ assertFalse(repo.canBrowse());
+ }
+
+ @Test
+ void nexusArtifactoryAndExplicitTypeCanBeBrowsed() {
+ MarketplaceRepository nexus =
+ new MarketplaceRepository("nexus",
"https://repository.example/repository/hop-plugins/";);
+ nexus.setBrowse(true);
+ assertTrue(nexus.canBrowse());
+
+ MarketplaceRepository artifactory =
+ new MarketplaceRepository(
+ "artifactory",
"https://artifactory.example/artifactory/hop-plugins/";);
+ artifactory.setBrowse(true);
+ assertTrue(artifactory.canBrowse());
+
+ MarketplaceRepository explicit = new MarketplaceRepository("jfrog",
"https://maven.example/m/";);
+ explicit.setBrowserType(MarketplaceRepository.BROWSER_JFROG);
+ explicit.setBrowse(true);
+ assertTrue(explicit.canBrowse());
+
+ explicit.setBrowse(false);
+ assertFalse(explicit.canBrowse());
+ }
}
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/HopHomeTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/HopHomeTest.java
index 4caf4c9ee9..a6c44b9c2f 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/HopHomeTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/HopHomeTest.java
@@ -83,6 +83,25 @@ class HopHomeTest {
}
}
+ @Test
+ void resolvePrefersWritablePluginFolderOverTheInstall() throws Exception {
+ Path image = tempDir.resolve("opt-hop");
+ Files.createDirectories(image.resolve("plugins"));
+ Path volume = tempDir.resolve("volume");
+ Files.createDirectories(volume.resolve("plugins"));
+ String previousUserDir = System.getProperty("user.dir");
+ String previousPluginFolders =
System.getProperty("HOP_PLUGIN_BASE_FOLDERS");
+ try {
+ System.setProperty("user.dir", image.toString());
+ System.setProperty(
+ "HOP_PLUGIN_BASE_FOLDERS", image.resolve("plugins") + "," +
volume.resolve("plugins"));
+ assertEquals(volume.toAbsolutePath().normalize(), HopHome.resolve());
+ } finally {
+ restoreProperty("user.dir", previousUserDir);
+ restoreProperty("HOP_PLUGIN_BASE_FOLDERS", previousPluginFolders);
+ }
+ }
+
@Test
void resolveUsesParentOfConfiguredConfigFolder() throws Exception {
Path hop = tempDir.resolve("hop-web");
diff --git
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
index a63e343918..ef904cee97 100644
---
a/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
+++
b/plugins/misc/marketplace/src/test/java/org/apache/hop/marketplace/install/PluginInstallerTest.java
@@ -25,23 +25,33 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import com.sun.net.httpserver.HttpServer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Base64;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import org.apache.hop.core.config.HopConfig;
+import org.apache.hop.core.config.plugin.ConfigFile;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.HopLogStore;
import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.marketplace.command.MarketplaceCommand;
import org.apache.hop.marketplace.config.MarketplaceConfig;
import org.apache.hop.marketplace.config.MarketplaceRepository;
import org.apache.hop.marketplace.resolve.MavenCoordinates;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import picocli.CommandLine;
class PluginInstallerTest {
@@ -343,6 +353,42 @@ class PluginInstallerTest {
}
}
+ @Test
+ void installCommandWithRepoUrlInstallsFromAdHocRepository() throws Exception
{
+ byte[] zipBytes = buildPluginZip();
+ HttpServer server = zipServer(zipBytes);
+ server.start();
+ try {
+ Path hopHome = tempDir.resolve("hop-adhoc-install");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ int port = server.getAddress().getPort();
+
+ String originalUserDir = System.getProperty("user.dir");
+ try {
+ System.setProperty("user.dir", hopHome.toAbsolutePath().toString());
+ MarketplaceCommand.InstallCommand cmd = new
MarketplaceCommand.InstallCommand();
+ CommandLine cl = new CommandLine(cmd);
+ cl.parseArgs("--repo-url", localUrl(port),
"org.apache.hop:hop-test-plugin:1.0.0");
+ cmd.run();
+
+ Path pluginJar = hopHome.resolve("plugins/tech/test/plugin.jar");
+ assertTrue(Files.isRegularFile(pluginJar));
+ assertTrue(
+ Files.isRegularFile(
+
hopHome.resolve(PluginInstaller.RECEIPTS_DIR).resolve("hop-test-plugin.json")));
+ assertEquals(
+ "corporate-repo",
+ PluginInstaller.readReceipt(hopHome,
"hop-test-plugin").getRepositoryId());
+ } finally {
+ if (originalUserDir != null) {
+ System.setProperty("user.dir", originalUserDir);
+ }
+ }
+ } finally {
+ server.stop(0);
+ }
+ }
+
@Test
void cancelDuringDownloadInstallsNothingAndDoesNotTryOtherRepositories()
throws Exception {
byte[] zipBytes = buildPluginZip();
@@ -389,6 +435,378 @@ class PluginInstallerTest {
}
}
+ @Test
+ void secondInstallSkipsWhenReceiptMatchesAndFilesRemain() throws Exception {
+ byte[] zipBytes = buildPluginZip();
+ AtomicInteger hits = new AtomicInteger();
+ HttpServer server = zipServer(zipBytes, hits);
+ server.start();
+ try {
+ Path hopHome = tempDir.resolve("hop-skip");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ MarketplaceConfig config =
localRepoConfig(server.getAddress().getPort());
+ PluginInstaller installer = new PluginInstaller(new LogChannel("test"),
hopHome, config);
+ MavenCoordinates coords = new MavenCoordinates("org.apache.hop",
"hop-test-plugin", "1.0.0");
+
+ installer.install(coords, true);
+ installer.install(coords, true);
+ assertEquals(
+ 1, hits.get(), "a matching receipt with its files present must not
download again");
+
+ Files.delete(hopHome.resolve("plugins/tech/test/plugin.jar"));
+ installer.install(coords, true);
+ assertEquals(2, hits.get(), "a missing receipt file must be installed
again");
+
assertTrue(Files.isRegularFile(hopHome.resolve("plugins/tech/test/plugin.jar")));
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void preferredRepositoryIsTriedFirstAndFallsBack() throws Exception {
+ byte[] zipBytes = buildPluginZip();
+ HttpServer server = zipServer(zipBytes);
+ server.start();
+ try {
+ int port = server.getAddress().getPort();
+ Path hopHome = tempDir.resolve("hop-preferred");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ MarketplaceConfig config = new MarketplaceConfig();
+ config.getRepositories().clear();
+ config
+ .getRepositories()
+ .add(
+ new MarketplaceRepository(
+ "corporate-repo", "http://127.0.0.1:"; + port + "/missing/",
false));
+ config.getRepositories().add(new MarketplaceRepository("central",
localUrl(port), true));
+
+ InstallReceipt receipt =
+ new PluginInstaller(new LogChannel("test"), hopHome, config)
+ .install(
+ new MavenCoordinates("org.apache.hop", "hop-test-plugin",
"1.0.0"),
+ true,
+ null,
+ "corporate-repo");
+ assertEquals("central", receipt.getRepositoryId());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void forcedRepositoryDoesNotFallBack() throws Exception {
+ byte[] zipBytes = buildPluginZip();
+ HttpServer server = zipServer(zipBytes);
+ server.start();
+ try {
+ int port = server.getAddress().getPort();
+ Path hopHome = tempDir.resolve("hop-forced");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ MarketplaceConfig config = new MarketplaceConfig();
+ config.getRepositories().clear();
+ config
+ .getRepositories()
+ .add(new MarketplaceRepository("only", "http://127.0.0.1:"; + port +
"/missing/", true));
+ config.getRepositories().add(new MarketplaceRepository("other",
localUrl(port), false));
+
+ assertThrows(
+ HopException.class,
+ () ->
+ new PluginInstaller(new LogChannel("test"), hopHome, config)
+ .install(
+ new MavenCoordinates("org.apache.hop",
"hop-test-plugin", "1.0.0"),
+ true,
+ "only",
+ null));
+
assertFalse(Files.exists(hopHome.resolve("plugins/tech/test/plugin.jar")));
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void installCommandWithRepoUrlUsesBasicAuthAndRejectsAnonymous() throws
Exception {
+ byte[] zipBytes = buildPluginZip();
+ AtomicInteger anonymous = new AtomicInteger();
+ HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
+ String path =
"/org/apache/hop/hop-test-plugin/1.0.0/hop-test-plugin-1.0.0.zip";
+ String expectedBasic =
+ "Basic "
+ +
Base64.getEncoder().encodeToString("admin:s3cret".getBytes(StandardCharsets.UTF_8));
+ server.createContext(
+ path,
+ exchange -> {
+ String auth = exchange.getRequestHeaders().getFirst("Authorization");
+ if (!expectedBasic.equals(auth)) {
+ anonymous.incrementAndGet();
+ exchange.sendResponseHeaders(401, -1);
+ exchange.close();
+ return;
+ }
+ exchange.getResponseHeaders().add("Content-Type", "application/zip");
+ exchange.sendResponseHeaders(200, zipBytes.length);
+ exchange.getResponseBody().write(zipBytes);
+ exchange.close();
+ });
+ server.start();
+ try {
+ int port = server.getAddress().getPort();
+ Path hopHome = tempDir.resolve("hop-basic-url");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ String originalUserDir = System.getProperty("user.dir");
+ try {
+ System.setProperty("user.dir", hopHome.toAbsolutePath().toString());
+ withIsolatedMarketplaceConfig(
+ () -> {
+ marketplaceEnv(Map.of());
+ MarketplaceCommand.InstallCommand anonymousCmd =
+ new MarketplaceCommand.InstallCommand();
+ CommandLine anonymousLine = new CommandLine(anonymousCmd);
+ anonymousLine.parseArgs(
+ "--repo-url", localUrl(port),
"org.apache.hop:hop-test-plugin:1.0.0");
+ PrintStream originalErr = System.err;
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ System.setErr(new PrintStream(err, true,
StandardCharsets.UTF_8));
+ try {
+ CommandLine.ExecutionException failure =
+ assertThrows(CommandLine.ExecutionException.class,
anonymousCmd::run);
+ assertTrue(failure.getMessage().contains("hop-test-plugin"),
failure.getMessage());
+ } finally {
+ System.setErr(originalErr);
+ }
+ assertTrue(
+ err.toString(StandardCharsets.UTF_8).contains("401"),
+ err.toString(StandardCharsets.UTF_8));
+
assertFalse(Files.exists(hopHome.resolve("plugins/tech/test/plugin.jar")));
+ assertTrue(anonymous.get() > 0);
+
+ marketplaceEnv(
+ Map.of(
+ "HOP_MARKETPLACE_CORPORATE_REPO_USERNAME",
+ "admin",
+ "HOP_MARKETPLACE_CORPORATE_REPO_PASSWORD",
+ "s3cret"));
+ MarketplaceCommand.InstallCommand cmd = new
MarketplaceCommand.InstallCommand();
+ CommandLine cl = new CommandLine(cmd);
+ cl.parseArgs(
+ "--repo-url",
+ localUrl(port),
+ "--repo-id",
+ "corporate-repo",
+ "--auth-type",
+ "basic",
+ "org.apache.hop:hop-test-plugin:1.0.0");
+ cmd.run();
+ });
+
assertTrue(Files.isRegularFile(hopHome.resolve("plugins/tech/test/plugin.jar")));
+ assertEquals(
+ "corporate-repo",
+ PluginInstaller.readReceipt(hopHome,
"hop-test-plugin").getRepositoryId());
+ } finally {
+ MarketplaceRepository.setEnvironmentForTesting(null);
+ if (originalUserDir != null) {
+ System.setProperty("user.dir", originalUserDir);
+ }
+ }
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void installCommandWithRepoUrlUsesBearerToken() throws Exception {
+ byte[] zipBytes = buildPluginZip();
+ HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
+ String path =
"/org/apache/hop/hop-test-plugin/1.0.0/hop-test-plugin-1.0.0.zip";
+ server.createContext(
+ path,
+ exchange -> {
+ String auth = exchange.getRequestHeaders().getFirst("Authorization");
+ if (!"Bearer s3cret-token".equals(auth)) {
+ exchange.sendResponseHeaders(401, -1);
+ exchange.close();
+ return;
+ }
+ exchange.getResponseHeaders().add("Content-Type", "application/zip");
+ exchange.sendResponseHeaders(200, zipBytes.length);
+ exchange.getResponseBody().write(zipBytes);
+ exchange.close();
+ });
+ server.start();
+ try {
+ int port = server.getAddress().getPort();
+ Path hopHome = tempDir.resolve("hop-token-url");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ String originalUserDir = System.getProperty("user.dir");
+ try {
+ System.setProperty("user.dir", hopHome.toAbsolutePath().toString());
+ withIsolatedMarketplaceConfig(
+ () -> {
+ marketplaceEnv(Map.of("HOP_MARKETPLACE_CORPORATE_REPO_TOKEN",
"s3cret-token"));
+ MarketplaceCommand.InstallCommand cmd = new
MarketplaceCommand.InstallCommand();
+ CommandLine cl = new CommandLine(cmd);
+ cl.parseArgs(
+ "--repo-url",
+ localUrl(port),
+ "--repo-id",
+ "corporate-repo",
+ "--auth-type",
+ "token",
+ "org.apache.hop:hop-test-plugin:1.0.0");
+ cmd.run();
+ });
+
assertTrue(Files.isRegularFile(hopHome.resolve("plugins/tech/test/plugin.jar")));
+ } finally {
+ MarketplaceRepository.setEnvironmentForTesting(null);
+ if (originalUserDir != null) {
+ System.setProperty("user.dir", originalUserDir);
+ }
+ }
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void installCommandRejectsShortNameWhenRepositoryCannotBeBrowsed() throws
Exception {
+ Path hopHome = tempDir.resolve("hop-short-name");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ String originalUserDir = System.getProperty("user.dir");
+ try {
+ System.setProperty("user.dir", hopHome.toAbsolutePath().toString());
+ withIsolatedMarketplaceConfig(
+ () -> {
+ marketplaceEnv(Map.of());
+ MarketplaceCommand.InstallCommand cmd = new
MarketplaceCommand.InstallCommand();
+ CommandLine cl = new CommandLine(cmd);
+ cl.parseArgs(
+ "--repo-url",
+ "http://127.0.0.1:9/maven/releases/";,
+ "--repo-type",
+ "maven",
+ "zz-no-such-plugin-8723:1.0.0");
+ CommandLine.ExecutionException failure =
+ assertThrows(CommandLine.ExecutionException.class, cmd::run);
+ assertTrue(failure.getMessage().contains("cannot be browsed"),
failure.getMessage());
+ assertTrue(
+ failure.getMessage().contains("groupId:artifactId:version"),
failure.getMessage());
+ });
+ } finally {
+ MarketplaceRepository.setEnvironmentForTesting(null);
+ if (originalUserDir != null) {
+ System.setProperty("user.dir", originalUserDir);
+ }
+ }
+ }
+
+ @Test
+ void installCommandWritesIntoConfiguredPluginFolder() throws Exception {
+ byte[] zipBytes = buildPluginZip();
+ HttpServer server = zipServer(zipBytes);
+ server.start();
+ try {
+ int port = server.getAddress().getPort();
+ Path image = tempDir.resolve("image-home");
+ Path volume = tempDir.resolve("volume-home");
+ Files.createDirectories(image.resolve("plugins"));
+ Files.createDirectories(volume.resolve("plugins"));
+ String originalUserDir = System.getProperty("user.dir");
+ String originalFolders = System.getProperty("HOP_PLUGIN_BASE_FOLDERS");
+ try {
+ System.setProperty("user.dir", image.toAbsolutePath().toString());
+ System.setProperty(
+ "HOP_PLUGIN_BASE_FOLDERS", image.resolve("plugins") + "," +
volume.resolve("plugins"));
+ withIsolatedMarketplaceConfig(
+ () -> {
+ MarketplaceCommand.InstallCommand cmd = new
MarketplaceCommand.InstallCommand();
+ CommandLine cl = new CommandLine(cmd);
+ cl.parseArgs("--repo-url", localUrl(port),
"org.apache.hop:hop-test-plugin:1.0.0");
+ cmd.run();
+ });
+
assertTrue(Files.isRegularFile(volume.resolve("plugins/tech/test/plugin.jar")));
+ assertTrue(Files.isRegularFile(volume.resolve("lib/core/shared.jar")));
+
assertFalse(Files.exists(image.resolve("plugins/tech/test/plugin.jar")));
+ } finally {
+ if (originalUserDir != null) {
+ System.setProperty("user.dir", originalUserDir);
+ }
+ if (originalFolders == null) {
+ System.clearProperty("HOP_PLUGIN_BASE_FOLDERS");
+ } else {
+ System.setProperty("HOP_PLUGIN_BASE_FOLDERS", originalFolders);
+ }
+ }
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void webLayoutInstallsSharedJarIntoWebInfLibAndSkipsWhenItRemains() throws
Exception {
+ byte[] zipBytes = buildPluginZip();
+ AtomicInteger hits = new AtomicInteger();
+ HttpServer server = zipServer(zipBytes, hits);
+ server.start();
+ try {
+ Path hopHome = tempDir.resolve("tomcat");
+ Files.createDirectories(hopHome.resolve("plugins"));
+ Path webLib = hopHome.resolve("webapps/ROOT/WEB-INF/lib");
+ Files.createDirectories(webLib);
+ MarketplaceConfig config =
localRepoConfig(server.getAddress().getPort());
+ PluginInstaller installer = new PluginInstaller(new LogChannel("test"),
hopHome, config);
+ MavenCoordinates coords = new MavenCoordinates("org.apache.hop",
"hop-test-plugin", "1.0.0");
+
+ installer.install(coords, true);
+ assertTrue(Files.isRegularFile(webLib.resolve("shared.jar")));
+ assertEquals("shared-lib",
Files.readString(webLib.resolve("shared.jar")));
+ assertFalse(Files.exists(hopHome.resolve("lib/core/shared.jar")));
+
assertTrue(Files.isRegularFile(hopHome.resolve("plugins/tech/test/plugin.jar")));
+
+ installer.install(coords, true);
+ assertEquals(1, hits.get(), "a shared jar in WEB-INF/lib still satisfies
the receipt");
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ private static void marketplaceEnv(Map<String, String> values) {
+ MarketplaceRepository.setEnvironmentForTesting(values::get);
+ }
+
+ private static void withIsolatedMarketplaceConfig(ThrowingRunnable action)
throws Exception {
+ HopConfig hopConfig = HopConfig.getInstance();
+ Field field = ConfigFile.class.getDeclaredField("configMap");
+ field.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map<String, Object> map = (Map<String, Object>) field.get(hopConfig);
+ Object previous = map.get(MarketplaceConfig.CONFIG_KEY);
+ Map<String, Object> closed = new LinkedHashMap<>();
+ closed.put("id", "closed");
+ closed.put("url", "http://127.0.0.1:1/";);
+ closed.put("enabled", true);
+ closed.put("primary", true);
+ closed.put("browse", false);
+ Map<String, Object> marketplace = new LinkedHashMap<>();
+ marketplace.put("enabled", true);
+ marketplace.put("groupId", "org.apache.hop");
+ marketplace.put("repositories", List.of(closed));
+ map.put(MarketplaceConfig.CONFIG_KEY, marketplace);
+ try {
+ action.run();
+ } finally {
+ if (previous == null) {
+ map.remove(MarketplaceConfig.CONFIG_KEY);
+ } else {
+ map.put(MarketplaceConfig.CONFIG_KEY, previous);
+ }
+ }
+ }
+
+ @FunctionalInterface
+ private interface ThrowingRunnable {
+ void run() throws Exception;
+ }
+
/** Captures the phase sequence and byte counts an install reports. */
private static class RecordingInstallListener implements IInstallListener {
private final List<Phase> phases = new ArrayList<>();
@@ -431,10 +849,17 @@ class PluginInstallerTest {
}
private static HttpServer zipServer(byte[] zipBytes) throws IOException {
+ return zipServer(zipBytes, null);
+ }
+
+ private static HttpServer zipServer(byte[] zipBytes, AtomicInteger hits)
throws IOException {
HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
server.createContext(
"/org/apache/hop/hop-test-plugin/1.0.0/hop-test-plugin-1.0.0.zip",
exchange -> {
+ if (hits != null) {
+ hits.incrementAndGet();
+ }
exchange.getResponseHeaders().add("Content-Type", "application/zip");
exchange.sendResponseHeaders(200, zipBytes.length);
exchange.getResponseBody().write(zipBytes);
