This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e35e01f330 Issue #8909 : Ship the lakehouse metadata jar with the
Spark engine (#8910)
e35e01f330 is described below
commit e35e01f3304e6166b65de7322678f776e0c66d12
Author: Matt Casters <[email protected]>
AuthorDate: Wed Oct 14 16:11:13 2026 +0200
Issue #8909 : Ship the lakehouse metadata jar with the Spark engine (#8910)
The marketplace classpath check installs hop-engines-spark onto the plain
client. The engine jar references org.apache.hop.lakehouse, which lives in
hop-tech-lakehouse and is not part of that client. Package that jar under
plugins/engines/spark/lib so the engine can resolve LakeCatalog and the lake
table metadata classes without the lakehouse plugin installed. The plugin
scanner skips lib/, so the transforms stay registered only by that plugin.
---
plugins/engines/spark/pom.xml | 8 +++++---
plugins/engines/spark/src/assembly/assembly.xml | 13 +++++++++++++
plugins/engines/spark/src/main/resources/dependencies.xml | 3 ++-
.../org/apache/hop/marketplace/optional-plugins.yaml | 4 ++--
4 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/plugins/engines/spark/pom.xml b/plugins/engines/spark/pom.xml
index 779eb9868e..035dc3b196 100644
--- a/plugins/engines/spark/pom.xml
+++ b/plugins/engines/spark/pom.xml
@@ -120,9 +120,11 @@
<scope>provided</scope>
</dependency>
- <!-- Lake table transform metadata and the catalog metadata type. Only
the plugin jar is
- used: at runtime dependencies.xml doesn't load the lakehouse lib/
folder either, and
- its unshaded Iceberg modules would clash with
iceberg-spark-runtime. -->
+ <!-- Lake table transform metadata and the catalog metadata type. Only
this jar is
+ packaged (under plugins/engines/spark/lib): dependencies.xml does
not load the
+ lakehouse plugin's lib/ either, and its unshaded Iceberg modules
would clash with
+ iceberg-spark-runtime. The copy in lib/ is what lets the engine
resolve these
+ classes when the lakehouse plugin itself is not installed. -->
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-tech-lakehouse</artifactId>
diff --git a/plugins/engines/spark/src/assembly/assembly.xml
b/plugins/engines/spark/src/assembly/assembly.xml
index 2fdfbd477e..63cf585442 100644
--- a/plugins/engines/spark/src/assembly/assembly.xml
+++ b/plugins/engines/spark/src/assembly/assembly.xml
@@ -42,6 +42,19 @@
</includes>
<outputDirectory>plugins/engines/spark</outputDirectory>
</dependencySet>
+ <!-- LakeCatalog and the lake table metadata classes. The lakehouse
plugin registers the
+ transforms; this copy is what the engine links against, so a
marketplace install of
+ Spark onto the plain client can start without that plugin. It
lives under lib/ so the
+ plugin scanner does not register the transforms twice.
Transitives stay out: the
+ unshaded Iceberg modules would clash with iceberg-spark-runtime.
-->
+ <dependencySet>
+ <scope>provided</scope>
+ <useTransitiveDependencies>false</useTransitiveDependencies>
+ <includes>
+ <include>org.apache.hop:hop-tech-lakehouse:jar</include>
+ </includes>
+ <outputDirectory>plugins/engines/spark/lib</outputDirectory>
+ </dependencySet>
<!-- Spark and transitive runtime jars for local[*] execution from Hop.
Jackson core/databind/annotations MUST NOT be packaged here:
HopURLClassLoader
is child-first, and dual ObjectMapper classes (plugin lib vs Hop
lib/core)
diff --git a/plugins/engines/spark/src/main/resources/dependencies.xml
b/plugins/engines/spark/src/main/resources/dependencies.xml
index c3b8fc7847..ecbc56a0dc 100644
--- a/plugins/engines/spark/src/main/resources/dependencies.xml
+++ b/plugins/engines/spark/src/main/resources/dependencies.xml
@@ -16,7 +16,8 @@
~
-->
<dependencies>
- <!-- Native handlers need transform metadata classes on the driver -->
+ <!-- Same metadata jar as the one packaged in lib/. When the lakehouse
plugin is installed,
+ the plugin class loader searches lib/ first, so the copy built with
this engine wins. -->
<folder>../../tech/lakehouse</folder>
<folder>../../transforms/memgroupby</folder>
<folder>../../transforms/mergejoin</folder>
diff --git
a/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
b/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
index b6d10c1205..e30bcc9e7d 100644
---
a/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
+++
b/plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml
@@ -251,8 +251,8 @@ plugins:
name: Lakehouse
category: Technology
description: >-
- Lake table input and output for Apache Iceberg. The native Spark engine
loads these
- transform classes from this plugin, so install it together with the
Spark engine.
+ Lake table input and output for Delta Lake and Apache Iceberg. Install
this plugin
+ with the Spark engine to use those transforms.
installPath: plugins/tech/lakehouse
modulePath: plugins/tech/lakehouse
- artifactId: hop-tech-databricks
