This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 59a6541ac6 Issue #8755 : Keep the PostgreSQL driver out of the Beam
plugin lib (#8864)
59a6541ac6 is described below
commit 59a6541ac67415f044dbf619aa1cdfd71c6d509e
Author: Matt Casters <[email protected]>
AuthorDate: Sun Oct 11 15:16:15 2026 +0200
Issue #8755 : Keep the PostgreSQL driver out of the Beam plugin lib (#8864)
---
assemblies/static/src/main/resources/hop | 2 ++
.../pages/pipeline/transforms/beamdebeziuminput.adoc | 7 ++++---
plugins/engines/beam/pom.xml | 17 +++++------------
plugins/engines/beam/src/assembly/assembly.xml | 15 +++++++--------
.../transforms/debezium/BeamDebeziumInputMetaTest.java | 6 ++++--
5 files changed, 22 insertions(+), 25 deletions(-)
diff --git a/assemblies/static/src/main/resources/hop
b/assemblies/static/src/main/resources/hop
index 5dc5f16522..4581032399 100755
--- a/assemblies/static/src/main/resources/hop
+++ b/assemblies/static/src/main/resources/hop
@@ -122,6 +122,8 @@ if [ -d "plugins/engines/beam/lib-beam" ]; then
CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib-beam/*"
fi
# snowflake-jdbc is here, not in lib/core: the jar embeds its own zstd classes.
+# This glob is the application classpath. Do not put a JDBC driver in this
folder:
+# DriverManager would return an app-classloader connection the database plugin
cannot cast.
if [ -d "plugins/engines/beam/lib" ]; then
CLASSPATH="${CLASSPATH}:plugins/engines/beam/lib/*"
fi
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamdebeziuminput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamdebeziuminput.adoc
index 79483aca4e..4ea4f8c55a 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamdebeziuminput.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamdebeziuminput.adoc
@@ -116,10 +116,11 @@ Do not use the limits to assume that a batch-only sink or
runner is supported.
== Dependencies and operational limitations
-The Beam plugin distributes `beam-sdks-java-io-debezium` 2.74.0, Debezium
PostgreSQL connector/core/API 3.1.3.Final, Kafka Connect API/transforms 3.9.0
and the Hop-managed PostgreSQL JDBC driver.
+The Beam plugin distributes `beam-sdks-java-io-debezium` 2.74.0, Debezium
PostgreSQL connector/core/API 3.1.3.Final and Kafka Connect API/transforms
3.9.0.
Beam DebeziumIO itself does *not* include database connectors.
-The bundled dependencies are Apache-2.0, and PostgreSQL JDBC is BSD-2-Clause.
-They are staged by the Beam plugin assembly into `lib/core`, which is included
in Hop's worker fat jar.
+The connector is packaged under `plugins/engines/beam/lib`. The PostgreSQL
JDBC driver is not packaged with the Beam plugin.
+The `hop` launcher puts that folder on the application classpath, and a driver
there breaks the PostgreSQL database plugin.
+Use the driver installed with the PostgreSQL database plugin (`lib/jdbc`). A
Beam worker that runs this connector outside that plugin needs the same driver
on the worker classpath.
For Custom, supply a compatible Debezium 3.1.3.Final connector and its
complete driver/runtime dependency set to the Hop classpath and all Beam
workers; when using Hop's fat-jar builder, additional jars can be placed in
`plugins/engines/beam/lib` before building the fat jar.
MySQL, SQL Server, Oracle, DB2 and their drivers are not bundled or
integration-tested by this transform.
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index f9d660b599..e035d83590 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -306,11 +306,11 @@
</exclusions>
</dependency>
<!-- DebeziumIO provides the CDC source but not a database connector.
- Packaged with its PostgreSQL driver under
plugins/engines/beam/lib, not lib/core.
- A driver on the application classpath breaks PostgreSQL Bulk
Loader.
- The driver is a direct dependency on purpose: the Beam assembly
exclude is not
- applied to transitive jars, so a driver pulled only through
Debezium still lands
- in lib/core. -->
+ The connector is packaged under plugins/engines/beam/lib. The
PostgreSQL driver
+ is not: the hop launcher adds that folder to the application
classpath, and a
+ driver there is the app-classloader PgConnection that PostgreSQL
Bulk Loader
+ cannot cast to the database plugin's PGConnection. The driver
stays in lib/jdbc.
+ This exclusion keeps Debezium from pulling it back into the Beam
runtime tree. -->
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-postgres</artifactId>
@@ -1665,13 +1665,6 @@
</exclusion>
</exclusions>
</dependency>
- <!-- Direct, not a Debezium transitive: the Beam assembly exclude does
not apply to
- transitive jars, and this driver must stay out of lib/core. -->
- <dependency>
- <groupId>org.postgresql</groupId>
- <artifactId>postgresql</artifactId>
- <version>42.7.12</version>
- </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-addsequence</artifactId>
diff --git a/plugins/engines/beam/src/assembly/assembly.xml
b/plugins/engines/beam/src/assembly/assembly.xml
index c088ad5156..df1edc03c7 100644
--- a/plugins/engines/beam/src/assembly/assembly.xml
+++ b/plugins/engines/beam/src/assembly/assembly.xml
@@ -91,13 +91,13 @@
Snowflake IO jar that loads it, off the shared lib/core
classpath. -->
<exclude>org.apache.beam:beam-sdks-java-io-snowflake</exclude>
<exclude>net.snowflake:snowflake-jdbc</exclude>
- <!-- debezium-connector-postgres used to pull
org.postgresql:postgresql
- transitively. Assembly excludes are not applied to
transitive jars, so that
- copy stayed in lib/core and DriverManager returned an
app-classloader
- PgConnection the database plugin cannot cast to its own
PGConnection
- (PostgreSQL Bulk Loader). The driver is now a direct
dependency and this
- exclude keeps it out of lib/core. The connector and the
driver are packaged
- under plugins/engines/beam/lib. -->
+ <!-- The hop launcher adds plugins/engines/beam/lib/* to the
application
+ classpath. A PostgreSQL driver in either lib/core or that
plugin lib is
+ loaded by the app classloader, and PostgreSQL Bulk Loader
cannot cast
+ that PgConnection to the database plugin's PGConnection.
The connector
+ is packaged under plugins/engines/beam/lib without the
driver. The
+ driver stays in lib/jdbc. This exclude is a backstop if a
dependency
+ pulls org.postgresql:postgresql back into the runtime
tree. -->
<exclude>io.debezium:debezium-connector-postgres</exclude>
<exclude>org.postgresql:postgresql</exclude>
</excludes>
@@ -111,7 +111,6 @@
<include>org.apache.beam:beam-sdks-java-io-snowflake:jar</include>
<include>net.snowflake:snowflake-jdbc:jar</include>
<include>io.debezium:debezium-connector-postgres:jar</include>
- <include>org.postgresql:postgresql:jar</include>
</includes>
<outputDirectory>plugins/engines/beam/lib</outputDirectory>
</dependencySet>
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/transforms/debezium/BeamDebeziumInputMetaTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/transforms/debezium/BeamDebeziumInputMetaTest.java
index a8e213449a..36fd869089 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/transforms/debezium/BeamDebeziumInputMetaTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/transforms/debezium/BeamDebeziumInputMetaTest.java
@@ -129,7 +129,7 @@ class BeamDebeziumInputMetaTest {
}
@Test
- void distributedPostgresConnectorAndDriverInstantiateWithoutOptionalJars()
throws Exception {
+ void distributedPostgresConnectorInstantiatesWithoutTheJdbcDriver() throws
Exception {
org.apache.kafka.connect.source.SourceConnector connector =
new io.debezium.connector.postgresql.PostgresConnector();
assertNotNull(connector.config());
@@ -138,7 +138,9 @@ class BeamDebeziumInputMetaTest {
io.debezium.data.SchemaUtil.copySchemaBasics(
org.apache.kafka.connect.data.Schema.STRING_SCHEMA));
assertNotNull(connector.taskClass().getConstructor().newInstance());
-
assertNotNull(Class.forName("org.postgresql.Driver").getConstructor().newInstance());
+ // The hop launcher puts plugins/engines/beam/lib on the application
classpath. The
+ // PostgreSQL driver must stay with the database plugin, not on this
module's classpath.
+ assertThrows(ClassNotFoundException.class, () ->
Class.forName("org.postgresql.Driver"));
}
@Test
