This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new d14bb2efc3 Add databricks driver download option, fixes #8058 (#8093)
d14bb2efc3 is described below
commit d14bb2efc380c38dea25889ec27eccb67fe2b97a
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Aug 25 15:23:46 2026 +0200
Add databricks driver download option, fixes #8058 (#8093)
---
.../ROOT/pages/database/databases/databricks.adoc | 3 +-
.../java/org/apache/hop/driver/DriverCatalog.java | 75 ++++++++-
.../apache/hop/driver/DriverInstallCommand.java | 69 +++++++-
.../hop/driver/DriverInstallCommandTest.java | 187 +++++++++++++++++++++
.../databricks/DatabricksDatabaseMeta.java | 15 ++
.../databricks/DatabricksDatabaseMetaTest.java | 17 ++
6 files changed, 355 insertions(+), 11 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/database/databases/databricks.adoc
b/docs/hop-user-manual/modules/ROOT/pages/database/databases/databricks.adoc
index 922b45c8c9..8687ec2cd0 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/database/databases/databricks.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/database/databases/databricks.adoc
@@ -28,7 +28,8 @@ WARNING: the database name field is not used in a Databricks
connection. Use the
|===
| Option | Info
|Type | Relational
-|Driver | https://docs.databricks.com/aws/en/integrations/jdbc/download[Driver
Link]
+|Driver | Not bundled - install the open source driver on demand, or get it
from https://docs.databricks.com/aws/en/integrations/jdbc/download[Databricks]
+|Install | `hop driver install databricks`
|Hop Dependencies | None
|Documentation |
https://docs.databricks.com/aws/en/integrations/jdbc/[Documentation Link]
|JDBC Url |
jdbc:databricks://<server-hostname>:443;httpPath=<http-path>[;<setting1>=<value1>;<setting2>=<value2>;<settingN>=<valueN>]
diff --git a/engine/src/main/java/org/apache/hop/driver/DriverCatalog.java
b/engine/src/main/java/org/apache/hop/driver/DriverCatalog.java
index 34b5ce8b0a..b4b531bdff 100644
--- a/engine/src/main/java/org/apache/hop/driver/DriverCatalog.java
+++ b/engine/src/main/java/org/apache/hop/driver/DriverCatalog.java
@@ -23,6 +23,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.stream.Stream;
import org.apache.hop.core.database.DatabasePluginType;
import org.apache.hop.core.database.DriverDownload;
import org.apache.hop.core.database.IDatabase;
@@ -38,29 +39,68 @@ import org.apache.hop.core.plugins.PluginRegistry;
public class DriverCatalog {
private final Map<String, DriverDefinition> driversById = new
LinkedHashMap<>();
+ private final Map<String, KnownDatabase> databasesById = new
LinkedHashMap<>();
private DriverCatalog() {
// use load()
}
+ /**
+ * A database plugin as the driver commands see it. Every database type ends
up here; only the
+ * ones declaring a {@link DriverDownload} also become a {@link
DriverDefinition}. It is what lets
+ * "no download for this database" be told apart from "no such database
type".
+ *
+ * @param id the command-line id: the database type lowercased, e.g. {@code
databricks}
+ * @param name the plugin's display name, e.g. {@code Databricks}
+ * @param driverClass the JDBC driver class the plugin expects, null when it
could not be asked
+ * @param classLoader the plugin's classloader, used to see whether that
driver class is there
+ */
+ public record KnownDatabase(String id, String name, String driverClass,
ClassLoader classLoader) {
+
+ /**
+ * @return true when the JDBC driver class is already loadable, i.e. the
driver ships with this
+ * Hop installation (bundled with the plugin, or installed earlier)
and there is nothing
+ * left to download.
+ */
+ public boolean isDriverAvailable() {
+ if (driverClass == null || driverClass.isBlank() || classLoader == null)
{
+ return false;
+ }
+ try {
+ classLoader.loadClass(driverClass);
+ return true;
+ } catch (Exception | LinkageError e) {
+ return false;
+ }
+ }
+ }
+
/** Build the catalog by scanning all registered database plugins for a
driver download. */
public static DriverCatalog load() {
DriverCatalog catalog = new DriverCatalog();
PluginRegistry registry = PluginRegistry.getInstance();
for (IPlugin plugin : registry.getPlugins(DatabasePluginType.class)) {
+ String databaseType = plugin.getIds()[0];
+ String id = databaseType.toLowerCase(Locale.ROOT);
try {
Object loaded = registry.loadClass(plugin);
if (loaded instanceof IDatabase database) {
+ String driverClass = driverClass(database);
+ catalog.databasesById.put(
+ id,
+ new KnownDatabase(
+ id, plugin.getName(), driverClass,
database.getClass().getClassLoader()));
DriverDownload download = database.getDriverDownload();
if (download != null) {
DriverDefinition definition =
- new DriverDefinition(
- plugin.getIds()[0], plugin.getName(),
driverClass(database), download);
+ new DriverDefinition(databaseType, plugin.getName(),
driverClass, download);
catalog.driversById.put(definition.getId(), definition);
}
}
} catch (Exception e) {
- // Skip any plugin that fails to load; it simply has no downloadable
driver here.
+ // The plugin failed to load, so it has no downloadable driver here -
but the id is still a
+ // database type Hop knows, which is worth saying instead of "unknown
driver id".
+ catalog.databasesById.putIfAbsent(id, new KnownDatabase(id,
plugin.getName(), null, null));
}
}
return catalog;
@@ -88,6 +128,35 @@ public class DriverCatalog {
return id == null ? null : driversById.get(id.toLowerCase(Locale.ROOT));
}
+ /**
+ * @return the database plugin known under this id/database type
(case-insensitive), whether or
+ * not it declares a driver download, or null when no such database type
is installed.
+ */
+ public KnownDatabase getDatabaseType(String id) {
+ return id == null ? null : databasesById.get(id.toLowerCase(Locale.ROOT));
+ }
+
+ /**
+ * Ids close enough to an unknown one to be worth offering as a "did you
mean", downloadable ones
+ * first - a typo or an id that only differs in its suffix ({@code postgres}
for {@code
+ * postgresql}) is the common case.
+ *
+ * @param id the id the user typed
+ * @return at most 5 candidate ids, closest first, empty when nothing looks
similar
+ */
+ public List<String> suggestIds(String id) {
+ if (id == null || id.isBlank()) {
+ return List.of();
+ }
+ String needle = id.toLowerCase(Locale.ROOT);
+ return Stream.concat(
+ databasesById.keySet().stream().filter(driversById::containsKey),
+ databasesById.keySet().stream().filter(known ->
!driversById.containsKey(known)))
+ .filter(known -> known.contains(needle) || needle.contains(known))
+ .limit(5)
+ .toList();
+ }
+
/**
* Find a catalog entry by Hop database plugin type, e.g. {@code ORACLE}
(matches {@code
* DatabaseMeta.getPluginId()}). The id and the database type share the same
key space.
diff --git
a/engine/src/main/java/org/apache/hop/driver/DriverInstallCommand.java
b/engine/src/main/java/org/apache/hop/driver/DriverInstallCommand.java
index b8295068ea..fb0128b0ad 100644
--- a/engine/src/main/java/org/apache/hop/driver/DriverInstallCommand.java
+++ b/engine/src/main/java/org/apache/hop/driver/DriverInstallCommand.java
@@ -32,6 +32,10 @@ import picocli.CommandLine.Parameters;
* <p>For restricted (Category X) drivers the install is refused unless {@code
--accept-license} is
* given: Apache Hop neither ships nor hosts these drivers; you obtain them
from the vendor / Maven
* Central under the vendor's license, at your own request.
+ *
+ * <p>Exit codes: 0 the driver is installed (or was already there), 2 the
vendor license still has
+ * to be accepted, 3 no database type goes by that id, 4 the database type
exists but declares no
+ * downloadable driver.
*/
@SuppressWarnings("java:S106")
@Command(
@@ -43,6 +47,7 @@ public class DriverInstallCommand implements
Callable<Integer> {
private static final int EXIT_OK = 0;
private static final int EXIT_LICENSE_REQUIRED = 2;
private static final int EXIT_UNKNOWN_DRIVER = 3;
+ private static final int EXIT_NO_DOWNLOAD = 4;
@Parameters(
index = "0",
@@ -82,9 +87,7 @@ public class DriverInstallCommand implements
Callable<Integer> {
DriverCatalog catalog = DriverCatalog.load();
DriverDefinition driver = catalog.get(driverId);
if (driver == null) {
- System.err.println("Unknown driver id: '" + driverId + "'.");
- System.err.println("Run 'hop driver list' to see the available
drivers.");
- return EXIT_UNKNOWN_DRIVER;
+ return reportNothingToInstall(catalog);
}
if (driver.isRestricted() && !acceptLicense) {
@@ -92,10 +95,7 @@ public class DriverInstallCommand implements
Callable<Integer> {
return EXIT_LICENSE_REQUIRED;
}
- File target =
- (targetFolder != null && !targetFolder.isBlank())
- ? new File(targetFolder).getAbsoluteFile()
- : DriverInstaller.defaultInstallFolder();
+ File target = resolveTarget();
String installVersion =
(version != null && !version.isBlank())
@@ -181,6 +181,61 @@ public class DriverInstallCommand implements
Callable<Integer> {
return EXIT_OK;
}
+ private File resolveTarget() {
+ return (targetFolder != null && !targetFolder.isBlank())
+ ? new File(targetFolder).getAbsoluteFile()
+ : DriverInstaller.defaultInstallFolder();
+ }
+
+ /**
+ * Explain why there is nothing to install. "Unknown driver id" is only true
when no database type
+ * goes by that id at all; when the type exists, what the user needs to hear
is whether its driver
+ * is already there or has to come from the vendor by hand.
+ */
+ private int reportNothingToInstall(DriverCatalog catalog) {
+ DriverCatalog.KnownDatabase database = catalog.getDatabaseType(driverId);
+
+ if (database == null) {
+ System.err.println(
+ "Unknown driver id: '"
+ + driverId
+ + "'. No database type in this Hop installation uses that id.");
+ List<String> suggestions = catalog.suggestIds(driverId);
+ if (!suggestions.isEmpty()) {
+ System.err.println("Did you mean: " + String.join(", ", suggestions) +
"?");
+ }
+ System.err.println("Run 'hop driver list' to see the drivers Hop can
download.");
+ return EXIT_UNKNOWN_DRIVER;
+ }
+
+ if (database.isDriverAvailable()) {
+ System.out.println(
+ "Nothing to install: the "
+ + database.name()
+ + " JDBC driver ("
+ + database.driverClass()
+ + ") is already available in this Hop installation.");
+ return EXIT_OK;
+ }
+
+ System.err.println(
+ "No driver download available for '" + driverId + "' (" +
database.name() + ").");
+ System.err.println(
+ "Hop knows this database type, but it declares no downloadable driver
- typically because");
+ System.err.println("the vendor does not publish it to a public Maven
repository.");
+ System.err.println();
+ System.err.println(
+ "Download the driver from the vendor and copy its jar(s) into "
+ + resolveTarget()
+ + ", then restart Hop.");
+ System.err.println(
+ "The documentation page for "
+ + database.name()
+ + " points at the vendor's download location.");
+ System.err.println("Run 'hop driver list' to see the drivers Hop can
download.");
+ return EXIT_NO_DOWNLOAD;
+ }
+
private void printLicenseNotice(DriverDefinition driver) {
System.out.println();
System.out.println(
diff --git
a/engine/src/test/java/org/apache/hop/driver/DriverInstallCommandTest.java
b/engine/src/test/java/org/apache/hop/driver/DriverInstallCommandTest.java
new file mode 100644
index 0000000000..5216a81484
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/driver/DriverInstallCommandTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.driver;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.database.DatabasePluginType;
+import org.apache.hop.core.database.DriverDownload;
+import org.apache.hop.core.database.IDatabase;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.plugins.IClassLoadingPlugin;
+import org.apache.hop.core.plugins.IPlugin;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import picocli.CommandLine;
+
+/**
+ * What {@code hop driver install <id>} says when it has nothing to install.
"Unknown driver id"
+ * only fits an id no database type goes by; a database type that simply
declares no download needs
+ * to be told apart from one whose driver already ships with Hop.
+ */
+class DriverInstallCommandTest {
+
+ private static final int EXIT_OK = 0;
+ private static final int EXIT_UNKNOWN_DRIVER = 3;
+ private static final int EXIT_NO_DOWNLOAD = 4;
+
+ /** A database plugin the registry accepts and whose class it asks the
plugin itself for. */
+ interface DatabasePluginMock extends IClassLoadingPlugin, IPlugin {}
+
+ private final List<DatabasePluginMock> registeredPlugins = new ArrayList<>();
+ private ByteArrayOutputStream out;
+ private ByteArrayOutputStream err;
+ private PrintStream originalOut;
+ private PrintStream originalErr;
+
+ @BeforeAll
+ static void setUpClass() throws HopException {
+ HopClientEnvironment.init();
+ }
+
+ @BeforeEach
+ void capture() {
+ originalOut = System.out;
+ originalErr = System.err;
+ out = new ByteArrayOutputStream();
+ err = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out, true, StandardCharsets.UTF_8));
+ System.setErr(new PrintStream(err, true, StandardCharsets.UTF_8));
+ }
+
+ @AfterEach
+ void restore() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ // The plugin registry is process-global, so a plugin registered here
would otherwise be
+ // visible to every later test in this JVM.
+ registeredPlugins.forEach(
+ plugin ->
PluginRegistry.getInstance().removePlugin(DatabasePluginType.class, plugin));
+ registeredPlugins.clear();
+ }
+
+ @Test
+ void anIdNoDatabaseTypeGoesByIsUnknown() throws HopException {
+ registerDatabase("TESTMANUAL", "Test Manual",
noDownload("com.example.NoSuchDriver"));
+
+ assertEquals(EXIT_UNKNOWN_DRIVER, install("nosuchdatabase"));
+ assertTrue(stderr().contains("Unknown driver id: 'nosuchdatabase'"),
stderr());
+ assertTrue(stderr().contains("hop driver list"), stderr());
+ }
+
+ @Test
+ void aNearMissGetsTheIdItAlmostTyped() throws HopException {
+ registerDatabase("TESTMANUAL", "Test Manual",
noDownload("com.example.NoSuchDriver"));
+
+ assertEquals(EXIT_UNKNOWN_DRIVER, install("testmanu"));
+ assertTrue(stderr().contains("Did you mean: testmanual?"), stderr());
+ }
+
+ @Test
+ void aDatabaseTypeWithoutADownloadSaysSoAndPointsAtTheJdbcFolder() throws
HopException {
+ registerDatabase("TESTMANUAL", "Test Manual",
noDownload("com.example.NoSuchDriver"));
+
+ assertEquals(EXIT_NO_DOWNLOAD, install("testmanual"));
+ String message = stderr();
+ assertTrue(message.contains("No driver download available for
'testmanual'"), message);
+ assertTrue(message.contains("Test Manual"), message);
+ // The whole point: tell the user where to put the jar they fetch from the
vendor themselves.
+
assertTrue(message.contains(DriverInstaller.defaultInstallFolder().getAbsolutePath()),
message);
+ assertFalse(message.contains("Unknown driver id"), message);
+ }
+
+ @Test
+ void aDriverThatAlreadyShipsWithHopIsNothingToInstall() throws HopException {
+ // Any class on this classpath stands in for a bundled JDBC driver: what
the command checks is
+ // whether the plugin's own classloader can load the driver class it names.
+ registerDatabase("TESTBUNDLED", "Test Bundled",
noDownload(DriverCatalog.class.getName()));
+
+ assertEquals(EXIT_OK, install("testbundled"));
+ assertTrue(stdout().contains("Nothing to install"), stdout());
+ assertTrue(stdout().contains("Test Bundled"), stdout());
+ }
+
+ @Test
+ void aDatabaseTypeWithADownloadStillTakesTheInstallPath() throws
HopException {
+ registerDatabase("TESTDOWNLOAD", "Test Download", withDownload());
+
+ // --target keeps the (restricted) driver from being fetched: the license
notice comes first.
+ assertEquals(2, install("testdownload"));
+ assertTrue(stdout().contains("--accept-license"), stdout());
+ }
+
+ // ------------------------------------------------------------------ helpers
+
+ private int install(String... args) {
+ return new CommandLine(new DriverInstallCommand()).execute(args);
+ }
+
+ private String stdout() {
+ return out.toString(StandardCharsets.UTF_8);
+ }
+
+ private String stderr() {
+ return err.toString(StandardCharsets.UTF_8);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void registerDatabase(String databaseType, String name, IDatabase
database)
+ throws HopException {
+ DatabasePluginMock plugin = mock(DatabasePluginMock.class);
+ when(plugin.getIds()).thenReturn(new String[] {databaseType});
+ when(plugin.getName()).thenReturn(name);
+ when(plugin.getMainType()).thenReturn((Class) IDatabase.class);
+ when(plugin.loadClass(IDatabase.class)).thenReturn(database);
+ when(plugin.matches(databaseType)).thenReturn(true);
+ PluginRegistry.getInstance().registerPlugin(DatabasePluginType.class,
plugin);
+ registeredPlugins.add(plugin);
+ }
+
+ private static IDatabase noDownload(String driverClass) {
+ IDatabase database = mock(IDatabase.class);
+ when(database.getDriverClass()).thenReturn(driverClass);
+ return database;
+ }
+
+ private static IDatabase withDownload() {
+ IDatabase database = mock(IDatabase.class);
+ when(database.getDriverClass()).thenReturn("com.example.NoSuchDriver");
+ when(database.getDriverDownload())
+ .thenReturn(
+ DriverDownload.builder()
+ .mavenCoordinate("com.example:example-jdbc")
+ .defaultVersion("1.0.0")
+ .licenseCategory("X")
+ .licenseName("Example License")
+ .build());
+ return database;
+ }
+}
diff --git
a/plugins/tech/databricks/src/main/java/org/apache/hop/database/databricks/DatabricksDatabaseMeta.java
b/plugins/tech/databricks/src/main/java/org/apache/hop/database/databricks/DatabricksDatabaseMeta.java
index a0e8f31899..b26ede7f69 100644
---
a/plugins/tech/databricks/src/main/java/org/apache/hop/database/databricks/DatabricksDatabaseMeta.java
+++
b/plugins/tech/databricks/src/main/java/org/apache/hop/database/databricks/DatabricksDatabaseMeta.java
@@ -27,6 +27,7 @@ import org.apache.hop.core.Const;
import org.apache.hop.core.database.BaseDatabaseMeta;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.database.DatabaseMetaPlugin;
+import org.apache.hop.core.database.DriverDownload;
import org.apache.hop.core.database.IDatabase;
import org.apache.hop.core.database.types.ColumnContext;
import org.apache.hop.core.database.types.DatabaseTypes;
@@ -213,6 +214,20 @@ public class DatabricksDatabaseMeta extends
BaseDatabaseMeta implements IDatabas
return "com.databricks.client.jdbc.Driver";
}
+ @Override
+ public DriverDownload getDriverDownload() {
+ return DriverDownload.builder()
+ .mavenCoordinate("com.databricks:databricks-jdbc")
+ .defaultVersion("3.4.2")
+ .licenseCategory("A")
+ .licenseName("Apache-2.0")
+
.licenseUrl("https://github.com/databricks/databricks-jdbc/blob/main/LICENSE";)
+ .vendor("Databricks")
+ .vendorUrl("https://github.com/databricks/databricks-jdbc";)
+ .notes("Open source Databricks JDBC driver (uber jar, ~39 MB)")
+ .build();
+ }
+
@Override
public String getURL(String hostname, String port, String databaseName)
throws HopDatabaseException {
diff --git
a/plugins/tech/databricks/src/test/java/org/apache/hop/database/databricks/DatabricksDatabaseMetaTest.java
b/plugins/tech/databricks/src/test/java/org/apache/hop/database/databricks/DatabricksDatabaseMetaTest.java
index 02b8e3b3b8..bbfa7aac97 100644
---
a/plugins/tech/databricks/src/test/java/org/apache/hop/database/databricks/DatabricksDatabaseMetaTest.java
+++
b/plugins/tech/databricks/src/test/java/org/apache/hop/database/databricks/DatabricksDatabaseMetaTest.java
@@ -17,9 +17,12 @@
package org.apache.hop.database.databricks;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.core.database.DriverDownload;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaBigNumber;
import org.apache.hop.core.row.value.ValueMetaBinary;
@@ -140,6 +143,20 @@ class DatabricksDatabaseMetaTest {
databaseMeta.getFieldDefinition(key, "ID", null, true, false, false));
}
+ /**
+ * Databricks bundles no driver, so the connection dialog's "Download
driver" button is the only
+ * way to get one - and that button only appears when the dialect declares a
download.
+ */
+ @Test
+ void aDownloadableDriverIsDeclared() {
+ DriverDownload download = nativeMeta.getDriverDownload();
+ assertNotNull(download);
+ assertEquals("com.databricks:databricks-jdbc",
download.getMavenCoordinate());
+ // The 2.x line is the proprietary Simba driver; only 3.x is Apache-2.0
and freely downloadable.
+ assertEquals("3", download.getDefaultVersion().split("\\.")[0]);
+ assertFalse(download.isRestricted());
+ }
+
@Test
void alterStatementsCarryTheColumnAndItsType() {
assertEquals(
