This is an automated email from the ASF dual-hosted git repository.
aminghadersohi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 5668faf18e6 fix(doris): quarter grain, SSL toggle, parameters URI,
error mapping and column types (#44718)
5668faf18e6 is described below
commit 5668faf18e6c89ae7408262c4061c08ce05e8f8f
Author: Amin Ghadersohi <[email protected]>
AuthorDate: Thu Oct 1 03:13:32 2026 +1000
fix(doris): quarter grain, SSL toggle, parameters URI, error mapping and
column types (#44718)
---
UPDATING.md | 29 ++-
.../databases/supported/apache-doris.mdx | 8 +
.../version-6.1.0/databases/supported/mysql.mdx | 18 ++
superset/db_engine_specs/doris.py | 111 +++++++++-
superset/db_engine_specs/mysql.py | 109 +++++++---
superset/translations/messages.pot | 12 ++
tests/unit_tests/db_engine_specs/test_doris.py | 227 ++++++++++++++++++++-
tests/unit_tests/db_engine_specs/test_mariadb.py | 83 ++++++++
tests/unit_tests/db_engine_specs/test_mysql.py | 85 +++++++-
9 files changed, 636 insertions(+), 46 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index 6197719c0ee..1b60597110f 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -67,12 +67,26 @@ format. Use a consistent timezone and precision for stored
strings and bounds.
able to provision a user regardless of `AUTH_USER_REGISTRATION`; it now
requires the same gate as `/register/`.
+### Doris SSL requests require TLS
+
+The Doris SSL toggle uses `ssl_mode=VERIFY_CA`. Saved `ssl_mode=REQUIRED` and
+`ssl=1` URLs also require TLS and follow the mysqlclient rules below:
`REQUIRED`
+is kept with Oracle libmysqlclient 5.7/8.x/9.x and upgraded to `VERIFY_CA` with
+MariaDB Connector/C or an unrecognized client, which can otherwise fall back to
+cleartext. Supply the trusted `ssl_ca` (or `connect_args.ssl.ca`) and a
+certificate valid for the connection hostname. Conflicting advanced settings
+fail closed. This shares MySQL's TLS normalization without changing Doris
+catalog/schema handling.
+
### MySQL SSL requests require TLS
-The SSL toggle (or ssl=1 in the URI) requires TLS. With mysqlclient, Oracle
-libmysqlclient 5.7/8.x/9.x uses ssl_mode=REQUIRED; MariaDB Connector/C and
-unrecognized client versions use VERIFY_CA to prevent cleartext fallback.
-Explicit VERIFY_CA and VERIFY_IDENTITY are retained.
+The SSL toggle (or ssl=1 in the URI) requires TLS, and so does a saved
+ssl_mode of REQUIRED, VERIFY_CA or VERIFY_IDENTITY without ssl=1. With
+mysqlclient, Oracle libmysqlclient 5.7/8.x/9.x uses ssl_mode=REQUIRED; MariaDB
+Connector/C and unrecognized client versions use VERIFY_CA to prevent cleartext
+fallback. Explicit VERIFY_CA and VERIFY_IDENTITY are retained. Contradictory
+options such as ssl_mode=DISABLED or ssl_disabled=True fail rather than
+cancelling the SSL request.
Existing saved connections with the toggle on are affected at upgrade, without
a
feature flag. Verification can fail for self-signed/default server certificates
@@ -106,8 +120,11 @@ native ssl dictionary). Superset passes those settings
through without enforcing
TLS; ensure the chosen driver configuration does not silently fall back to
cleartext.
-Connections using the separate MariaDB engine (mariadb:// URIs) and other
-MySQL-compatible engines keep their existing SSL handling.
+Connections using the separate MariaDB engine (mariadb:// URIs and its drivers)
+get the same handling. With MariaDB Connector/Python
+(mariadb+mariadbconnector://), the toggle keeps ssl=True and enables
+ssl_verify_cert=True. Other MySQL-compatible engines such as OceanBase and
+StarRocks keep their existing SSL handling.
### Version history retention setting
diff --git
a/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/apache-doris.mdx
b/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/apache-doris.mdx
index a023eb06430..50dbf483892 100644
---
a/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/apache-doris.mdx
+++
b/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/apache-doris.mdx
@@ -161,3 +161,11 @@ export const databaseInfo = {
};
<DatabasePage name="Apache Doris" database={databaseInfo} />
+
+## TLS connections
+
+The SSL toggle uses certificate verification (`ssl_mode=VERIFY_CA`). Configure
a
+trusted CA with the driver's `ssl_ca` URI option or a native `ssl` dictionary
in
+`connect_args`. Editing a URI with `ssl_mode=REQUIRED` keeps the SSL toggle
enabled
+and upgrades it to `VERIFY_CA` when saved. An explicit `VERIFY_IDENTITY` mode
is
+preserved, including hostname verification, along with other URI query options.
diff --git
a/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/mysql.mdx
b/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/mysql.mdx
index ab2f34e3d6a..df15c9af7c4 100644
--- a/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/mysql.mdx
+++ b/docs/user_docs_versioned_docs/version-6.1.0/databases/supported/mysql.mdx
@@ -191,3 +191,21 @@ export const databaseInfo = {
};
<DatabasePage name="MySQL" database={databaseInfo} />
+
+## TLS connections
+
+The **SSL** toggle (or `ssl=1` in the connection URI) requires TLS rather than
+allowing an unencrypted fallback, and so does a URI or `connect_args` value of
+`ssl_mode=REQUIRED` (or stronger) without the toggle. With mysqlclient linked
+against Oracle libmysqlclient 5.7/8.x/9.x, this uses `ssl_mode=REQUIRED`.
+`REQUIRED` alone is not sufficient with MariaDB Connector/C, so with that or an
+unrecognized client library it becomes `ssl_mode=VERIFY_CA`: configure the
+appropriate `ssl_ca` for a private CA and use a certificate valid for the
+connection hostname. Explicit `VERIFY_CA` and `VERIFY_IDENTITY` are retained.
+
+With `mysql+mysqlconnector://` or `mysql+pymysql://`, the toggle enables
+certificate verification. A private CA must be supplied as `ssl_ca`; do not
+turn off verification to work around an untrusted certificate. PyMySQL 1.2 or
+newer is required for the toggle. Use individual SSL options rather than a
+nested `ssl` dictionary with PyMySQL. Conflicting options that disable TLS or
+verification are rejected.
diff --git a/superset/db_engine_specs/doris.py
b/superset/db_engine_specs/doris.py
index 9d256478326..e84471b3694 100644
--- a/superset/db_engine_specs/doris.py
+++ b/superset/db_engine_specs/doris.py
@@ -20,15 +20,23 @@ from re import Pattern
from typing import Any, Callable, Optional
from urllib import parse
-from flask_babel import gettext as __
+from flask_babel import gettext as __, lazy_gettext as _
from sqlalchemy import Float, Integer, Numeric, String, TEXT, text, types
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.type_api import TypeEngine
-from superset.db_engine_specs.base import DatabaseCategory
-from superset.db_engine_specs.mysql import MySQLEngineSpec
+from superset.constants import TimeGrain
+from superset.databases.utils import make_url_safe
+from superset.db_engine_specs.base import BasicParametersType, DatabaseCategory
+from superset.db_engine_specs.mysql import (
+ MYSQL_SSL_MODE_REQUIRED,
+ MYSQL_SSL_MODE_VERIFY_CA,
+ MYSQL_SSL_MODE_VERIFY_IDENTITY,
+ MySQLEngineSpec,
+ require_mysql_tls,
+)
from superset.errors import SupersetErrorType
from superset.models.core import Database
from superset.utils.core import GenericDataType
@@ -40,17 +48,29 @@ DEFAULT_SCHEMA = "information_schema"
CONNECTION_ACCESS_DENIED_REGEX = re.compile(
"Access denied for user '(?P<username>.*?)'"
)
+# The client library (libmysqlclient or MariaDB Connector/C) words these; the
+# MariaDB one omits the server name ("Can't connect to server on ...").
CONNECTION_INVALID_HOSTNAME_REGEX = re.compile(
- "Unknown Doris server host '(?P<hostname>.*?)'"
+ "Unknown (?:MySQL |Doris )?server host '(?P<hostname>.*?)'"
)
CONNECTION_UNKNOWN_DATABASE_REGEX = re.compile("Unknown database
'(?P<database>.*?)'")
CONNECTION_HOST_DOWN_REGEX = re.compile(
- "Can't connect to Doris server on '(?P<hostname>.*?)'"
+ "Can't connect to (?:MySQL |Doris )?server on '(?P<hostname>.*?)'"
)
SYNTAX_ERROR_REGEX = re.compile(
"check the manual that corresponds to your MySQL server "
"version for the right syntax to use near '(?P<server_error>.*)"
)
+# Doris' own parser: "mismatched input 'SELEC' expecting ..."
+DORIS_SYNTAX_ERROR_REGEX = re.compile(
+ r"(?:mismatched input|extraneous input|no viable alternative at input)"
+ r" '(?P<server_error>.*?)'"
+)
+TABLE_DOES_NOT_EXIST_REGEX = re.compile(r"Table \[(?P<table_name>.*?)\] does
not exist")
+SCHEMA_DOES_NOT_EXIST_REGEX = re.compile(
+ r"Database \[(?P<schema_name>.*?)\] does not exist"
+)
+COLUMN_DOES_NOT_EXIST_REGEX = re.compile("Unknown column
'(?P<column_name>.*?)'")
logger = logging.getLogger(__name__)
@@ -116,7 +136,8 @@ class DorisEngineSpec(MySQLEngineSpec):
sqlalchemy_uri_placeholder = (
"doris://user:password@host:port/catalog.db[?key=value&key=value...]"
)
- encryption_parameters = {"ssl": "0"}
+ # REQUIRED can fall back with MariaDB Connector/C; verification fails
closed.
+ encryption_parameters = {"ssl_mode": MYSQL_SSL_MODE_VERIFY_CA}
supports_dynamic_schema = True
supports_catalog = supports_dynamic_catalog = True
# while technically supported by Doris, this generates invalid table
identifiers
@@ -127,6 +148,13 @@ class DorisEngineSpec(MySQLEngineSpec):
# until someone confirms the same expressions against a live Doris
instance.
_extended_aggregations: dict[str, Callable[[ColumnElement],
ColumnElement]] = {}
+ _time_grain_expressions = {
+ **MySQLEngineSpec._time_grain_expressions,
+ # Doris rejects MySQL's ``+ INTERVAL n QUARTER``
+ TimeGrain.QUARTER: "MAKEDATE(YEAR({col}), 1) "
+ "+ INTERVAL (QUARTER({col}) - 1) * 3 MONTH",
+ }
+
metadata = {
"description": (
"Apache Doris is a high-performance real-time analytical database."
@@ -245,6 +273,19 @@ class DorisEngineSpec(MySQLEngineSpec):
String(),
GenericDataType.STRING,
),
+ (
+ re.compile(r"^variant", re.IGNORECASE),
+ types.JSON(),
+ GenericDataType.STRING,
+ ),
+ (
+ re.compile(r"^ipv[46]$", re.IGNORECASE),
+ String(),
+ GenericDataType.STRING,
+ ),
+ # MySQL protocol type names, as reported for SQL Lab result columns
+ # (NEWDECIMAL, TINY, SHORT, BLOB, ...)
+ *MySQLEngineSpec.column_type_mappings,
)
custom_errors: dict[Pattern[str], tuple[str, SupersetErrorType, dict[str,
Any]]] = {
@@ -276,8 +317,63 @@ class DorisEngineSpec(MySQLEngineSpec):
SupersetErrorType.SYNTAX_ERROR,
{},
),
+ DORIS_SYNTAX_ERROR_REGEX: (
+ _(
+ 'Please check your query for syntax errors near
"%(server_error)s". '
+ "Then, try running your query again."
+ ),
+ SupersetErrorType.SYNTAX_ERROR,
+ {},
+ ),
+ TABLE_DOES_NOT_EXIST_REGEX: (
+ _('The table "%(table_name)s" does not exist.'),
+ SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR,
+ {},
+ ),
+ SCHEMA_DOES_NOT_EXIST_REGEX: (
+ _('The schema "%(schema_name)s" does not exist.'),
+ SupersetErrorType.SCHEMA_DOES_NOT_EXIST_ERROR,
+ {},
+ ),
+ COLUMN_DOES_NOT_EXIST_REGEX: (
+ _('We can\'t seem to resolve the column "%(column_name)s".'),
+ SupersetErrorType.COLUMN_DOES_NOT_EXIST_ERROR,
+ {},
+ ),
}
+ @classmethod
+ def build_sqlalchemy_uri(
+ cls,
+ parameters: BasicParametersType,
+ encrypted_extra: Optional[dict[str, str]] = None,
+ ) -> str:
+ """Build a Doris URI while preserving explicit hostname
verification."""
+ uri = make_url_safe(super().build_sqlalchemy_uri(parameters,
encrypted_extra))
+ if (
+ parameters.get("query", {}).get("ssl_mode")
+ == MYSQL_SSL_MODE_VERIFY_IDENTITY
+ ):
+ uri = uri.update_query_dict({"ssl_mode":
MYSQL_SSL_MODE_VERIFY_IDENTITY})
+ # ``engine+default_driver`` would be ``pydoris+pydoris``, which no
+ # SQLAlchemy entry point provides; ``doris`` is the dialect's scheme.
+ return
uri.set(drivername="doris").render_as_string(hide_password=False)
+
+ @classmethod
+ def get_parameters_from_uri(
+ cls, uri: str, encrypted_extra: Optional[dict[str, Any]] = None
+ ) -> BasicParametersType:
+ """Recognize legacy TLS requests without losing hostname
verification."""
+ url = make_url_safe(uri)
+ if url.query.get("ssl_mode") == MYSQL_SSL_MODE_REQUIRED:
+ url = url.update_query_dict(cls.encryption_parameters)
+ parameters = super().get_parameters_from_uri(
+ url.render_as_string(hide_password=False), encrypted_extra
+ )
+ if url.query.get("ssl_mode") == MYSQL_SSL_MODE_VERIFY_IDENTITY:
+ parameters["encryption"] = True
+ return parameters
+
@classmethod
def adjust_engine_params(
cls,
@@ -300,7 +396,8 @@ class DorisEngineSpec(MySQLEngineSpec):
database = ".".join(part for part in (catalog, schema) if part)
uri = uri.set(database=database)
- return uri, connect_args
+ # pydoris is a mysqlclient dialect, whatever scheme the URI uses.
+ return require_mysql_tls(uri, connect_args, driver="mysqldb")
@classmethod
def get_default_catalog(cls, database: Database) -> str:
diff --git a/superset/db_engine_specs/mysql.py
b/superset/db_engine_specs/mysql.py
index 0b1eb32c329..fdd055d84e3 100644
--- a/superset/db_engine_specs/mysql.py
+++ b/superset/db_engine_specs/mysql.py
@@ -79,6 +79,15 @@ SYNTAX_ERROR_REGEX = re.compile(
"version for the right syntax to use near '(?P<server_error>.*)"
)
+MYSQL_SSL_MODE_REQUIRED = "REQUIRED"
+MYSQL_SSL_MODE_VERIFY_CA = "VERIFY_CA"
+MYSQL_SSL_MODE_VERIFY_IDENTITY = "VERIFY_IDENTITY"
+MYSQL_SSL_REQUIRED_MODES = (
+ MYSQL_SSL_MODE_REQUIRED,
+ MYSQL_SSL_MODE_VERIFY_CA,
+ MYSQL_SSL_MODE_VERIFY_IDENTITY,
+)
+
def _mysql_bool_option(options: dict[str, Any], key: str) -> Optional[bool]:
"""Parse a boolean driver option, treating a blank value as unset."""
@@ -116,16 +125,21 @@ def _require_pymysql_tls(query: dict[str, Any], args:
dict[str, Any]) -> None:
)
-def _require_mysql_verified_tls(
- driver: str, query: dict[str, Any], args: dict[str, Any]
-) -> None:
- """Use required verification on drivers without an encryption-only mode."""
- # The drivers test ssl_disabled for truthiness, so a URL string such as
+def _reject_mysql_ssl_disabled(query: dict[str, Any], args: dict[str, Any]) ->
None:
+ """Fail on a true ssl_disabled and drop values that do not disable TLS."""
+ # Drivers test ssl_disabled for truthiness, so a URL string such as
# "false" would disable TLS. Parse it here and drop non-disabling values.
for source in (query, args):
if _mysql_bool_option(source, "ssl_disabled"):
raise ValueError("MySQL SSL request conflicts with ssl_disabled")
source.pop("ssl_disabled", None)
+
+
+def _require_mysql_verified_tls(
+ driver: str, query: dict[str, Any], args: dict[str, Any]
+) -> None:
+ """Use required verification on drivers without an encryption-only mode."""
+ _reject_mysql_ssl_disabled(query, args)
# Connector/Python has no REQUIRED mode: certificate verification is
# necessary to prevent its opportunistic fallback to cleartext.
if _mysql_bool_option({**query, **args}, "ssl_verify_cert") is False:
@@ -135,54 +149,80 @@ def _require_mysql_verified_tls(
args["ssl_verify_cert"] = True
-def _mysql_ssl_requested(value: Any) -> bool:
- """Parse scalar requests; leave native SSL dictionaries to the driver."""
- if isinstance(value, (str, bool, int)):
- return asbool(value)
- if value is not None and not isinstance(value, dict):
- raise ValueError("Invalid MySQL ssl option")
- return False
-
-
def _mysqlclient_ssl_mode(options: dict[str, Any]) -> str:
"""Select fail-closed TLS semantics for the linked client library."""
# mysqlclient maps REQUIRED to opportunistic TLS with MariaDB
# Connector/C. Verification modes fail closed on both client libraries.
- mode = options.get("ssl_mode", "REQUIRED")
- if mode not in ("REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"):
+ mode = options.get("ssl_mode", MYSQL_SSL_MODE_REQUIRED)
+ if mode not in MYSQL_SSL_REQUIRED_MODES:
raise ValueError("MySQL SSL request conflicts with ssl_mode")
- if mode == "REQUIRED":
+ if mode == MYSQL_SSL_MODE_REQUIRED:
client_info = import_module("MySQLdb").get_client_info()
# Only recognized Oracle clients have fail-closed REQUIRED semantics.
# MariaDB Connector/C (3.x) and unknown clients require verification.
if not re.match(r"^(?:5\.7|8\.\d+|9\.\d+)\.", client_info):
- mode = "VERIFY_CA"
+ mode = MYSQL_SSL_MODE_VERIFY_CA
return mode
+def _require_mysqlclient_tls(query: dict[str, Any], args: dict[str, Any]) ->
None:
+ """Select an ssl_mode that fails closed with the linked client library."""
+ _reject_mysql_ssl_disabled(query, args)
+ args["ssl_mode"] = _mysqlclient_ssl_mode({**query, **args})
+
+
+def _require_mariadb_connector_tls(
+ options: dict[str, Any], args: dict[str, Any]
+) -> None:
+ """Keep MariaDB Connector/Python's fail-closed ssl flag and verify."""
+ if _mysql_bool_option(options, "ssl_verify_cert") is False:
+ raise ValueError("MySQL SSL request requires ssl_verify_cert")
+ args["ssl"] = True
+ args["ssl_verify_cert"] = True
+
+
+def _mysql_ssl_requested(value: Any) -> bool:
+ """Parse scalar requests; leave native SSL dictionaries to the driver."""
+ if isinstance(value, (str, bool, int)):
+ return asbool(value)
+ if value is not None and not isinstance(value, dict):
+ raise ValueError("Invalid MySQL ssl option")
+ return False
+
+
def require_mysql_tls(
- uri: URL, connect_args: dict[str, Any]
+ uri: URL, connect_args: dict[str, Any], driver: Optional[str] = None
) -> tuple[URL, dict[str, Any]]:
- """Consume scalar ``ssl`` requests without weakening native TLS
settings."""
- if uri.get_backend_name() != "mysql":
- return uri, connect_args
+ """Require TLS for ``ssl`` or ``ssl_mode`` requests without weakening them.
+
+ Options are selected by DBAPI driver, so any MySQL-protocol dialect can use
+ this. Pass ``driver`` when the URL's dialect is not importable or does not
+ report the underlying MySQL driver.
+ """
query = dict(uri.query)
args = dict(connect_args)
# A true URL request cannot be cancelled by an advanced connect argument.
requested = _mysql_ssl_requested(query.get("ssl"))
requested = _mysql_ssl_requested(args.get("ssl")) or requested
+ # A saved ssl_mode that already requires TLS is a request in its own right,
+ # so REQUIRED goes through the same fail-closed normalization.
+ requested = requested or any(
+ options.get("ssl_mode") in MYSQL_SSL_REQUIRED_MODES for options in
(query, args)
+ )
if not requested:
return uri, connect_args
for options in (query, args):
if isinstance(options.get("ssl"), (str, bool, int)):
options.pop("ssl")
- driver = uri.get_driver_name()
+ driver = driver or uri.get_driver_name()
options = {**query, **args}
if driver == "mysqldb":
- args["ssl_mode"] = _mysqlclient_ssl_mode(options)
+ _require_mysqlclient_tls(query, args)
elif driver in ("mysqlconnector", "pymysql"):
_require_mysql_verified_tls(driver, query, args)
+ elif driver == "mariadbconnector":
+ _require_mariadb_connector_tls(options, args)
elif driver == "auroradataapi":
# The Data API is only reachable over HTTPS and takes no ssl argument.
pass
@@ -191,6 +231,12 @@ def require_mysql_tls(
return uri.set(query=query), args
+# Engines whose connections are passed through ``require_mysql_tls``. Other
+# MySQL-protocol specs opt in by listing their engine here, or by calling the
+# function from their own ``adjust_engine_params``.
+MYSQL_TLS_ENGINES = frozenset({"mysql", "mariadb"})
+
+
class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
engine = "mysql"
engine_name = "MySQL"
@@ -237,7 +283,8 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
"connection_string": "mysql://{username}:{password}@{host}/{database}",
"default_port": 3306,
"notes": (
- "The SSL toggle (or ssl=1 in the URI) requires TLS. With
mysqlclient, "
+ "The SSL toggle (or ssl=1 in the URI) requires TLS, as does a
saved "
+ "ssl_mode of REQUIRED or stronger. With mysqlclient, "
"Oracle libmysqlclient 5.7/8.x/9.x uses ssl_mode=REQUIRED; MariaDB
"
"Connector/C and unrecognized client versions use VERIFY_CA to
prevent "
"cleartext fallback. Explicit VERIFY_CA and VERIFY_IDENTITY are
retained. "
@@ -250,7 +297,8 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
"Connector/Python and PyMySQL enable ssl_verify_cert=True. PyMySQL
"
"requires version 1.2 or newer; use individual ssl_ca, ssl_cert
and "
"ssl_key options instead of a nested ssl dictionary with the
toggle. "
- "Options that disable TLS or required verification are rejected. "
+ "Options that disable TLS (ssl_mode=DISABLED, ssl_disabled=True)
or "
+ "required verification are rejected. "
"Standard Aurora MySQL connections intentionally follow the same
rules, "
"including IAM connections. For certificate verification, install
the "
"Amazon RDS CA bundle on every web and worker node and set ssl_ca
to that "
@@ -268,8 +316,11 @@ class MySQLEngineSpec(BasicParametersMixin,
BaseEngineSpec):
"driver-supported native ssl dictionary). Superset passes those
settings "
"through without enforcing TLS; ensure the chosen driver
configuration "
"does not silently fall back to cleartext. "
- "Connections using the separate MariaDB engine (mariadb:// URIs)
and "
- "other MySQL-compatible engines keep their existing SSL handling."
+ "Connections using the separate MariaDB engine (mariadb:// URIs)
get "
+ "the same handling; with MariaDB Connector/Python the toggle keeps
"
+ "ssl=True and enables ssl_verify_cert=True. Other MySQL-compatible
"
+ "engines such as OceanBase and StarRocks keep their existing SSL "
+ "handling."
),
"parameters": {
"username": "Database username",
@@ -593,7 +644,7 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec):
if schema:
uri = uri.set(database=parse.quote(schema, safe=""))
- if cls.engine == "mysql":
+ if cls.engine in MYSQL_TLS_ENGINES:
return require_mysql_tls(uri, new_connect_args)
return uri, new_connect_args
diff --git a/superset/translations/messages.pot
b/superset/translations/messages.pot
index 151695e6a36..77ba4898875 100644
--- a/superset/translations/messages.pot
+++ b/superset/translations/messages.pot
@@ -14673,6 +14673,10 @@ msgid ""
"run this query."
msgstr ""
+#, python-format
+msgid "The schema \"%(schema_name)s\" does not exist."
+msgstr ""
+
#, python-format
msgid ""
"The schema \"%(schema_name)s\" does not exist. A valid schema must be "
@@ -14730,6 +14734,10 @@ msgid ""
" this query."
msgstr ""
+#, python-format
+msgid "The table \"%(table_name)s\" does not exist."
+msgstr ""
+
#, python-format
msgid ""
"The table \"%(table_name)s\" does not exist. A valid table must be used "
@@ -16924,6 +16932,10 @@ msgid ""
"%(location)s."
msgstr ""
+#, python-format
+msgid "We can't seem to resolve the column \"%(column_name)s\"."
+msgstr ""
+
#, python-format
msgid "We have the following keys: %s"
msgstr ""
diff --git a/tests/unit_tests/db_engine_specs/test_doris.py
b/tests/unit_tests/db_engine_specs/test_doris.py
index 1e0a2b083cc..25e63ed1254 100644
--- a/tests/unit_tests/db_engine_specs/test_doris.py
+++ b/tests/unit_tests/db_engine_specs/test_doris.py
@@ -16,7 +16,7 @@
# under the License.
from typing import Any, Optional
-from unittest.mock import MagicMock, Mock
+from unittest.mock import MagicMock, Mock, patch
import pytest
from pytest_mock import MockerFixture
@@ -280,3 +280,228 @@ def test_get_catalog_names(
# Verify the returned catalog names
assert catalogs == expected_result
+
+
[email protected](
+ "native_type,generic_type",
+ [
+ ("variant", GenericDataType.STRING),
+ ("ipv4", GenericDataType.STRING),
+ ("ipv6", GenericDataType.STRING),
+ # MySQL protocol type names reported for SQL Lab result columns
+ ("NEWDECIMAL", GenericDataType.NUMERIC),
+ ("TINY", GenericDataType.NUMERIC),
+ ("SHORT", GenericDataType.NUMERIC),
+ ("BLOB", GenericDataType.STRING),
+ ],
+)
+def test_get_column_spec_extra_types(
+ native_type: str, generic_type: GenericDataType
+) -> None:
+ """Doris-only and MySQL-protocol result types map to a generic type."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ spec = DorisEngineSpec.get_column_spec(native_type)
+ assert spec is not None
+ assert spec.generic_type == generic_type
+
+
+def test_quarter_time_grain_avoids_interval_quarter() -> None:
+ """The quarter grain avoids ``INTERVAL n QUARTER``, which Doris rejects."""
+ from superset.constants import TimeGrain
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ expression = DorisEngineSpec._time_grain_expressions[TimeGrain.QUARTER]
+ assert expression == (
+ "MAKEDATE(YEAR({col}), 1) + INTERVAL (QUARTER({col}) - 1) * 3 MONTH"
+ )
+ assert "INTERVAL 1 QUARTER" not in expression
+
+
[email protected](
+ "message,error_type",
+ [
+ (
+ "(2002, \"Can't connect to server on '127.0.0.1' (115)\")",
+ "CONNECTION_HOST_DOWN_ERROR",
+ ),
+ (
+ "(2003, \"Can't connect to MySQL server on 'db' (111)\")",
+ "CONNECTION_HOST_DOWN_ERROR",
+ ),
+ (
+ "(2005, \"Unknown server host 'no-such-host.invalid' (-2)\")",
+ "CONNECTION_INVALID_HOSTNAME_ERROR",
+ ),
+ (
+ "(1105, \"errCode = 2, detailMessage = \\nmismatched input 'SELEC'
"
+ "expecting {<EOF>, ';'}\")",
+ "SYNTAX_ERROR",
+ ),
+ (
+ "(1105, 'errCode = 2, detailMessage = Table [missing] does not
exist "
+ "in database [db].(line 1, pos 14)')",
+ "TABLE_DOES_NOT_EXIST_ERROR",
+ ),
+ (
+ "(1105, 'errCode = 2, detailMessage = Database [nodb] does not
exist."
+ "(line 1, pos 14)')",
+ "SCHEMA_DOES_NOT_EXIST_ERROR",
+ ),
+ (
+ "(1105, \"errCode = 2, detailMessage = Unknown column 'nope' in "
+ "'table list' in PROJECT clause(line 1, pos 7)\")",
+ "COLUMN_DOES_NOT_EXIST_ERROR",
+ ),
+ (
+ "(1045, \"Access denied for user '[email protected]' (using password:
YES)\")",
+ "CONNECTION_ACCESS_DENIED_ERROR",
+ ),
+ (
+ "(1049, \"errCode = 2, detailMessage = Unknown database 'nodb'\")",
+ "CONNECTION_UNKNOWN_DATABASE_ERROR",
+ ),
+ ],
+)
+def test_extract_errors(message: str, error_type: str) -> None:
+ """Driver errors map to the corresponding Superset error types."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ errors = DorisEngineSpec.extract_errors(Exception(message))
+ assert errors[0].error_type.name == error_type
+
+
+def test_build_sqlalchemy_uri() -> None:
+ """The parameters form builds a valid Doris URI with optional verified
TLS."""
+ from superset.db_engine_specs.base import BasicParametersType
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ parameters: BasicParametersType = {
+ "username": "user",
+ "password": "p@ss",
+ "host": "doris.example.com",
+ "port": 9030,
+ "database": "internal.db",
+ "query": {},
+ }
+ encrypted = make_url(
+ DorisEngineSpec.build_sqlalchemy_uri({**parameters, "encryption":
True})
+ )
+ assert encrypted.drivername == "doris"
+ assert dict(encrypted.query) == {"ssl_mode": "VERIFY_CA"}
+ assert encrypted.password == "p@ss" # noqa: S105
+
+ plain = make_url(
+ DorisEngineSpec.build_sqlalchemy_uri({**parameters, "encryption":
False})
+ )
+ assert plain.drivername == "doris"
+ assert dict(plain.query) == {}
+
+ round_trip = DorisEngineSpec.get_parameters_from_uri(
+ encrypted.render_as_string(hide_password=False)
+ )
+ assert round_trip["encryption"] is True
+ assert round_trip["query"] == {}
+
+
[email protected]("client_info", ["3.3.17", "8.4.6"])
[email protected]("source", ["toggle", "ssl=1", "ssl_mode=REQUIRED"])
+def test_doris_tls_request_uses_verification(source: str, client_info: str) ->
None:
+ """
+ The toggle always verifies. Scalar SSL and legacy REQUIRED requests keep
+ REQUIRED only with an Oracle client, which does not fall back to cleartext.
+ """
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ if source == "toggle":
+ uri = make_url(
+ DorisEngineSpec.build_sqlalchemy_uri(
+ {
+ "username": "root",
+ "host": "localhost",
+ "port": 9030,
+ "database": "internal.db",
+ "encryption": True,
+ }
+ )
+ )
+ else:
+ uri = make_url(f"doris://root@localhost/internal.db?{source}")
+ with patch("superset.db_engine_specs.mysql.import_module") as module:
+ module.return_value.get_client_info.return_value = client_info
+ url, args = DorisEngineSpec.adjust_engine_params(
+ uri, {}, catalog="external", schema="other"
+ )
+ oracle_required = source != "toggle" and client_info == "8.4.6"
+ expected_mode = "REQUIRED" if oracle_required else "VERIFY_CA"
+ assert args.get("ssl_mode", url.query.get("ssl_mode")) == expected_mode
+ assert url.drivername == "doris"
+ assert url.database == "external.other"
+
+
[email protected]("mode", ["DISABLED", "PREFERRED", None])
+def test_doris_toggle_cannot_be_cancelled(mode: Optional[str]) -> None:
+ """Connect arguments cannot cancel a REQUIRED request in the URI."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ with pytest.raises(ValueError, match="conflicts"):
+ DorisEngineSpec.adjust_engine_params(
+ make_url("doris://root@localhost/internal.db?ssl_mode=REQUIRED"),
+ {"ssl_mode": mode},
+ )
+
+
+def test_doris_tls_preserves_verified_mode_and_ca() -> None:
+ """Explicit hostname verification and the native CA dictionary survive."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ uri =
make_url("doris://root@localhost/internal.db?ssl_mode=VERIFY_IDENTITY")
+ _, args = DorisEngineSpec.adjust_engine_params(uri, {"ssl": {"ca":
"/ca.pem"}})
+ assert args["ssl_mode"] == "VERIFY_IDENTITY"
+ assert args["ssl"] == {"ca": "/ca.pem"}
+
+
+def test_unrequested_doris_connection_unchanged() -> None:
+ """Connections without a TLS request retain their URI and arguments."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ uri = make_url("doris://root@localhost/internal.db")
+ assert DorisEngineSpec.adjust_engine_params(uri, {}) == (uri, {})
+
+
[email protected]("mode", ["REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"])
+def test_doris_tls_parameters_round_trip(mode: str) -> None:
+ """Editing TLS URIs keeps encryption enabled and preserves stronger
modes."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ parameters = DorisEngineSpec.get_parameters_from_uri(
+
f"doris://user:p%40ss@localhost:9030/internal.db?ssl_mode={mode}&charset=utf8mb4"
+ )
+ assert parameters["encryption"] is True
+ expected_mode = "VERIFY_IDENTITY" if mode == "VERIFY_IDENTITY" else
"VERIFY_CA"
+ expected_query = {"charset": "utf8mb4"}
+ if mode == "VERIFY_IDENTITY":
+ expected_query["ssl_mode"] = mode
+ assert parameters["query"] == expected_query
+ uri = make_url(DorisEngineSpec.build_sqlalchemy_uri(parameters))
+ assert uri.query == {"ssl_mode": expected_mode, "charset": "utf8mb4"}
+ assert uri.password == "p@ss" # noqa: S105
+ assert parameters["encryption"] is True
+ assert parameters["query"] == expected_query
+
+
[email protected]("mode", ["REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"])
+def test_doris_tls_preserves_uri_query(mode: str) -> None:
+ """TLS normalization preserves unrelated and repeated URI query options."""
+ from superset.db_engine_specs.doris import DorisEngineSpec
+
+ uri = make_url(
+ f"doris://root@localhost/internal.db?ssl_mode={mode}"
+ "&charset=utf8mb4&ssl_ca=%2Fca.pem&option=first&option=second"
+ )
+ with patch("superset.db_engine_specs.mysql.import_module") as module:
+ # MariaDB Connector/C, which needs REQUIRED upgraded to VERIFY_CA.
+ module.return_value.get_client_info.return_value = "3.3.17"
+ url, args = DorisEngineSpec.adjust_engine_params(uri, {})
+ assert url == uri
+ assert args["ssl_mode"] == ("VERIFY_CA" if mode == "REQUIRED" else mode)
diff --git a/tests/unit_tests/db_engine_specs/test_mariadb.py
b/tests/unit_tests/db_engine_specs/test_mariadb.py
index d4ff0ffef50..6a8c56a0adb 100644
--- a/tests/unit_tests/db_engine_specs/test_mariadb.py
+++ b/tests/unit_tests/db_engine_specs/test_mariadb.py
@@ -15,6 +15,12 @@
# specific language governing permissions and limitations
# under the License.
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy.engine.url import make_url
+
from superset.db_engine_specs.mariadb import MariaDBEngineSpec
from superset.db_engine_specs.mysql import MySQLEngineSpec
@@ -33,3 +39,80 @@ def test_mariadb_inherits_extended_aggregations() -> None:
assert MariaDBEngineSpec.get_extended_aggregation_func("VAR_SAMP") is not
None
# Same as MySQL, MEDIAN is not supported.
assert MariaDBEngineSpec.get_extended_aggregation_func("MEDIAN") is None
+
+
[email protected](
+ "client_info,expected_mode",
+ [("3.3.17", "VERIFY_CA"), ("8.4.6", "REQUIRED")],
+)
[email protected]("source", ["toggle", "ssl=1", "ssl_mode=REQUIRED"])
+def test_mariadb_tls_request_uses_verification(
+ source: str, client_info: str, expected_mode: str
+) -> None:
+ """MariaDB URIs get the same fail-closed TLS as MySQL under mysqlclient."""
+ if source == "toggle":
+ uri = make_url(
+ MariaDBEngineSpec.build_sqlalchemy_uri(
+ {
+ "username": "user",
+ "host": "localhost",
+ "port": 3306,
+ "database": "db",
+ "encryption": True,
+ },
+ {},
+ )
+ )
+ else:
+ uri = make_url(f"mariadb://user@localhost/db?{source}")
+ assert uri.get_backend_name() == "mariadb"
+ with patch("superset.db_engine_specs.mysql.import_module") as module:
+ module.return_value.get_client_info.return_value = client_info
+ url, args = MariaDBEngineSpec.adjust_engine_params(uri, {},
schema="other")
+ options = dict(url.query, **args)
+ assert options["ssl_mode"] == expected_mode
+ assert "ssl" not in options
+ assert url.database == "other"
+
+
[email protected]("options", [{"ssl_mode": "DISABLED"}, {"ssl_mode":
None}])
+def test_mariadb_tls_request_cannot_be_cancelled(options: dict[str, Any]) ->
None:
+ """An ssl_mode that disables or clears TLS conflicts with the SSL
request."""
+ with pytest.raises(ValueError, match="conflicts with ssl_mode"):
+ MariaDBEngineSpec.adjust_engine_params(
+ make_url("mariadb://localhost/db?ssl=1"), options
+ )
+
+
+def test_mariadb_pymysql_tls_request_requires_verification() -> None:
+ """PyMySQL MariaDB SSL requests verify the certificate and keep the CA."""
+ url, args = MariaDBEngineSpec.adjust_engine_params(
+ make_url("mariadb+pymysql://localhost/db?ssl=1&ssl_ca=/ca.pem"), {}
+ )
+ assert args["ssl_verify_cert"] is True
+ assert args["ssl_ca"] == "/ca.pem"
+ assert "ssl" not in url.query
+
+
+def test_mariadb_connector_tls_request_requires_verification() -> None:
+ """MariaDB Connector/Python keeps its fail-closed ssl flag and verifies."""
+ url, args = MariaDBEngineSpec.adjust_engine_params(
+ make_url("mariadb+mariadbconnector://localhost/db?ssl=1"), {}
+ )
+ assert "ssl" not in url.query
+ assert args["ssl"] is True
+ assert args["ssl_verify_cert"] is True
+
+ with pytest.raises(ValueError, match="requires ssl_verify_cert"):
+ MariaDBEngineSpec.adjust_engine_params(
+ make_url("mariadb+mariadbconnector://localhost/db?ssl=1"),
+ {"ssl_verify_cert": False},
+ )
+
+
+def test_unrequested_mariadb_connection_unchanged() -> None:
+ """Without an SSL request the URI and connect_args are returned
untouched."""
+ uri = make_url("mariadb://localhost/db")
+ url, args = MariaDBEngineSpec.adjust_engine_params(uri, {})
+ assert url == uri
+ assert "ssl_mode" not in args
diff --git a/tests/unit_tests/db_engine_specs/test_mysql.py
b/tests/unit_tests/db_engine_specs/test_mysql.py
index d6dc20722b8..287ca8b7d48 100644
--- a/tests/unit_tests/db_engine_specs/test_mysql.py
+++ b/tests/unit_tests/db_engine_specs/test_mysql.py
@@ -742,6 +742,87 @@ def test_pymysql_hostname_check_conflict_fails_closed() ->
None:
)
[email protected](
+ "client_info,expected_mode",
+ [("3.3.17", "VERIFY_CA"), ("8.4.6", "REQUIRED")],
+)
[email protected]("source", ["uri", "connect_args"])
+def test_saved_required_mode_is_a_tls_request(
+ source: str, client_info: str, expected_mode: str
+) -> None:
+ """A bare ssl_mode=REQUIRED is enforced without also needing ssl=1."""
+ from superset.db_engine_specs.mysql import MySQLEngineSpec
+
+ uri = make_url("mysql://localhost/db")
+ args: dict[str, Any] = {}
+ if source == "uri":
+ uri = uri.update_query_dict({"ssl_mode": "REQUIRED"})
+ else:
+ args = {"ssl_mode": "REQUIRED"}
+ with patch("superset.db_engine_specs.mysql.import_module") as module:
+ module.return_value.get_client_info.return_value = client_info
+ _, connect_args = MySQLEngineSpec.adjust_engine_params(uri, args)
+ assert connect_args["ssl_mode"] == expected_mode
+
+
[email protected]("mode", ["DISABLED", "PREFERRED"])
+def test_non_required_mode_is_not_a_tls_request(mode: str) -> None:
+ """Modes that do not require TLS are left to the driver without a
request."""
+ from superset.db_engine_specs.mysql import MySQLEngineSpec
+
+ uri = make_url(f"mysql://localhost/db?ssl_mode={mode}")
+ url, args = MySQLEngineSpec.adjust_engine_params(uri, {})
+ assert url == uri
+ assert "ssl_mode" not in args
+
+
+def test_require_mysql_tls_uses_explicit_driver() -> None:
+ """Compatible dialects select options by driver, not by URL backend."""
+ from superset.db_engine_specs.mysql import require_mysql_tls
+
+ uri = make_url("doris://localhost/db?ssl=1")
+ with patch("superset.db_engine_specs.mysql.import_module") as module:
+ module.return_value.get_client_info.return_value = "3.3.17"
+ url, args = require_mysql_tls(uri, {}, driver="mysqldb")
+ assert url.drivername == "doris"
+ assert "ssl" not in url.query
+ assert args["ssl_mode"] == "VERIFY_CA"
+
+
[email protected]("source", ["uri", "connect_args"])
+def test_mysqlclient_ssl_request_rejects_ssl_disabled(source: str) -> None:
+ """ssl_disabled=True cannot cancel a mysqlclient SSL request."""
+ from superset.db_engine_specs.mysql import MySQLEngineSpec
+
+ uri = make_url("mysql://localhost/db?ssl=1")
+ args: dict[str, Any] = {}
+ if source == "uri":
+ uri = uri.update_query_dict({"ssl_disabled": "true"})
+ else:
+ args = {"ssl_disabled": True}
+ with pytest.raises(ValueError, match="conflicts with ssl_disabled"):
+ MySQLEngineSpec.adjust_engine_params(uri, args)
+
+
[email protected]("value", ["false", "0", False])
+def test_mysqlclient_ssl_request_drops_false_ssl_disabled(value: str | bool)
-> None:
+ """A false ssl_disabled is dropped; REQUIRED is upgraded for MariaDB
clients."""
+ from superset.db_engine_specs.mysql import MySQLEngineSpec
+
+ uri = make_url("mysql://localhost/db?ssl_mode=REQUIRED")
+ if isinstance(value, str):
+ uri = uri.update_query_dict({"ssl_disabled": value})
+ args: dict[str, Any] = {}
+ else:
+ args = {"ssl_disabled": value}
+ with patch("superset.db_engine_specs.mysql.import_module") as module:
+ module.return_value.get_client_info.return_value = "3.3.17"
+ url, connect_args = MySQLEngineSpec.adjust_engine_params(uri, args)
+ assert "ssl_disabled" not in url.query
+ assert "ssl_disabled" not in connect_args
+ assert connect_args["ssl_mode"] == "VERIFY_CA"
+
+
def test_pymysql_old_version_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
"""PyMySQL before 1.2 must not reach its silent cleartext fallback."""
from superset.db_engine_specs.mysql import MySQLEngineSpec
@@ -806,9 +887,7 @@ def test_ssl_request_rejects_invalid_ssl_value(value: Any)
-> None:
)
[email protected](
- "driver", ["mysql+cymysql", "mysql+mariadbconnector", "mysql+aiomysql"]
-)
[email protected]("driver", ["mysql+cymysql", "mysql+aiomysql"])
def test_ssl_request_unsupported_driver_fails_closed(driver: str) -> None:
"""Drivers without known fail-closed TLS options must not receive ssl=1."""
from superset.db_engine_specs.mysql import MySQLEngineSpec
