This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new dc0e306a24c fix(elasticsearch): classify byte, short, half_float,
scaled_float and unsigned_long columns (#44713)
dc0e306a24c is described below
commit dc0e306a24c48bbcb67931dd68e39c1f09982d46
Author: Amin Ghadersohi <[email protected]>
AuthorDate: Thu Oct 1 12:49:48 2026 +1000
fix(elasticsearch): classify byte, short, half_float, scaled_float and
unsigned_long columns (#44713)
---
superset/db_engine_specs/elasticsearch.py | 25 +++++++++++++
.../db_engine_specs/test_elasticsearch.py | 41 ++++++++++++++++++++++
2 files changed, 66 insertions(+)
diff --git a/superset/db_engine_specs/elasticsearch.py
b/superset/db_engine_specs/elasticsearch.py
index 048e086b7dd..f67e677d638 100644
--- a/superset/db_engine_specs/elasticsearch.py
+++ b/superset/db_engine_specs/elasticsearch.py
@@ -31,12 +31,35 @@ from superset.db_engine_specs.exceptions import (
SupersetDBAPIOperationalError,
SupersetDBAPIProgrammingError,
)
+from superset.utils.core import GenericDataType
if TYPE_CHECKING:
from superset.models.core import Database
logger = logging.getLogger()
+# Elasticsearch/OpenSearch field types that the default column type mappings
+# do not recognize. DOUBLE, FLOAT, INTEGER, LONG, BOOLEAN and DATETIME are
+# already covered by the defaults. Like the defaults, the patterns anchor only
+# at the start, so a parameterized name such as SCALED_FLOAT(100) still
matches.
+FIELD_TYPE_MAPPINGS = (
+ (
+ re.compile(r"^(byte|short)", re.IGNORECASE),
+ types.SmallInteger(),
+ GenericDataType.NUMERIC,
+ ),
+ (
+ re.compile(r"^(half_float|scaled_float)", re.IGNORECASE),
+ types.Float(),
+ GenericDataType.NUMERIC,
+ ),
+ (
+ re.compile(r"^unsigned_long", re.IGNORECASE),
+ types.BigInteger(),
+ GenericDataType.NUMERIC,
+ ),
+)
+
def _fetch_page_via_cursor(
database: Database,
@@ -146,6 +169,7 @@ class ElasticSearchEngineSpec(BaseEngineSpec): # pylint:
disable=abstract-metho
allows_subqueries = True
allows_sql_comments = False
supports_offset = False
+ column_type_mappings = FIELD_TYPE_MAPPINGS
metadata = {
"description": (
@@ -325,6 +349,7 @@ class OpenDistroEngineSpec(BaseEngineSpec): # pylint:
disable=abstract-method
allows_subqueries = True
allows_sql_comments = False
supports_offset = False
+ column_type_mappings = FIELD_TYPE_MAPPINGS
_time_grain_expressions = {
None: "{col}",
diff --git a/tests/unit_tests/db_engine_specs/test_elasticsearch.py
b/tests/unit_tests/db_engine_specs/test_elasticsearch.py
index 49a8353d9ef..7af6cc2db4f 100644
--- a/tests/unit_tests/db_engine_specs/test_elasticsearch.py
+++ b/tests/unit_tests/db_engine_specs/test_elasticsearch.py
@@ -470,6 +470,47 @@ def
test_opendistro_fetch_data_with_cursor_uses_opendistro_endpoints() -> None:
assert calls[1][0][1] == "/_opendistro/_sql/close"
[email protected](
+ "native_type,sqla_type,generic_type",
+ [
+ ("BYTE", "SmallInteger", "NUMERIC"),
+ ("SHORT", "SmallInteger", "NUMERIC"),
+ ("HALF_FLOAT", "Float", "NUMERIC"),
+ ("SCALED_FLOAT", "Float", "NUMERIC"),
+ ("SCALED_FLOAT(100)", "Float", "NUMERIC"),
+ ("UNSIGNED_LONG", "BigInteger", "NUMERIC"),
+ ("DOUBLE", None, "NUMERIC"),
+ ("FLOAT", None, "NUMERIC"),
+ ("INTEGER", None, "NUMERIC"),
+ ("LONG", None, "NUMERIC"),
+ ("BOOLEAN", None, "BOOLEAN"),
+ ("DATETIME", None, "TEMPORAL"),
+ ("STRING", None, "STRING"),
+ ],
+)
[email protected](
+ "spec_name", ["ElasticSearchEngineSpec", "OpenDistroEngineSpec"]
+)
+def test_field_types_are_classified(
+ spec_name: str, native_type: str, sqla_type: Optional[str], generic_type:
str
+) -> None:
+ """
+ Every numeric field type a reflected Elasticsearch/OpenSearch column can
+ report gets a generic type, so it can be used as a numeric column.
+ """
+ from sqlalchemy import types
+
+ from superset.db_engine_specs import elasticsearch
+ from superset.utils.core import GenericDataType
+
+ spec = getattr(elasticsearch, spec_name)
+ column_spec = spec.get_column_spec(native_type)
+ assert column_spec is not None, native_type
+ assert column_spec.generic_type == GenericDataType[generic_type]
+ if sqla_type: # None: already covered by the default mappings
+ assert isinstance(column_spec.sqla_type, getattr(types, sqla_type))
+
+
def test_opendistro_fetch_data_with_cursor_reads_jdbc_format_pages() -> None:
"""
The OpenSearch SQL plugin answers in its JDBC format: ``schema`` and
