This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f84eb2ab6f Make "placeholder" of ODBC configurable in UI (#36000)
f84eb2ab6f is described below
commit f84eb2ab6fe777938f85a5fbb2a0b8a6dc07b9bc
Author: jarnedemunter1 <[email protected]>
AuthorDate: Thu Dec 21 21:55:26 2023 +0100
Make "placeholder" of ODBC configurable in UI (#36000)
---------
Co-authored-by: EXM492 <[email protected]>
Co-authored-by: EXM492 <[email protected]>
Co-authored-by: Joffrey Bienvenu <[email protected]>
---
airflow/providers/common/sql/hooks/sql.py | 12 +++++++-----
airflow/providers/common/sql/hooks/sql.pyi | 1 +
airflow/providers/odbc/hooks/odbc.py | 18 +++++++++++++++++-
airflow/providers/postgres/hooks/postgres.py | 5 ++---
airflow/providers/presto/hooks/presto.py | 5 ++++-
airflow/providers/sqlite/hooks/sqlite.py | 5 ++++-
airflow/providers/trino/hooks/trino.py | 5 ++++-
tests/providers/odbc/hooks/test_odbc.py | 5 +++++
8 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/airflow/providers/common/sql/hooks/sql.py
b/airflow/providers/common/sql/hooks/sql.py
index bb85dedc1c..0d4b112b32 100644
--- a/airflow/providers/common/sql/hooks/sql.py
+++ b/airflow/providers/common/sql/hooks/sql.py
@@ -143,8 +143,6 @@ class DbApiHook(BaseHook):
connector: ConnectorProtocol | None = None
# Override with db-specific query to check connection
_test_connection_sql = "select 1"
- # Override with the db-specific value used for placeholders
- placeholder: str = "%s"
def __init__(self, *args, schema: str | None = None, log_sql: bool = True,
**kwargs):
super().__init__()
@@ -163,6 +161,11 @@ class DbApiHook(BaseHook):
self.__schema = schema
self.log_sql = log_sql
self.descriptions: list[Sequence[Sequence] | None] = []
+ self._placeholder: str = "%s"
+
+ @property
+ def placeholder(self) -> str:
+ return self._placeholder
def get_conn(self):
"""Return a connection object."""
@@ -463,8 +466,7 @@ class DbApiHook(BaseHook):
"""Return a cursor."""
return self.get_conn().cursor()
- @classmethod
- def _generate_insert_sql(cls, table, values, target_fields, replace,
**kwargs) -> str:
+ def _generate_insert_sql(self, table, values, target_fields, replace,
**kwargs) -> str:
"""
Generate the INSERT SQL statement.
@@ -477,7 +479,7 @@ class DbApiHook(BaseHook):
:return: The generated INSERT or REPLACE SQL statement
"""
placeholders = [
- cls.placeholder,
+ self.placeholder,
] * len(values)
if target_fields:
diff --git a/airflow/providers/common/sql/hooks/sql.pyi
b/airflow/providers/common/sql/hooks/sql.pyi
index 41bd6ebf47..58d6b874d8 100644
--- a/airflow/providers/common/sql/hooks/sql.pyi
+++ b/airflow/providers/common/sql/hooks/sql.pyi
@@ -53,6 +53,7 @@ class DbApiHook(BaseForDbApiHook):
placeholder: str
log_sql: Incomplete
descriptions: Incomplete
+ _placeholder: str
def __init__(self, *args, schema: Union[str, None] = ..., log_sql: bool =
..., **kwargs) -> None: ...
def get_conn(self): ...
def get_uri(self) -> str: ...
diff --git a/airflow/providers/odbc/hooks/odbc.py
b/airflow/providers/odbc/hooks/odbc.py
index d84933dc29..a913626a00 100644
--- a/airflow/providers/odbc/hooks/odbc.py
+++ b/airflow/providers/odbc/hooks/odbc.py
@@ -25,6 +25,8 @@ import pyodbc
from airflow.providers.common.sql.hooks.sql import DbApiHook
from airflow.utils.helpers import merge_dicts
+DEFAULT_ODBC_PLACEHOLDERS = frozenset({"%s", "?"})
+
class OdbcHook(DbApiHook):
"""
@@ -161,7 +163,7 @@ class OdbcHook(DbApiHook):
if self.connection.port:
conn_str += f"PORT={self.connection.port};"
- extra_exclude = {"driver", "dsn", "connect_kwargs",
"sqlalchemy_scheme"}
+ extra_exclude = {"driver", "dsn", "connect_kwargs",
"sqlalchemy_scheme", "placeholder"}
extra_params = {
k: v for k, v in self.connection.extra_dejson.items() if
k.lower() not in extra_exclude
}
@@ -198,6 +200,20 @@ class OdbcHook(DbApiHook):
conn = pyodbc.connect(self.odbc_connection_string,
**self.connect_kwargs)
return conn
+ @property
+ def placeholder(self):
+ placeholder = self.connection.extra_dejson.get("placeholder")
+ if placeholder in DEFAULT_ODBC_PLACEHOLDERS:
+ return placeholder
+ else:
+ self.log.warning(
+ "Placeholder defined in Connection '%s' is not listed in
'DEFAULT_ODBC_PLACEHOLDERS' "
+ "and got ignored. Falling back to the default placeholder
'%s'.",
+ placeholder,
+ self._placeholder,
+ )
+ return self._placeholder
+
def get_uri(self) -> str:
"""URI invoked in
:meth:`~airflow.providers.common.sql.hooks.sql.DbApiHook.get_sqlalchemy_engine`."""
quoted_conn_str = quote_plus(self.odbc_connection_string)
diff --git a/airflow/providers/postgres/hooks/postgres.py
b/airflow/providers/postgres/hooks/postgres.py
index 15472a3037..76ce4dacef 100644
--- a/airflow/providers/postgres/hooks/postgres.py
+++ b/airflow/providers/postgres/hooks/postgres.py
@@ -275,9 +275,8 @@ class PostgresHook(DbApiHook):
pk_columns = [row[0] for row in self.get_records(sql, (schema, table))]
return pk_columns or None
- @classmethod
def _generate_insert_sql(
- cls, table: str, values: tuple[str, ...], target_fields:
Iterable[str], replace: bool, **kwargs
+ self, table: str, values: tuple[str, ...], target_fields:
Iterable[str], replace: bool, **kwargs
) -> str:
"""Generate the INSERT SQL statement.
@@ -292,7 +291,7 @@ class PostgresHook(DbApiHook):
:return: The generated INSERT or REPLACE SQL statement
"""
placeholders = [
- cls.placeholder,
+ self.placeholder,
] * len(values)
replace_index = kwargs.get("replace_index")
diff --git a/airflow/providers/presto/hooks/presto.py
b/airflow/providers/presto/hooks/presto.py
index 4cfb2b7601..611bac37c5 100644
--- a/airflow/providers/presto/hooks/presto.py
+++ b/airflow/providers/presto/hooks/presto.py
@@ -84,7 +84,10 @@ class PrestoHook(DbApiHook):
default_conn_name = "presto_default"
conn_type = "presto"
hook_name = "Presto"
- placeholder = "?"
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._placeholder: str = "?"
def get_conn(self) -> Connection:
"""Returns a connection object."""
diff --git a/airflow/providers/sqlite/hooks/sqlite.py
b/airflow/providers/sqlite/hooks/sqlite.py
index 9fede39423..12316de5bd 100644
--- a/airflow/providers/sqlite/hooks/sqlite.py
+++ b/airflow/providers/sqlite/hooks/sqlite.py
@@ -30,7 +30,10 @@ class SqliteHook(DbApiHook):
default_conn_name = "sqlite_default"
conn_type = "sqlite"
hook_name = "Sqlite"
- placeholder = "?"
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._placeholder: str = "?"
def get_conn(self) -> sqlite3.dbapi2.Connection:
"""Returns a sqlite connection object."""
diff --git a/airflow/providers/trino/hooks/trino.py
b/airflow/providers/trino/hooks/trino.py
index 3369af13f0..7547b6bf95 100644
--- a/airflow/providers/trino/hooks/trino.py
+++ b/airflow/providers/trino/hooks/trino.py
@@ -87,9 +87,12 @@ class TrinoHook(DbApiHook):
conn_type = "trino"
hook_name = "Trino"
query_id = ""
- placeholder = "?"
_test_connection_sql = "select 1"
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._placeholder: str = "?"
+
def get_conn(self) -> Connection:
"""Returns a connection object."""
db = self.get_connection(self.trino_conn_id) # type:
ignore[attr-defined]
diff --git a/tests/providers/odbc/hooks/test_odbc.py
b/tests/providers/odbc/hooks/test_odbc.py
index 3740a5654c..8c0fbee40f 100644
--- a/tests/providers/odbc/hooks/test_odbc.py
+++ b/tests/providers/odbc/hooks/test_odbc.py
@@ -275,6 +275,11 @@ class TestOdbcHook:
assert "have supplied 'driver' via connection extra but it
will not be used" in caplog.text
assert driver == "Blah driver"
+ def test_placeholder_config_from_extra(self):
+ conn_params = dict(extra=json.dumps(dict(placeholder="?")))
+ hook = self.get_hook(conn_params=conn_params)
+ assert hook.placeholder == "?"
+
def test_database(self):
hook = self.get_hook(hook_params=dict(database="abc"))
assert hook.database == "abc"