Taragolis commented on code in PR #30093:
URL: https://github.com/apache/airflow/pull/30093#discussion_r1136957908
##########
airflow/providers/amazon/aws/hooks/redshift_data.py:
##########
@@ -190,3 +198,198 @@ def get_table_primary_key(
token = response["NextToken"]
return pk_columns or None
+
+
+class RedshiftDataAsyncHook(AwsBaseAsyncHook):
+ """
+ RedshiftDataHook inherits from AwsBaseHook to connect with AWS redshift
+ by using boto3 client_type as redshift-data we can interact with redshift
cluster database
+ and execute the query
+
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is None or empty then the default boto3 behaviour is used. If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ maintained on each worker node).
+ :param verify: Whether or not to verify SSL certificates.
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
+ :param region_name: AWS region_name. If not specified then the default
boto3 behaviour is used.
+ :param client_type: boto3.client client_type. Eg 's3', 'emr' etc
+ :param resource_type: boto3.resource resource_type. Eg 'dynamodb' etc
+ :param config: Configuration for botocore client.
+
(https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html)
+ :param poll_interval: polling period in seconds to check for the status
+ """
+
+ def __init__(self, *args: Any, poll_interval: int = 0, **kwargs: Any) ->
None:
+ aws_connection_type: str = "redshift-data"
+ try:
+ # for apache-airflow-providers-amazon>=3.0.0
+ kwargs["client_type"] = aws_connection_type
+ kwargs["resource_type"] = aws_connection_type
+ super().__init__(*args, **kwargs)
+ except ValueError:
+ # for apache-airflow-providers-amazon>=4.1.0
+ kwargs["client_type"] = aws_connection_type
+ super().__init__(*args, **kwargs)
+ self.client_type = aws_connection_type
+ self.poll_interval = poll_interval
+
+ def get_conn_params(self) -> dict[str, str | int]:
+ """Helper method to retrieve connection args"""
+ if not self.aws_conn_id:
+ raise AirflowException("Required connection details is missing !")
+
+ connection_object = self.get_connection(self.aws_conn_id)
+ extra_config = connection_object.extra_dejson
+
+ conn_params: dict[str, str | int] = {}
+
+ if "db_user" in extra_config:
+ conn_params["db_user"] = extra_config.get("db_user", None)
+ else:
+ raise AirflowException("Required db user is missing !")
+
+ if "database" in extra_config:
+ conn_params["database"] = extra_config.get("database", None)
+ elif connection_object.schema:
+ conn_params["database"] = connection_object.schema
+ else:
+ raise AirflowException("Required Database name is missing !")
+
+ if "access_key_id" in extra_config or "aws_access_key_id" in
extra_config:
+ conn_params["aws_access_key_id"] = (
+ extra_config["access_key_id"]
+ if "access_key_id" in extra_config
+ else extra_config["aws_access_key_id"]
+ )
+ conn_params["aws_secret_access_key"] = (
+ extra_config["secret_access_key"]
+ if "secret_access_key" in extra_config
+ else extra_config["aws_secret_access_key"]
+ )
+ elif connection_object.login:
+ conn_params["aws_access_key_id"] = connection_object.login
+ conn_params["aws_secret_access_key"] = connection_object.password
+ else:
+ raise AirflowException("Required access_key_id,
aws_secret_access_key")
+
+ if "region" in extra_config or "region_name" in extra_config:
+ self.log.info("Retrieving region_name from
Connection.extra_config['region_name']")
+ conn_params["region_name"] = (
+ extra_config["region"] if "region" in extra_config else
extra_config["region_name"]
+ )
+ else:
+ raise AirflowException("Required Region name is missing !")
+
+ if "aws_session_token" in extra_config:
+ self.log.info(
+ "session token retrieved from extra, please note you are
responsible for renewing these.",
+ )
+ conn_params["aws_session_token"] =
extra_config["aws_session_token"]
+
+ if "cluster_identifier" in extra_config:
+ self.log.info("Retrieving cluster_identifier from
Connection.extra_config['cluster_identifier']")
+ conn_params["cluster_identifier"] =
extra_config["cluster_identifier"]
+ else:
+ raise AirflowException("Required Cluster identifier is missing !")
+
+ return conn_params
+
+ def execute_query(
+ self, sql: dict[Any, Any] | Iterable[Any], params: ParamsDict |
dict[Any, Any]
+ ) -> tuple[list[str], dict[str, str]]:
+ """
+ Runs an SQL statement, which can be data manipulation language (DML)
+ or data definition language (DDL)
+
+ :param sql: list of query ids
+ """
+ if not sql:
+ raise AirflowException("SQL query is None.")
+ try:
+ if isinstance(sql, str):
+ split_statements_tuple = split_statements(StringIO(sql))
+ sql = [sql_string for sql_string, _ in split_statements_tuple
if sql_string]
+
+ try:
+ # for apache-airflow-providers-amazon>=3.0.0
+ client = self.get_conn()
+ except ValueError:
+ # for apache-airflow-providers-amazon>=4.1.0
+ self.resource_type = None
Review Comment:
???
##########
airflow/providers/amazon/aws/hooks/redshift_data.py:
##########
@@ -190,3 +198,198 @@ def get_table_primary_key(
token = response["NextToken"]
return pk_columns or None
+
+
+class RedshiftDataAsyncHook(AwsBaseAsyncHook):
+ """
+ RedshiftDataHook inherits from AwsBaseHook to connect with AWS redshift
+ by using boto3 client_type as redshift-data we can interact with redshift
cluster database
+ and execute the query
+
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is None or empty then the default boto3 behaviour is used. If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ maintained on each worker node).
+ :param verify: Whether or not to verify SSL certificates.
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
+ :param region_name: AWS region_name. If not specified then the default
boto3 behaviour is used.
+ :param client_type: boto3.client client_type. Eg 's3', 'emr' etc
+ :param resource_type: boto3.resource resource_type. Eg 'dynamodb' etc
+ :param config: Configuration for botocore client.
+
(https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html)
+ :param poll_interval: polling period in seconds to check for the status
+ """
+
+ def __init__(self, *args: Any, poll_interval: int = 0, **kwargs: Any) ->
None:
+ aws_connection_type: str = "redshift-data"
+ try:
+ # for apache-airflow-providers-amazon>=3.0.0
+ kwargs["client_type"] = aws_connection_type
+ kwargs["resource_type"] = aws_connection_type
Review Comment:
This is not valid to set both `client_type` and `resource_type` and also
`aibotocore` do not support resources
##########
airflow/providers/amazon/aws/hooks/redshift_data.py:
##########
@@ -190,3 +198,198 @@ def get_table_primary_key(
token = response["NextToken"]
return pk_columns or None
+
+
+class RedshiftDataAsyncHook(AwsBaseAsyncHook):
+ """
+ RedshiftDataHook inherits from AwsBaseHook to connect with AWS redshift
+ by using boto3 client_type as redshift-data we can interact with redshift
cluster database
+ and execute the query
+
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is None or empty then the default boto3 behaviour is used. If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ maintained on each worker node).
+ :param verify: Whether or not to verify SSL certificates.
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
+ :param region_name: AWS region_name. If not specified then the default
boto3 behaviour is used.
+ :param client_type: boto3.client client_type. Eg 's3', 'emr' etc
+ :param resource_type: boto3.resource resource_type. Eg 'dynamodb' etc
+ :param config: Configuration for botocore client.
+
(https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html)
+ :param poll_interval: polling period in seconds to check for the status
+ """
+
+ def __init__(self, *args: Any, poll_interval: int = 0, **kwargs: Any) ->
None:
+ aws_connection_type: str = "redshift-data"
Review Comment:
This connection not provided by Amazon Provider
##########
airflow/providers/amazon/aws/hooks/redshift_data.py:
##########
@@ -190,3 +198,198 @@ def get_table_primary_key(
token = response["NextToken"]
return pk_columns or None
+
+
+class RedshiftDataAsyncHook(AwsBaseAsyncHook):
+ """
+ RedshiftDataHook inherits from AwsBaseHook to connect with AWS redshift
+ by using boto3 client_type as redshift-data we can interact with redshift
cluster database
+ and execute the query
+
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is None or empty then the default boto3 behaviour is used. If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ maintained on each worker node).
+ :param verify: Whether or not to verify SSL certificates.
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
+ :param region_name: AWS region_name. If not specified then the default
boto3 behaviour is used.
+ :param client_type: boto3.client client_type. Eg 's3', 'emr' etc
+ :param resource_type: boto3.resource resource_type. Eg 'dynamodb' etc
+ :param config: Configuration for botocore client.
+
(https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html)
+ :param poll_interval: polling period in seconds to check for the status
+ """
+
+ def __init__(self, *args: Any, poll_interval: int = 0, **kwargs: Any) ->
None:
+ aws_connection_type: str = "redshift-data"
+ try:
+ # for apache-airflow-providers-amazon>=3.0.0
+ kwargs["client_type"] = aws_connection_type
+ kwargs["resource_type"] = aws_connection_type
+ super().__init__(*args, **kwargs)
+ except ValueError:
+ # for apache-airflow-providers-amazon>=4.1.0
+ kwargs["client_type"] = aws_connection_type
+ super().__init__(*args, **kwargs)
+ self.client_type = aws_connection_type
+ self.poll_interval = poll_interval
+
+ def get_conn_params(self) -> dict[str, str | int]:
+ """Helper method to retrieve connection args"""
+ if not self.aws_conn_id:
+ raise AirflowException("Required connection details is missing !")
Review Comment:
-
https://airflow.apache.org/docs/apache-airflow-providers-amazon/7.3.0/connections/aws.html#per-service-configuration
- https://github.com/apache/airflow/pull/28706#discussion_r1062824388
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]