SameerMesiah97 commented on code in PR #63035:
URL: https://github.com/apache/airflow/pull/63035#discussion_r3142359551
##########
providers/amazon/src/airflow/providers/amazon/aws/executors/aws_lambda/lambda_executor.py:
##########
@@ -49,29 +51,39 @@
from airflow.executors import workloads
from airflow.models.taskinstance import TaskInstance
+ if AIRFLOW_V_3_3_PLUS:
+ from airflow.executors.workloads.types import WorkloadKey as
_WorkloadKey
+
+ WorkloadKey: TypeAlias = _WorkloadKey
+ else:
+ WorkloadKey: TypeAlias = TaskInstanceKey # type: ignore[no-redef,
misc]
+
class AwsLambdaExecutor(BaseExecutor):
"""
- An Airflow Executor that submits tasks to AWS Lambda asynchronously.
+ An Airflow Executor that submits workloads (tasks and callbacks) to AWS
Lambda asynchronously.
When execute_async() is called, the executor invokes a specified AWS
Lambda function (asynchronously)
- with a payload that includes the task command and a unique task key.
+ with a payload that includes the workload command and a unique workload
key.
The Lambda function writes its result directly to an SQS queue, which is
then polled by this executor
- to update task state in Airflow.
+ to update workload state in Airflow.
"""
supports_multi_team: bool = True
+ if AIRFLOW_V_3_3_PLUS:
+ supports_callbacks: bool = True
+
if TYPE_CHECKING and AIRFLOW_V_3_0_PLUS:
# In the v3 path, we store workloads, not commands as strings.
- # TODO: TaskSDK: move this type change into BaseExecutor
- queued_tasks: dict[TaskInstanceKey, workloads.All] # type:
ignore[assignment]
+ # TODO: TaskSDK: move this type change into BaseExecutor.
+ queued_tasks: dict[WorkloadKey, workloads.All] # type:
ignore[assignment]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.pending_tasks: deque = deque()
- self.running_tasks: dict[str, TaskInstanceKey] = {}
+ self.pending_workloads: deque = deque()
+ self.running_workloads: dict[str, WorkloadKey] = {}
Review Comment:
Backwards compatibility shims have been added..
##########
providers/amazon/src/airflow/providers/amazon/aws/executors/aws_lambda/lambda_executor.py:
##########
@@ -49,29 +51,39 @@
from airflow.executors import workloads
from airflow.models.taskinstance import TaskInstance
+ if AIRFLOW_V_3_3_PLUS:
+ from airflow.executors.workloads.types import WorkloadKey as
_WorkloadKey
+
+ WorkloadKey: TypeAlias = _WorkloadKey
+ else:
+ WorkloadKey: TypeAlias = TaskInstanceKey # type: ignore[no-redef,
misc]
+
class AwsLambdaExecutor(BaseExecutor):
"""
- An Airflow Executor that submits tasks to AWS Lambda asynchronously.
+ An Airflow Executor that submits workloads (tasks and callbacks) to AWS
Lambda asynchronously.
When execute_async() is called, the executor invokes a specified AWS
Lambda function (asynchronously)
- with a payload that includes the task command and a unique task key.
+ with a payload that includes the workload command and a unique workload
key.
The Lambda function writes its result directly to an SQS queue, which is
then polled by this executor
- to update task state in Airflow.
+ to update workload state in Airflow.
"""
supports_multi_team: bool = True
+ if AIRFLOW_V_3_3_PLUS:
+ supports_callbacks: bool = True
+
if TYPE_CHECKING and AIRFLOW_V_3_0_PLUS:
# In the v3 path, we store workloads, not commands as strings.
- # TODO: TaskSDK: move this type change into BaseExecutor
- queued_tasks: dict[TaskInstanceKey, workloads.All] # type:
ignore[assignment]
+ # TODO: TaskSDK: move this type change into BaseExecutor.
+ queued_tasks: dict[WorkloadKey, workloads.All] # type:
ignore[assignment]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.pending_tasks: deque = deque()
- self.running_tasks: dict[str, TaskInstanceKey] = {}
+ self.pending_workloads: deque = deque()
+ self.running_workloads: dict[str, WorkloadKey] = {}
Review Comment:
Backwards compatibility shims have been added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
