vincbeck commented on code in PR #75907:
URL: https://github.com/apache/airflow/pull/75907#discussion_r4412947098
##########
providers/amazon/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor.py:
##########
@@ -143,6 +149,31 @@ def __init__(self, *args, **kwargs):
self.container_name = self.conf.get(CONFIG_GROUP_NAME,
AllEcsConfigKeys.CONTAINER_NAME)
self.attempts_since_last_successful_connection = 0
+ # ``ExecutorConf`` has no ``getint`` before Airflow 3.2.
+ self.run_task_concurrency = int(
+ self.conf.get(
+ CONFIG_GROUP_NAME,
+ AllEcsConfigKeys.RUN_TASK_CONCURRENCY,
+
fallback=CONFIG_DEFAULTS[AllEcsConfigKeys.RUN_TASK_CONCURRENCY],
+ )
+ )
+ if self.run_task_concurrency < 0:
+ raise ValueError(
+ f"{AllEcsConfigKeys.RUN_TASK_CONCURRENCY} must be 0 (no limit)
or more, "
+ f"got {self.run_task_concurrency}"
+ )
+ # With a concurrency of 1, RunTask calls are made one by one in the
scheduler loop. With 0, there is
+ # no limit: the executor never has more than ``parallelism`` workloads
to launch at once, and the
+ # pool only starts the threads that the calls in flight need.
+ self._run_task_threads = 0
+ self._run_task_pool: ThreadPoolExecutor | None = None
+ if self.run_task_concurrency != 1:
+ self._run_task_threads = self.run_task_concurrency or
self.parallelism
+ self._run_task_pool = ThreadPoolExecutor(
+ max_workers=self._run_task_threads,
thread_name_prefix="ecs-run-task"
+ )
Review Comment:
Fair, let's remove 0 as value for now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
