Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W merged PR #66520: URL: https://github.com/apache/airflow/pull/66520 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3310035283
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+DR_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selectors_used = sum([args.run_id is not None, args.partition_key is not
None, has_range])
+if selectors_used == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if selectors_used > 1:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+
+if args.date is not None:
+if args.start_date is not None or args.end_date is not None:
+raise SystemExit("--date cannot be combined with --start-date /
--end-date.")
+raw = args.date
+parts = raw.split("~", 1)
+if len(parts) != 2 or not parts[0].strip() or not parts[1].strip():
+raise SystemExit("--date must be in the form 'a~b', e.g.
'2026-01-01~2026-01-31'.")
+try:
+args.start_date = parsedate(parts[0].strip())
+args.end_date = parsedate(parts[1].strip())
+except ValueError:
+raise SystemExit("--date must be in the form 'a~b', e.g.
'2026-01-01~2026-01-31'.")
+
+if args.end_date is not None:
+args.end_date = args.end_date.replace(hour=23, minute=59, second=59,
microsecond=99)
+
+stmt = select(DagRun).where(DagRun.dag_id == args.dag_id)
+if args.run_id:
+stmt = stmt.where(DagRun.run_id == args.run_id)
+elif args.partition_key is not None:
+stmt = stmt.where(DagRun.partition_key == args.partition_key)
+else:
+stmt = stmt.where(or_(DagRun.partition_key.is_not(None),
DagRun.partition_date.is_not(None)))
+if args.start_date is not None:
+stmt = stmt.where(DagRun.partition_date >= args.start_date)
+if args.end_date is not None:
+stmt = stmt.where(DagRun.partition_date <= args.end_date)
+stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id)
+
+clear_tis = bool(args.clear_task_instances)
+cleared = 0
+processed_any = False
+
+# For --clear-task-instances: buffer run_ids, flush in DR_CHUNK_SIZE
batches.
+# task_instances is a lazy-select relationship; avoid N+1 by issuing one
explicit
+# SELECT per chunk instead of accessing run.task_instances directly.
+ti_buffer_run_ids: list[str] = []
+tis_cleared_total = 0
+runs_for_ti_total = 0
+tis_dry_total = 0
+runs_for_ti_dry = 0
+
+for run in session.scalars(stmt).yield_per(100):
+processed_any = True
+fields_already_cleared = run.partition_key is None and
run.partition_date is None
+if fields_already_cleared and not clear_tis:
+print(f"DagRun {run.run_id}: already cleared, skipping.")
+continue
+if not fields_already_cleared:
+print(
+f"DagRun {run.run_id}: "
+f"partition_key={run.partition_key!r} -> None, "
+f"partition_date={run.partition_date.isoformat() if
run.part
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3310025413
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+DR_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selectors_used = sum([args.run_id is not None, args.partition_key is not
None, has_range])
+if selectors_used == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if selectors_used > 1:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+
+if args.date is not None:
+if args.start_date is not None or args.end_date is not None:
+raise SystemExit("--date cannot be combined with --start-date /
--end-date.")
+raw = args.date
+parts = raw.split("~", 1)
+if len(parts) != 2 or not parts[0].strip() or not parts[1].strip():
+raise SystemExit("--date must be in the form 'a~b', e.g.
'2026-01-01~2026-01-31'.")
+try:
+args.start_date = parsedate(parts[0].strip())
+args.end_date = parsedate(parts[1].strip())
Review Comment:
updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3302918242
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+DR_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selectors_used = sum([args.run_id is not None, args.partition_key is not
None, has_range])
+if selectors_used == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if selectors_used > 1:
Review Comment:
you're right! probably some leftover 🤦♂️
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+DR_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selectors_used = sum([args.run_id is not None, args.partition_key is not
None, has_range])
+if selectors_used == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if selectors_used > 1:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+
+if args.date is not None:
+if args.start_date is not None or args.end_date is not None:
+raise SystemExit("--date cannot be combined with --start-date /
--end-date.")
+raw = args.date
+parts = raw.split("~", 1)
+if len(parts) != 2 or not parts[0].strip()
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
jason810496 commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3302862130
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+DR_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selectors_used = sum([args.run_id is not None, args.partition_key is not
None, has_range])
+if selectors_used == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if selectors_used > 1:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+
+if args.date is not None:
+if args.start_date is not None or args.end_date is not None:
+raise SystemExit("--date cannot be combined with --start-date /
--end-date.")
+raw = args.date
+parts = raw.split("~", 1)
+if len(parts) != 2 or not parts[0].strip() or not parts[1].strip():
+raise SystemExit("--date must be in the form 'a~b', e.g.
'2026-01-01~2026-01-31'.")
+try:
+args.start_date = parsedate(parts[0].strip())
+args.end_date = parsedate(parts[1].strip())
Review Comment:
I just double checked with claude that the `parsedate` here could bypass the
validation with `2026-01-02T01:00:00~2026-01-02T10:00:00` as input.
Perhaps having strict validation for the `-dd-mm` format?
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+DR_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3298636996
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+TI_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selected = [
+("--run-id", args.run_id is not None),
+("--partition-key", args.partition_key is not None),
+("a partition_date range", has_range),
+]
+chosen = [name for name, on in selected if on]
+if len(chosen) == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if len(chosen) > 1:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+
+if args.date is not None:
+if args.start_date is not None or args.end_date is not None:
+raise SystemExit("--date cannot be combined with --start-date /
--end-date.")
+raw = args.date
+parts = raw.split("~", 1)
+if len(parts) != 2 or not parts[0].strip() or not parts[1].strip():
+raise SystemExit("--date must be in the form 'a~b', e.g.
'2026-01-01~2026-01-31'.")
+try:
+args.start_date = parsedate(parts[0].strip())
+args.end_date = parsedate(parts[1].strip())
+except ValueError:
+raise SystemExit("--date must be in the form 'a~b', e.g.
'2026-01-01~2026-01-31'.")
+
+if args.end_date is not None:
+args.end_date = args.end_date.replace(hour=23, minute=59, second=59,
microsecond=99)
+
+stmt = select(DagRun).where(DagRun.dag_id == args.dag_id)
+if args.run_id:
+stmt = stmt.where(DagRun.run_id == args.run_id)
+elif args.partition_key is not None:
+stmt = stmt.where(DagRun.partition_key == args.partition_key)
+else:
+stmt = stmt.where(or_(DagRun.partition_key.is_not(None),
DagRun.partition_date.is_not(None)))
+if args.start_date is not None:
+stmt = stmt.where(DagRun.partition_date >= args.start_date)
+if args.end_date is not None:
+stmt = stmt.where(DagRun.partition_date <= args.end_date)
+stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id)
+
+clear_tis = bool(args.clear_task_instances)
+cleared = 0
+processed_any = False
+
+# For --clear-task-instances: buffer run_ids, flush in TI_CHUNK_SIZE
batches.
+# task_instances is a lazy-select relationship; avoid N+1 by issuing one
explicit
+# SELECT per chunk instead of accessing run.task_instances directly.
+ti_buffer_run_ids: list[str] = []
+tis_cleared_total = 0
+runs_for_ti_total = 0
+tis_dry_total = 0
+runs_for_ti_dry = 0
+
+for run in session.scalars(stmt).yield_per(100):
+processed_any = True
+fields_already_cleared = run.partition_key is None and
run.partition_date is None
+if fields_already_cleared and not clear_tis:
+print(f"DagRun {run.run_id}: already cleared, skipping.")
+continue
+if not fields_already_cleared:
+print(
+f"DagRun {run.run_id}: "
+
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3298256323
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+TI_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selected = [
+("--run-id", args.run_id is not None),
+("--partition-key", args.partition_key is not None),
+("a partition_date range", has_range),
+]
+chosen = [name for name, on in selected if on]
+if len(chosen) == 0:
Review Comment:
Sounds good! aligned with the `sum([...])` pattern from #66004
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
jason810496 commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3296731932
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+TI_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selected = [
+("--run-id", args.run_id is not None),
+("--partition-key", args.partition_key is not None),
+("a partition_date range", has_range),
+]
+chosen = [name for name, on in selected if on]
+if len(chosen) == 0:
Review Comment:
Would it be better to align the validation as
https://github.com/apache/airflow/pull/66004 by using just `sum`? Since the
`selected` and `chosen` list variables aren't used in rest of the processing.
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow._shared.timezones.timezone import parse as parsedate
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance, clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+TI_CHUNK_SIZE = 500
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None or
args.date is not None
+selected = [
+("--run-id", args.run_id is not None),
+("--partition-key", args.partition_key is not None),
+("a partition_date range", has_range),
+]
+chosen = [name for name, on in selected if on]
+if len(chosen) == 0:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+if len(chosen) > 1:
+raise SystemExit(
+"Specify exactly one of --run-id, --partition-key, or a
partition_date range "
+"(--start-date/--end-date or --date)."
+)
+
+if args.date is not None:
+if args.start_date is not None or args.end
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520: URL: https://github.com/apache/airflow/pull/66520#discussion_r3287785148 ## airflow-core/src/airflow/cli/commands/partition_command.py: ## @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Partitions sub-commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from airflow.models.dagrun import DagRun +from airflow.models.taskinstance import clear_task_instances +from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded +from airflow.utils.session import NEW_SESSION, provide_session + +if TYPE_CHECKING: +from sqlalchemy.orm import Session + + +@cli_utils.action_cli +@providers_configuration_loaded +@provide_session +def clear(args, *, session: Session = NEW_SESSION) -> None: +"""Clear the partition_key and partition_date of matching DagRuns.""" +has_range = args.start_date is not None or args.end_date is not None +if not args.run_id and not has_range: +raise SystemExit( +"Specify either --run-id for a single DagRun, or a partition_date range " +"via --start-date and/or --end-date." +) +if args.run_id and has_range: +raise SystemExit( +"--run-id cannot be combined with a partition_date range (--start-date / --end-date)." +) Review Comment: yes, it's now suppors * --run-id (single run) * --partition-key (every run with that exact key) * --partition-date-start / --partition-date-end (inclusive partition_date window; runs with NULL partition_date are never matched; -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520: URL: https://github.com/apache/airflow/pull/66520#discussion_r3282317382 ## airflow-core/src/airflow/cli/commands/partition_command.py: ## @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Partitions sub-commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from airflow.models.dagrun import DagRun +from airflow.models.taskinstance import clear_task_instances +from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded +from airflow.utils.session import NEW_SESSION, provide_session + +if TYPE_CHECKING: +from sqlalchemy.orm import Session + + +@cli_utils.action_cli +@providers_configuration_loaded +@provide_session +def clear(args, *, session: Session = NEW_SESSION) -> None: +"""Clear the partition_key and partition_date of matching DagRuns.""" +has_range = args.start_date is not None or args.end_date is not None +if not args.run_id and not has_range: +raise SystemExit( +"Specify either --run-id for a single DagRun, or a partition_date range " +"via --start-date and/or --end-date." +) +if args.run_id and has_range: +raise SystemExit( +"--run-id cannot be combined with a partition_date range (--start-date / --end-date)." +) + +stmt = select(DagRun).where(DagRun.dag_id == args.dag_id) +if args.run_id: +stmt = stmt.where(DagRun.run_id == args.run_id) +else: +stmt = stmt.where(or_(DagRun.partition_key.is_not(None), DagRun.partition_date.is_not(None))) +if args.start_date is not None: +stmt = stmt.where(DagRun.partition_date >= args.start_date) +if args.end_date is not None: +stmt = stmt.where(DagRun.partition_date <= args.end_date) +stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id) + +runs = session.scalars(stmt).all() Review Comment: changed it to `yield_per` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520: URL: https://github.com/apache/airflow/pull/66520#discussion_r3282317382 ## airflow-core/src/airflow/cli/commands/partition_command.py: ## @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Partitions sub-commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from airflow.models.dagrun import DagRun +from airflow.models.taskinstance import clear_task_instances +from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded +from airflow.utils.session import NEW_SESSION, provide_session + +if TYPE_CHECKING: +from sqlalchemy.orm import Session + + +@cli_utils.action_cli +@providers_configuration_loaded +@provide_session +def clear(args, *, session: Session = NEW_SESSION) -> None: +"""Clear the partition_key and partition_date of matching DagRuns.""" +has_range = args.start_date is not None or args.end_date is not None +if not args.run_id and not has_range: +raise SystemExit( +"Specify either --run-id for a single DagRun, or a partition_date range " +"via --start-date and/or --end-date." +) +if args.run_id and has_range: +raise SystemExit( +"--run-id cannot be combined with a partition_date range (--start-date / --end-date)." +) + +stmt = select(DagRun).where(DagRun.dag_id == args.dag_id) +if args.run_id: +stmt = stmt.where(DagRun.run_id == args.run_id) +else: +stmt = stmt.where(or_(DagRun.partition_key.is_not(None), DagRun.partition_date.is_not(None))) +if args.start_date is not None: +stmt = stmt.where(DagRun.partition_date >= args.start_date) +if args.end_date is not None: +stmt = stmt.where(DagRun.partition_date <= args.end_date) +stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id) + +runs = session.scalars(stmt).all() Review Comment: yep, chunk it to 500 TIs -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
Lee-W commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3282024973
##
airflow-core/src/airflow/cli/cli_config.py:
##
@@ -1018,6 +1018,36 @@ def string_lower_type(val):
ARG_ASSET_URI = Arg(("--uri",), default="", help="Asset URI")
ARG_ASSET_ALIAS = Arg(("--alias",), default=False, action="store_true",
help="Show asset alias")
+# partitions clear
+ARG_PARTITIONS_CLEAR_DAG_ID = Arg(
+("-d", "--dag-id"),
+help="The id of the Dag whose DagRun partition fields should be cleared",
+required=True,
+)
+ARG_PARTITIONS_CLEAR_START_DATE = Arg(
+("-s", "--start-date"),
+help="Only clear DagRuns whose partition_date is on or after this date",
+type=parsedate,
+)
+ARG_PARTITIONS_CLEAR_END_DATE = Arg(
+("-e", "--end-date"),
+help="Only clear DagRuns whose partition_date is on or before this date",
+type=parsedate,
+)
Review Comment:
sounds good. it now supports
1. --date 2025-01-01~2025-01-31
2. --start-date 2025-01-01 --end-date 2025-01-31
3. --partition-key
yes, syntax like `--date 2025-01-01~2025-01-31`, clear the fields till
`2025-01-31` not inclueing `2025-02-01`
Yes, `--date 2025-01-01~2025-01-31` (or `--end-date 2025-01-31`) clears the
fields through 2025-01-31, excluding 2025-02-01. tested by
`test_date_range_end_of_day_clamp`. (01-03 is still there)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
uranusjr commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3272996988
##
airflow-core/src/airflow/cli/cli_config.py:
##
@@ -1018,6 +1018,36 @@ def string_lower_type(val):
ARG_ASSET_URI = Arg(("--uri",), default="", help="Asset URI")
ARG_ASSET_ALIAS = Arg(("--alias",), default=False, action="store_true",
help="Show asset alias")
+# partitions clear
+ARG_PARTITIONS_CLEAR_DAG_ID = Arg(
+("-d", "--dag-id"),
+help="The id of the Dag whose DagRun partition fields should be cleared",
+required=True,
+)
+ARG_PARTITIONS_CLEAR_START_DATE = Arg(
+("-s", "--start-date"),
+help="Only clear DagRuns whose partition_date is on or after this date",
+type=parsedate,
+)
+ARG_PARTITIONS_CLEAR_END_DATE = Arg(
+("-e", "--end-date"),
+help="Only clear DagRuns whose partition_date is on or before this date",
+type=parsedate,
+)
Review Comment:
I wonder if these can be merged into e.g. `--date 2025-01-01~2025-01-31`.
By the way, if I want to clear the entire January, what should the end date
be? `2025-01-31` would only clear until midnight 31st (right?) but `2026-02-01`
would incorrectly include midnight 1st Februrary. This is likely not what users
expect.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
uranusjr commented on code in PR #66520: URL: https://github.com/apache/airflow/pull/66520#discussion_r3272965833 ## airflow-core/src/airflow/cli/commands/partition_command.py: ## @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Partitions sub-commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from airflow.models.dagrun import DagRun +from airflow.models.taskinstance import clear_task_instances +from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded +from airflow.utils.session import NEW_SESSION, provide_session + +if TYPE_CHECKING: +from sqlalchemy.orm import Session + + +@cli_utils.action_cli +@providers_configuration_loaded +@provide_session +def clear(args, *, session: Session = NEW_SESSION) -> None: +"""Clear the partition_key and partition_date of matching DagRuns.""" +has_range = args.start_date is not None or args.end_date is not None +if not args.run_id and not has_range: +raise SystemExit( +"Specify either --run-id for a single DagRun, or a partition_date range " +"via --start-date and/or --end-date." +) +if args.run_id and has_range: +raise SystemExit( +"--run-id cannot be combined with a partition_date range (--start-date / --end-date)." +) + +stmt = select(DagRun).where(DagRun.dag_id == args.dag_id) +if args.run_id: +stmt = stmt.where(DagRun.run_id == args.run_id) +else: +stmt = stmt.where(or_(DagRun.partition_key.is_not(None), DagRun.partition_date.is_not(None))) +if args.start_date is not None: +stmt = stmt.where(DagRun.partition_date >= args.start_date) +if args.end_date is not None: +stmt = stmt.where(DagRun.partition_date <= args.end_date) +stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id) + +runs = session.scalars(stmt).all() Review Comment: This can potentially be a lot. Maybe we should iterate over the query without calling `.all()` (which would minimize max memory usage) instead. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
uranusjr commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3272975197
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None
+if not args.run_id and not has_range:
+raise SystemExit(
+"Specify either --run-id for a single DagRun, or a partition_date
range "
+"via --start-date and/or --end-date."
+)
+if args.run_id and has_range:
+raise SystemExit(
+"--run-id cannot be combined with a partition_date range
(--start-date / --end-date)."
+)
+
+stmt = select(DagRun).where(DagRun.dag_id == args.dag_id)
+if args.run_id:
+stmt = stmt.where(DagRun.run_id == args.run_id)
+else:
+stmt = stmt.where(or_(DagRun.partition_key.is_not(None),
DagRun.partition_date.is_not(None)))
+if args.start_date is not None:
+stmt = stmt.where(DagRun.partition_date >= args.start_date)
+if args.end_date is not None:
+stmt = stmt.where(DagRun.partition_date <= args.end_date)
+stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id)
+
+runs = session.scalars(stmt).all()
+if not runs:
+print(f"No matching DagRuns found for dag_id={args.dag_id}.")
+return
+
+clear_tis = bool(args.clear_task_instances)
+cleared = 0
+runs_for_ti_clear: list[DagRun] = []
+for run in runs:
+fields_already_cleared = run.partition_key is None and
run.partition_date is None
+if fields_already_cleared and not clear_tis:
+print(f"DagRun {run.run_id}: already cleared, skipping.")
+continue
+if not fields_already_cleared:
+print(
+f"DagRun {run.run_id}: "
+f"partition_key={run.partition_key!r} -> None, "
+f"partition_date={run.partition_date.isoformat() if
run.partition_date else None} -> None"
+)
+if not args.dry_run:
+run.partition_key = None
+run.partition_date = None
+cleared += 1
+if clear_tis:
+runs_for_ti_clear.append(run)
+
+if clear_tis and runs_for_ti_clear:
+tis = [ti for run in runs_for_ti_clear for ti in run.task_instances]
Review Comment:
This can potentially be HUGE (some people have giant dags). This needs to
somehow be lazy, and perhaps chunked.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
uranusjr commented on code in PR #66520:
URL: https://github.com/apache/airflow/pull/66520#discussion_r3272975197
##
airflow-core/src/airflow/cli/commands/partition_command.py:
##
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Partitions sub-commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select
+
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import clear_task_instances
+from airflow.utils import cli as cli_utils
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.session import NEW_SESSION, provide_session
+
+if TYPE_CHECKING:
+from sqlalchemy.orm import Session
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+@provide_session
+def clear(args, *, session: Session = NEW_SESSION) -> None:
+"""Clear the partition_key and partition_date of matching DagRuns."""
+has_range = args.start_date is not None or args.end_date is not None
+if not args.run_id and not has_range:
+raise SystemExit(
+"Specify either --run-id for a single DagRun, or a partition_date
range "
+"via --start-date and/or --end-date."
+)
+if args.run_id and has_range:
+raise SystemExit(
+"--run-id cannot be combined with a partition_date range
(--start-date / --end-date)."
+)
+
+stmt = select(DagRun).where(DagRun.dag_id == args.dag_id)
+if args.run_id:
+stmt = stmt.where(DagRun.run_id == args.run_id)
+else:
+stmt = stmt.where(or_(DagRun.partition_key.is_not(None),
DagRun.partition_date.is_not(None)))
+if args.start_date is not None:
+stmt = stmt.where(DagRun.partition_date >= args.start_date)
+if args.end_date is not None:
+stmt = stmt.where(DagRun.partition_date <= args.end_date)
+stmt = stmt.order_by(DagRun.partition_date, DagRun.run_id)
+
+runs = session.scalars(stmt).all()
+if not runs:
+print(f"No matching DagRuns found for dag_id={args.dag_id}.")
+return
+
+clear_tis = bool(args.clear_task_instances)
+cleared = 0
+runs_for_ti_clear: list[DagRun] = []
+for run in runs:
+fields_already_cleared = run.partition_key is None and
run.partition_date is None
+if fields_already_cleared and not clear_tis:
+print(f"DagRun {run.run_id}: already cleared, skipping.")
+continue
+if not fields_already_cleared:
+print(
+f"DagRun {run.run_id}: "
+f"partition_key={run.partition_key!r} -> None, "
+f"partition_date={run.partition_date.isoformat() if
run.partition_date else None} -> None"
+)
+if not args.dry_run:
+run.partition_key = None
+run.partition_date = None
+cleared += 1
+if clear_tis:
+runs_for_ti_clear.append(run)
+
+if clear_tis and runs_for_ti_clear:
+tis = [ti for run in runs_for_ti_clear for ti in run.task_instances]
Review Comment:
This can potentially be HUGE (some people have giant dags). This needs to
somehow be lazy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
uranusjr commented on code in PR #66520: URL: https://github.com/apache/airflow/pull/66520#discussion_r3272954545 ## airflow-core/src/airflow/cli/commands/partition_command.py: ## @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Partitions sub-commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from airflow.models.dagrun import DagRun +from airflow.models.taskinstance import clear_task_instances +from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded +from airflow.utils.session import NEW_SESSION, provide_session + +if TYPE_CHECKING: +from sqlalchemy.orm import Session + + +@cli_utils.action_cli +@providers_configuration_loaded +@provide_session +def clear(args, *, session: Session = NEW_SESSION) -> None: +"""Clear the partition_key and partition_date of matching DagRuns.""" +has_range = args.start_date is not None or args.end_date is not None +if not args.run_id and not has_range: +raise SystemExit( +"Specify either --run-id for a single DagRun, or a partition_date range " +"via --start-date and/or --end-date." +) +if args.run_id and has_range: +raise SystemExit( +"--run-id cannot be combined with a partition_date range (--start-date / --end-date)." +) Review Comment: I wonder if we should allow clearing by partition_key (and or partition_date?) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] feat(cli): add `partitions clear` to reset DagRun partition fields [airflow]
potiuk commented on PR #66520: URL: https://github.com/apache/airflow/pull/66520#issuecomment-4413599701 Rebase is needed I guess. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
