This is an automated email from the ASF dual-hosted git repository.
ashb pushed a change to branch provider-log-ti-id
in repository https://gitbox.apache.org/repos/asf/airflow.git
from a8198a9f4cb Look up task logs by task try UUID
add 592cf3ffdd2 Use the current user in the HITL detail tests instead of
hardcoding (#74184)
add 8331bb485c5 Read the msgraph connection extra with
get_async_extra_dejson (#74161)
add b156582855c Fix OpenSearch provider log pagination (#73947)
add c8b136358ce Allow OpensearchTaskHandler and OpensearchRemoteLogIO to
take empty username and password (#71692)
add cc94831e213 Fix docstring cross-references that Sphinx 9 cannot
resolve (#74169)
add b58cfa912a9 Add an API endpoint and UI tab comparing two stored Dag
versions (#73322)
add d1d5b1af26e Adapt GCP BigtableHook to run in private cloud (#73181)
add b3674d43be0 `SparkSubmitHook`: Mask `_mask_cmd` secrets in linear time
(#71711)
add 8d5814c09d5 Bump the react group across 1 directory with 4 updates
(#73334)
add 7e448f70af5 Use async database access for Variable key listing (#73407)
add 523567e47d0 Add tests for Google Go module subprocess utility (#73268)
add 4a766db2b03 Bump the typescript group across 1 directory with 4
updates (#72994)
add 7fad35d9c99 Add deadline alert coverage to Dag endpoint tests (#72296)
add 6f5d5d06b17 Align standard provider tests with source paths (#73267)
add c2f600f6ec6 Add airflowctl tasks state command (#71206)
add 86b8a521837 Bind the AWS auth manager SAML response to the browser
that started the login (#73698)
add cefc0c3239d Bump the java-sdk-dependency-updates group across 1
directory with 3 updates (#74108)
add 2a274ab3c93 Refuse nested secret ids looked up with no team in
multi-team mode (#73789)
add 8055f6a8717 Skip creating an empty hook lineage collector for
OpenLineage events (#74190)
add 38fefc1ca95 Fix invalid cron expression in OpenLineage system test Dag
(#74187)
add 27f827fcdce Honor the polling interval for deferred Kubernetes jobs
(#74143)
add 3bd4df6b2da Fix Snowflake SQL API wait_for_query timing out on
finished queries (#73761)
add f74a685978c Use configured default queue for Celery executor callbacks
(#73552)
add ad3bb1f676c Replace coordinator artifact roots with
task_handler_bundle_name (#73970)
add 1c3f78e1d57 Validate a serialized Dag and fill its unset settings from
the Airflow config (#74041)
add a44fd6d1f31 Allow explicit runs to drain a paused Dag (#73763)
add 40832901c59 Improve LocalExecutor bookkeeping: correctly add tasks to
the `running` list (#74142)
add 5487c080bea Add Azure Blob Storage support to the DataFusion object
storage layer (#73374)
add 206486b18f6 Merge branch 'main' into provider-log-ti-id
No new revisions were added by this update.
Summary of changes:
.agents/skills/airflow-java-sdk/SKILL.md | 11 +-
airflow-core/adr/lang-sdk/0004-dag-parsing.md | 12 +-
.../0013-persisted-task-handler-bindings.md | 2 +-
.../authoring-and-scheduling/language-sdks/go.rst | 93 ++--
.../language-sdks/index.rst | 6 +-
.../language-sdks/java.rst | 112 ++--
.../language-sdks/typescript.rst | 85 +--
airflow-core/docs/core-concepts/backfill.rst | 7 +-
airflow-core/docs/core-concepts/dags.rst | 13 +
airflow-core/docs/security/api_permissions_ref.rst | 4 +
airflow-core/docs/security/security_model.rst | 9 +
airflow-core/newsfragments/72407.significant.rst | 10 +-
airflow-core/newsfragments/73970.significant.rst | 5 +
.../api_fastapi/core_api/datamodels/backfills.py | 8 +
.../api_fastapi/core_api/datamodels/dag_run.py | 7 +
.../core_api/datamodels/dag_versions.py | 158 ++++++
.../core_api/openapi/v2-rest-api-generated.yaml | 301 +++++++++++
.../api_fastapi/core_api/routes/public/assets.py | 10 +
.../core_api/routes/public/backfills.py | 17 +-
.../api_fastapi/core_api/routes/public/dag_run.py | 11 +
.../core_api/routes/public/dag_versions.py | 98 +++-
.../api_fastapi/execution_api/routes/variables.py | 10 +-
.../src/airflow/config_templates/config.yml | 17 +-
.../src/airflow/executors/local_executor.py | 75 ++-
airflow-core/src/airflow/models/backfill.py | 3 +
airflow-core/src/airflow/models/dag.py | 16 +
.../src/airflow/serialization/dag_version_diff.py | 3 +-
airflow-core/src/airflow/serialization/schema.json | 13 +-
.../airflow/serialization/serialized_objects.py | 96 ++++
.../src/airflow/ui/openapi-gen/queries/common.ts | 18 +
.../ui/openapi-gen/queries/ensureQueryData.ts | 38 ++
.../src/airflow/ui/openapi-gen/queries/prefetch.ts | 38 ++
.../src/airflow/ui/openapi-gen/queries/queries.ts | 38 ++
.../src/airflow/ui/openapi-gen/queries/suspense.ts | 38 ++
.../airflow/ui/openapi-gen/requests/schemas.gen.ts | 206 +++++++
.../ui/openapi-gen/requests/services.gen.ts | 72 ++-
.../airflow/ui/openapi-gen/requests/types.gen.ts | 155 ++++++
airflow-core/src/airflow/ui/package.json | 16 +-
airflow-core/src/airflow/ui/pnpm-lock.yaml | 591 +++++++++------------
.../ui/public/i18n/locales/ar/components.json | 4 +-
.../ui/public/i18n/locales/ca/components.json | 4 +-
.../ui/public/i18n/locales/de/components.json | 4 +-
.../ui/public/i18n/locales/el/components.json | 4 +-
.../ui/public/i18n/locales/en/components.json | 16 +-
.../src/airflow/ui/public/i18n/locales/en/dag.json | 63 +++
.../ui/public/i18n/locales/es/components.json | 4 +-
.../ui/public/i18n/locales/fr/components.json | 4 +-
.../ui/public/i18n/locales/he/components.json | 4 +-
.../ui/public/i18n/locales/hi/components.json | 4 +-
.../ui/public/i18n/locales/hu/components.json | 4 +-
.../ui/public/i18n/locales/it/components.json | 4 +-
.../ui/public/i18n/locales/ja/components.json | 4 +-
.../ui/public/i18n/locales/ko/components.json | 4 +-
.../ui/public/i18n/locales/nl/components.json | 4 +-
.../ui/public/i18n/locales/pl/components.json | 4 +-
.../ui/public/i18n/locales/pt/components.json | 4 +-
.../ui/public/i18n/locales/ru/components.json | 4 +-
.../ui/public/i18n/locales/th/components.json | 4 +-
.../ui/public/i18n/locales/tr/components.json | 4 +-
.../ui/public/i18n/locales/zh-CN/components.json | 4 +-
.../ui/public/i18n/locales/zh-TW/components.json | 4 +-
.../components/DagActions/RunBackfillForm.test.tsx | 96 +++-
.../src/components/DagActions/RunBackfillForm.tsx | 35 +-
.../TriggerDag/PausedDagOptions.test.tsx | 250 +++++++++
.../src/components/TriggerDag/PausedDagOptions.tsx | 123 +++++
.../src/components/TriggerDag/TriggerDAGButton.tsx | 4 -
.../components/TriggerDag/TriggerDAGForm.test.tsx | 78 ++-
.../src/components/TriggerDag/TriggerDAGForm.tsx | 32 +-
.../components/TriggerDag/TriggerDAGModal.test.tsx | 238 +++++++++
.../src/components/TriggerDag/TriggerDAGModal.tsx | 21 +-
.../airflow/ui/src/components/TriggerDag/types.ts | 3 +
.../src/components/VersionCompareSelect.test.tsx | 82 +++
.../Code => components}/VersionCompareSelect.tsx | 4 +-
.../src/airflow/ui/src/constants/searchParams.ts | 3 +
airflow-core/src/airflow/ui/src/constants/tab.ts | 1 +
.../ui/src/layouts/Details/DetailsLayout.tsx | 1 -
.../src/pages/Asset/CreateAssetEventModal.test.tsx | 161 +++++-
.../ui/src/pages/Asset/CreateAssetEventModal.tsx | 20 +-
.../src/airflow/ui/src/pages/Dag/Code/Code.tsx | 2 +-
airflow-core/src/airflow/ui/src/pages/Dag/Dag.tsx | 3 +-
.../pages/Dag/Versions/VersionDiff.i18n.test.tsx | 83 +++
.../ui/src/pages/Dag/Versions/VersionDiff.test.tsx | 275 ++++++++++
.../ui/src/pages/Dag/Versions/VersionDiff.tsx | 197 +++++++
.../ui/src/pages/Dag/Versions/Versions.test.tsx | 227 ++++++++
.../airflow/ui/src/pages/Dag/Versions/Versions.tsx | 182 +++++++
.../Details/Gantt => pages/Dag/Versions}/index.ts | 3 +-
.../ui/src/pages/DagsList/DagCardActions.tsx | 1 -
.../src/airflow/ui/src/pages/DagsList/DagsList.tsx | 1 -
.../ui/src/queries/useCreateBackfill.test.tsx | 99 ++++
.../airflow/ui/src/queries/useCreateBackfill.ts | 39 +-
.../src/airflow/ui/src/queries/useTrigger.test.tsx | 98 ++++
.../src/airflow/ui/src/queries/useTrigger.ts | 31 +-
airflow-core/src/airflow/ui/src/router.tsx | 6 +
.../tests/unit/always/test_project_structure.py | 3 -
airflow-core/tests/unit/api_fastapi/conftest.py | 13 +
.../core_api/routes/public/test_assets.py | 48 +-
.../core_api/routes/public/test_backfills.py | 72 ++-
.../core_api/routes/public/test_dag_run.py | 75 ++-
.../core_api/routes/public/test_dag_versions.py | 296 +++++++++++
.../core_api/routes/public/test_dags.py | 175 +++++-
.../core_api/routes/public/test_hitl.py | 3 +-
.../tests/unit/executors/test_local_executor.py | 455 +++++++++++++++-
.../executors/test_local_executor_check_workers.py | 5 +-
airflow-core/tests/unit/jobs/test_scheduler_job.py | 9 +-
airflow-core/tests/unit/models/test_backfill.py | 133 ++++-
airflow-core/tests/unit/models/test_dag.py | 40 ++
airflow-core/tests/unit/models/test_dagrun.py | 6 +-
.../tests/unit/models/test_serialized_dag.py | 4 +-
.../unit/serialization/test_dag_serialization.py | 215 ++++++++
.../unit/serialization/test_dag_version_diff.py | 1 -
.../airflowctl_tests/test_airflowctl_commands.py | 2 +
airflow-ctl/docs/images/command_hashes.txt | 2 +-
airflow-ctl/docs/images/output_tasks.svg | 78 +--
.../src/airflowctl/api/datamodels/generated.py | 178 +++++++
airflow-ctl/src/airflowctl/api/operations.py | 7 +-
airflow-ctl/src/airflowctl/ctl/cli_config.py | 17 +
.../src/airflowctl/ctl/commands/task_command.py | 53 +-
.../tests/airflow_ctl/api/test_operations.py | 54 +-
.../airflow_ctl/ctl/commands/test_task_command.py | 275 +++++++++-
.../tests/airflow_e2e_tests/conftest.py | 56 +-
contributing-docs/30_new_language_sdk.rst | 8 +-
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
dev/breeze/tests/test_selective_checks.py | 15 +-
docs/spelling_wordlist.txt | 3 +
generated/known_airflow_exceptions.txt | 2 +-
go-sdk/README.md | 39 +-
go-sdk/example/bundle/Justfile | 5 +-
go-sdk/schema/dag-schema.json | 13 +-
java-sdk/README.md | 30 +-
java-sdk/buildSrc/build.gradle.kts | 2 +-
java-sdk/gradle/verification-metadata.xml | 410 +++++++-------
.../apache/airflow/sdk/plugin/AirflowSdkPlugin.kt | 4 +-
java-sdk/scala_spark_example/README.md | 6 +-
java-sdk/sdk/build.gradle.kts | 4 +-
java-sdk/sdk/schema/dag-schema.json | 13 +-
kubernetes-tests/lang_sdk/README.md | 14 +-
kubernetes-tests/lang_sdk/config/values.yaml | 14 +-
.../lang_sdk/pod_templates/lang_sdk_golang.yaml | 3 +-
.../lang_sdk/pod_templates/lang_sdk_java.yaml | 3 +-
kubernetes-tests/lang_sdk/stage_artifacts.py | 8 +-
providers/akeyless/docs/changelog.rst | 7 +
.../airflow/providers/akeyless/secrets/akeyless.py | 28 +-
.../tests/unit/akeyless/secrets/test_akeyless.py | 27 +-
.../docs/auth-manager/setup/identity-center.rst | 36 ++
providers/amazon/provider.yaml | 15 +
.../providers/amazon/aws/auth_manager/constants.py | 1 +
.../amazon/aws/auth_manager/routes/login.py | 200 ++++++-
.../providers/amazon/aws/executors/batch/utils.py | 6 +-
.../airflow/providers/amazon/get_provider_info.py | 7 +
.../amazon/aws/auth_manager/routes/test_login.py | 359 +++++++++++--
.../providers/apache/spark/hooks/spark_submit.py | 98 +++-
.../unit/apache/spark/hooks/test_spark_submit.py | 118 ++++
.../providers/celery/executors/celery_executor.py | 4 +-
.../unit/celery/executors/test_celery_executor.py | 29 +-
.../providers/cncf/kubernetes/triggers/job.py | 4 +-
.../unit/cncf/kubernetes/triggers/test_job.py | 13 +-
.../ai/docs/operators/llm_schema_compare.rst | 2 +-
providers/common/ai/docs/operators/llm_sql.rst | 2 +-
providers/common/ai/docs/toolsets/datafusion.rst | 2 +-
providers/common/sql/docs/operators.rst | 60 ++-
.../sql/src/airflow/providers/common/sql/config.py | 3 +
.../providers/common/sql/datafusion/engine.py | 185 +++++++
.../sql/datafusion/object_storage_provider.py | 36 +-
.../common/sql/example_dags/example_analytics.py | 13 +
.../unit/common/sql/datafusion/test_engine.py | 447 ++++++++++++++++
.../sql/datafusion/test_object_storage_provider.py | 48 ++
.../sql/tests/unit/common/sql/test_config.py | 1 +
providers/fab/docs/auth-manager/access-control.rst | 4 +
.../providers/google/cloud/hooks/bigtable.py | 24 +
.../providers/google/cloud/hooks/compute.py | 3 -
.../google/cloud/hooks/vertex_ai/auto_ml.py | 2 +-
.../tests/unit/google/cloud/hooks/test_bigtable.py | 26 +
.../tests/unit/google/test_go_module_utils.py | 73 +++
providers/hashicorp/docs/changelog.rst | 7 +
.../airflow/providers/hashicorp/secrets/vault.py | 31 +-
.../tests/unit/hashicorp/secrets/test_vault.py | 55 ++
providers/microsoft/azure/pyproject.toml | 2 +-
.../providers/microsoft/azure/hooks/msgraph.py | 13 +-
.../unit/microsoft/azure/hooks/test_msgraph.py | 17 +
.../providers/openlineage/extractors/manager.py | 31 ++
...schedule_multiple_cron_trigger_timetable_dag.py | 2 +-
...hedule_multiple_cron_trigger_timetable_dag.json | 12 +-
.../unit/openlineage/extractors/test_manager.py | 71 ++-
.../providers/opensearch/log/os_task_handler.py | 32 +-
.../opensearch/log/test_os_remote_log_io.py | 18 +
.../unit/opensearch/log/test_os_task_handler.py | 102 +++-
.../providers/snowflake/hooks/snowflake_sql_api.py | 12 +-
.../unit/snowflake/hooks/test_snowflake_sql_api.py | 35 +-
.../tests/unit/standard/operators/test_empty.py | 56 ++
...ternal_task_sensor.py => test_external_task.py} | 32 ++
.../ci/lang_sdk_serialization/serialize_python.py | 23 +-
.../prek/check_contextmanager_class_decorators.py | 1 +
.../test_check_contextmanager_class_decorators.py | 80 +++
task-sdk/docs/deferred-vs-async-operators.rst | 2 +-
task-sdk/docs/executable-bundle-spec.rst | 31 +-
task-sdk/docs/ts-bundle-spec.rst | 26 +-
.../airflow/sdk/coordinators/_bundle_metadata.py | 45 +-
.../src/airflow/sdk/coordinators/_subprocess.py | 131 +----
.../sdk/coordinators/executable/coordinator.py | 30 +-
.../airflow/sdk/coordinators/java/coordinator.py | 34 +-
.../airflow/sdk/coordinators/node/coordinator.py | 31 +-
.../src/airflow/sdk/execution_time/coordinator.py | 21 +-
.../coordinators/executable/test_coordinator.py | 73 +--
.../task_sdk/coordinators/java/test_coordinator.py | 96 ++--
.../task_sdk/coordinators/node/test_coordinator.py | 45 +-
.../tests/task_sdk/coordinators/test_subprocess.py | 194 ++-----
.../task_sdk/execution_time/test_coordinator.py | 85 ++-
ts-sdk/README.md | 23 +-
ts-sdk/example/README.md | 14 +-
ts-sdk/schema/dag-schema.json | 13 +-
210 files changed, 9508 insertions(+), 1756 deletions(-)
create mode 100644 airflow-core/newsfragments/73970.significant.rst
create mode 100644
airflow-core/src/airflow/ui/src/components/TriggerDag/PausedDagOptions.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/components/TriggerDag/PausedDagOptions.tsx
create mode 100644
airflow-core/src/airflow/ui/src/components/TriggerDag/TriggerDAGModal.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/components/VersionCompareSelect.test.tsx
rename airflow-core/src/airflow/ui/src/{pages/Dag/Code =>
components}/VersionCompareSelect.tsx (97%)
create mode 100644
airflow-core/src/airflow/ui/src/pages/Dag/Versions/VersionDiff.i18n.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/pages/Dag/Versions/VersionDiff.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/pages/Dag/Versions/VersionDiff.tsx
create mode 100644
airflow-core/src/airflow/ui/src/pages/Dag/Versions/Versions.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/pages/Dag/Versions/Versions.tsx
copy airflow-core/src/airflow/ui/src/{layouts/Details/Gantt =>
pages/Dag/Versions}/index.ts (95%)
create mode 100644
airflow-core/src/airflow/ui/src/queries/useCreateBackfill.test.tsx
create mode 100644 airflow-core/src/airflow/ui/src/queries/useTrigger.test.tsx
create mode 100644 providers/google/tests/unit/google/test_go_module_utils.py
create mode 100644
providers/standard/tests/unit/standard/operators/test_empty.py
rename
providers/standard/tests/unit/standard/sensors/{test_external_task_sensor.py =>
test_external_task.py} (98%)
create mode 100644
scripts/tests/ci/prek/test_check_contextmanager_class_decorators.py
