This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to tag nightly-master in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! *** from 11eb649 (commit) to d305876 (commit) from 11eb649 Fix to make y-axis of Tries chart visible (#10071) add 27e637f Bugfix: Error in SSHOperator when command is None (#11361) add 7541c88 Allways use Airlfow db in FAB (#11364) add 7f674c6 Use only-if-needed upgrade strategy for PRs (#11363) add 5605d10 Fix DagBag bug when a dag has invalid schedule_interval (#11344) add 422b61a Adding ElastiCache Hook for creating, describing and deleting replication groups (#8701) add 8baf657 Fix regression in DataflowTemplatedJobStartOperator (#11167) add d2754ef Strict type check for Microsoft (#11359) add fe0bf6e Reduce "start-up" time for tasks in CeleryExecutor (#11372) add ff1a2aa Set start_date, end_date & duration for tasks failing without DagRun (#11358) add eb5fea7 Replace nuke with useful information on error page (#11346) add 49aad02 Users can specify sub-secrets and paths k8spodop (#11369) add 29a145c Add capability of adding service account annotations to Helm Chart (#11387) add e198077 Add pypirc initialization (#11386) add 73b9163 Fully support running more than one scheduler concurrently (#10956) add d752575 Revert "Revert "Adds --install-wheels flag to breeze command line (#11317)" (#11348)" (#11356) add 7b0a2f5 Replaced basestring with str in the Exasol hook (#11360) add 298052f [airflow/providers/cncf/kubernetes] correct hook methods name (#11008) add 3164025 Fix airflow_local_settings.py showing up as directory (#10999) add 39fc961 Fix case of JavaScript. (#10957) add 6fe020e Add tests for Custom cluster policy (#11381) add b7404b0 KubernetesPodOperator should retry log tailing in case of interruption (#11325) add 8640fb6 fix tests (#11368) add 0497390 Constraints and PIP packages can be installed from local sources (#11382) add 401a579 Push and schedule duplicates are not cancelled. (#11397) add 7507183 Remove redundant parentheses from Python files (#10967) add a34f5ee Fixes automated upgrade to latest constraints. (#11399) add 076fe88 Fixes cancelling of too many workflows. (#11403) add 0620aaa Fix spelling (#11401) add 7959df9 Fix spelling (#11404) add f9dddd5 Workarounds "unknown blob" issue by introducing retries (#11411) add 45d33db Add capability of customising PyPI sources (#11385) add 9416bed Moving the test to quarantine. (#11405) add bd204bb Optionally set null marker in csv exports in BaseSQLToGCSOperator (#11409) add 4de8f85 Fixes SHA used for cancel-workflow-action (#11400) add 5bc5994 Split tests to more sub-types (#11402) add 686e0ee Fix incorrect typing, remove hardcoded argument values and improve code in AzureContainerInstancesOperator (#11408) add ce2f19d Fix constraints generation script (#11412) add b786327 Fix spelling in CeleryExecutor (#11407) add 7e40a32 Add more info about dag_concurrency (#11300) add 42a23d1 Update MySQLToS3Operator's s3_bucket to template_fields (#10778) add c3e3405 Change prefix of AwsDynamoDB hook module (#11209) add 1845cd1 Strict type check for google ads and cloud hooks (#11390) add 9a01ce0 Mutual SSL added in PGBouncer configuration in the Chart (#11384) add fb8b1e5 Merge Airflow and Backport Packages preparation instructions (#11310) add 2c7e343 Fix syntax highlightling for concurrency in configurations doc (#11438) add a155db1 Fix typo in airflow/utils/dag_processing.py (#11445) add 369bbf0 Selective tests - depends on files changed in the commit. (#11417) add eaaad31 Fix correct Sphinx return type for DagFileProcessorProcess.result (#11444) add d8d13fa Use augmented assignment (#11449) add d305876 Remove redundant None provided as default to dict.get() (#11448) No new revisions were added by this update. Summary of changes: .dockerignore | 5 + .github/workflows/build-images-workflow-run.yml | 55 +- .github/workflows/ci.yml | 144 +- .github/workflows/codeql-cancel.yml | 3 +- .gitignore | 2 + BREEZE.rst | 77 +- CONTRIBUTING.rst | 4 +- Dockerfile | 32 +- Dockerfile.ci | 5 +- IMAGES.rst | 25 +- TESTING.rst | 57 +- airflow/api/common/experimental/trigger_dag.py | 1 + .../api_connexion/endpoints/connection_endpoint.py | 2 +- airflow/api_connexion/endpoints/log_endpoint.py | 2 +- airflow/api_connexion/exceptions.py | 2 +- airflow/api_connexion/security.py | 2 +- airflow/cli/cli_parser.py | 2 +- airflow/cli/commands/scheduler_command.py | 1 - airflow/config_templates/config.yml | 45 + airflow/config_templates/default_airflow.cfg | 29 + .../config_templates/default_webserver_config.py | 5 - airflow/contrib/hooks/aws_dynamodb_hook.py | 6 +- airflow/executors/base_executor.py | 33 +- airflow/executors/celery_executor.py | 41 + airflow/executors/celery_kubernetes_executor.py | 22 +- airflow/jobs/scheduler_job.py | 1225 +++++----- airflow/kubernetes/pod_launcher.py | 56 +- airflow/kubernetes/secret.py | 17 +- ...606e2_add_scheduling_decision_to_dagrun_and_.py | 88 + airflow/models/dag.py | 301 ++- airflow/models/dagbag.py | 38 +- airflow/models/dagrun.py | 124 +- airflow/models/pool.py | 19 +- airflow/models/serialized_dag.py | 34 +- airflow/models/taskinstance.py | 35 +- airflow/operators/python.py | 2 +- .../cloud => amazon/aws}/ADDITIONAL_INFO.md | 11 +- airflow/providers/amazon/aws/hooks/aws_dynamodb.py | 56 +- .../aws/hooks/{aws_dynamodb.py => dynamodb.py} | 0 .../aws/hooks/elasticache_replication_group.py | 282 +++ .../providers/amazon/aws/hooks/step_function.py | 2 +- .../amazon/aws/transfers/dynamodb_to_s3.py | 2 +- .../amazon/aws/transfers/hive_to_dynamodb.py | 2 +- .../providers/amazon/aws/transfers/mysql_to_s3.py | 7 +- .../providers/apache/spark/hooks/spark_submit.py | 6 +- .../providers/cncf/kubernetes/hooks/kubernetes.py | 12 +- .../cncf/kubernetes/operators/kubernetes_pod.py | 24 +- .../cncf/kubernetes/operators/spark_kubernetes.py | 2 +- .../cncf/kubernetes/sensors/spark_kubernetes.py | 2 +- airflow/providers/exasol/hooks/exasol.py | 5 +- airflow/providers/google/ads/hooks/ads.py | 5 +- airflow/providers/google/ads/operators/ads.py | 4 +- .../providers/google/ads/transfers/ads_to_gcs.py | 4 +- .../_internal_client/secret_manager_client.py | 2 +- airflow/providers/google/cloud/hooks/bigquery.py | 53 +- .../providers/google/cloud/hooks/cloud_build.py | 6 +- .../google/cloud/hooks/cloud_memorystore.py | 6 +- airflow/providers/google/cloud/hooks/cloud_sql.py | 10 +- .../cloud/hooks/cloud_storage_transfer_service.py | 38 +- airflow/providers/google/cloud/hooks/compute.py | 18 +- .../providers/google/cloud/hooks/datacatalog.py | 56 +- airflow/providers/google/cloud/hooks/dataflow.py | 79 +- airflow/providers/google/cloud/hooks/datafusion.py | 4 +- airflow/providers/google/cloud/hooks/dataproc.py | 8 +- airflow/providers/google/cloud/hooks/datastore.py | 36 +- airflow/providers/google/cloud/hooks/functions.py | 12 +- airflow/providers/google/cloud/hooks/gcs.py | 113 +- airflow/providers/google/cloud/hooks/gdm.py | 6 +- .../providers/google/cloud/hooks/life_sciences.py | 8 +- airflow/providers/google/cloud/hooks/mlengine.py | 26 +- .../google/cloud/hooks/natural_language.py | 16 +- airflow/providers/google/cloud/hooks/pubsub.py | 12 +- .../providers/google/cloud/hooks/stackdriver.py | 4 +- airflow/providers/google/cloud/hooks/tasks.py | 4 +- airflow/providers/google/cloud/hooks/translate.py | 4 +- airflow/providers/google/cloud/hooks/vision.py | 36 +- .../providers/google/cloud/operators/dataflow.py | 13 +- .../providers/google/cloud/operators/functions.py | 4 +- .../providers/google/cloud/operators/mlengine.py | 6 +- .../providers/google/cloud/transfers/sql_to_gcs.py | 6 + .../google/suite/transfers/gcs_to_sheets.py | 2 +- .../microsoft/azure/log/wasb_task_handler.py | 22 +- .../microsoft/azure/operators/adls_list.py | 4 +- airflow/providers/microsoft/azure/operators/adx.py | 6 +- .../microsoft/azure/operators/azure_batch.py | 4 +- .../azure/operators/azure_container_instances.py | 89 +- .../microsoft/azure/operators/azure_cosmos.py | 3 +- .../microsoft/azure/operators/wasb_delete_blob.py | 4 +- .../microsoft/azure/secrets/azure_key_vault.py | 4 +- .../microsoft/azure/sensors/azure_cosmos.py | 3 +- airflow/providers/microsoft/azure/sensors/wasb.py | 8 +- .../microsoft/azure/transfers/azure_blob_to_gcs.py | 2 +- .../microsoft/azure/transfers/file_to_wasb.py | 4 +- .../microsoft/azure/transfers/local_to_adls.py | 2 +- .../azure/transfers/oracle_to_azure_data_lake.py | 4 +- airflow/providers/microsoft/mssql/hooks/mssql.py | 18 +- .../providers/microsoft/mssql/operators/mssql.py | 14 +- airflow/providers/microsoft/winrm/hooks/winrm.py | 43 +- .../providers/microsoft/winrm/operators/winrm.py | 28 +- airflow/providers/oracle/hooks/oracle.py | 8 +- airflow/providers/postgres/hooks/postgres.py | 2 +- airflow/providers/redis/hooks/redis.py | 2 +- airflow/providers/salesforce/hooks/salesforce.py | 2 +- airflow/providers/sendgrid/utils/emailer.py | 2 +- airflow/providers/sftp/hooks/sftp.py | 2 +- airflow/providers/snowflake/hooks/snowflake.py | 2 +- airflow/providers/ssh/operators/ssh.py | 2 +- airflow/sensors/smart_sensor_operator.py | 12 +- airflow/settings.py | 14 + airflow/stats.py | 58 + airflow/ti_deps/deps/not_previously_skipped_dep.py | 2 +- airflow/utils/asciiart.py | 59 - airflow/utils/callback_requests.py | 101 + airflow/utils/dag_processing.py | 166 +- airflow/utils/decorators.py | 2 +- airflow/utils/sqlalchemy.py | 124 ++ airflow/www/app.py | 1 + airflow/www/templates/airflow/traceback.html | 9 +- airflow/www/views.py | 21 +- backport_packages/README.md | 173 ++ backport_packages/remove_old_releases.py | 96 + breeze | 102 +- breeze-complete | 16 +- chart/README.md | 4 +- chart/templates/_helpers.yaml | 18 + chart/templates/configmap.yaml | 2 +- .../templates/pgbouncer/pgbouncer-deployment.yaml | 22 + .../scheduler/scheduler-serviceaccount.yaml | 6 + ...ret.yaml => pgbouncer-certificates-secret.yaml} | 16 +- .../webserver/webserver-serviceaccount.yaml | 6 + chart/templates/workers/worker-serviceaccount.yaml | 8 +- chart/values.schema.json | 39 + chart/values.yaml | 17 +- dev/README.md | 1332 ++++++++--- docker-context-files/README.md | 31 + docs/logging-monitoring/metrics.rst | 15 +- docs/operators-and-hooks-ref.rst | 8 +- docs/production-deployment.rst | 79 + docs/spelling_wordlist.txt | 1 + docs/templates/layout.html | 2 +- kubernetes_tests/test_kubernetes_pod_operator.py | 41 +- scripts/ci/docker-compose/base.yml | 8 +- scripts/ci/libraries/_build_images.sh | 22 +- scripts/ci/libraries/_docker.env | 5 + scripts/ci/libraries/_initialization.sh | 46 +- scripts/ci/libraries/_parameters.sh | 28 +- scripts/ci/libraries/_push_pull_remove_images.sh | 67 +- scripts/ci/selective_tests.sh | 350 +++ scripts/ci/testing/ci_run_airflow_testing.sh | 9 +- .../ci/tools/ci_check_if_tests_should_be_run.sh | 56 - scripts/ci/tools/ci_count_changed_files.sh | 63 - scripts/in_container/entrypoint_ci.sh | 148 +- scripts/in_container/run_ci_tests.sh | 57 +- scripts/in_container/run_generate_constraints.sh | 5 +- tests/api/client/test_local_client.py | 14 +- tests/api/common/experimental/test_trigger_dag.py | 4 + tests/api_connexion/schemas/test_error_schema.py | 2 +- tests/bats/test_breeze_complete.bats | 8 + tests/cli/commands/test_cheat_sheet_command.py | 2 +- tests/cli/commands/test_connection_command.py | 22 +- tests/cli/test_cli_parser.py | 8 +- tests/{ => core}/test_config_templates.py | 0 tests/{ => core}/test_configuration.py | 0 tests/{ => core}/test_core.py | 0 tests/{ => core}/test_core_to_contrib.py | 0 tests/{ => core}/test_example_dags.py | 0 tests/{ => core}/test_example_dags_system.py | 0 .../test_impersonation_tests.py} | 0 tests/{ => core}/test_local_settings.py | 24 + tests/{ => core}/test_logging_config.py | 0 tests/{ => core}/test_project_structure.py | 38 +- tests/{ => core}/test_sentry.py | 0 tests/{ => core}/test_sqlalchemy_config.py | 3 +- tests/{ => core}/test_stats.py | 18 +- tests/dags/test_zip.zip | Bin 2204 -> 2977 bytes tests/deprecated_classes.py | 2 +- tests/executors/test_celery_executor.py | 22 +- tests/jobs/test_backfill_job.py | 24 +- tests/jobs/test_local_task_job.py | 3 +- tests/jobs/test_scheduler_job.py | 2355 ++++++++++---------- tests/kubernetes/models/test_secret.py | 23 + tests/kubernetes/test_pod_launcher.py | 42 +- tests/models/test_dag.py | 566 +++-- tests/models/test_dagbag.py | 1 + tests/models/test_dagrun.py | 131 +- .../{test_aws_dynamodb.py => test_dynamodb.py} | 2 +- .../hooks/test_elasticache_replication_group.py | 210 ++ tests/providers/amazon/aws/hooks/test_s3.py | 4 +- .../amazon/aws/hooks/test_step_function.py | 4 +- .../amazon/aws/transfers/test_hive_to_dynamodb.py | 2 +- .../providers/google/cloud/hooks/test_dataflow.py | 47 + .../providers/google/cloud/hooks/test_mlengine.py | 4 +- .../google/cloud/operators/test_dataflow.py | 2 + .../google/cloud/transfers/test_sql_to_gcs.py | 23 + .../operators/test_azure_container_instances.py | 114 + .../providers/salesforce/hooks/test_salesforce.py | 2 +- tests/providers/ssh/operators/test_ssh.py | 1 + tests/sensors/test_external_task_sensor.py | 2 +- tests/serialization/test_dag_serialization.py | 6 +- .../task/task_runner/test_standard_task_runner.py | 7 +- tests/test_utils/mock_executor.py | 3 +- tests/test_utils/mock_plugins.py | 2 +- .../operators/postgres_local_executor.cfg | 2 +- tests/test_utils/perf/perf_kit/__init__.py | 10 +- tests/test_utils/perf/perf_kit/python.py | 2 +- tests/test_utils/perf/perf_kit/sqlalchemy.py | 2 +- tests/test_utils/perf/sql_queries.py | 2 +- tests/ti_deps/deps/test_runnable_exec_date_dep.py | 19 +- tests/utils/test_dag_processing.py | 119 +- tests/utils/test_sqlalchemy.py | 14 +- tests/www/test_views.py | 23 +- 211 files changed, 7652 insertions(+), 3594 deletions(-) create mode 100644 airflow/migrations/versions/98271e7606e2_add_scheduling_decision_to_dagrun_and_.py copy airflow/providers/{google/cloud => amazon/aws}/ADDITIONAL_INFO.md (61%) copy airflow/providers/amazon/aws/hooks/{aws_dynamodb.py => dynamodb.py} (100%) create mode 100644 airflow/providers/amazon/aws/hooks/elasticache_replication_group.py delete mode 100644 airflow/utils/asciiart.py create mode 100644 airflow/utils/callback_requests.py create mode 100644 backport_packages/README.md create mode 100644 backport_packages/remove_old_releases.py copy chart/templates/secrets/{pgbouncer-stats-secret.yaml => pgbouncer-certificates-secret.yaml} (69%) create mode 100644 docker-context-files/README.md create mode 100755 scripts/ci/selective_tests.sh delete mode 100755 scripts/ci/tools/ci_check_if_tests_should_be_run.sh delete mode 100755 scripts/ci/tools/ci_count_changed_files.sh rename tests/{ => core}/test_config_templates.py (100%) rename tests/{ => core}/test_configuration.py (100%) rename tests/{ => core}/test_core.py (100%) rename tests/{ => core}/test_core_to_contrib.py (100%) rename tests/{ => core}/test_example_dags.py (100%) rename tests/{ => core}/test_example_dags_system.py (100%) rename tests/{test_impersonation.py => core/test_impersonation_tests.py} (100%) rename tests/{ => core}/test_local_settings.py (85%) rename tests/{ => core}/test_logging_config.py (100%) rename tests/{ => core}/test_project_structure.py (82%) rename tests/{ => core}/test_sentry.py (100%) rename tests/{ => core}/test_sqlalchemy_config.py (96%) rename tests/{ => core}/test_stats.py (93%) rename tests/providers/amazon/aws/hooks/{test_aws_dynamodb.py => test_dynamodb.py} (96%) create mode 100644 tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py