This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 3a4f517 (commit)
to 7dd7be3 (commit)
from 3a4f517 fix task instance modal open performance issue (#10764)
add b746f33 Removes stable tests from quarantine (#10768)
add 10ce311 Deprecate using global as the default region in Google
Dataproc operators and hooks (#10772)
add 2934220 Always return a list from S3Hook list methods (#10774)
add 4de67a6 Move dev docker images to airflow registry (#9652)
add 3c3342f Add unit test for AzureCosmosDocumentSensor (#10765)
add 078bfaf Extract missing gcs_to_local example DAG from gcs example
(#10767)
add 961131d All files in providers package heve unit tests (#10799)
add cd0cc4c Check that all pre-commits are synchronized code<>docs
(#10789)
add 417ab30 Add section for official source code (#10678)
add aaf56f9 Refactor official source section to use bullets (#10801)
add 43303f1 Make script/ci/images Google Shell Guide compatible (#10745)
add 4f07463 Make script/ci/kubernetes Google Shell Guide Compatible
(#10746)
add 71e1d09 Fixed wrong "-e" on md5 file status check (#10803)
add 3c6fdd8 Make ci/backport_packages Google Shell guide compliant
(#10733)
add c60fccc Fix integration tests being accidentally excluded (#10807)
add 2811851 Move Impersonation test back to quarantine (#10809)
add 7fd65d7 Don't include kubernetes_tests/ and backport_packages/ in our
wheel (#10805)
add ff41361 Add task logging handler to airflow info command (#10771)
add ac943c9 [AIRFLOW-3964][AIP-17] Consolidate and de-dup sensor tasks
using Smart Sensor (#5499)
add 20481c3 Add pod_override setting for KubernetesExecutor (#10756)
add 004e1d8 Update log exception to reflect rename of execute_helper
(#10819)
add 9549274 Upgrade black to 20.8b1 (#10818)
add 7edfac9 Add connection caching to KubernetesHook (#10447)
add 40939dc Make airflow testing Google Shell Guide compatible (#10813)
add 409ebc1 Make scripts/ci/tools Google Shell Guide Compatible (#10811)
add 7adae24 Add documentation for preparing database for Airflow (#10413)
add a356656 Make dockerfiles Google Shell Guide Compliant (#10734)
add 9f23407 Fix doc errors introduced in #10413 (#10832)
add dfcb7af Fix typos in the stackdriver howto document (#10834)
add 9528f15 Fix doc errors introduced in #10413 (#10833)
add cc851c7 Fix grammar in UPDATING.md (#10841)
add e1e0485 Remove k8s dependency from serialization (#10831)
add 0d38cf1 Fix typo in docs/howto/operator/amazon/aws/emr.rst (#10844)
add ee8b02a Add missing assert call in test_dbapi_hook.py (#10842)
add f920957 Fix and remove some more typos from spelling_wordlist.txt
(#10845)
add ce66bc9 Add test for Health Endpoint when there is an exception
(#10846)
add eaf4f67 Add USC Graduate School to INTHEWILD.md (#10843)
add ff72327 Move parse_once to quarantine (#10857)
add 67af97a Rename "Beyond the Horizon" section and refactor content
(#10802)
add 63b6e53 Detect orphaned task instances by SchedulerJob id and
heartbeat (#10729)
add 1a95361 Fix and unquarantine
TestDagFileProcessorAgent.test_parse_once (#10862)
add e773f8b Refactor content to a markdown table (#10863)
add 68cc727 Add on_kill method to DataprocSubmitJobOperator (#10847)
add 59e8341 Add new lint check to now allow realtive imports (#10825)
add b9dc3c5 Added Plexus as an Airflow provider (#10591)
add c58d606 Update qubole_hook to not remove pool as an arg for
qubole_operator (#10820)
add 3e91da5 fix typo in firebase/example_filestore DAG (#10875)
add 5cc8e98 Remove unused type comment that upsets mypy (#10877)
add d93c1fd Disables --warn-unused-ignore flag for mypy (#10880)
add 56bd9b7 Modify helm chart to use pod_template_file (#10872)
add 41a6273 Add on_kill method to BigQueryInsertJobOperator (#10866)
add 030582a Fix `breeze -i` error (#10887)
add 47e592e Flag --start-airflow for breeze (#10837)
add 7dd7be3 Bump prismjs from 1.20.0 to 1.21.0 in /airflow/www (#10234)
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci.yml | 8 +-
.pre-commit-config.yaml | 21 +-
BREEZE.rst | 19 +-
CHANGELOG.txt | 8 +-
CONTRIBUTING.rst | 2 +-
INSTALL | 2 +-
INTHEWILD.md | 1 +
README.md | 77 +--
STATIC_CODE_CHECKS.rst | 46 +-
TESTING.rst | 12 +
UPDATING.md | 10 +-
airflow/api_connexion/endpoints/health_endpoint.py | 5 +-
.../api_connexion/endpoints/variable_endpoint.py | 7 +-
airflow/api_connexion/schemas/error_schema.py | 4 +-
airflow/cli/commands/dag_command.py | 5 +-
airflow/cli/commands/info_command.py | 31 +-
airflow/config_templates/config.yml | 33 +
airflow/config_templates/default_airflow.cfg | 15 +
.../example_kubernetes_executor_config.py | 100 ++-
airflow/exceptions.py | 7 +
airflow/executors/kubernetes_executor.py | 5 +-
airflow/jobs/backfill_job.py | 78 ++-
airflow/jobs/base_job.py | 94 +--
airflow/jobs/scheduler_job.py | 69 +-
airflow/kubernetes/pod_generator.py | 68 +-
airflow/kubernetes/pod_launcher.py | 3 +-
airflow/macros/__init__.py | 2 +-
... => b247b1e3d1ed_add_queued_by_job_id_to_ti.py} | 22 +-
.../e38be357a868_update_schema_for_smart_sensor.py | 92 +++
airflow/models/__init__.py | 1 +
airflow/models/baseoperator.py | 7 +
airflow/models/dagbag.py | 12 +-
airflow/models/dagrun.py | 9 +-
airflow/models/sensorinstance.py | 166 +++++
airflow/models/taskinstance.py | 134 +++-
.../amazon/aws/example_dags/example_datasync_1.py | 4 +-
.../amazon/aws/example_dags/example_ecs_fargate.py | 7 +-
.../amazon/aws/example_dags/example_s3_bucket.py | 12 +-
airflow/providers/amazon/aws/hooks/base_aws.py | 11 +-
airflow/providers/amazon/aws/hooks/batch_client.py | 5 +-
airflow/providers/amazon/aws/hooks/redshift.py | 3 +-
airflow/providers/amazon/aws/hooks/s3.py | 29 +-
airflow/providers/amazon/aws/operators/batch.py | 4 +-
airflow/providers/amazon/aws/operators/datasync.py | 6 +-
.../amazon/aws/operators/ec2_start_instance.py | 4 +-
.../amazon/aws/operators/ec2_stop_instance.py | 4 +-
.../amazon/aws/secrets/secrets_manager.py | 8 +-
.../amazon/aws/transfers/dynamodb_to_s3.py | 4 +-
.../providers/amazon/aws/transfers/s3_to_sftp.py | 3 +-
.../apache/druid/transfers/hive_to_druid.py | 5 +-
.../providers/apache/hive/operators/hive_stats.py | 10 +-
.../apache/hive/sensors/metastore_partition.py | 1 +
.../apache/hive/sensors/named_hive_partition.py | 10 +
.../apache/livy/example_dags/example_livy.py | 9 +-
.../apache/pig/example_dags/example_pig.py | 7 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 27 +-
.../databricks/example_dags/example_databricks.py | 4 +-
.../providers/databricks/operators/databricks.py | 8 +-
.../providers/elasticsearch/log/es_task_handler.py | 40 +-
airflow/providers/facebook/ads/hooks/ads.py | 11 +-
.../providers/google/ads/transfers/ads_to_gcs.py | 5 +-
.../_internal_client/secret_manager_client.py | 3 +-
.../cloud/example_dags/example_automl_tables.py | 18 +-
.../example_dags/example_automl_translation.py | 5 +-
.../example_automl_video_intelligence_tracking.py | 3 +-
.../cloud/example_dags/example_bigquery_dts.py | 4 +-
.../example_dags/example_bigquery_operations.py | 29 +-
.../cloud/example_dags/example_bigquery_queries.py | 19 +-
.../cloud/example_dags/example_bigquery_sensors.py | 5 +-
.../google/cloud/example_dags/example_bigtable.py | 20 +-
.../google/cloud/example_dags/example_cloud_sql.py | 13 +-
.../cloud/example_dags/example_cloud_sql_query.py | 5 +-
.../example_cloud_storage_transfer_service_gcp.py | 3 +-
.../google/cloud/example_dags/example_dataflow.py | 23 +-
.../cloud/example_dags/example_datafusion.py | 27 +-
.../google/cloud/example_dags/example_dataprep.py | 9 +-
.../google/cloud/example_dags/example_datastore.py | 23 +-
.../google/cloud/example_dags/example_dlp.py | 13 +-
.../example_dags/example_facebook_ads_to_gcs.py | 19 +-
.../google/cloud/example_dags/example_gcs.py | 15 +-
..._postgres_to_gcs.py => example_gcs_to_local.py} | 30 +-
.../example_dags/example_kubernetes_engine.py | 10 +-
.../cloud/example_dags/example_life_sciences.py | 12 +-
.../cloud/example_dags/example_local_to_gcs.py | 5 +-
.../google/cloud/example_dags/example_mlengine.py | 27 +-
.../google/cloud/example_dags/example_pubsub.py | 10 +-
.../cloud/example_dags/example_sheets_to_gcs.py | 4 +-
.../google/cloud/example_dags/example_spanner.py | 12 +-
.../cloud/example_dags/example_stackdriver.py | 18 +-
.../example_dags/example_video_intelligence.py | 6 +-
airflow/providers/google/cloud/hooks/automl.py | 36 +-
airflow/providers/google/cloud/hooks/bigquery.py | 54 +-
.../providers/google/cloud/hooks/bigquery_dts.py | 4 +-
airflow/providers/google/cloud/hooks/bigtable.py | 4 +-
.../providers/google/cloud/hooks/cloud_build.py | 4 +-
.../google/cloud/hooks/cloud_memorystore.py | 8 +-
airflow/providers/google/cloud/hooks/cloud_sql.py | 12 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 4 +-
airflow/providers/google/cloud/hooks/compute.py | 23 +-
.../providers/google/cloud/hooks/datacatalog.py | 4 +-
airflow/providers/google/cloud/hooks/dataflow.py | 4 +-
airflow/providers/google/cloud/hooks/datafusion.py | 54 +-
airflow/providers/google/cloud/hooks/dataproc.py | 10 +-
airflow/providers/google/cloud/hooks/datastore.py | 4 +-
airflow/providers/google/cloud/hooks/dlp.py | 4 +-
airflow/providers/google/cloud/hooks/functions.py | 17 +-
airflow/providers/google/cloud/hooks/gcs.py | 4 +-
airflow/providers/google/cloud/hooks/gdm.py | 4 +-
airflow/providers/google/cloud/hooks/kms.py | 4 +-
.../google/cloud/hooks/kubernetes_engine.py | 4 +-
.../providers/google/cloud/hooks/life_sciences.py | 8 +-
airflow/providers/google/cloud/hooks/mlengine.py | 52 +-
.../google/cloud/hooks/natural_language.py | 4 +-
airflow/providers/google/cloud/hooks/pubsub.py | 16 +-
.../providers/google/cloud/hooks/secret_manager.py | 4 +-
airflow/providers/google/cloud/hooks/spanner.py | 45 +-
.../providers/google/cloud/hooks/speech_to_text.py | 4 +-
.../providers/google/cloud/hooks/stackdriver.py | 4 +-
airflow/providers/google/cloud/hooks/tasks.py | 28 +-
.../providers/google/cloud/hooks/text_to_speech.py | 4 +-
airflow/providers/google/cloud/hooks/translate.py | 4 +-
.../google/cloud/hooks/video_intelligence.py | 4 +-
airflow/providers/google/cloud/hooks/vision.py | 11 +-
.../providers/google/cloud/log/gcs_task_handler.py | 6 +-
.../google/cloud/log/stackdriver_task_handler.py | 3 +-
airflow/providers/google/cloud/operators/automl.py | 69 +-
.../providers/google/cloud/operators/bigquery.py | 39 +-
.../providers/google/cloud/operators/bigtable.py | 34 +-
.../providers/google/cloud/operators/datafusion.py | 36 +-
.../providers/google/cloud/operators/dataprep.py | 8 +-
.../providers/google/cloud/operators/dataproc.py | 48 +-
.../providers/google/cloud/operators/datastore.py | 64 +-
airflow/providers/google/cloud/operators/dlp.py | 150 +++-
airflow/providers/google/cloud/operators/gcs.py | 6 +-
.../providers/google/cloud/operators/mlengine.py | 5 +-
.../google/cloud/operators/natural_language.py | 12 +-
airflow/providers/google/cloud/operators/pubsub.py | 4 +-
.../providers/google/cloud/operators/spanner.py | 30 +-
.../google/cloud/operators/speech_to_text.py | 3 +-
.../google/cloud/operators/stackdriver.py | 5 +-
airflow/providers/google/cloud/operators/tasks.py | 65 +-
.../google/cloud/operators/text_to_speech.py | 6 +-
.../providers/google/cloud/operators/translate.py | 5 +-
.../google/cloud/operators/translate_speech.py | 6 +-
.../google/cloud/operators/video_intelligence.py | 9 +-
airflow/providers/google/cloud/operators/vision.py | 83 ++-
.../providers/google/cloud/sensors/bigquery_dts.py | 3 +-
airflow/providers/google/cloud/sensors/bigtable.py | 5 +-
.../sensors/cloud_storage_transfer_service.py | 3 +-
airflow/providers/google/cloud/sensors/pubsub.py | 4 +-
.../google/cloud/transfers/facebook_ads_to_gcs.py | 5 +-
.../google/cloud/transfers/gcs_to_sftp.py | 15 +-
.../google/cloud/transfers/sftp_to_gcs.py | 11 +-
.../google/cloud/transfers/sheets_to_gcs.py | 10 +-
.../google/cloud/utils/credentials_provider.py | 8 +-
.../google/cloud/utils/field_validator.py | 3 +-
.../providers/google/common/hooks/discovery_api.py | 4 +-
.../firebase/example_dags/example_firestore.py | 2 +-
.../providers/google/firebase/hooks/firestore.py | 4 +-
.../example_dags/example_analytics.py | 4 +-
.../example_dags/example_campaign_manager.py | 18 +-
.../example_dags/example_display_video.py | 12 +-
.../google/marketing_platform/hooks/analytics.py | 11 +-
.../marketing_platform/hooks/campaign_manager.py | 11 +-
.../marketing_platform/hooks/display_video.py | 20 +-
.../google/marketing_platform/hooks/search_ads.py | 9 +-
.../marketing_platform/operators/analytics.py | 34 +-
.../operators/campaign_manager.py | 4 +-
.../marketing_platform/operators/display_video.py | 13 +-
.../suite/example_dags/example_gcs_to_sheets.py | 4 +-
.../google/suite/example_dags/example_sheets.py | 4 +-
airflow/providers/google/suite/hooks/drive.py | 4 +-
airflow/providers/google/suite/hooks/sheets.py | 4 +-
.../google/suite/transfers/gcs_to_sheets.py | 8 +-
airflow/providers/hashicorp/hooks/vault.py | 4 +-
airflow/providers/http/hooks/http.py | 5 +-
.../microsoft/azure/log/wasb_task_handler.py | 4 +-
.../azure/operators/azure_container_instances.py | 9 +-
.../microsoft/azure/transfers/file_to_wasb.py | 5 +-
.../hooks => airflow/providers/plexus}/__init__.py | 0
.../providers/plexus/example_dags}/__init__.py | 0
.../example_dags/example_plexus.py} | 45 +-
.../providers/plexus}/hooks/__init__.py | 0
airflow/providers/plexus/hooks/plexus.py | 73 ++
.../providers/plexus/operators}/__init__.py | 0
airflow/providers/plexus/operators/job.py | 162 +++++
airflow/providers/postgres/hooks/postgres.py | 7 +-
.../qubole/example_dags/example_qubole.py | 8 +-
airflow/providers/qubole/hooks/qubole.py | 2 +-
airflow/providers/qubole/operators/qubole_check.py | 4 +-
.../segment/operators/segment_track_event.py | 4 +-
airflow/providers/sftp/operators/sftp.py | 3 +-
airflow/providers/slack/hooks/slack.py | 5 +-
.../snowflake/example_dags/example_snowflake.py | 12 +-
airflow/providers/ssh/hooks/ssh.py | 8 +-
.../example_dags/example_yandexcloud_dataproc.py | 37 +-
.../providers/yandex/hooks/yandexcloud_dataproc.py | 3 +-
.../yandex/operators/yandexcloud_dataproc.py | 24 +-
airflow/sensors/base_sensor_operator.py | 74 +-
airflow/sensors/smart_sensor_operator.py | 764 +++++++++++++++++++++
airflow/serialization/enums.py | 1 +
airflow/serialization/serialized_objects.py | 13 +
.../smart_sensor_dags}/__init__.py | 0
airflow/smart_sensor_dags/smart_sensor_group.py | 63 ++
airflow/utils/file.py | 16 +-
airflow/utils/log/file_processor_handler.py | 15 +-
airflow/utils/log/file_task_handler.py | 22 +-
airflow/utils/log/log_reader.py | 3 +-
airflow/utils/state.py | 25 +-
airflow/www/static/css/graph.css | 4 +
airflow/www/static/css/tree.css | 4 +
airflow/www/templates/airflow/ti_log.html | 28 +-
airflow/www/yarn.lock | 6 +-
breeze | 30 +-
breeze-complete | 6 +-
chart/files/pod-template-file.yaml | 105 +++
chart/templates/_helpers.yaml | 4 +
chart/templates/configmap.yaml | 8 +
.../templates/scheduler/scheduler-deployment.yaml | 4 +
...duler_test.yaml => pod-template-file_test.yaml} | 39 +-
chart/values.yaml | 29 +-
docs/autoapi_templates/index.rst | 4 +
docs/concepts.rst | 9 +
docs/executor/kubernetes.rst | 29 +
docs/howto/initialize-database.rst | 51 +-
docs/howto/operator/amazon/aws/emr.rst | 2 +-
docs/howto/operator/google/cloud/stackdriver.rst | 18 +-
.../operator/google/transfer/gcs_to_local.rst | 2 +-
docs/img/smart_sensor_architecture.png | Bin 0 -> 80325 bytes
docs/img/smart_sensor_single_task_execute_flow.png | Bin 0 -> 75462 bytes
docs/index.rst | 1 +
docs/logging-monitoring/logging-tasks.rst | 48 +-
docs/logging-monitoring/metrics.rst | 5 +
docs/operators-and-hooks-ref.rst | 10 +
docs/smart-sensor.rst | 86 +++
docs/spelling_wordlist.txt | 7 -
kubernetes_tests/test_kubernetes_pod_operator.py | 2 +-
.../ci_prepare_and_test_backport_packages.sh | 6 +-
scripts/ci/docker-compose/base.yml | 2 +
scripts/ci/docker-compose/integration-kerberos.yml | 2 +-
scripts/ci/docker-compose/integration-openldap.yml | 4 +-
scripts/ci/dockerfiles/apache-rat/Dockerfile | 58 ++
.../ci/dockerfiles/apache-rat}/build_and_push.sh | 19 +-
scripts/ci/dockerfiles/bats/build_and_push.sh | 10 +
scripts/ci/dockerfiles/krb5-kdc-server/Dockerfile | 64 ++
.../dockerfiles/krb5-kdc-server}/build_and_push.sh | 15 +-
.../krb5-kdc-server/krb-conf/client}/krb5.conf | 9 +-
.../krb5-kdc-server/krb-conf/server/kadm5.acl | 1 +
.../krb5-kdc-server/krb-conf/server/kdc.conf} | 24 +-
.../dockerfiles/krb5-kdc-server/start_kdc.sh} | 33 +-
.../dockerfiles/krb5-kdc-server/supervisord.conf | 40 ++
scripts/ci/dockerfiles/stress/Dockerfile | 34 +
.../ci/dockerfiles/stress}/build_and_push.sh | 18 +-
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 6 +-
scripts/ci/kubernetes/ci_run_helm_testing.sh | 9 +-
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 32 +-
scripts/ci/libraries/_initialization.sh | 8 +-
scripts/ci/libraries/_md5sum.sh | 2 +-
.../ci/pre_commit/pre_commit_check_pre_commits.sh | 66 ++
scripts/ci/static_checks/check_license.sh | 2 +-
scripts/ci/testing/ci_run_airflow_testing.sh | 33 +-
.../ci/tools/ci_check_if_tests_should_be_run.sh | 15 +-
scripts/ci/tools/ci_count_changed_files.sh | 12 +-
scripts/in_container/check_environment.sh | 39 ++
scripts/in_container/entrypoint_ci.sh | 9 +
setup.cfg | 2 +-
setup.py | 7 +-
.../endpoints/test_config_endpoint.py | 16 +-
.../endpoints/test_connection_endpoint.py | 13 +-
tests/api_connexion/endpoints/test_dag_endpoint.py | 67 +-
.../endpoints/test_dag_run_endpoint.py | 44 +-
.../endpoints/test_dag_source_endpoint.py | 3 +-
.../endpoints/test_event_log_endpoint.py | 48 +-
.../endpoints/test_extra_link_endpoint.py | 13 +-
.../endpoints/test_health_endpoint.py | 10 +-
tests/api_connexion/endpoints/test_log_endpoint.py | 14 +-
.../api_connexion/endpoints/test_pool_endpoint.py | 34 +-
.../api_connexion/endpoints/test_task_endpoint.py | 10 +-
.../endpoints/test_variable_endpoint.py | 59 +-
.../api_connexion/endpoints/test_xcom_endpoint.py | 39 +-
tests/api_connexion/schemas/test_config_schema.py | 19 +-
.../schemas/test_connection_schema.py | 8 +-
tests/api_connexion/schemas/test_dag_run_schema.py | 27 +-
tests/api_connexion/schemas/test_health_schema.py | 5 +-
tests/api_connexion/schemas/test_task_schema.py | 9 +-
tests/api_connexion/schemas/test_version_schema.py | 5 +-
tests/api_connexion/schemas/test_xcom_schema.py | 5 +-
tests/cli/commands/test_dag_command.py | 2 -
tests/cli/commands/test_info_command.py | 20 +
tests/cli/commands/test_task_command.py | 1 -
tests/conftest.py | 18 +
tests/executors/test_dask_executor.py | 3 -
tests/hooks/test_dbapi_hook.py | 4 +-
tests/jobs/test_backfill_job.py | 95 ++-
tests/jobs/test_local_task_job.py | 2 -
tests/jobs/test_scheduler_job.py | 196 +++---
tests/kubernetes/pod.yaml | 2 +-
tests/kubernetes/test_pod_generator.py | 50 +-
tests/kubernetes/test_worker_configuration.py | 1 +
tests/models/test_dagbag.py | 8 +-
tests/models/test_sensorinstance.py | 46 ++
.../amazon/aws/hooks/test_aws_dynamodb.py | 4 +-
tests/providers/amazon/aws/hooks/test_base_aws.py | 13 +-
.../amazon/aws/hooks/test_batch_client.py | 19 +-
.../amazon/aws/hooks/test_batch_waiters.py | 12 +-
tests/providers/amazon/aws/hooks/test_datasync.py | 3 +-
tests/providers/amazon/aws/hooks/test_ec2.py | 15 +-
tests/providers/amazon/aws/hooks/test_s3.py | 4 +-
tests/providers/amazon/aws/hooks/test_sagemaker.py | 19 +-
.../amazon/aws/log/test_cloudwatch_task_handler.py | 9 +-
.../amazon/aws/log/test_s3_task_handler.py | 11 +-
.../providers/amazon/aws/operators/test_athena.py | 48 +-
tests/providers/amazon/aws/operators/test_batch.py | 9 +-
.../amazon/aws/operators/test_cloud_formation.py | 4 +-
.../amazon/aws/operators/test_datasync.py | 39 +-
.../aws/operators/test_ec2_start_instance.py | 10 +-
.../amazon/aws/operators/test_ec2_stop_instance.py | 10 +-
tests/providers/amazon/aws/operators/test_ecs.py | 5 +-
.../amazon/aws/operators/test_ecs_system.py | 15 +-
.../aws/operators/test_emr_create_job_flow.py | 6 +-
.../amazon/aws/operators/test_s3_bucket.py | 10 +-
.../aws/operators/test_sagemaker_endpoint.py | 5 +-
.../amazon/aws/operators/test_sagemaker_model.py | 5 +-
.../aws/operators/test_sagemaker_processing.py | 5 +-
.../aws/operators/test_sagemaker_transform.py | 9 +-
.../amazon/aws/operators/test_sagemaker_tuning.py | 6 +-
.../amazon/aws/sensors/test_ec2_instance_state.py | 31 +-
.../providers/amazon/aws/sensors/test_emr_base.py | 20 +-
tests/providers/amazon/aws/sensors/test_s3_key.py | 11 +-
.../amazon/aws/sensors/test_sagemaker_endpoint.py | 16 +-
.../amazon/aws/sensors/test_sagemaker_training.py | 4 +-
.../amazon/aws/sensors/test_sagemaker_transform.py | 16 +-
.../amazon/aws/sensors/test_sagemaker_tuning.py | 16 +-
.../amazon/aws/transfers/test_dynamodb_to_s3.py | 9 +-
.../amazon/aws/transfers/test_hive_to_dynamodb.py | 8 +-
.../amazon/aws/transfers/test_redshift_to_s3.py | 11 +-
.../apache/cassandra/sensors/test_table.py | 8 +-
tests/providers/apache/hdfs/hooks/test_hdfs.py | 21 +-
.../providers/apache/hdfs/sensors/test_web_hdfs.py | 12 +-
.../apache/hive/operators/test_hive_stats.py | 40 +-
.../apache/hive/transfers/test_vertica_to_hive.py | 10 +-
tests/providers/apache/kylin/hooks/test_kylin.py | 5 +-
.../providers/apache/spark/hooks/test_spark_sql.py | 6 +-
.../apache/spark/hooks/test_spark_submit.py | 5 +-
.../cncf/kubernetes/hooks/test_kubernetes.py | 4 +-
.../kubernetes/sensors/test_spark_kubernetes.py | 65 +-
.../providers/databricks/hooks/test_databricks.py | 10 +-
.../databricks/operators/test_databricks.py | 12 +-
tests/providers/datadog/hooks/test_datadog.py | 17 +-
tests/providers/docker/operators/test_docker.py | 12 +-
.../elasticsearch/log/test_es_task_handler.py | 20 +-
tests/providers/exasol/hooks/test_exasol.py | 6 +-
tests/providers/google/ads/operators/test_ads.py | 3 +-
.../google/ads/transfers/test_ads_to_gcs.py | 3 +-
tests/providers/google/cloud/hooks/test_automl.py | 31 +-
.../providers/google/cloud/hooks/test_bigquery.py | 128 +++-
.../google/cloud/hooks/test_bigquery_dts.py | 4 +-
.../providers/google/cloud/hooks/test_bigtable.py | 16 +-
.../google/cloud/hooks/test_cloud_memorystore.py | 8 +-
.../providers/google/cloud/hooks/test_cloud_sql.py | 98 ++-
tests/providers/google/cloud/hooks/test_compute.py | 21 +-
.../google/cloud/hooks/test_datacatalog.py | 16 +-
.../providers/google/cloud/hooks/test_dataflow.py | 53 +-
.../google/cloud/hooks/test_datafusion.py | 21 +-
.../providers/google/cloud/hooks/test_dataprep.py | 26 +-
.../providers/google/cloud/hooks/test_dataproc.py | 14 +
.../providers/google/cloud/hooks/test_datastore.py | 23 +-
tests/providers/google/cloud/hooks/test_dlp.py | 109 ++-
.../providers/google/cloud/hooks/test_functions.py | 8 +-
tests/providers/google/cloud/hooks/test_gcs.py | 3 +-
tests/providers/google/cloud/hooks/test_gdm.py | 12 +-
tests/providers/google/cloud/hooks/test_kms.py | 14 +-
.../providers/google/cloud/hooks/test_mlengine.py | 25 +-
.../google/cloud/hooks/test_natural_language.py | 24 +-
tests/providers/google/cloud/hooks/test_pubsub.py | 9 +-
tests/providers/google/cloud/hooks/test_spanner.py | 3 +-
.../google/cloud/hooks/test_stackdriver.py | 31 +-
tests/providers/google/cloud/hooks/test_tasks.py | 50 +-
tests/providers/google/cloud/hooks/test_vision.py | 3 +-
.../cloud/log/test_stackdriver_task_handler.py | 8 +-
.../google/cloud/operators/test_automl.py | 42 +-
.../google/cloud/operators/test_bigquery.py | 138 +++-
.../cloud/operators/test_bigquery_dts_system.py | 4 +-
.../google/cloud/operators/test_bigtable.py | 111 ++-
.../google/cloud/operators/test_cloud_build.py | 4 +-
.../cloud/operators/test_cloud_memorystore.py | 32 +-
.../google/cloud/operators/test_cloud_sql.py | 96 ++-
.../test_cloud_storage_transfer_service.py | 69 +-
.../google/cloud/operators/test_compute.py | 171 ++++-
.../google/cloud/operators/test_datacatalog.py | 66 +-
.../google/cloud/operators/test_datafusion.py | 30 +-
.../google/cloud/operators/test_dataprep_system.py | 4 +-
.../google/cloud/operators/test_dataproc.py | 44 +-
.../google/cloud/operators/test_datastore.py | 39 +-
tests/providers/google/cloud/operators/test_dlp.py | 145 ++--
.../google/cloud/operators/test_functions.py | 87 ++-
tests/providers/google/cloud/operators/test_gcs.py | 9 +-
.../google/cloud/operators/test_life_sciences.py | 6 +-
.../google/cloud/operators/test_mlengine.py | 77 ++-
.../google/cloud/operators/test_mlengine_utils.py | 9 +-
.../google/cloud/operators/test_pubsub.py | 21 +-
.../google/cloud/operators/test_spanner.py | 73 +-
.../google/cloud/operators/test_speech_to_text.py | 3 +-
.../google/cloud/operators/test_stackdriver.py | 10 +-
.../providers/google/cloud/operators/test_tasks.py | 74 +-
.../google/cloud/operators/test_text_to_speech.py | 6 +-
.../google/cloud/operators/test_translate.py | 3 +-
.../cloud/operators/test_translate_speech.py | 18 +-
.../cloud/operators/test_video_intelligence.py | 9 +-
.../google/cloud/operators/test_vision.py | 68 +-
.../google/cloud/sensors/test_bigtable.py | 17 +-
.../providers/google/cloud/sensors/test_pubsub.py | 23 +-
.../cloud/transfers/test_facebook_ads_to_gcs.py | 11 +-
.../google/cloud/transfers/test_gcs_to_gcs.py | 12 +-
.../google/cloud/transfers/test_gcs_to_local.py | 5 +-
.../test_gcs_to_local_system.py} | 21 +-
.../google/cloud/transfers/test_gcs_to_sftp.py | 8 +-
.../cloud/transfers/test_gcs_to_sftp_system.py | 6 +-
.../google/cloud/transfers/test_presto_to_gcs.py | 8 +-
.../google/cloud/transfers/test_s3_to_gcs.py | 4 +-
.../google/cloud/transfers/test_sftp_to_gcs.py | 13 +-
.../google/cloud/transfers/test_sheets_to_gcs.py | 13 +-
.../providers/google/cloud/utils/base_gcp_mock.py | 10 +-
.../cloud/utils/test_credentials_provider.py | 51 +-
.../google/cloud/utils/test_field_sanitizer.py | 9 +-
.../google/cloud/utils/test_field_validator.py | 33 +-
.../utils/test_mlengine_prediction_summary.py | 21 +-
.../google/common/hooks/test_base_google.py | 21 +-
.../common/utils/test_id_token_credentials.py | 10 +-
.../marketing_platform/hooks/test_analytics.py | 18 +-
.../hooks/test_campaign_manager.py | 5 +-
.../marketing_platform/hooks/test_display_video.py | 10 +-
.../marketing_platform/hooks/test_search_ads.py | 5 +-
.../marketing_platform/operators/test_analytics.py | 30 +-
.../operators/test_campaign_manager.py | 55 +-
.../operators/test_display_video.py | 69 +-
.../operators/test_search_ads.py | 15 +-
.../sensors/test_campaign_manager.py | 5 +-
.../sensors/test_display_video.py | 14 +-
.../marketing_platform/sensors/test_search_ads.py | 5 +-
.../google/suite/transfers/test_gcs_to_gdrive.py | 6 +-
.../google/suite/transfers/test_gcs_to_sheets.py | 12 +-
tests/providers/grpc/operators/test_grpc.py | 10 +-
.../_internal_client/test_vault_client.py | 17 +-
tests/providers/hashicorp/hooks/test_vault.py | 30 +-
tests/providers/hashicorp/secrets/test_vault.py | 14 +-
tests/providers/http/hooks/test_http.py | 5 +-
tests/providers/jenkins/hooks/test_jenkins.py | 2 +-
.../jenkins/operators/test_jenkins_job_trigger.py | 45 +-
tests/providers/jira/operators/test_jira.py | 5 +-
tests/providers/jira/sensors/test_jira.py | 5 +-
.../microsoft/azure/hooks/test_azure_batch.py | 28 +-
.../microsoft/azure/log/test_wasb_task_handler.py | 10 +-
.../microsoft/azure/sensors/test_azure_cosmos.py | 55 ++
.../microsoft/mssql/operators/test_mssql.py | 10 +-
tests/providers/mysql/hooks/test_mysql.py | 21 +-
tests/providers/mysql/operators/test_mysql.py | 26 +-
.../mysql/transfers/test_presto_to_mysql.py | 6 +-
.../mysql/transfers/test_vertica_to_mysql.py | 10 +-
tests/providers/odbc/hooks/test_odbc.py | 10 +-
tests/providers/pagerduty/hooks/test_pagerduty.py | 22 +-
.../{zendesk/hooks => plexus}/__init__.py | 0
tests/providers/plexus/hooks/test_plexus.py | 63 ++
.../hooks => plexus/operators}/__init__.py | 0
tests/providers/plexus/operators/test_job.py | 189 +++++
tests/providers/postgres/hooks/test_postgres.py | 33 +-
tests/providers/presto/hooks/test_presto.py | 7 +-
tests/providers/qubole/operators/test_qubole.py | 9 +-
.../providers/salesforce/hooks/test_salesforce.py | 3 +-
.../segment/operators/test_segment_track_event.py | 9 +-
.../singularity/operators/test_singularity.py | 77 ++-
tests/providers/slack/operators/test_slack.py | 9 +-
tests/providers/ssh/hooks/test_ssh.py | 11 +-
tests/providers/ssh/operators/test_ssh.py | 18 +-
tests/providers/vertica/hooks/test_vertica.py | 7 +-
.../yandex/operators/test_yandexcloud_dataproc.py | 58 +-
tests/sensors/test_smart_sensor_operator.py | 326 +++++++++
tests/sensors/test_timeout_sensor.py | 3 -
tests/serialization/test_dag_serialization.py | 27 +-
tests/test_config_templates.py | 3 +-
tests/test_project_structure.py | 22 +-
tests/utils/log/test_log_reader.py | 58 +-
tests/utils/test_dag_processing.py | 1 +
tests/utils/test_log_handlers.py | 2 +-
tests/utils/test_process_utils.py | 2 +-
tests/www/test_views.py | 23 +-
486 files changed, 9270 insertions(+), 2115 deletions(-)
copy
airflow/migrations/versions/{5e7d17757c7a_add_pid_field_to_taskinstance.py =>
b247b1e3d1ed_add_queued_by_job_id_to_ti.py} (64%)
create mode 100644
airflow/migrations/versions/e38be357a868_update_schema_for_smart_sensor.py
create mode 100644 airflow/models/sensorinstance.py
copy airflow/providers/google/cloud/example_dags/{example_postgres_to_gcs.py
=> example_gcs_to_local.py} (54%)
copy {tests/providers/zendesk/hooks => airflow/providers/plexus}/__init__.py
(100%)
copy {tests/providers/zendesk/hooks =>
airflow/providers/plexus/example_dags}/__init__.py (100%)
copy airflow/providers/{docker/example_dags/example_docker_swarm.py =>
plexus/example_dags/example_plexus.py} (58%)
copy {tests/providers/zendesk => airflow/providers/plexus}/hooks/__init__.py
(100%)
create mode 100644 airflow/providers/plexus/hooks/plexus.py
copy {tests/providers/zendesk/hooks =>
airflow/providers/plexus/operators}/__init__.py (100%)
create mode 100644 airflow/providers/plexus/operators/job.py
create mode 100644 airflow/sensors/smart_sensor_operator.py
copy {tests/www/api/experimental => airflow/smart_sensor_dags}/__init__.py
(100%)
create mode 100644 airflow/smart_sensor_dags/smart_sensor_group.py
create mode 100644 chart/files/pod-template-file.yaml
copy chart/tests/{git-sync-scheduler_test.yaml => pod-template-file_test.yaml}
(84%)
create mode 100644 docs/img/smart_sensor_architecture.png
create mode 100644 docs/img/smart_sensor_single_task_execute_flow.png
create mode 100644 docs/smart-sensor.rst
create mode 100644 scripts/ci/dockerfiles/apache-rat/Dockerfile
copy {chart/dockerfiles/statsd-exporter =>
scripts/ci/dockerfiles/apache-rat}/build_and_push.sh (71%)
create mode 100644 scripts/ci/dockerfiles/krb5-kdc-server/Dockerfile
copy {chart/dockerfiles/pgbouncer =>
scripts/ci/dockerfiles/krb5-kdc-server}/build_and_push.sh (76%)
copy scripts/{in_container/krb5 =>
ci/dockerfiles/krb5-kdc-server/krb-conf/client}/krb5.conf (88%)
copy airflow/cli/__init__.py =>
scripts/ci/dockerfiles/krb5-kdc-server/krb-conf/server/kadm5.acl (97%)
copy scripts/{in_container/krb5/krb5.conf =>
ci/dockerfiles/krb5-kdc-server/krb-conf/server/kdc.conf} (67%)
copy scripts/{in_container/_in_container_script_init.sh =>
ci/dockerfiles/krb5-kdc-server/start_kdc.sh} (53%)
mode change 100755 => 100644
create mode 100644 scripts/ci/dockerfiles/krb5-kdc-server/supervisord.conf
create mode 100644 scripts/ci/dockerfiles/stress/Dockerfile
copy {chart/dockerfiles/pgbouncer =>
scripts/ci/dockerfiles/stress}/build_and_push.sh (72%)
create mode 100755 scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
create mode 100644 tests/models/test_sensorinstance.py
copy tests/providers/google/cloud/{operators/test_gcs_system.py =>
transfers/test_gcs_to_local_system.py} (74%)
create mode 100644 tests/providers/microsoft/azure/sensors/test_azure_cosmos.py
copy tests/providers/{zendesk/hooks => plexus}/__init__.py (100%)
create mode 100644 tests/providers/plexus/hooks/test_plexus.py
copy tests/providers/{zendesk/hooks => plexus/operators}/__init__.py (100%)
create mode 100644 tests/providers/plexus/operators/test_job.py
create mode 100644 tests/sensors/test_smart_sensor_operator.py
