This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from e9ecf0a (commit)
to ff5dccc (commit)
from e9ecf0a Improve test for the next_execution cli command (#9058)
add 3435e6e Improve SchedulerJob code style (#9018)
add 05e4c39 Provide_context coma is added only when there is not one
already (#9064)
add 93eda7a Remove not-existing files in pylint_todo.txt (#9073)
add 7f026d1 Remove Travis CI badge from README (#9074)
add 735bf45 Test that DagFileProcessor can operate against on a
Serialized DAG (#8739)
add db42974 Adds hive as extra in pyhive (#9075)
add 88c8edb Prevents failure on fixing permissions for files with space
in it (#9076)
add 2b1dc1b Support properties in plugins (#9002)
add a3fc8be Add metric for job start/end task run (#8680)
add 29eb68b Create guide for Dataproc Operators (#9037)
add 93b8f3e Test queries when number of active DAG Run is not zero (#9082)
add c002b25 Add displaying multiple dates in airflow next_execution
command (#9072)
add a85d228 Update Breeze Documentation to have WSL 2 Instructions
instead of WSL 1 (#9057)
add a92eb84 Shorten command help and move long help to command
description (#9070)
add c970053 Improve TestCliConfig in local environment (#9085)
add 7c0e6ed Don't create empty modules for plugins (#9078)
add 7898525 Add BigQueryInsertJobOperator (#8868)
add 55b9b8f #8525 Add SQL Branch Operator (#8942)
add 20f8982 Add OpenAPI specification (II) (#8721)
add 93e924d Enable configurable git sync depth (#9094)
add 3dd81b7 Don't reuse MY_DIR in breeze to mean different folder from
ci/_utils.sh (#9098)
add b7b4846 Updated missing parameters for docker image building (#9039)
add a6216a7 You can push with Breeze as separate command and to cache
(#8976)
add 87a4a0a Allow using Airflow with Flask CLI (#9030)
add 9e0ccde Produce less verbose output when building docker mount
options (#9103)
add 17adcea Fix handling of subprocess error handling in
s3_file_transform and gcs (#9106)
add 67379d1 Add fudament for API based on connexion (#8149)
add 5f038d0 Add filepaths for API label in BoringCyborg Bot (#9116)
add 11d726d Add query count test for SchedulerJob (#9088)
add 10796cb Remove Hive/Hadoop/Java dependency from unit tests (#9029)
add 2b45d8f Move TestDagFileProcessorQueriesCount to quarantine (#9119)
add 1c9374d Add snowflake to slack operator (#9023)
add ff5dccc Kubernetes Cluster is started on host not in the container
(#8265)
No new revisions were added by this update.
Summary of changes:
.github/boring-cyborg.yml | 6 +
.github/workflows/ci.yml | 41 +-
.gitignore | 4 +-
.pre-commit-config.yaml | 13 +-
BREEZE.rst | 375 ++-
CONTRIBUTING.rst | 1 +
Dockerfile.ci | 67 +-
IMAGES.rst | 28 +-
MANIFEST.in | 1 +
README.md | 4 +-
TESTING.rst | 230 +-
.../hooks => airflow/api_connexion}/__init__.py | 0
.../api_connexion/endpoints}/__init__.py | 0
.../api_connexion/endpoints/config_endpoint.py | 12 +-
.../endpoints/connection_endpoint.py} | 40 +-
.../endpoints/dag_endpoint.py} | 35 +-
.../endpoints/dag_run_endpoint.py} | 47 +-
.../api_connexion/endpoints/dag_source_endpoint.py | 12 +-
.../api_connexion/endpoints/event_log_endpoint.py | 17 +-
.../api_connexion/endpoints/extra_link_endpoint.py | 12 +-
.../api_connexion/endpoints/health_endpoint.py | 15 +-
.../endpoints/import_errror_endpoint.py} | 26 +-
.../api_connexion/endpoints/log_endpoint.py | 12 +-
.../endpoints/pool_endpoint.py} | 40 +-
.../api_connexion/endpoints/task_endpoint.py | 17 +-
.../endpoints/task_instance_endpoint.py} | 33 +-
.../endpoints/variable_endpoint.py} | 40 +-
.../endpoints/xcom_endpoint.py} | 40 +-
airflow/api_connexion/openapi/v1.yaml | 2426 ++++++++++++++++++++
airflow/cli/cli_parser.py | 110 +-
airflow/cli/commands/dag_command.py | 13 +-
airflow/cli/commands/role_command.py | 6 +-
airflow/cli/commands/sync_perm_command.py | 4 +-
airflow/cli/commands/user_command.py | 14 +-
airflow/cli/commands/webserver_command.py | 2 +-
airflow/config_templates/airflow_local_settings.py | 2 +-
airflow/config_templates/config.yml | 8 +
airflow/config_templates/default_airflow.cfg | 4 +
airflow/executors/executor_loader.py | 7 +
airflow/executors/kubernetes_executor.py | 2 +
airflow/jobs/base_job.py | 5 +-
airflow/jobs/scheduler_job.py | 300 ++-
airflow/kubernetes/refresh_config.py | 4 +-
airflow/kubernetes/volume_mount.py | 2 +-
airflow/kubernetes/worker_configuration.py | 2 +-
airflow/models/dagbag.py | 3 +-
airflow/models/serialized_dag.py | 5 +-
airflow/models/taskinstance.py | 3 +
airflow/operators/sql_branch_operator.py | 175 ++
airflow/plugins_manager.py | 66 +-
.../amazon/aws/operators/s3_file_transform.py | 2 +-
airflow/providers/apache/hive/operators/hive.py | 14 +
.../apache/hive/sensors/hive_partition.py | 2 +-
airflow/providers/dependencies.json | 3 +
.../cloud/example_dags/example_bigquery_queries.py | 33 +-
.../google/cloud/example_dags/example_dataproc.py | 29 +-
airflow/providers/google/cloud/hooks/bigquery.py | 76 +-
.../providers/google/cloud/operators/bigquery.py | 82 +
airflow/providers/google/cloud/operators/gcs.py | 2 +-
.../snowflake/example_dags/example_snowflake.py | 32 +-
.../snowflake/operators/snowflake_to_slack.py | 152 ++
airflow/www/app.py | 85 +-
airflow/www/security.py | 7 +-
airflow/www/views.py | 21 +-
backport_packages/refactor_backport_packages.py | 5 +-
breeze | 350 ++-
breeze-complete | 25 +-
docs/howto/operator/gcp/bigquery.rst | 37 +-
docs/howto/operator/gcp/dataproc.rst | 188 ++
docs/metrics.rst | 2 +
docs/operators-and-hooks-ref.rst | 8 +-
docs/plugins.rst | 5 +-
images/docker_expose_daemon.png | Bin 72072 -> 0 bytes
images/docker_wsl_integration.png | Bin 0 -> 36171 bytes
.../kubernetes => kubernetes_tests}/__init__.py | 0
.../test_kubernetes_executor.py | 98 +-
.../test_kubernetes_pod_operator.py | 169 +-
requirements/requirements-python3.6.txt | 71 +-
requirements/requirements-python3.7.txt | 67 +-
requirements/setup-3.6.md5 | 2 +-
requirements/setup-3.7.md5 | 2 +-
scripts/ci/_utils.sh | 629 ++++-
...lint_todo.sh => ci_deploy_app_to_kubernetes.sh} | 34 +-
scripts/ci/ci_fix_ownership.sh | 9 +
...oduction_images.sh => ci_load_image_to_kind.sh} | 17 +-
...ges.sh => ci_perform_kind_cluster_operation.sh} | 7 +-
scripts/ci/ci_run_airflow_testing.sh | 33 +-
scripts/ci/ci_run_kubernetes_tests.sh | 105 +
scripts/ci/docker-compose/base.yml | 1 -
scripts/ci/docker-compose/local.yml | 1 +
scripts/ci/docker-compose/runtime-kubernetes.yml | 25 -
scripts/ci/in_container/_in_container_utils.sh | 32 +-
scripts/ci/in_container/entrypoint_ci.sh | 72 +-
.../ci/in_container/kubernetes/app/deploy_app.sh | 206 --
.../app/templates/init_git_sync.template.yaml | 36 -
.../kubernetes/docker/airflow-test-env-init.sh | 24 -
.../ci/in_container/kubernetes/docker/bootstrap.sh | 27 -
.../kubernetes/docker/rebuild_airflow_image.sh | 74 -
.../in_container/kubernetes/setup_kind_cluster.sh | 187 --
scripts/ci/in_container/run_ci_tests.sh | 4 -
.../kubernetes/app/postgres.yaml | 0
.../{in_container => }/kubernetes/app/secrets.yaml | 0
.../kubernetes/app/templates/airflow.template.yaml | 66 +-
.../app/templates/configmaps.template.yaml | 1 +
.../app/templates/init_git_sync.template.yaml | 35 +-
.../{in_container => }/kubernetes/app/volumes.yaml | 0
.../docker/airflow-test-env-init-dags.sh} | 19 +-
.../kubernetes/docker/airflow-test-env-init-db.sh | 26 +-
scripts/ci/kubernetes/docker/bootstrap.sh | 74 +
.../kubernetes/kind-cluster-conf.yaml | 0
scripts/ci/pre_commit_local_yml_mounts.sh | 26 +-
scripts/ci/pylint_todo.txt | 6 -
setup.py | 6 +-
.../zendesk/hooks => api_connexion}/__init__.py | 0
.../hooks => api_connexion/endpoints}/__init__.py | 0
.../endpoints/test_config_endpoint.py} | 27 +-
.../endpoints/test_connection_endpoint.py | 66 +
tests/api_connexion/endpoints/test_dag_endpoint.py | 59 +
.../endpoints/test_dag_run_endpoint.py | 66 +
.../endpoints/test_dag_source_endpoint.py | 50 +-
.../endpoints/test_event_log_endpoint.py} | 36 +-
.../endpoints/test_extra_link_endpoint.py} | 29 +-
.../endpoints/test_health_endpoint.py} | 26 +-
.../endpoints/test_import_errror_endpoint.py | 52 +
.../endpoints/test_log_endpoint.py} | 29 +-
.../api_connexion/endpoints/test_pool_endpoint.py | 66 +
.../endpoints/test_task_endpoint.py} | 36 +-
.../endpoints/test_task_instance_endpoint.py | 61 +
.../endpoints/test_variable_endpoint.py | 66 +
.../api_connexion/endpoints/test_xcom_endpoint.py | 76 +
tests/bats/test_local_mounts.bats | 27 +-
tests/cli/commands/test_celery_command.py | 5 +-
tests/cli/commands/test_config_command.py | 4 +-
tests/cli/commands/test_dag_command.py | 39 +-
tests/cli/commands/test_role_command.py | 3 +-
tests/cli/commands/test_sync_perm_command.py | 19 +-
tests/cli/commands/test_task_command.py | 5 +-
tests/cli/commands/test_user_command.py | 3 +-
tests/cli/test_cli_parser.py | 7 +
tests/conftest.py | 37 -
tests/jobs/test_scheduler_job.py | 281 ++-
tests/models/test_serialized_dag.py | 2 -
tests/models/test_taskinstance.py | 25 +-
tests/operators/test_sql_branch_operator.py | 476 ++++
tests/plugins/test_plugins_manager.py | 36 +-
.../providers/apache/hdfs/sensors/test_webhdfs.py | 6 +-
tests/providers/apache/hive/hooks/test_hive.py | 462 +++-
tests/providers/apache/hive/operators/test_hive.py | 123 +-
.../apache/hive/operators/test_hive_stats.py | 57 +-
.../apache/hive/operators/test_hive_to_mysql.py | 88 +-
.../apache/hive/operators/test_hive_to_samba.py | 42 +-
.../apache/hive/operators/test_mysql_to_hive.py | 291 ++-
.../apache/hive/sensors/test_hive_partition.py | 7 +-
.../hive/sensors/test_metastore_partition.py | 6 +-
.../hive/sensors/test_named_hive_partition.py | 54 +-
.../kubernetes/operators/test_kubernetes_pod.py | 134 ++
.../operators/test_spark_kubernetes_system.py | 2 -
.../google/cloud/operators/test_bigquery.py | 91 +-
.../snowflake/operators/test_snowflake_to_slack.py | 80 +
tests/runtime/__init__.py | 17 -
tests/test_project_structure.py | 2 -
tests/test_utils/mock_hooks.py | 96 +
tests/test_utils/mock_operators.py | 8 +
tests/test_utils/mock_process.py | 87 +
.../www/api/experimental/test_dag_runs_endpoint.py | 2 +-
tests/www/api/experimental/test_endpoints.py | 3 +-
.../api/experimental/test_kerberos_endpoints.py | 2 +-
tests/www/test_app.py | 28 +-
tests/www/test_utils.py | 12 +-
tests/www/test_views.py | 6 +-
170 files changed, 8590 insertions(+), 2512 deletions(-)
copy {tests/providers/zendesk/hooks => airflow/api_connexion}/__init__.py
(100%)
copy {tests/providers/zendesk/hooks =>
airflow/api_connexion/endpoints}/__init__.py (100%)
copy tests/test_utils/fake_datetime.py =>
airflow/api_connexion/endpoints/config_endpoint.py (77%)
copy airflow/{utils/net.py => api_connexion/endpoints/connection_endpoint.py}
(53%)
copy airflow/{utils/net.py => api_connexion/endpoints/dag_endpoint.py} (56%)
copy airflow/{utils/net.py => api_connexion/endpoints/dag_run_endpoint.py}
(51%)
copy tests/test_utils/fake_datetime.py =>
airflow/api_connexion/endpoints/dag_source_endpoint.py (76%)
copy tests/test_utils/fake_datetime.py =>
airflow/api_connexion/endpoints/event_log_endpoint.py (69%)
copy tests/test_utils/fake_datetime.py =>
airflow/api_connexion/endpoints/extra_link_endpoint.py (76%)
copy tests/bats/test_empty_test.bats =>
airflow/api_connexion/endpoints/health_endpoint.py (79%)
copy airflow/{utils/net.py =>
api_connexion/endpoints/import_errror_endpoint.py} (63%)
copy tests/test_utils/fake_datetime.py =>
airflow/api_connexion/endpoints/log_endpoint.py (77%)
copy airflow/{utils/net.py => api_connexion/endpoints/pool_endpoint.py} (56%)
copy tests/test_utils/fake_datetime.py =>
airflow/api_connexion/endpoints/task_endpoint.py (69%)
copy airflow/{utils/net.py =>
api_connexion/endpoints/task_instance_endpoint.py} (57%)
copy airflow/{utils/net.py => api_connexion/endpoints/variable_endpoint.py}
(54%)
copy airflow/{utils/net.py => api_connexion/endpoints/xcom_endpoint.py} (54%)
create mode 100644 airflow/api_connexion/openapi/v1.yaml
create mode 100644 airflow/operators/sql_branch_operator.py
create mode 100644 airflow/providers/snowflake/operators/snowflake_to_slack.py
create mode 100644 docs/howto/operator/gcp/dataproc.rst
delete mode 100755 images/docker_expose_daemon.png
create mode 100644 images/docker_wsl_integration.png
rename {tests/runtime/kubernetes => kubernetes_tests}/__init__.py (100%)
rename {tests/runtime/kubernetes =>
kubernetes_tests}/test_kubernetes_executor.py (76%)
rename {tests/runtime/kubernetes =>
kubernetes_tests}/test_kubernetes_pod_operator.py (85%)
copy scripts/ci/{ci_refresh_pylint_todo.sh => ci_deploy_app_to_kubernetes.sh}
(61%)
copy scripts/ci/{ci_build_production_images.sh => ci_load_image_to_kind.sh}
(61%)
copy scripts/ci/{ci_build_production_images.sh =>
ci_perform_kind_cluster_operation.sh} (89%)
create mode 100755 scripts/ci/ci_run_kubernetes_tests.sh
delete mode 100644 scripts/ci/docker-compose/runtime-kubernetes.yml
delete mode 100755 scripts/ci/in_container/kubernetes/app/deploy_app.sh
delete mode 100644
scripts/ci/in_container/kubernetes/app/templates/init_git_sync.template.yaml
delete mode 100755
scripts/ci/in_container/kubernetes/docker/airflow-test-env-init.sh
delete mode 100755 scripts/ci/in_container/kubernetes/docker/bootstrap.sh
delete mode 100755
scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh
delete mode 100755 scripts/ci/in_container/kubernetes/setup_kind_cluster.sh
rename scripts/ci/{in_container => }/kubernetes/app/postgres.yaml (100%)
rename scripts/ci/{in_container => }/kubernetes/app/secrets.yaml (100%)
rename scripts/ci/{in_container =>
}/kubernetes/app/templates/airflow.template.yaml (70%)
rename scripts/ci/{in_container =>
}/kubernetes/app/templates/configmaps.template.yaml (99%)
copy tests/providers/amazon/aws/operators/test_emr_system.py =>
scripts/ci/kubernetes/app/templates/init_git_sync.template.yaml (57%)
rename scripts/ci/{in_container => }/kubernetes/app/volumes.yaml (100%)
copy scripts/ci/{in_container/_in_container_script_init.sh =>
kubernetes/docker/airflow-test-env-init-dags.sh} (72%)
copy hooks/push => scripts/ci/kubernetes/docker/airflow-test-env-init-db.sh
(64%)
create mode 100755 scripts/ci/kubernetes/docker/bootstrap.sh
rename scripts/ci/{in_container => }/kubernetes/kind-cluster-conf.yaml (100%)
copy tests/{providers/zendesk/hooks => api_connexion}/__init__.py (100%)
copy tests/{providers/zendesk/hooks => api_connexion/endpoints}/__init__.py
(100%)
copy tests/{cli/commands/test_version_command.py =>
api_connexion/endpoints/test_config_endpoint.py} (61%)
create mode 100644 tests/api_connexion/endpoints/test_connection_endpoint.py
create mode 100644 tests/api_connexion/endpoints/test_dag_endpoint.py
create mode 100644 tests/api_connexion/endpoints/test_dag_run_endpoint.py
copy airflow/utils/weight_rule.py =>
tests/api_connexion/endpoints/test_dag_source_endpoint.py (51%)
copy tests/{cli/commands/test_version_command.py =>
api_connexion/endpoints/test_event_log_endpoint.py} (51%)
copy tests/{cli/commands/test_version_command.py =>
api_connexion/endpoints/test_extra_link_endpoint.py} (59%)
copy tests/{providers/amazon/aws/hooks/test_sqs.py =>
api_connexion/endpoints/test_health_endpoint.py} (62%)
create mode 100644 tests/api_connexion/endpoints/test_import_errror_endpoint.py
copy tests/{cli/commands/test_version_command.py =>
api_connexion/endpoints/test_log_endpoint.py} (59%)
create mode 100644 tests/api_connexion/endpoints/test_pool_endpoint.py
copy tests/{cli/commands/test_version_command.py =>
api_connexion/endpoints/test_task_endpoint.py} (50%)
create mode 100644 tests/api_connexion/endpoints/test_task_instance_endpoint.py
create mode 100644 tests/api_connexion/endpoints/test_variable_endpoint.py
create mode 100644 tests/api_connexion/endpoints/test_xcom_endpoint.py
create mode 100644 tests/operators/test_sql_branch_operator.py
create mode 100644
tests/providers/cncf/kubernetes/operators/test_kubernetes_pod.py
create mode 100644
tests/providers/snowflake/operators/test_snowflake_to_slack.py
delete mode 100644 tests/runtime/__init__.py
create mode 100644 tests/test_utils/mock_hooks.py
create mode 100644 tests/test_utils/mock_process.py
