This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-8-test by this push: new bf85dbe9b3 Speed up autocompletion of Breeze by simplifying provider state (#36499) bf85dbe9b3 is described below commit bf85dbe9b3ff7d10a91c8e71bdb254445b9fc193 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sat Dec 30 19:16:15 2023 +0100 Speed up autocompletion of Breeze by simplifying provider state (#36499) Some recent changes, adding removed and suspended state for breeze caused significant slow-down of autocompletion retrieval - as it turned out, because we loaded and parsed all provider yaml files during auto-completion - in order to determine list of providers available for some commands. We already planned to replace the several states (suspended, not-ready, removed) with a single state field - by doing it and addding the field to pre-commit generated "provider_dependencies.json" we could switch to parsing the single provider_dependencies.json file and retrieve provider list from there following the state stored in that json file. This also simplifies state management following the recently added state diagram by following the same state lifecycle: "not-ready" -> "ready" -> "suspended" -> "removed" --- PROVIDERS.rst | 2 +- airflow/provider.yaml.schema.json | 22 +- airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 90 +++---- airflow/providers/airbyte/provider.yaml | 6 +- airflow/providers/alibaba/provider.yaml | 6 +- airflow/providers/amazon/provider.yaml | 15 +- airflow/providers/apache/beam/provider.yaml | 6 +- airflow/providers/apache/cassandra/provider.yaml | 6 +- airflow/providers/apache/drill/provider.yaml | 5 +- airflow/providers/apache/druid/provider.yaml | 5 +- airflow/providers/apache/flink/provider.yaml | 7 +- airflow/providers/apache/hdfs/provider.yaml | 11 +- airflow/providers/apache/hive/provider.yaml | 6 +- airflow/providers/apache/impala/provider.yaml | 5 +- airflow/providers/apache/kafka/provider.yaml | 6 +- airflow/providers/apache/kylin/provider.yaml | 10 +- airflow/providers/apache/livy/provider.yaml | 5 +- airflow/providers/apache/pig/provider.yaml | 5 +- airflow/providers/apache/pinot/provider.yaml | 5 +- airflow/providers/apache/spark/provider.yaml | 6 +- airflow/providers/apache/sqoop/provider.yaml | 3 +- airflow/providers/apprise/provider.yaml | 6 +- airflow/providers/arangodb/provider.yaml | 6 +- airflow/providers/asana/provider.yaml | 6 +- airflow/providers/atlassian/jira/provider.yaml | 5 +- airflow/providers/celery/provider.yaml | 6 +- airflow/providers/cloudant/provider.yaml | 6 +- airflow/providers/cncf/kubernetes/provider.yaml | 7 +- airflow/providers/cohere/provider.yaml | 6 +- airflow/providers/common/io/provider.yaml | 4 +- airflow/providers/common/sql/provider.yaml | 6 +- airflow/providers/daskexecutor/provider.yaml | 3 +- airflow/providers/databricks/provider.yaml | 7 +- airflow/providers/datadog/provider.yaml | 6 +- airflow/providers/dbt/cloud/provider.yaml | 6 +- airflow/providers/dingding/provider.yaml | 5 +- airflow/providers/discord/provider.yaml | 5 +- airflow/providers/docker/provider.yaml | 6 +- airflow/providers/elasticsearch/provider.yaml | 6 +- airflow/providers/exasol/provider.yaml | 8 +- airflow/providers/facebook/provider.yaml | 5 +- airflow/providers/ftp/provider.yaml | 5 +- airflow/providers/github/provider.yaml | 6 +- airflow/providers/google/provider.yaml | 8 +- airflow/providers/grpc/provider.yaml | 6 +- airflow/providers/hashicorp/provider.yaml | 6 +- airflow/providers/http/provider.yaml | 5 +- airflow/providers/imap/provider.yaml | 5 +- airflow/providers/influxdb/provider.yaml | 5 +- airflow/providers/jdbc/provider.yaml | 6 +- airflow/providers/jenkins/provider.yaml | 6 +- airflow/providers/microsoft/azure/provider.yaml | 7 +- airflow/providers/microsoft/mssql/provider.yaml | 5 +- airflow/providers/microsoft/psrp/provider.yaml | 5 +- airflow/providers/microsoft/winrm/provider.yaml | 5 +- airflow/providers/mongo/provider.yaml | 5 +- airflow/providers/mysql/provider.yaml | 5 +- airflow/providers/neo4j/provider.yaml | 5 +- airflow/providers/odbc/provider.yaml | 8 +- airflow/providers/openai/provider.yaml | 5 +- airflow/providers/openfaas/provider.yaml | 5 +- airflow/providers/openlineage/provider.yaml | 8 +- airflow/providers/opensearch/provider.yaml | 6 +- airflow/providers/opsgenie/provider.yaml | 5 +- airflow/providers/oracle/provider.yaml | 6 +- airflow/providers/pagerduty/provider.yaml | 6 +- airflow/providers/papermill/provider.yaml | 5 +- airflow/providers/pgvector/provider.yaml | 5 +- airflow/providers/pinecone/provider.yaml | 6 +- airflow/providers/plexus/provider.yaml | 3 +- airflow/providers/postgres/provider.yaml | 6 +- airflow/providers/presto/provider.yaml | 6 +- airflow/providers/redis/provider.yaml | 5 +- airflow/providers/salesforce/provider.yaml | 6 +- airflow/providers/samba/provider.yaml | 5 +- airflow/providers/segment/provider.yaml | 5 +- airflow/providers/sendgrid/provider.yaml | 5 +- airflow/providers/sftp/provider.yaml | 6 +- airflow/providers/singularity/provider.yaml | 5 +- airflow/providers/slack/provider.yaml | 6 +- airflow/providers/smtp/provider.yaml | 6 +- airflow/providers/snowflake/provider.yaml | 8 +- airflow/providers/sqlite/provider.yaml | 6 +- airflow/providers/ssh/provider.yaml | 6 +- airflow/providers/tableau/provider.yaml | 5 +- airflow/providers/tabular/provider.yaml | 6 +- airflow/providers/telegram/provider.yaml | 5 +- airflow/providers/trino/provider.yaml | 6 +- airflow/providers/vertica/provider.yaml | 5 +- airflow/providers/weaviate/provider.yaml | 8 +- airflow/providers/yandex/provider.yaml | 6 +- airflow/providers/zendesk/provider.yaml | 6 +- dev/breeze/src/airflow_breeze/breeze.py | 2 + .../commands/release_management_commands.py | 6 +- dev/breeze/src/airflow_breeze/utils/packages.py | 53 ++-- dev/breeze/tests/test_packages.py | 2 +- docs/exts/docs_build/package_filter.py | 2 +- docs/exts/provider_yaml_utils.py | 2 +- generated/provider_dependencies.json | 297 ++++++++++++++------- ...put_release-management_generate-constraints.svg | 6 +- ...output_sbom_generate-providers-requirements.svg | 68 ++--- ...output_sbom_generate-providers-requirements.txt | 2 +- .../pre_commit/pre_commit_check_provider_docs.py | 2 +- .../pre_commit_update_providers_dependencies.py | 15 +- .../in_container/run_provider_yaml_files_check.py | 25 +- setup.py | 2 + tests/always/test_example_dags.py | 2 +- 107 files changed, 685 insertions(+), 439 deletions(-) diff --git a/PROVIDERS.rst b/PROVIDERS.rst index a9bdc2b7ab..d8c3360190 100644 --- a/PROVIDERS.rst +++ b/PROVIDERS.rst @@ -254,7 +254,7 @@ process can be suspended. This means: -* The provider's status is set to "suspended" +* The provider's state in ``provider.yaml`` is set to "suspended" * No new releases of the provider will be made until the problem with dependencies is solved * Sources of the provider remain in the repository for now (in the future we might add process to remove them) * No new changes will be accepted for the provider (other than the ones that fix the dependencies) diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index d811f7318d..2d439b7fda 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -21,17 +21,15 @@ "type": "string" } }, - "suspended": { - "description": "If set to true, the provider is suspended and it's not a candidate for release nor contributes dependencies to constraint calculations/CI image. Tests are excluded.", - "type:": "boolean" - }, - "removed": { - "description": "If set to true, the provider is also removed and will be soon removed from the code", - "type:": "boolean" - }, - "not-ready": { - "description": "If set to true, the provider is not included by default in release commands - for example when provider release or documentation is being prepared (not-ready providers are enabled in CI by default)", - "type:": "boolean" + "state": { + "description": "State of provider: might be not-ready, regular, suspended, removed.", + "type:": "string", + "enum": [ + "not-ready", + "ready", + "suspended", + "removed" + ] }, "dependencies": { "description": "Dependencies that should be added to the provider", @@ -455,7 +453,7 @@ "name", "package-name", "description", - "suspended", + "state", "source-date-epoch", "dependencies", "versions" diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 27ae7ef456..9170786aff 100644 --- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -55,51 +55,51 @@ If you still have doubts about building your provider, we recommend that you rea open a issue on GitHub so the community can help you. The folders are optional: example_dags, hooks, links, logs, notifications, operators, secrets, sensors, transfers, -triggers, waiters (and the list changes continuously). +triggers (and the list changes continuously). .. code-block:: bash airflow/ - ├── providers/<NEW_PROVIDER>/ - │ ├── __init__.py - │ ├── example_dags/ - │ │ ├── __init__.py - │ │ └── example_<NEW_PROVIDER>.py - │ ├── executors/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - │ ├── hooks/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - │ ├── operators/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - .... - │ ├── transfers/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - │ └── triggers/ - │ ├── __init__.py - │ └── <NEW_PROVIDER>.py - └── tests/providers/<NEW_PROVIDER>/ - ├── __init__.py - ├── executors/ - │ ├── __init__.py - │ └── test_<NEW_PROVIDER>.py - ├── hooks/ - │ ├── __init__.py - │ └── test_<NEW_PROVIDER>.py - ├── operators/ - │ ├── __init__.py - │ ├── test_<NEW_PROVIDER>.py - │ └── test_<NEW_PROVIDER>_system.py - ... - ├── transfers/ - │ ├── __init__.py - │ └── test_<NEW_PROVIDER>.py - └── triggers/ - ├── __init__.py - └── test_<NEW_PROVIDER>.py + ├── providers/<NEW_PROVIDER>/ + │ ├── __init__.py + │ ├── executors/ + │ │ ├── __init__.py + │ │ └── *.py + │ ├── hooks/ + │ │ ├── __init__.py + │ │ └── *.py + │ ├── operators/ + │ │ ├── __init__.py + │ │ └── *.py + │ ├── transfers/ + │ │ ├── __init__.py + │ │ └── *.py + │ └── triggers/ + │ ├── __init__.py + │ └── *.py + └── tests + ├── providers/<NEW_PROVIDER>/ + │ ├── __init__.py + │ ├── executors/ + │ │ ├── __init__.py + │ │ └── test_*.py + │ ├── hooks/ + │ │ ├── __init__.py + │ │ └── test_*>.py + │ ├── operators/ + │ │ ├── __init__.py + │ │ ├── test_*.py + │ ... + │ ├── transfers/ + │ │ ├── __init__.py + │ │ └── test_*.py + │ └── triggers/ + │ ├── __init__.py + │ └── test_*.py + └── system/providers/<NEW_PROVIDER>/ + ├── __init__.py + └── example_*.py + Considering that you have already transferred your provider's code to the above structure, it will now be necessary to create unit tests for each component you created. The example below I have already set up an environment using @@ -107,7 +107,7 @@ breeze and I'll run unit tests for my Hook. .. code-block:: bash - root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers/<NEW_PROVIDER>/hook/<NEW_PROVIDER>.py + root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers/<NEW_PROVIDER>/hook/test_*.py Adding chicken-egg providers ---------------------------- @@ -467,7 +467,7 @@ As of April 2023, we have the possibility to suspend individual providers, so th back dependencies for Airflow and other providers. The process of suspending providers is described in `description of the process <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#suspending-releases-for-providers>`_ -Technically, suspending a provider is done by setting ``suspended : true``, in the provider.yaml of the +Technically, suspending a provider is done by setting ``state: suspended``, in the provider.yaml of the provider. This should be followed by committing the change and either automatically or manually running pre-commit checks that will either update derived configuration files or ask you to update them manually. Note that you might need to run pre-commit several times until all the static checks pass, @@ -509,10 +509,10 @@ Removing providers When removing providers from Airflow code, we need to make one last release where we mark the provider as removed - in documentation and in description of the PyPI package. In order to that release manager has to -add "removed: true" flag in the provider yaml file and include the provider in the next wave of the +add "state: removed" flag in the provider yaml file and include the provider in the next wave of the providers (and then remove all the code and documentation related to the provider). -The "removed: true" flag will cause the provider to be available for the following commands (note that such +The "removed: removed" flag will cause the provider to be available for the following commands (note that such provider has to be explicitly added as selected to the package - such provider will not be included in the available list of providers or when documentation is built unless --include-removed-providers flag is used): diff --git a/airflow/providers/airbyte/provider.yaml b/airflow/providers/airbyte/provider.yaml index 01b7baa603..c973844dd7 100644 --- a/airflow/providers/airbyte/provider.yaml +++ b/airflow/providers/airbyte/provider.yaml @@ -21,9 +21,11 @@ name: Airbyte description: | `Airbyte <https://airbyte.com/>`__ -suspended: false -source-date-epoch: 1700827428 +state: ready +source-date-epoch: 1703288099 versions: + - 3.5.1 + - 3.5.0 - 3.4.0 - 3.3.2 - 3.3.1 diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml index df25ac135c..0c03ce6480 100644 --- a/airflow/providers/alibaba/provider.yaml +++ b/airflow/providers/alibaba/provider.yaml @@ -21,9 +21,11 @@ name: Alibaba description: | Alibaba Cloud integration (including `Alibaba Cloud <https://www.alibabacloud.com/>`__). -suspended: false -source-date-epoch: 1700827429 +state: ready +source-date-epoch: 1703747466 versions: + - 2.7.1 + - 2.7.0 - 2.6.0 - 2.5.3 - 2.5.2 diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index aa123ed7c9..d86e02877c 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -21,9 +21,12 @@ name: Amazon description: | Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__). -suspended: false -source-date-epoch: 1700827430 +state: ready +source-date-epoch: 1703747484 versions: + - 8.15.0 + - 8.14.0 + - 8.13.0 - 8.12.0 - 8.11.0 - 8.10.0 @@ -81,11 +84,13 @@ dependencies: # same minimum version specified. Boto3 1.28.0 has been released on July 6 2023. We should also make sure we # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version - # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + # that supported boto3 1.28. + # NOTE!!! BEFORE botocore 1.33 BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS # NOTE!!! Make sure to update _MIN_BOTO3_VERSION in setup.py when you update it here - boto3>=1.28.0 - # NOTE!!! BOTOCORE version is always shifted by 3 MINOR VERSIONS from boto3 - # See https://github.com/boto/boto3/issues/2702 + # NOTE!!! BEFORE botocore 1.33 version is always shifted by 3 MINOR VERSIONS from boto3, see: + # - https://github.com/boto/boto3/issues/2702 + # - https://github.com/boto/boto3/blob/a3e710df58676c2567b8023dbe09025ad5e0e50f/.changes/1.33.0.json#L13-L15 - botocore>=1.31.0 - inflection>=0.5.1 # Allow a wider range of watchtower versions for flexibility among users diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index 1001e156d9..0facc9cd45 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -21,9 +21,11 @@ name: Apache Beam description: | `Apache Beam <https://beam.apache.org/>`__. -suspended: false -source-date-epoch: 1700827432 +state: ready +source-date-epoch: 1703288103 versions: + - 5.5.0 + - 5.4.0 - 5.3.0 - 5.2.3 - 5.2.2 diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml index 807bc09aea..96827667f3 100644 --- a/airflow/providers/apache/cassandra/provider.yaml +++ b/airflow/providers/apache/cassandra/provider.yaml @@ -21,9 +21,11 @@ name: Apache Cassandra description: | `Apache Cassandra <https://cassandra.apache.org/>`__. -suspended: false -source-date-epoch: 1700827433 +state: ready +source-date-epoch: 1703288104 versions: + - 3.4.1 + - 3.4.0 - 3.3.0 - 3.2.1 - 3.2.0 diff --git a/airflow/providers/apache/drill/provider.yaml b/airflow/providers/apache/drill/provider.yaml index b0c113c275..950917e411 100644 --- a/airflow/providers/apache/drill/provider.yaml +++ b/airflow/providers/apache/drill/provider.yaml @@ -21,9 +21,10 @@ name: Apache Drill description: | `Apache Drill <https://drill.apache.org/>`__. -suspended: false -source-date-epoch: 1700827433 +state: ready +source-date-epoch: 1703288105 versions: + - 2.6.0 - 2.5.0 - 2.4.4 - 2.4.3 diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index f909cc6b8b..67ab94fe7a 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -21,9 +21,10 @@ name: Apache Druid description: | `Apache Druid <https://druid.apache.org/>`__. -suspended: false -source-date-epoch: 1700827434 +state: ready +source-date-epoch: 1703288106 versions: + - 3.7.0 - 3.6.0 - 3.5.0 - 3.4.2 diff --git a/airflow/providers/apache/flink/provider.yaml b/airflow/providers/apache/flink/provider.yaml index 6e4fce8e57..db6f7f224e 100644 --- a/airflow/providers/apache/flink/provider.yaml +++ b/airflow/providers/apache/flink/provider.yaml @@ -21,9 +21,10 @@ name: Apache Flink description: | `Apache Flink <https://flink.apache.org/>`__ -suspended: false -source-date-epoch: 1700827435 +state: ready +source-date-epoch: 1703288107 versions: + - 1.3.0 - 1.2.0 - 1.1.3 - 1.1.2 @@ -41,7 +42,7 @@ integrations: - integration-name: Apache Flink external-doc-url: https://github.com/apache/flink-kubernetes-operator how-to-guide: - - /docs/apache-airflow-providers-apache-flink/operators.rst # TODO + - /docs/apache-airflow-providers-apache-flink/operators.rst logo: /integration-logos/kubernetes/FlinkOnK8s.png tags: [apache] diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml index 911c8871ea..99155692b3 100644 --- a/airflow/providers/apache/hdfs/provider.yaml +++ b/airflow/providers/apache/hdfs/provider.yaml @@ -22,9 +22,12 @@ description: | `Hadoop Distributed File System (HDFS) <https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html>`__ and `WebHDFS <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html>`__. -suspended: false -source-date-epoch: 1700827436 +state: ready +source-date-epoch: 1703747531 versions: + - 4.3.2 + - 4.3.1 + - 4.3.0 - 4.2.0 - 4.1.1 - 4.1.0 @@ -69,3 +72,7 @@ hooks: - integration-name: WebHDFS python-modules: - airflow.providers.apache.hdfs.hooks.webhdfs + +connection-types: + - hook-class-name: airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook + connection-type: webhdfs diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index f279ced9d0..a313d7c053 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -21,9 +21,11 @@ name: Apache Hive description: | `Apache Hive <https://hive.apache.org/>`__ -suspended: false -source-date-epoch: 1700827437 +state: ready +source-date-epoch: 1703288109 versions: + - 6.4.0 + - 6.3.0 - 6.2.0 - 6.1.6 - 6.1.5 diff --git a/airflow/providers/apache/impala/provider.yaml b/airflow/providers/apache/impala/provider.yaml index 5b1153add2..b6cc987088 100644 --- a/airflow/providers/apache/impala/provider.yaml +++ b/airflow/providers/apache/impala/provider.yaml @@ -21,9 +21,10 @@ name: Apache Impala description: | `Apache Impala <https://impala.apache.org/>`__. -suspended: false -source-date-epoch: 1700827438 +state: ready +source-date-epoch: 1703288110 versions: + - 1.3.0 - 1.2.1 - 1.2.0 - 1.1.3 diff --git a/airflow/providers/apache/kafka/provider.yaml b/airflow/providers/apache/kafka/provider.yaml index 1ffba48c0f..5c6a98dd8b 100644 --- a/airflow/providers/apache/kafka/provider.yaml +++ b/airflow/providers/apache/kafka/provider.yaml @@ -19,11 +19,13 @@ package-name: apache-airflow-providers-apache-kafka name: Apache Kafka -suspended: false -source-date-epoch: 1700827439 +state: ready +source-date-epoch: 1703288110 description: | `Apache Kafka <https://kafka.apache.org/>`__ versions: + - 1.3.1 + - 1.3.0 - 1.2.0 - 1.1.2 - 1.1.1 diff --git a/airflow/providers/apache/kylin/provider.yaml b/airflow/providers/apache/kylin/provider.yaml index ee49cae30e..6bcb1799de 100644 --- a/airflow/providers/apache/kylin/provider.yaml +++ b/airflow/providers/apache/kylin/provider.yaml @@ -21,9 +21,11 @@ name: Apache Kylin description: | `Apache Kylin <https://kylin.apache.org/>`__ -suspended: false -source-date-epoch: 1700827439 +state: ready +source-date-epoch: 1703747544 versions: + - 3.5.0 + - 3.4.0 - 3.3.0 - 3.2.1 - 3.2.0 @@ -56,3 +58,7 @@ hooks: - integration-name: Apache Kylin python-modules: - airflow.providers.apache.kylin.hooks.kylin + +connection-types: + - hook-class-name: airflow.providers.apache.kylin.hooks.kylin.KylinHook + connection-type: kylin diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml index bb1461d1fc..da48cf78cc 100644 --- a/airflow/providers/apache/livy/provider.yaml +++ b/airflow/providers/apache/livy/provider.yaml @@ -21,9 +21,10 @@ name: Apache Livy description: | `Apache Livy <https://livy.apache.org/>`__ -suspended: false -source-date-epoch: 1700827440 +state: ready +source-date-epoch: 1703288112 versions: + - 3.7.0 - 3.6.0 - 3.5.4 - 3.5.3 diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml index 8582be1c64..77ed3301fa 100644 --- a/airflow/providers/apache/pig/provider.yaml +++ b/airflow/providers/apache/pig/provider.yaml @@ -21,9 +21,10 @@ name: Apache Pig description: | `Apache Pig <https://pig.apache.org/>`__ -suspended: false -source-date-epoch: 1700827441 +state: ready +source-date-epoch: 1703288113 versions: + - 4.3.0 - 4.2.0 - 4.1.2 - 4.1.1 diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index e290de70b7..f90e12da60 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -21,9 +21,10 @@ name: Apache Pinot description: | `Apache Pinot <https://pinot.apache.org/>`__ -suspended: false -source-date-epoch: 1700827442 +state: ready +source-date-epoch: 1703288114 versions: + - 4.3.0 - 4.2.0 - 4.1.4 - 4.1.3 diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 0c95deb67d..fd5547f6a3 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -21,9 +21,11 @@ name: Apache Spark description: | `Apache Spark <https://spark.apache.org/>`__ -suspended: false -source-date-epoch: 1700827443 +state: ready +source-date-epoch: 1703288115 versions: + - 4.6.0 + - 4.5.0 - 4.4.0 - 4.3.0 - 4.2.0 diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml index d6bc9b6c60..0c84e84efe 100644 --- a/airflow/providers/apache/sqoop/provider.yaml +++ b/airflow/providers/apache/sqoop/provider.yaml @@ -21,8 +21,7 @@ name: Apache Sqoop description: | `Apache Sqoop <https://sqoop.apache.org/>`__ -removed: true -suspended: true +state: removed source-date-epoch: 1701983361 versions: - 4.2.0 diff --git a/airflow/providers/apprise/provider.yaml b/airflow/providers/apprise/provider.yaml index 5d5df2360e..0dfb2fe6f8 100644 --- a/airflow/providers/apprise/provider.yaml +++ b/airflow/providers/apprise/provider.yaml @@ -23,10 +23,12 @@ name: Apprise description: | `Apprise <https://github.com/caronc/apprise>`__ -suspended: false -source-date-epoch: 1700827445 +state: ready +source-date-epoch: 1703288116 versions: + - 1.2.1 + - 1.2.0 - 1.1.0 - 1.0.2 - 1.0.1 diff --git a/airflow/providers/arangodb/provider.yaml b/airflow/providers/arangodb/provider.yaml index dc26c153a4..bd6d75b8e5 100644 --- a/airflow/providers/arangodb/provider.yaml +++ b/airflow/providers/arangodb/provider.yaml @@ -25,9 +25,11 @@ dependencies: - apache-airflow>=2.6.0 - python-arango>=7.3.2 -suspended: false -source-date-epoch: 1700827446 +state: ready +source-date-epoch: 1703288117 versions: + - 2.4.1 + - 2.4.0 - 2.3.0 - 2.2.2 - 2.2.1 diff --git a/airflow/providers/asana/provider.yaml b/airflow/providers/asana/provider.yaml index fc4780a70b..07b764af56 100644 --- a/airflow/providers/asana/provider.yaml +++ b/airflow/providers/asana/provider.yaml @@ -21,9 +21,11 @@ name: Asana description: | `Asana <https://asana.com/>`__ -suspended: false -source-date-epoch: 1700827446 +state: ready +source-date-epoch: 1703288118 versions: + - 2.4.1 + - 2.4.0 - 2.3.0 - 2.2.2 - 2.2.1 diff --git a/airflow/providers/atlassian/jira/provider.yaml b/airflow/providers/atlassian/jira/provider.yaml index b43c503b04..cdc2638161 100644 --- a/airflow/providers/atlassian/jira/provider.yaml +++ b/airflow/providers/atlassian/jira/provider.yaml @@ -21,9 +21,10 @@ name: Atlassian Jira description: | `Atlassian Jira <https://www.atlassian.com/software/jira/>`__ -suspended: false -source-date-epoch: 1700827447 +state: ready +source-date-epoch: 1703288118 versions: + - 2.4.0 - 2.3.0 - 2.2.0 - 2.1.1 diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index b9ce8c6720..962fe57c9d 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -21,9 +21,11 @@ name: Celery description: | `Celery <https://docs.celeryq.dev/en/stable/>`__ -suspended: false -source-date-epoch: 1700827448 +state: ready +source-date-epoch: 1703288119 versions: + - 3.5.1 + - 3.5.0 - 3.4.1 - 3.4.0 - 3.3.4 diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index 4bd2b8fa85..5bdd756e7a 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -21,9 +21,11 @@ name: IBM Cloudant description: | `IBM Cloudant <https://www.ibm.com/cloud/cloudant>`__ -suspended: false -source-date-epoch: 1700827449 +state: ready +source-date-epoch: 1703288120 versions: + - 3.4.1 + - 3.4.0 - 3.3.0 - 3.2.1 - 3.2.0 diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index d59206a3ce..a54edd6f3b 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -21,9 +21,12 @@ name: Kubernetes description: | `Kubernetes <https://kubernetes.io/>`__ -suspended: false -source-date-epoch: 1700827450 +state: ready +source-date-epoch: 1703747586 versions: + - 7.13.0 + - 7.12.0 + - 7.11.0 - 7.10.0 - 7.9.0 - 7.8.0 diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml index efe330e9cd..01313be518 100644 --- a/airflow/providers/cohere/provider.yaml +++ b/airflow/providers/cohere/provider.yaml @@ -23,10 +23,12 @@ name: Cohere description: | `Cohere <https://docs.cohere.com/docs>`__ -suspended: false -source-date-epoch: 1700827451 +state: ready +source-date-epoch: 1703288123 versions: + - 1.1.1 + - 1.1.0 - 1.0.0 integrations: diff --git a/airflow/providers/common/io/provider.yaml b/airflow/providers/common/io/provider.yaml index 975df0d340..bcb7418520 100644 --- a/airflow/providers/common/io/provider.yaml +++ b/airflow/providers/common/io/provider.yaml @@ -21,8 +21,8 @@ name: Common IO description: | ``Common IO Provider`` -suspended: false -source-date-epoch: 1700827452 +state: ready +source-date-epoch: 1703288123 versions: - 1.1.0 - 1.0.1 diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index 06a9f6177a..676a72727c 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -21,9 +21,11 @@ name: Common SQL description: | `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__ -suspended: false -source-date-epoch: 1700827452 +state: ready +source-date-epoch: 1703288124 versions: + - 1.10.0 + - 1.9.0 - 1.8.1 - 1.8.0 - 1.7.2 diff --git a/airflow/providers/daskexecutor/provider.yaml b/airflow/providers/daskexecutor/provider.yaml index 369143bcd9..388d31c30b 100644 --- a/airflow/providers/daskexecutor/provider.yaml +++ b/airflow/providers/daskexecutor/provider.yaml @@ -21,8 +21,7 @@ name: Dask Executor description: | `Dask <https://www.dask.org/>`__ -removed: true -suspended: true +state: removed source-date-epoch: 1700827453 versions: - 1.1.1 diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index 033836dad2..de7829f1cf 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -21,9 +21,10 @@ name: Databricks description: | `Databricks <https://databricks.com/>`__ -suspended: false -source-date-epoch: 1700827454 +state: ready +source-date-epoch: 1703288125 versions: + - 6.0.0 - 5.0.1 - 5.0.0 - 4.7.0 @@ -58,7 +59,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - - apache-airflow-providers-common-sql>=1.8.1 + - apache-airflow-providers-common-sql>=1.10.0 - requests>=2.27,<3 # The connector 2.9.0 released on Aug 10, 2023 has a bug that it does not properly declare urllib3 and # it needs to be excluded. See https://github.com/databricks/databricks-sql-python/issues/190 diff --git a/airflow/providers/datadog/provider.yaml b/airflow/providers/datadog/provider.yaml index bf99c54812..a958527e1e 100644 --- a/airflow/providers/datadog/provider.yaml +++ b/airflow/providers/datadog/provider.yaml @@ -21,9 +21,11 @@ name: Datadog description: | `Datadog <https://www.datadoghq.com/>`__ -suspended: false -source-date-epoch: 1700827455 +state: ready +source-date-epoch: 1703288126 versions: + - 3.5.1 + - 3.5.0 - 3.4.0 - 3.3.2 - 3.3.1 diff --git a/airflow/providers/dbt/cloud/provider.yaml b/airflow/providers/dbt/cloud/provider.yaml index c20fad8213..ba6237264d 100644 --- a/airflow/providers/dbt/cloud/provider.yaml +++ b/airflow/providers/dbt/cloud/provider.yaml @@ -21,9 +21,11 @@ name: dbt Cloud description: | `dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__ -suspended: false -source-date-epoch: 1700827456 +state: ready +source-date-epoch: 1703288127 versions: + - 3.5.1 + - 3.5.0 - 3.4.1 - 3.4.0 - 3.3.0 diff --git a/airflow/providers/dingding/provider.yaml b/airflow/providers/dingding/provider.yaml index 72af5d4b90..3647e9d4d8 100644 --- a/airflow/providers/dingding/provider.yaml +++ b/airflow/providers/dingding/provider.yaml @@ -21,9 +21,10 @@ name: Dingding description: | `DingTalk <https://www.dingtalk.com/>`__ -suspended: false -source-date-epoch: 1700827457 +state: ready +source-date-epoch: 1703288128 versions: + - 3.4.0 - 3.3.0 - 3.2.1 - 3.2.0 diff --git a/airflow/providers/discord/provider.yaml b/airflow/providers/discord/provider.yaml index c3db68a963..a98539f07c 100644 --- a/airflow/providers/discord/provider.yaml +++ b/airflow/providers/discord/provider.yaml @@ -21,9 +21,10 @@ name: Discord description: | `Discord <https://discord.com/>`__ -suspended: false -source-date-epoch: 1700827457 +state: ready +source-date-epoch: 1703288129 versions: + - 3.5.0 - 3.4.1 - 3.4.0 - 3.3.1 diff --git a/airflow/providers/docker/provider.yaml b/airflow/providers/docker/provider.yaml index 4ea3c7b644..4f04a065a6 100644 --- a/airflow/providers/docker/provider.yaml +++ b/airflow/providers/docker/provider.yaml @@ -21,9 +21,11 @@ name: Docker description: | `Docker <https://www.docker.com/>`__ -suspended: false -source-date-epoch: 1700827458 +state: ready +source-date-epoch: 1703288130 versions: + - 3.9.1 + - 3.9.0 - 3.8.2 - 3.8.1 - 3.8.0 diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index 596ac662db..208a5e57fb 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -21,9 +21,11 @@ name: Elasticsearch description: | `Elasticsearch <https://www.elastic.co/elasticsearch>`__ -suspended: false -source-date-epoch: 1700827460 +state: ready +source-date-epoch: 1703747600 versions: + - 5.3.1 + - 5.3.0 - 5.2.0 - 5.1.1 - 5.1.0 diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index 7396f03932..f47d0f7d43 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -21,9 +21,11 @@ name: Exasol description: | `Exasol <https://www.exasol.com/>`__ -suspended: false -source-date-epoch: 1700827461 +state: ready +source-date-epoch: 1703288132 versions: + - 4.4.1 + - 4.4.0 - 4.3.0 - 4.2.5 - 4.2.4 @@ -51,7 +53,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-common-sql>=1.10.0 - pyexasol>=0.5.1 - pandas>=0.17.1 diff --git a/airflow/providers/facebook/provider.yaml b/airflow/providers/facebook/provider.yaml index caa4d3cb88..00fb98142a 100644 --- a/airflow/providers/facebook/provider.yaml +++ b/airflow/providers/facebook/provider.yaml @@ -21,9 +21,10 @@ name: Facebook description: | `Facebook Ads <https://www.facebook.com/about/ads>`__ -suspended: false -source-date-epoch: 1700827462 +state: ready +source-date-epoch: 1703288134 versions: + - 3.4.0 - 3.3.0 - 3.2.2 - 3.2.1 diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml index e9b1260d0d..8dc441635b 100644 --- a/airflow/providers/ftp/provider.yaml +++ b/airflow/providers/ftp/provider.yaml @@ -21,9 +21,10 @@ name: File Transfer Protocol (FTP) description: | `File Transfer Protocol (FTP) <https://tools.ietf.org/html/rfc114>`__ -suspended: false -source-date-epoch: 1700827463 +state: ready +source-date-epoch: 1703288135 versions: + - 3.7.0 - 3.6.1 - 3.6.0 - 3.5.2 diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml index 49c508130e..61d24d2e93 100644 --- a/airflow/providers/github/provider.yaml +++ b/airflow/providers/github/provider.yaml @@ -28,9 +28,11 @@ dependencies: # See https://github.com/PyGithub/PyGithub/issues/2436. - PyGithub!=1.58 -suspended: false -source-date-epoch: 1700827464 +state: ready +source-date-epoch: 1703288136 versions: + - 2.5.1 + - 2.5.0 - 2.4.0 - 2.3.2 - 2.3.1 diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index c47f7dd663..d7aca84d81 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -28,9 +28,11 @@ description: | - `Google Marketing Platform <https://marketingplatform.google.com/>`__ - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite) -suspended: false -source-date-epoch: 1700827465 +state: ready +source-date-epoch: 1703747618 versions: + - 10.13.1 + - 10.13.0 - 10.12.0 - 10.11.1 - 10.11.0 @@ -86,7 +88,7 @@ dependencies: - asgiref>=3.5.2 - gcloud-aio-auth>=4.0.0,<5.0.0 - gcloud-aio-bigquery>=6.1.2 - - gcloud-aio-storage + - gcloud-aio-storage>=9.0.0 - gcsfs>=2023.10.0 - google-ads>=22.1.0 - google-api-core>=2.11.0 diff --git a/airflow/providers/grpc/provider.yaml b/airflow/providers/grpc/provider.yaml index b2004bd45c..cf190109c4 100644 --- a/airflow/providers/grpc/provider.yaml +++ b/airflow/providers/grpc/provider.yaml @@ -21,9 +21,11 @@ name: gRPC description: | `gRPC <https://grpc.io/>`__ -suspended: false -source-date-epoch: 1700827466 +state: ready +source-date-epoch: 1703288139 versions: + - 3.4.1 + - 3.4.0 - 3.3.0 - 3.2.2 - 3.2.1 diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml index a559c3fa72..ce2b3846b4 100644 --- a/airflow/providers/hashicorp/provider.yaml +++ b/airflow/providers/hashicorp/provider.yaml @@ -21,9 +21,11 @@ name: Hashicorp description: | Hashicorp including `Hashicorp Vault <https://www.vaultproject.io/>`__ -suspended: false -source-date-epoch: 1700827467 +state: ready +source-date-epoch: 1703288140 versions: + - 3.6.1 + - 3.6.0 - 3.5.0 - 3.4.3 - 3.4.2 diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index 4296522462..d193955808 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -21,9 +21,10 @@ name: Hypertext Transfer Protocol (HTTP) description: | `Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__ -suspended: false -source-date-epoch: 1700827468 +state: ready +source-date-epoch: 1703288141 versions: + - 4.8.0 - 4.7.0 - 4.6.0 - 4.5.2 diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index caadd8db3f..00a6fdb995 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -22,9 +22,10 @@ name: Internet Message Access Protocol (IMAP) description: | `Internet Message Access Protocol (IMAP) <https://tools.ietf.org/html/rfc3501>`__ -suspended: false -source-date-epoch: 1700827469 +state: ready +source-date-epoch: 1703288142 versions: + - 3.5.0 - 3.4.0 - 3.3.2 - 3.3.1 diff --git a/airflow/providers/influxdb/provider.yaml b/airflow/providers/influxdb/provider.yaml index 24fc5437d6..6820b4cfeb 100644 --- a/airflow/providers/influxdb/provider.yaml +++ b/airflow/providers/influxdb/provider.yaml @@ -28,9 +28,10 @@ dependencies: - influxdb-client>=1.19.0 - requests>=2.26.0 -suspended: false -source-date-epoch: 1700827470 +state: ready +source-date-epoch: 1703288143 versions: + - 2.4.0 - 2.3.0 - 2.2.3 - 2.2.2 diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index e8e4aa735e..1acd7ee944 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -21,9 +21,11 @@ name: Java Database Connectivity (JDBC) description: | `Java Database Connectivity (JDBC) <https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/>`__ -suspended: false -source-date-epoch: 1700827471 +state: ready +source-date-epoch: 1703288143 versions: + - 4.2.1 + - 4.2.0 - 4.1.0 - 4.0.2 - 4.0.1 diff --git a/airflow/providers/jenkins/provider.yaml b/airflow/providers/jenkins/provider.yaml index 1d7248ed3f..90124220d1 100644 --- a/airflow/providers/jenkins/provider.yaml +++ b/airflow/providers/jenkins/provider.yaml @@ -21,9 +21,11 @@ name: Jenkins description: | `Jenkins <https://jenkins.io/>`__ -suspended: false -source-date-epoch: 1700827472 +state: ready +source-date-epoch: 1703288144 versions: + - 3.5.1 + - 3.5.0 - 3.4.0 - 3.3.2 - 3.3.1 diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index be2a8635cc..fc3aefaf9c 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -19,9 +19,12 @@ package-name: apache-airflow-providers-microsoft-azure name: Microsoft Azure description: | `Microsoft Azure <https://azure.microsoft.com/>`__ -suspended: false -source-date-epoch: 1700827474 +state: ready +source-date-epoch: 1703747626 versions: + - 8.5.1 + - 8.5.0 + - 8.4.0 - 8.3.0 - 8.2.0 - 8.1.0 diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index 264bdde941..2a56bc9a2b 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -21,9 +21,10 @@ name: Microsoft SQL Server (MSSQL) description: | `Microsoft SQL Server (MSSQL) <https://www.microsoft.com/sql-server/>`__ -suspended: false -source-date-epoch: 1700827475 +state: ready +source-date-epoch: 1703288147 versions: + - 3.6.0 - 3.5.0 - 3.4.2 - 3.4.1 diff --git a/airflow/providers/microsoft/psrp/provider.yaml b/airflow/providers/microsoft/psrp/provider.yaml index fa2fa33e9a..6d81aa851f 100644 --- a/airflow/providers/microsoft/psrp/provider.yaml +++ b/airflow/providers/microsoft/psrp/provider.yaml @@ -23,9 +23,10 @@ description: | `PowerShell Remoting Protocol (PSRP) <https://docs.microsoft.com/openspecs/windows_protocols/ms-psrp/>`__. -suspended: false -source-date-epoch: 1700827476 +state: ready +source-date-epoch: 1703288148 versions: + - 2.5.0 - 2.4.0 - 2.3.2 - 2.3.1 diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/airflow/providers/microsoft/winrm/provider.yaml index 995f903022..4a4794557e 100644 --- a/airflow/providers/microsoft/winrm/provider.yaml +++ b/airflow/providers/microsoft/winrm/provider.yaml @@ -21,9 +21,10 @@ name: Windows Remote Management (WinRM) description: | `Windows Remote Management (WinRM) <https://docs.microsoft.com/windows/win32/winrm/portal>`__ -suspended: false -source-date-epoch: 1700827477 +state: ready +source-date-epoch: 1703288149 versions: + - 3.4.0 - 3.3.0 - 3.2.2 - 3.2.1 diff --git a/airflow/providers/mongo/provider.yaml b/airflow/providers/mongo/provider.yaml index f93682308b..9e2bea6859 100644 --- a/airflow/providers/mongo/provider.yaml +++ b/airflow/providers/mongo/provider.yaml @@ -21,9 +21,10 @@ name: MongoDB description: | `MongoDB <https://www.mongodb.com/>`__ -suspended: false -source-date-epoch: 1700827478 +state: ready +source-date-epoch: 1703288150 versions: + - 3.5.0 - 3.4.0 - 3.3.0 - 3.2.2 diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index 9bd4e6590b..9712ec8676 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -21,9 +21,10 @@ name: MySQL description: | `MySQL <https://www.mysql.com/>`__ -suspended: false -source-date-epoch: 1700827479 +state: ready +source-date-epoch: 1703288151 versions: + - 5.5.0 - 5.4.0 - 5.3.1 - 5.3.0 diff --git a/airflow/providers/neo4j/provider.yaml b/airflow/providers/neo4j/provider.yaml index 24d9842991..2f0ace0688 100644 --- a/airflow/providers/neo4j/provider.yaml +++ b/airflow/providers/neo4j/provider.yaml @@ -21,9 +21,10 @@ name: Neo4j description: | `Neo4j <https://neo4j.com/>`__ -suspended: false -source-date-epoch: 1700827480 +state: ready +source-date-epoch: 1703288152 versions: + - 3.5.0 - 3.4.0 - 3.3.3 - 3.3.2 diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 17fa87e82b..1f5c0f9fd6 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -21,9 +21,11 @@ name: ODBC description: | `ODBC <https://github.com/mkleehammer/pyodbc/wiki>`__ -suspended: false -source-date-epoch: 1700827480 +state: ready +source-date-epoch: 1703288153 versions: + - 4.4.0 + - 4.3.0 - 4.2.0 - 4.1.0 - 4.0.0 @@ -44,7 +46,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - - apache-airflow-providers-common-sql>=1.8.1 + - apache-airflow-providers-common-sql>=1.10.0 - pyodbc integrations: diff --git a/airflow/providers/openai/provider.yaml b/airflow/providers/openai/provider.yaml index 571d58a90f..5f384a3bf2 100644 --- a/airflow/providers/openai/provider.yaml +++ b/airflow/providers/openai/provider.yaml @@ -23,10 +23,11 @@ name: OpenAI description: | `OpenAI <https://platform.openai.com/docs/introduction>`__ -suspended: false -source-date-epoch: 1700827481 +state: ready +source-date-epoch: 1703288154 versions: + - 1.1.0 - 1.0.1 - 1.0.0 diff --git a/airflow/providers/openfaas/provider.yaml b/airflow/providers/openfaas/provider.yaml index cddb927cd7..6caf4fd0b7 100644 --- a/airflow/providers/openfaas/provider.yaml +++ b/airflow/providers/openfaas/provider.yaml @@ -21,9 +21,10 @@ name: OpenFaaS description: | `OpenFaaS <https://www.openfaas.com/>`__ -suspended: false -source-date-epoch: 1700827482 +state: ready +source-date-epoch: 1703288154 versions: + - 3.4.0 - 3.3.0 - 3.2.1 - 3.2.0 diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index ffeab0884a..7c490c5a98 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -21,9 +21,11 @@ name: OpenLineage Airflow description: | `OpenLineage <https://openlineage.io/>`__ -suspended: false -source-date-epoch: 1700827483 +state: ready +source-date-epoch: 1703288155 versions: + - 1.3.1 + - 1.3.0 - 1.2.1 - 1.2.0 - 1.1.1 @@ -84,7 +86,7 @@ config: Semicolon separated paths to custom OpenLineage extractors. type: string example: full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass - default: "" + default: ~ version_added: ~ config_path: description: | diff --git a/airflow/providers/opensearch/provider.yaml b/airflow/providers/opensearch/provider.yaml index 88616a953b..3fc9a21766 100644 --- a/airflow/providers/opensearch/provider.yaml +++ b/airflow/providers/opensearch/provider.yaml @@ -21,9 +21,11 @@ name: OpenSearch description: | `OpenSearch <https://opensearch.org/>`__ -suspended: false -source-date-epoch: 1700827483 +state: ready +source-date-epoch: 1703288156 versions: + - 1.1.1 + - 1.1.0 - 1.0.0 dependencies: diff --git a/airflow/providers/opsgenie/provider.yaml b/airflow/providers/opsgenie/provider.yaml index 9838a0dced..892f1d1b56 100644 --- a/airflow/providers/opsgenie/provider.yaml +++ b/airflow/providers/opsgenie/provider.yaml @@ -21,9 +21,10 @@ name: Opsgenie description: | `Opsgenie <https://www.atlassian.com/software/opsgenie>`__ -suspended: false -source-date-epoch: 1700827484 +state: ready +source-date-epoch: 1703288156 versions: + - 5.4.0 - 5.3.0 - 5.2.0 - 5.1.1 diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index 28a392c3bd..89b6480d0e 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -21,9 +21,11 @@ name: Oracle description: | `Oracle <https://www.oracle.com/en/database/>`__ -suspended: false -source-date-epoch: 1700827485 +state: ready +source-date-epoch: 1703288157 versions: + - 3.9.1 + - 3.9.0 - 3.8.0 - 3.7.4 - 3.7.3 diff --git a/airflow/providers/pagerduty/provider.yaml b/airflow/providers/pagerduty/provider.yaml index 612f4942ec..b12bb1a106 100644 --- a/airflow/providers/pagerduty/provider.yaml +++ b/airflow/providers/pagerduty/provider.yaml @@ -21,9 +21,11 @@ name: Pagerduty description: | `Pagerduty <https://www.pagerduty.com/>`__ -suspended: false -source-date-epoch: 1700827486 +state: ready +source-date-epoch: 1703288158 versions: + - 3.5.1 + - 3.5.0 - 3.4.0 - 3.3.1 - 3.3.0 diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml index 34d125cb18..1f20ecf359 100644 --- a/airflow/providers/papermill/provider.yaml +++ b/airflow/providers/papermill/provider.yaml @@ -21,9 +21,10 @@ name: Papermill description: | `Papermill <https://github.com/nteract/papermill>`__ -suspended: false -source-date-epoch: 1700827486 +state: ready +source-date-epoch: 1703288159 versions: + - 3.6.0 - 3.5.0 - 3.4.0 - 3.2.1 diff --git a/airflow/providers/pgvector/provider.yaml b/airflow/providers/pgvector/provider.yaml index e61b64c4ff..9727fb4160 100644 --- a/airflow/providers/pgvector/provider.yaml +++ b/airflow/providers/pgvector/provider.yaml @@ -23,10 +23,11 @@ name: pgvector description: | `pgvector <https://github.com/pgvector/pgvector>`__ -suspended: false -source-date-epoch: 1700827487 +state: ready +source-date-epoch: 1703288160 versions: + - 1.1.0 - 1.0.0 integrations: diff --git a/airflow/providers/pinecone/provider.yaml b/airflow/providers/pinecone/provider.yaml index 4b26dd6281..80a304abd8 100644 --- a/airflow/providers/pinecone/provider.yaml +++ b/airflow/providers/pinecone/provider.yaml @@ -23,10 +23,12 @@ name: Pinecone description: | `Pinecone <https://docs.pinecone.io/docs/overview>`__ -suspended: false -source-date-epoch: 1700827487 +state: ready +source-date-epoch: 1703288160 versions: + - 1.1.1 + - 1.1.0 - 1.0.0 integrations: diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml index 64752b5fd5..a5fb1f8e07 100644 --- a/airflow/providers/plexus/provider.yaml +++ b/airflow/providers/plexus/provider.yaml @@ -21,8 +21,7 @@ name: Plexus description: | `Plexus <https://plexus.corescientific.com/>`__ -removed: true -suspended: true +state: removed source-date-epoch: 1701983408 versions: - 3.4.0 diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 4d31b7bd4c..c6e4338978 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -21,9 +21,11 @@ name: PostgreSQL description: | `PostgreSQL <https://www.postgresql.org/>`__ -suspended: false -source-date-epoch: 1700827489 +state: ready +source-date-epoch: 1703288162 versions: + - 5.10.0 + - 5.9.0 - 5.8.0 - 5.7.1 - 5.7.0 diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index 0f49ba1e5f..432354167a 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -21,9 +21,11 @@ name: Presto description: | `Presto <https://prestodb.io/>`__ -suspended: false -source-date-epoch: 1700827490 +state: ready +source-date-epoch: 1703288163 versions: + - 5.4.0 + - 5.3.0 - 5.2.1 - 5.2.0 - 5.1.4 diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index 1b54a9a3ad..96fafc3c8f 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -21,9 +21,10 @@ name: Redis description: | `Redis <https://redis.io/>`__ -suspended: false -source-date-epoch: 1700827491 +state: ready +source-date-epoch: 1703288164 versions: + - 3.5.0 - 3.4.1 - 3.4.0 - 3.3.2 diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml index f4822306a4..c65e83e1c2 100644 --- a/airflow/providers/salesforce/provider.yaml +++ b/airflow/providers/salesforce/provider.yaml @@ -21,9 +21,11 @@ name: Salesforce description: | `Salesforce <https://www.salesforce.com/>`__ -suspended: false -source-date-epoch: 1700827492 +state: ready +source-date-epoch: 1703288165 versions: + - 5.6.1 + - 5.6.0 - 5.5.1 - 5.5.0 - 5.4.3 diff --git a/airflow/providers/samba/provider.yaml b/airflow/providers/samba/provider.yaml index 44ec1f5723..1277cd194f 100644 --- a/airflow/providers/samba/provider.yaml +++ b/airflow/providers/samba/provider.yaml @@ -21,9 +21,10 @@ name: Samba description: | `Samba <https://www.samba.org/>`__ -suspended: false -source-date-epoch: 1700827493 +state: ready +source-date-epoch: 1703288166 versions: + - 4.4.0 - 4.3.0 - 4.2.2 - 4.2.1 diff --git a/airflow/providers/segment/provider.yaml b/airflow/providers/segment/provider.yaml index 95d0f50740..ac7174a523 100644 --- a/airflow/providers/segment/provider.yaml +++ b/airflow/providers/segment/provider.yaml @@ -21,9 +21,10 @@ name: Segment description: | `Segment <https://segment.com/>`__ -suspended: false -source-date-epoch: 1700827494 +state: ready +source-date-epoch: 1703288167 versions: + - 3.4.0 - 3.3.0 - 3.2.1 - 3.2.0 diff --git a/airflow/providers/sendgrid/provider.yaml b/airflow/providers/sendgrid/provider.yaml index c7b6159f0d..ed74aa6992 100644 --- a/airflow/providers/sendgrid/provider.yaml +++ b/airflow/providers/sendgrid/provider.yaml @@ -25,9 +25,10 @@ dependencies: - apache-airflow>=2.6.0 - sendgrid>=6.0.0 -suspended: false -source-date-epoch: 1700827494 +state: ready +source-date-epoch: 1703288168 versions: + - 3.4.0 - 3.3.0 - 3.2.2 - 3.2.1 diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml index 72750ba324..a91bec8791 100644 --- a/airflow/providers/sftp/provider.yaml +++ b/airflow/providers/sftp/provider.yaml @@ -21,9 +21,11 @@ name: SFTP description: | `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__ -suspended: false -source-date-epoch: 1700827495 +state: ready +source-date-epoch: 1703288169 versions: + - 4.8.1 + - 4.8.0 - 4.7.0 - 4.6.1 - 4.6.0 diff --git a/airflow/providers/singularity/provider.yaml b/airflow/providers/singularity/provider.yaml index 04cf595e73..a84c5ffa90 100644 --- a/airflow/providers/singularity/provider.yaml +++ b/airflow/providers/singularity/provider.yaml @@ -21,9 +21,10 @@ name: Singularity description: | `Singularity <https://sylabs.io/guides/latest/user-guide/>`__ -suspended: false -source-date-epoch: 1700827496 +state: ready +source-date-epoch: 1703288170 versions: + - 3.4.0 - 3.3.0 - 3.2.2 - 3.2.1 diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index 24ee456a86..ede4202c6b 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -24,9 +24,11 @@ description: | - `Slack API <https://api.slack.com/>`__ - `Slack Incoming Webhook <https://api.slack.com/messaging/webhooks>`__ -suspended: false -source-date-epoch: 1700827497 +state: ready +source-date-epoch: 1703288171 versions: + - 8.5.1 + - 8.5.0 - 8.4.0 - 8.3.0 - 8.2.0 diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml index 76da8dd3aa..58619a1a5a 100644 --- a/airflow/providers/smtp/provider.yaml +++ b/airflow/providers/smtp/provider.yaml @@ -22,9 +22,11 @@ name: Simple Mail Transfer Protocol (SMTP) description: | `Simple Mail Transfer Protocol (SMTP) <https://tools.ietf.org/html/rfc5321>`__ -suspended: false -source-date-epoch: 1700827498 +state: ready +source-date-epoch: 1703288172 versions: + - 1.6.0 + - 1.5.0 - 1.4.1 - 1.4.0 - 1.3.2 diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index d8e6513229..6716386632 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -21,9 +21,11 @@ name: Snowflake description: | `Snowflake <https://www.snowflake.com/>`__ -suspended: false -source-date-epoch: 1700827499 +state: ready +source-date-epoch: 1703288173 versions: + - 5.2.1 + - 5.2.0 - 5.1.2 - 5.1.1 - 5.1.0 @@ -66,7 +68,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-common-sql>=1.10.0 - snowflake-connector-python>=2.7.8 - snowflake-sqlalchemy>=1.1.0 diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index 8a87cce91a..753b433552 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -21,9 +21,11 @@ name: SQLite description: | `SQLite <https://www.sqlite.org/>`__ -suspended: false -source-date-epoch: 1700827501 +state: ready +source-date-epoch: 1703288175 versions: + - 3.7.0 + - 3.6.0 - 3.5.0 - 3.4.3 - 3.4.2 diff --git a/airflow/providers/ssh/provider.yaml b/airflow/providers/ssh/provider.yaml index 0a5ce29943..90797ce4ac 100644 --- a/airflow/providers/ssh/provider.yaml +++ b/airflow/providers/ssh/provider.yaml @@ -21,9 +21,11 @@ name: SSH description: | `Secure Shell (SSH) <https://tools.ietf.org/html/rfc4251>`__ -suspended: false -source-date-epoch: 1700827502 +state: ready +source-date-epoch: 1703288176 versions: + - 3.10.0 + - 3.9.0 - 3.8.1 - 3.8.0 - 3.7.3 diff --git a/airflow/providers/tableau/provider.yaml b/airflow/providers/tableau/provider.yaml index 401bddcd14..4c35a77f7a 100644 --- a/airflow/providers/tableau/provider.yaml +++ b/airflow/providers/tableau/provider.yaml @@ -21,9 +21,10 @@ name: Tableau description: | `Tableau <https://www.tableau.com/>`__ -suspended: false -source-date-epoch: 1700827503 +state: ready +source-date-epoch: 1703288177 versions: + - 4.4.0 - 4.3.0 - 4.2.2 - 4.2.1 diff --git a/airflow/providers/tabular/provider.yaml b/airflow/providers/tabular/provider.yaml index 42053cf534..7b161a41d2 100644 --- a/airflow/providers/tabular/provider.yaml +++ b/airflow/providers/tabular/provider.yaml @@ -21,9 +21,11 @@ name: Tabular description: | `Tabular <https://tabular.io/>`__ -suspended: false -source-date-epoch: 1700827503 +state: ready +source-date-epoch: 1703288178 versions: + - 1.4.1 + - 1.4.0 - 1.3.0 - 1.2.1 - 1.2.0 diff --git a/airflow/providers/telegram/provider.yaml b/airflow/providers/telegram/provider.yaml index 683c2377c3..d5f4c00d1a 100644 --- a/airflow/providers/telegram/provider.yaml +++ b/airflow/providers/telegram/provider.yaml @@ -21,9 +21,10 @@ name: Telegram description: | `Telegram <https://telegram.org/>`__ -suspended: false -source-date-epoch: 1700827504 +state: ready +source-date-epoch: 1703288178 versions: + - 4.3.0 - 4.2.0 - 4.1.1 - 4.1.0 diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index a98a3b5ab8..5542a0983f 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -21,9 +21,11 @@ name: Trino description: | `Trino <https://trino.io/>`__ -suspended: false -source-date-epoch: 1700827505 +state: ready +source-date-epoch: 1703288179 versions: + - 5.6.0 + - 5.5.0 - 5.4.1 - 5.4.0 - 5.3.1 diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index 1b96c12b07..f0fb2bde70 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -21,9 +21,10 @@ name: Vertica description: | `Vertica <https://www.vertica.com/>`__ -suspended: false -source-date-epoch: 1700827506 +state: ready +source-date-epoch: 1703288181 versions: + - 3.7.0 - 3.6.0 - 3.5.2 - 3.5.1 diff --git a/airflow/providers/weaviate/provider.yaml b/airflow/providers/weaviate/provider.yaml index 015fa66c6f..6559c85677 100644 --- a/airflow/providers/weaviate/provider.yaml +++ b/airflow/providers/weaviate/provider.yaml @@ -23,10 +23,13 @@ name: Weaviate description: | `Weaviate <https://weaviate.io/developers/weaviate>`__ -suspended: false -source-date-epoch: 1700827507 +state: ready +source-date-epoch: 1703747660 versions: + - 1.3.0 + - 1.2.0 + - 1.1.0 - 1.0.0 integrations: @@ -39,6 +42,7 @@ integrations: dependencies: - apache-airflow>=2.6.0 - weaviate-client>=3.24.2 + - pandas>=0.17.1 hooks: - integration-name: Weaviate diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index 1ee8964b47..01d8a8faa1 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -20,9 +20,11 @@ package-name: apache-airflow-providers-yandex name: Yandex description: | Yandex including `Yandex.Cloud <https://cloud.yandex.com/>`__ -suspended: false -source-date-epoch: 1700827507 +state: ready +source-date-epoch: 1703288182 versions: + - 3.7.1 + - 3.7.0 - 3.6.0 - 3.5.0 - 3.4.0 diff --git a/airflow/providers/zendesk/provider.yaml b/airflow/providers/zendesk/provider.yaml index bf65368d21..d5a20c7020 100644 --- a/airflow/providers/zendesk/provider.yaml +++ b/airflow/providers/zendesk/provider.yaml @@ -21,9 +21,11 @@ name: Zendesk description: | `Zendesk <https://www.zendesk.com/>`__ -suspended: false -source-date-epoch: 1700827508 +state: ready +source-date-epoch: 1703747680 versions: + - 4.6.0 + - 4.5.0 - 4.4.0 - 4.3.2 - 4.3.1 diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index 3217a3af80..a2135a84d3 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -41,6 +41,7 @@ from airflow_breeze.commands.sbom_commands import sbom # noqa from airflow_breeze.commands.setup_commands import setup # noqa from airflow_breeze.commands.testing_commands import group_for_testing # noqa + main.add_command(group_for_testing) main.add_command(kubernetes_group) main.add_command(ci_group) @@ -50,5 +51,6 @@ main.add_command(setup) main.add_command(release_management) main.add_command(sbom) + if __name__ == "__main__": main() diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 7525a353ff..27ceaae9c0 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -450,7 +450,7 @@ def prepare_provider_documentation( except PrepareReleaseDocsUserQuitException: break else: - if provider_metadata.get("removed"): + if provider_metadata["state"] == "removed": removed_packages.append(provider_id) else: success_packages.append(provider_id) @@ -481,12 +481,12 @@ def basic_provider_checks(provider_package_id: str) -> dict[str, Any]: if not provider_metadata: get_console().print(f"[error]The package {provider_package_id} is not a provider package. Exiting[/]") sys.exit(1) - if provider_metadata.get("removed", False): + if provider_metadata["state"] == "removed": get_console().print( f"[warning]The package: {provider_package_id} is scheduled for removal, but " f"since you asked for it, it will be built [/]\n" ) - elif provider_metadata.get("suspended"): + elif provider_metadata.get("state") == "suspended": get_console().print(f"[warning]The package: {provider_package_id} is suspended " f"skipping it [/]\n") raise PackageSuspendedException() return provider_metadata diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 59fe46f2e2..49626a4668 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -156,6 +156,7 @@ def refresh_provider_metadata_with_provider_id(provider_id: str): refresh_provider_metadata_from_yaml_file(provider_yaml_path) +@lru_cache(maxsize=1) def get_provider_packages_metadata() -> dict[str, dict[str, Any]]: """ Load all data from providers files @@ -207,11 +208,7 @@ def get_provider_info_dict(provider_id: str) -> dict[str, Any]: @lru_cache def get_suspended_provider_ids() -> list[str]: - return [ - provider_id - for provider_id, provider_metadata in get_provider_packages_metadata().items() - if provider_metadata.get("suspended", False) - ] + return get_available_packages(include_suspended=True, include_regular=False) @lru_cache @@ -221,20 +218,12 @@ def get_suspended_provider_folders() -> list[str]: @lru_cache def get_removed_provider_ids() -> list[str]: - return [ - provider_id - for provider_id, provider_metadata in get_provider_packages_metadata().items() - if provider_metadata.get("removed", False) - ] + return get_available_packages(include_removed=True, include_regular=False) @lru_cache def get_not_ready_provider_ids() -> list[str]: - return [ - provider_id - for provider_id, provider_metadata in get_provider_packages_metadata().items() - if provider_metadata.get("not-ready", False) - ] + return get_available_packages(include_not_ready=True, include_regular=False) def get_provider_requirements(provider_id: str) -> list[str]: @@ -249,6 +238,7 @@ def get_available_packages( include_suspended: bool = False, include_removed: bool = False, include_not_ready: bool = False, + include_regular: bool = True, ) -> list[str]: """ Return provider ids for all packages that are available currently (not suspended). @@ -256,28 +246,33 @@ def get_available_packages( :rtype: object :param include_suspended: whether the suspended packages should be included :param include_removed: whether the removed packages should be included - :param include_not_ready: whether the not-ready ppackages should be included + :param include_not_ready: whether the not-ready packages should be included + :param include_regular: whether the regular packages should be included :param include_non_provider_doc_packages: whether the non-provider doc packages should be included (packages like apache-airflow, helm-chart, docker-stack) :param include_all_providers: whether "all-providers" should be included ni the list. """ - provider_ids: list[str] = list(json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()).keys()) - available_packages = [] - not_ready_provider_ids = get_not_ready_provider_ids() - if not include_not_ready: - provider_ids = [ - provider_id for provider_id in provider_ids if provider_id not in not_ready_provider_ids - ] + provider_dependencies = json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()) + + valid_states = set() + if include_not_ready: + valid_states.add("not-ready") + if include_regular: + valid_states.add("ready") + if include_suspended: + valid_states.add("suspended") + if include_removed: + valid_states.add("removed") + available_packages: list[str] = [ + provider_id + for provider_id, provider_dependencies in provider_dependencies.items() + if provider_dependencies["state"] in valid_states + ] if include_non_provider_doc_packages: available_packages.extend(REGULAR_DOC_PACKAGES) if include_all_providers: available_packages.append("all-providers") - available_packages.extend(provider_ids) - if include_suspended: - available_packages.extend(get_suspended_provider_ids()) - if include_removed: - available_packages.extend(get_removed_provider_ids()) return sorted(set(available_packages)) @@ -499,7 +494,7 @@ def get_provider_details(provider_id: str) -> ProviderPackageDetails: versions=provider_info["versions"], excluded_python_versions=provider_info.get("excluded-python-versions") or [], plugins=plugins, - removed=provider_info.get("removed", False), + removed=provider_info["state"] == "removed", ) diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index f50b8e2c34..059bcad808 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -340,7 +340,7 @@ def test_get_provider_info_dict(): assert provider_info_dict["name"] == "Amazon" assert provider_info_dict["package-name"] == "apache-airflow-providers-amazon" assert "Amazon" in provider_info_dict["description"] - assert provider_info_dict["suspended"] is False + assert provider_info_dict["state"] == "ready" assert provider_info_dict["filesystems"] == ["airflow.providers.amazon.aws.fs.s3"] assert len(provider_info_dict["versions"]) > 45 assert len(provider_info_dict["dependencies"]) > 10 diff --git a/docs/exts/docs_build/package_filter.py b/docs/exts/docs_build/package_filter.py index d1e497e15c..26dae9ff1c 100644 --- a/docs/exts/docs_build/package_filter.py +++ b/docs/exts/docs_build/package_filter.py @@ -31,7 +31,7 @@ def get_removed_provider_ids() -> list[str]: removed_provider_ids = [] for provider_path in PROVIDERS_DIR.rglob("provider.yaml"): provider_yaml = yaml.safe_load(provider_path.read_text()) - if provider_yaml.get("removed"): + if provider_yaml["state"] == "removed": removed_provider_ids.append( provider_yaml["package-name"][len("apache-airflow-providers-") :].replace("-", ".") ) diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index 2a49cf4652..d417ba259d 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -69,7 +69,7 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: raise Exception(f"Unable to parse: {provider_yaml_path}.") - if provider["suspended"] and not include_suspended: + if provider["state"] == "suspended" and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) provider["python-module"] = _filepath_to_module(provider_yaml_dir) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 8d2f31048b..ada5928fc5 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -7,7 +7,8 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "alibaba": { "deps": [ @@ -17,7 +18,8 @@ "oss2>=2.14.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "amazon": { "deps": [ @@ -48,7 +50,8 @@ "salesforce", "ssh" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.beam": { "deps": [ @@ -58,7 +61,8 @@ "cross-providers-deps": [ "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.cassandra": { "deps": [ @@ -66,7 +70,8 @@ "cassandra-driver>=3.13.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.drill": { "deps": [ @@ -77,7 +82,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.druid": { "deps": [ @@ -89,7 +95,8 @@ "apache.hive", "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.flink": { "deps": [ @@ -100,7 +107,8 @@ "cross-providers-deps": [ "cncf.kubernetes" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.hdfs": { "deps": [ @@ -108,7 +116,8 @@ "hdfs[avro,dataframe,kerberos]>=2.0.4" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.hive": { "deps": [ @@ -128,7 +137,8 @@ "samba", "vertica" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.impala": { "deps": [ @@ -138,7 +148,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.kafka": { "deps": [ @@ -147,7 +158,8 @@ "confluent-kafka>=1.8.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.kylin": { "deps": [ @@ -155,7 +167,8 @@ "kylinpy>=2.6" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.livy": { "deps": [ @@ -167,14 +180,16 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.pig": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.pinot": { "deps": [ @@ -185,7 +200,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.spark": { "deps": [ @@ -196,7 +212,16 @@ "cross-providers-deps": [ "cncf.kubernetes" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" + }, + "apache.sqoop": { + "deps": [ + "apache-airflow>=2.6.0" + ], + "cross-providers-deps": [], + "excluded-python-versions": [], + "state": "removed" }, "apprise": { "deps": [ @@ -204,7 +229,8 @@ "apprise" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "arangodb": { "deps": [ @@ -212,7 +238,8 @@ "python-arango>=7.3.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "asana": { "deps": [ @@ -220,7 +247,8 @@ "asana>=0.10,<4.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "atlassian.jira": { "deps": [ @@ -228,7 +256,8 @@ "atlassian-python-api>=1.14.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "celery": { "deps": [ @@ -240,7 +269,8 @@ "cross-providers-deps": [ "cncf.kubernetes" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "cloudant": { "deps": [ @@ -248,7 +278,8 @@ "cloudant>=2.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "cncf.kubernetes": { "deps": [ @@ -261,7 +292,8 @@ "kubernetes_asyncio>=18.20.1,<25" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "cohere": { "deps": [ @@ -269,14 +301,16 @@ "cohere>=4.27" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "common.io": { "deps": [ "apache-airflow>=2.8.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "common.sql": { "deps": [ @@ -286,12 +320,24 @@ "cross-providers-deps": [ "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" + }, + "daskexecutor": { + "deps": [ + "apache-airflow>=2.5.0", + "cloudpickle>=1.4.1", + "dask>=2.9.0,!=2022.10.1,!=2023.5.0", + "distributed>=2.11.1,!=2023.5.0" + ], + "cross-providers-deps": [], + "excluded-python-versions": [], + "state": "removed" }, "databricks": { "deps": [ "aiohttp>=3.6.3, <4", - "apache-airflow-providers-common-sql>=1.8.1", + "apache-airflow-providers-common-sql>=1.10.0", "apache-airflow>=2.6.0", "databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0", "requests>=2.27,<3" @@ -299,7 +345,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "datadog": { "deps": [ @@ -307,7 +354,8 @@ "datadog>=0.14.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "dbt.cloud": { "deps": [ @@ -320,7 +368,8 @@ "http", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "dingding": { "deps": [ @@ -330,7 +379,8 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "discord": { "deps": [ @@ -340,7 +390,8 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "docker": { "deps": [ @@ -349,7 +400,8 @@ "python-dotenv>=0.21.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "elasticsearch": { "deps": [ @@ -360,11 +412,12 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "exasol": { "deps": [ - "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-common-sql>=1.10.0", "apache-airflow>=2.6.0", "pandas>=0.17.1", "pyexasol>=0.5.1" @@ -372,7 +425,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "facebook": { "deps": [ @@ -380,7 +434,8 @@ "facebook-business>=6.0.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "ftp": { "deps": [ @@ -389,7 +444,8 @@ "cross-providers-deps": [ "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "github": { "deps": [ @@ -397,7 +453,8 @@ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "google": { "deps": [ @@ -407,7 +464,7 @@ "asgiref>=3.5.2", "gcloud-aio-auth>=4.0.0,<5.0.0", "gcloud-aio-bigquery>=6.1.2", - "gcloud-aio-storage", + "gcloud-aio-storage>=9.0.0", "gcsfs>=2023.10.0", "google-ads>=22.1.0", "google-api-core>=2.11.0", @@ -479,7 +536,8 @@ "ssh", "trino" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "grpc": { "deps": [ @@ -489,7 +547,8 @@ "grpcio>=1.15.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "hashicorp": { "deps": [ @@ -499,7 +558,8 @@ "cross-providers-deps": [ "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "http": { "deps": [ @@ -510,14 +570,16 @@ "requests_toolbelt" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "imap": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "influxdb": { "deps": [ @@ -526,7 +588,8 @@ "requests>=2.26.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "jdbc": { "deps": [ @@ -537,7 +600,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "jenkins": { "deps": [ @@ -545,7 +609,8 @@ "python-jenkins>=1.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.azure": { "deps": [ @@ -577,7 +642,8 @@ "oracle", "sftp" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.mssql": { "deps": [ @@ -588,7 +654,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.psrp": { "deps": [ @@ -596,7 +663,8 @@ "pypsrp>=0.8.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.winrm": { "deps": [ @@ -604,7 +672,8 @@ "pywinrm>=0.4" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "mongo": { "deps": [ @@ -613,7 +682,8 @@ "pymongo>=3.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "mysql": { "deps": [ @@ -630,7 +700,8 @@ "trino", "vertica" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "neo4j": { "deps": [ @@ -638,18 +709,20 @@ "neo4j>=4.2.1" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "odbc": { "deps": [ - "apache-airflow-providers-common-sql>=1.8.1", + "apache-airflow-providers-common-sql>=1.10.0", "apache-airflow>=2.6.0", "pyodbc" ], "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "openai": { "deps": [ @@ -657,14 +730,16 @@ "openai[datalib]>=1.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "openfaas": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "openlineage": { "deps": [ @@ -677,7 +752,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "opensearch": { "deps": [ @@ -685,7 +761,8 @@ "opensearch-py>=2.2.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "opsgenie": { "deps": [ @@ -693,7 +770,8 @@ "opsgenie-sdk>=2.1.5" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "oracle": { "deps": [ @@ -704,7 +782,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "pagerduty": { "deps": [ @@ -712,7 +791,8 @@ "pdpyras>=4.1.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "papermill": { "deps": [ @@ -722,7 +802,8 @@ "scrapbook[all]" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "pgvector": { "deps": [ @@ -734,7 +815,8 @@ "common.sql", "postgres" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "pinecone": { "deps": [ @@ -742,7 +824,17 @@ "pinecone-client>=2.2.4" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" + }, + "plexus": { + "deps": [ + "apache-airflow>=2.6.0", + "arrow>=0.16.0" + ], + "cross-providers-deps": [], + "excluded-python-versions": [], + "state": "removed" }, "postgres": { "deps": [ @@ -755,7 +847,8 @@ "common.sql", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "presto": { "deps": [ @@ -768,7 +861,8 @@ "common.sql", "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "redis": { "deps": [ @@ -776,7 +870,8 @@ "redis>=4.5.2,<5.0.0,!=4.5.5" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "salesforce": { "deps": [ @@ -785,7 +880,8 @@ "simple-salesforce>=1.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "samba": { "deps": [ @@ -795,7 +891,8 @@ "cross-providers-deps": [ "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "segment": { "deps": [ @@ -803,7 +900,8 @@ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "sendgrid": { "deps": [ @@ -811,7 +909,8 @@ "sendgrid>=6.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "sftp": { "deps": [ @@ -823,7 +922,8 @@ "openlineage", "ssh" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "singularity": { "deps": [ @@ -831,7 +931,8 @@ "spython>=0.0.56" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "slack": { "deps": [ @@ -842,18 +943,20 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "smtp": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "snowflake": { "deps": [ - "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-common-sql>=1.10.0", "apache-airflow>=2.6.0", "snowflake-connector-python>=2.7.8", "snowflake-sqlalchemy>=1.1.0" @@ -862,7 +965,8 @@ "common.sql", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "sqlite": { "deps": [ @@ -872,7 +976,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "ssh": { "deps": [ @@ -881,7 +986,8 @@ "sshtunnel>=0.3.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "tableau": { "deps": [ @@ -889,14 +995,16 @@ "tableauserverclient" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "tabular": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "telegram": { "deps": [ @@ -904,7 +1012,8 @@ "python-telegram-bot>=20.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "trino": { "deps": [ @@ -918,7 +1027,8 @@ "google", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "vertica": { "deps": [ @@ -929,15 +1039,18 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "weaviate": { "deps": [ "apache-airflow>=2.6.0", + "pandas>=0.17.1", "weaviate-client>=3.24.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "yandex": { "deps": [ @@ -945,7 +1058,8 @@ "yandexcloud>=0.228.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "zendesk": { "deps": [ @@ -953,6 +1067,7 @@ "zenpy>=2.0.24" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" } } diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg index 82651b9a58..b8bc2f4b31 100644 --- a/images/breeze/output_release-management_generate-constraints.svg +++ b/images/breeze/output_release-management_generate-constraints.svg @@ -177,9 +177,9 @@ </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="312.8" textLength="97.6" clip-path="url(#breeze- [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breeze-release-mana [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r5" x="414.8" y="361.6" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">[default: constraints-source-providers]              &# [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">airflow_version >= current_version and should be installed in CI from locall [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">packages with >= current_version.dev0            &# [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">current_version and should be installed in CI from locally built packages  [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">current_version.dev0                  & [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="459.2" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-18)">(TEXT)                     [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="85.4" clip-path="url(#breeze- [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)"> diff --git a/images/breeze/output_sbom_generate-providers-requirements.svg b/images/breeze/output_sbom_generate-providers-requirements.svg index cc6e4a41e6..cb90b20960 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.svg +++ b/images/breeze/output_sbom_generate-providers-requirements.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 1026.0" xmlns="http://www.w3.org/2000/svg"> +<svg class="rich-terminal" viewBox="0 0 1482 1050.4" xmlns="http://www.w3.org/2000/svg"> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-sbom-generate-providers-requirements-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="975.0" /> + <rect x="0" y="0" width="1463.0" height="999.4" /> </clipPath> <clipPath id="breeze-sbom-generate-providers-requirements-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -162,9 +162,12 @@ <clipPath id="breeze-sbom-generate-providers-requirements-line-38"> <rect x="0" y="928.7" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-sbom-generate-providers-requirements-line-39"> + <rect x="0" y="953.1" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1024" rx="8"/><text class="breeze-sbom-generate-providers-requirements-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: sbom generate-providers-requirements</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1048.4" rx="8"/><text class="breeze-sbom-generate-providers-requirements-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: sbom generate-providers-requirements</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -185,35 +188,36 @@ </text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-8)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="215.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-8)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="215.2" textLength="109.8" clip-path="url(#breeze-sbom-generate-pr [...] </text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-9)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="239.6" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-9)">(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apach [...] </text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-10)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="264" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-10)">apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin |&# [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-11)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="288.4" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-11)">apache.livy | apache.pig | apache.pinot | apache.spark | apprise | arangodb | asa [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-12)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="312.8" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-12)">atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | commo [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-13)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="337.2" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-13)">databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="361.6" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">facebook | ftp | github | google | grpc | hashicorp | http | imap  [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="386" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">| microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="410.4" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">| odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="434.8" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">papermill | pgvector | pinecone | postgres | presto | redis | salesforce |&# [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="459.2" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="483.6" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">telegram | trino | vertica | weaviate | yandex | zendesk)      [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-20)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-20)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="508" textLength="109.8" clip-path="url(#breeze-sbom-generate-provid [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-21)">│</text><text class="breeze-sbom-generate-providers-requirements-r1" x="292.8" y="532.4" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-21)">value to account for the most recent version of the provider     & [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-22)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="556.8" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-22)">(TEXT)                       [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-23)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-23)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="581.2" textLength="73.2" clip-path="url(#breeze-sbom-generate-p [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="605.6" textLength="1464" clip-path="url(#breeze-sbom-generate-providers-requirements-line-24)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-sbom-generate-providers-requirements-r1" x="1464" y="605.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-24)"> -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="630" textLength="24.4" clip-path="url(#breeze-sbom-generate-providers-requirements-line-25)">╭─</text><text class="breeze-sbom-generate-providers-requirements-r5" x="24.4" y="630" textLength="219.6" clip-path="url(#breeze-sbom-generate-providers-requirements-line-25)"> Parallel running </text><text class="breeze-sbom-generate-providers-requirements-r5" x="244" y="630" textLength="1195.6" clip-path [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-26)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="654.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-26)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="654.4" textLength="48.8" clip-path="url(#breeze-sbom-generate-p [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-27)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="678.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-27)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="678.8" textLength="146.4" clip-path="url(#breeze-sbom-generate- [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-28)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="378.2" y="703.2" textLength="915" clip-path="url(#breeze-sbom-generate-providers-requirements-line-28)">(INTEGER RANGE)                     [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-29)">│</text><text class="breeze-sbom-generate-providers-requirements-r5" x="378.2" y="727.6" textLength="915" clip-path="url(#breeze-sbom-generate-providers-requirements-line-29)">[default: 4; 1<=x<=8]                   [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-30)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-30)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="752" textLength="61" clip-path="url(#breeze-sbom-generate-providers [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-31)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="776.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-31)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="776.4" textLength="73.2" clip-path="url(#breeze-sbom-generate-p [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-32)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-32)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="800.8" textLength="97.6" clip-path="url(#breeze-sbom-generate-p [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="825.2" textLength="1464" clip-path="url(#breeze-sbom-generate-providers-requirements-line-33)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-sbom-generate-providers-requirements-r1" x="1464" y="825.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-33)"> -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="849.6" textLength="24.4" clip-path="url(#breeze-sbom-generate-providers-requirements-line-34)">╭─</text><text class="breeze-sbom-generate-providers-requirements-r5" x="24.4" y="849.6" textLength="195.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-34)"> Common options </text><text class="breeze-sbom-generate-providers-requirements-r5" x="219.6" y="849.6" textLength="1220" clip- [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-35)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="874" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-35)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="874" textLength="97.6" clip-path="url(#breeze-sbom-generate-provide [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="898.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-36)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="898.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-36)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="898.4" textLength="48.8" clip-path="url(#breeze-sbom-generate-p [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-37)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="922.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-37)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="922.8" textLength="85.4" clip-path="url(#breeze-sbom-generate-p [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="947.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-38)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="947.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-38)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="947.2" textLength="61" clip-path="url(#breeze-sbom-generate-pro [...] -</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="971.6" textLength="1464" clip-path="url(#breeze-sbom-generate-providers-requirements-line-39)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-sbom-generate-providers-requirements-r1" x="1464" y="971.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-39)"> +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-11)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="288.4" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-11)">apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise |  [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-12)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="312.8" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-12)">asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-13)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="337.2" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-13)">| daskexecutor | databricks | datadog | dbt.cloud | dingding | discord | doc [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="361.6" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">elasticsearch | exasol | facebook | ftp | github | google | grpc | hash [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="386" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp |  [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="410.4" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | op [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="434.8" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">| opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus&# [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="459.2" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">| redis | salesforce | samba | segment | sendgrid | sftp | singularity  [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="483.6" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">snowflake | sqlite | ssh | tableau | tabular | telegram | trino | verti [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-20)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="508" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-20)">| zendesk)                      [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-21)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="532.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-21)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="532.4" textLength="109.8" clip-path="url(#breeze-sbom-generate- [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-22)">│</text><text class="breeze-sbom-generate-providers-requirements-r1" x="292.8" y="556.8" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-22)">value to account for the most recent version of the provider     & [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-23)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="581.2" textLength="1146.8" clip-path="url(#breeze-sbom-generate-providers-requirements-line-23)">(TEXT)                       [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-24)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-24)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="605.6" textLength="73.2" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="630" textLength="1464" clip-path="url(#breeze-sbom-generate-providers-requirements-line-25)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-sbom-generate-providers-requirements-r1" x="1464" y="630" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-25)"> +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="654.4" textLength="24.4" clip-path="url(#breeze-sbom-generate-providers-requirements-line-26)">╭─</text><text class="breeze-sbom-generate-providers-requirements-r5" x="24.4" y="654.4" textLength="219.6" clip-path="url(#breeze-sbom-generate-providers-requirements-line-26)"> Parallel running </text><text class="breeze-sbom-generate-providers-requirements-r5" x="244" y="654.4" textLength="1195.6" cli [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-27)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="678.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-27)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="678.8" textLength="48.8" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-28)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-28)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="703.2" textLength="146.4" clip-path="url(#breeze-sbom-generate- [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-29)">│</text><text class="breeze-sbom-generate-providers-requirements-r6" x="378.2" y="727.6" textLength="915" clip-path="url(#breeze-sbom-generate-providers-requirements-line-29)">(INTEGER RANGE)                     [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-30)">│</text><text class="breeze-sbom-generate-providers-requirements-r5" x="378.2" y="752" textLength="915" clip-path="url(#breeze-sbom-generate-providers-requirements-line-30)">[default: 4; 1<=x<=8]                   [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-31)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="776.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-31)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="776.4" textLength="61" clip-path="url(#breeze-sbom-generate-pro [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-32)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-32)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="800.8" textLength="73.2" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-33)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="825.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-33)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="825.2" textLength="97.6" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="849.6" textLength="1464" clip-path="url(#breeze-sbom-generate-providers-requirements-line-34)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-sbom-generate-providers-requirements-r1" x="1464" y="849.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-34)"> +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="874" textLength="24.4" clip-path="url(#breeze-sbom-generate-providers-requirements-line-35)">╭─</text><text class="breeze-sbom-generate-providers-requirements-r5" x="24.4" y="874" textLength="195.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-35)"> Common options </text><text class="breeze-sbom-generate-providers-requirements-r5" x="219.6" y="874" textLength="1220" clip-path=" [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="898.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-36)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="898.4" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-36)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="898.4" textLength="97.6" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-37)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="922.8" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-37)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="922.8" textLength="48.8" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="947.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-38)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="947.2" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-38)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="947.2" textLength="85.4" clip-path="url(#breeze-sbom-generate-p [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="971.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-39)">│</text><text class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="971.6" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-39)">-</text><text class="breeze-sbom-generate-providers-requirements-r4" x="36.6" y="971.6" textLength="61" clip-path="url(#breeze-sbom-generate-pro [...] +</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0" y="996" textLength="1464" clip-path="url(#breeze-sbom-generate-providers-requirements-line-40)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-sbom-generate-providers-requirements-r1" x="1464" y="996" textLength="12.2" clip-path="url(#breeze-sbom-generate-providers-requirements-line-40)"> </text> </g> </g> diff --git a/images/breeze/output_sbom_generate-providers-requirements.txt b/images/breeze/output_sbom_generate-providers-requirements.txt index f696cff211..1af97158bb 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.txt +++ b/images/breeze/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -2dbd6d5c101420d1a788b3cd9325d892 +344968a060e00ace19738cca47d70641 diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_docs.py b/scripts/ci/pre_commit/pre_commit_check_provider_docs.py index 7814581100..0681b02287 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_docs.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_docs.py @@ -94,7 +94,7 @@ def find_all_providers(): for provider_file in AIRFLOW_PROVIDERS_DIR.rglob("provider.yaml"): provider_name = str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace(os.sep, ".") provider_info = yaml.safe_load(provider_file.read_text()) - if not provider_info["suspended"]: + if provider_info["state"] != "suspended": ALL_PROVIDERS[provider_name] = provider_info diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index a3056df3a1..5fa638505f 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -97,10 +97,9 @@ def find_all_providers_and_provider_files(): os.sep, "." ) provider_info = yaml.safe_load(provider_file.read_text()) - if not provider_info["suspended"]: - ALL_PROVIDERS[provider_name] = provider_info - else: + if provider_info["state"] == "suspended": suspended_paths.append(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR).as_posix()) + ALL_PROVIDERS[provider_name] = provider_info path = Path(root, filename) if path.is_file() and path.name.endswith(".py"): ALL_PROVIDER_FILES.append(Path(root, filename)) @@ -174,6 +173,9 @@ def check_if_different_provider_used(file_path: Path) -> None: ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider) +STATES: dict[str, str] = {} + + if __name__ == "__main__": find_all_providers_and_provider_files() num_files = len(ALL_PROVIDER_FILES) @@ -182,8 +184,8 @@ if __name__ == "__main__": for file in ALL_PROVIDER_FILES: check_if_different_provider_used(file) for provider, provider_yaml_content in ALL_PROVIDERS.items(): - if not provider_yaml_content.get("suspended"): - ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"]) + ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"]) + STATES[provider] = provider_yaml_content["state"] if warnings: console.print("[yellow]Warnings!\n") for warning in warnings: @@ -194,7 +196,7 @@ if __name__ == "__main__": for error in errors: console.print(f"[red] {error}") console.print(f"[bright_blue]Total: {len(errors)} errors.") - unique_sorted_dependencies: dict[str, dict[str, list[str]]] = defaultdict(dict) + unique_sorted_dependencies: dict[str, dict[str, list[str] | str]] = defaultdict(dict) for key in sorted(ALL_DEPENDENCIES.keys()): unique_sorted_dependencies[key]["deps"] = sorted(ALL_DEPENDENCIES[key]["deps"]) unique_sorted_dependencies[key]["cross-providers-deps"] = sorted( @@ -202,6 +204,7 @@ if __name__ == "__main__": ) excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions") unique_sorted_dependencies[key]["excluded-python-versions"] = excluded_versions or [] + unique_sorted_dependencies[key]["state"] = STATES[key] if errors: console.print() console.print("[red]Errors found during verification. Exiting!") diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index aaffa0e2f8..a4a1098c8e 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -114,7 +114,7 @@ def _load_package_data(package_paths: Iterable[str]): jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: raise Exception(f"Unable to parse: {rel_path}.") - if not provider.get("suspended"): + if provider["state"] not in ["suspended", "removed"]: result[rel_path] = provider else: suspended_providers.add(provider["package-name"]) @@ -557,7 +557,7 @@ def check_doc_files(yaml_files: dict[str, dict]) -> tuple[int, int]: op["how-to-guide"] for op in provider["transfers"] if "how-to-guide" in op ) if suspended_providers: - console.print("[yellow]Suspended providers:[/]") + console.print("[yellow]Suspended/Removed providers:[/]") console.print(suspended_providers) expected_doc_files = itertools.chain( @@ -680,26 +680,6 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]): return num_providers, num_errors -@run_check("Checking remove flag only set for suspended providers") -def check_removed_flag_only_set_for_suspended_providers(yaml_files: dict[str, dict]): - num_errors = 0 - num_providers = 0 - for package_info in yaml_files.values(): - num_providers += 1 - package_name = package_info["package-name"] - suspended = package_info["suspended"] - removed = package_info.get("removed", False) - if removed and not suspended: - errors.append( - f"The provider {package_name} has removed set to True in their provider.yaml file " - f"but suspended flag is set to false. You should only set removed flag in order to " - f"prepare last release for a provider that has been previously suspended. " - f"[yellow]How to fix it[/]: Please suspend the provider first before removing it." - ) - num_errors += 1 - return num_providers, num_errors - - if __name__ == "__main__": ProvidersManager().initialize_providers_configuration() architecture = Architecture.get_current() @@ -726,7 +706,6 @@ if __name__ == "__main__": check_notification_classes(all_parsed_yaml_files) check_unique_provider_name(all_parsed_yaml_files) check_providers_have_all_documentation_files(all_parsed_yaml_files) - check_removed_flag_only_set_for_suspended_providers(all_parsed_yaml_files) if all_files_loaded: # Only check those if all provider files are loaded diff --git a/setup.py b/setup.py index ee60c88520..66931a066e 100644 --- a/setup.py +++ b/setup.py @@ -105,6 +105,8 @@ def fill_provider_dependencies() -> dict[str, dict[str, list[str]]]: dependencies = json.load(f) provider_dict = {} for key, value in dependencies.items(): + if value["state"] in ["suspended", "removed"]: + continue if value.get(DEPS): apply_pypi_suffix_to_airflow_packages(value[DEPS]) if CURRENT_PYTHON_VERSION not in value["excluded-python-versions"] or skip_python_version_check: diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 3687832fdc..aadf4e5f91 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -40,7 +40,7 @@ def get_suspended_providers_folders() -> list[str]: suspended_providers = [] for provider_path in AIRFLOW_PROVIDERS_ROOT.rglob("provider.yaml"): provider_yaml = yaml.safe_load(provider_path.read_text()) - if provider_yaml.get("suspended"): + if provider_yaml["state"] == "suspended": suspended_providers.append( provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT) .as_posix()