This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 6937ae7647 Speed up autocompletion of Breeze by simplifying provider state (#36499) 6937ae7647 is described below commit 6937ae76476b3bc869ef912d000bcc94ad642db1 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sat Dec 30 19:16:15 2023 +0100 Speed up autocompletion of Breeze by simplifying provider state (#36499) Some recent changes, adding removed and suspended state for breeze caused significant slow-down of autocompletion retrieval - as it turned out, because we loaded and parsed all provider yaml files during auto-completion - in order to determine list of providers available for some commands. We already planned to replace the several states (suspended, not-ready, removed) with a single state field - by doing it and addding the field to pre-commit generated "provider_dependencies.json" we could switch to parsing the single provider_dependencies.json file and retrieve provider list from there following the state stored in that json file. This also simplifies state management following the recently added state diagram by following the same state lifecycle: "not-ready" -> "ready" -> "suspended" -> "removed" --- PROVIDERS.rst | 2 +- airflow/provider.yaml.schema.json | 22 +- airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 90 +++---- airflow/providers/airbyte/provider.yaml | 2 +- airflow/providers/alibaba/provider.yaml | 2 +- airflow/providers/amazon/provider.yaml | 2 +- airflow/providers/apache/beam/provider.yaml | 2 +- airflow/providers/apache/cassandra/provider.yaml | 2 +- airflow/providers/apache/drill/provider.yaml | 2 +- airflow/providers/apache/druid/provider.yaml | 2 +- airflow/providers/apache/flink/provider.yaml | 2 +- airflow/providers/apache/hdfs/provider.yaml | 2 +- airflow/providers/apache/hive/provider.yaml | 2 +- airflow/providers/apache/impala/provider.yaml | 2 +- airflow/providers/apache/kafka/provider.yaml | 2 +- airflow/providers/apache/kylin/provider.yaml | 2 +- airflow/providers/apache/livy/provider.yaml | 2 +- airflow/providers/apache/pig/provider.yaml | 2 +- airflow/providers/apache/pinot/provider.yaml | 2 +- airflow/providers/apache/spark/provider.yaml | 2 +- airflow/providers/apprise/provider.yaml | 2 +- airflow/providers/arangodb/provider.yaml | 2 +- airflow/providers/asana/provider.yaml | 2 +- airflow/providers/atlassian/jira/provider.yaml | 2 +- airflow/providers/celery/provider.yaml | 2 +- airflow/providers/cloudant/provider.yaml | 2 +- airflow/providers/cncf/kubernetes/provider.yaml | 2 +- airflow/providers/cohere/provider.yaml | 2 +- airflow/providers/common/io/provider.yaml | 2 +- airflow/providers/common/sql/provider.yaml | 2 +- airflow/providers/databricks/provider.yaml | 2 +- airflow/providers/datadog/provider.yaml | 2 +- airflow/providers/dbt/cloud/provider.yaml | 2 +- airflow/providers/dingding/provider.yaml | 2 +- airflow/providers/discord/provider.yaml | 2 +- airflow/providers/docker/provider.yaml | 2 +- airflow/providers/elasticsearch/provider.yaml | 2 +- airflow/providers/exasol/provider.yaml | 2 +- airflow/providers/fab/provider.yaml | 4 +- airflow/providers/facebook/provider.yaml | 2 +- airflow/providers/ftp/provider.yaml | 2 +- airflow/providers/github/provider.yaml | 2 +- airflow/providers/google/provider.yaml | 2 +- airflow/providers/grpc/provider.yaml | 2 +- airflow/providers/hashicorp/provider.yaml | 2 +- airflow/providers/http/provider.yaml | 2 +- airflow/providers/imap/provider.yaml | 2 +- airflow/providers/influxdb/provider.yaml | 2 +- airflow/providers/jdbc/provider.yaml | 2 +- airflow/providers/jenkins/provider.yaml | 2 +- airflow/providers/microsoft/azure/provider.yaml | 2 +- airflow/providers/microsoft/mssql/provider.yaml | 2 +- airflow/providers/microsoft/psrp/provider.yaml | 2 +- airflow/providers/microsoft/winrm/provider.yaml | 2 +- airflow/providers/mongo/provider.yaml | 2 +- airflow/providers/mysql/provider.yaml | 2 +- airflow/providers/neo4j/provider.yaml | 2 +- airflow/providers/odbc/provider.yaml | 2 +- airflow/providers/openai/provider.yaml | 2 +- airflow/providers/openfaas/provider.yaml | 2 +- airflow/providers/openlineage/provider.yaml | 2 +- airflow/providers/opensearch/provider.yaml | 2 +- airflow/providers/opsgenie/provider.yaml | 2 +- airflow/providers/oracle/provider.yaml | 2 +- airflow/providers/pagerduty/provider.yaml | 2 +- airflow/providers/papermill/provider.yaml | 2 +- airflow/providers/pgvector/provider.yaml | 2 +- airflow/providers/pinecone/provider.yaml | 2 +- airflow/providers/postgres/provider.yaml | 2 +- airflow/providers/presto/provider.yaml | 2 +- airflow/providers/redis/provider.yaml | 2 +- airflow/providers/salesforce/provider.yaml | 2 +- airflow/providers/samba/provider.yaml | 2 +- airflow/providers/segment/provider.yaml | 2 +- airflow/providers/sendgrid/provider.yaml | 2 +- airflow/providers/sftp/provider.yaml | 2 +- airflow/providers/singularity/provider.yaml | 2 +- airflow/providers/slack/provider.yaml | 2 +- airflow/providers/smtp/provider.yaml | 2 +- airflow/providers/snowflake/provider.yaml | 2 +- airflow/providers/sqlite/provider.yaml | 2 +- airflow/providers/ssh/provider.yaml | 2 +- airflow/providers/tableau/provider.yaml | 2 +- airflow/providers/tabular/provider.yaml | 2 +- airflow/providers/telegram/provider.yaml | 2 +- airflow/providers/trino/provider.yaml | 2 +- airflow/providers/vertica/provider.yaml | 2 +- airflow/providers/weaviate/provider.yaml | 2 +- airflow/providers/yandex/provider.yaml | 2 +- airflow/providers/zendesk/provider.yaml | 2 +- .../commands/release_management_commands.py | 6 +- dev/breeze/src/airflow_breeze/utils/packages.py | 53 ++--- dev/breeze/tests/test_packages.py | 2 +- docs/exts/docs_build/package_filter.py | 2 +- docs/exts/provider_yaml_utils.py | 2 +- generated/provider_dependencies.json | 261 ++++++++++++++------- ...put_release-management_generate-constraints.svg | 6 +- .../pre_commit/pre_commit_check_provider_docs.py | 2 +- .../pre_commit_update_providers_dependencies.py | 15 +- .../in_container/run_provider_yaml_files_check.py | 23 +- tests/always/test_example_dags.py | 2 +- 101 files changed, 362 insertions(+), 302 deletions(-) diff --git a/PROVIDERS.rst b/PROVIDERS.rst index a9bdc2b7ab..d8c3360190 100644 --- a/PROVIDERS.rst +++ b/PROVIDERS.rst @@ -254,7 +254,7 @@ process can be suspended. This means: -* The provider's status is set to "suspended" +* The provider's state in ``provider.yaml`` is set to "suspended" * No new releases of the provider will be made until the problem with dependencies is solved * Sources of the provider remain in the repository for now (in the future we might add process to remove them) * No new changes will be accepted for the provider (other than the ones that fix the dependencies) diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index 1873efbee3..de7d6ce503 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -21,17 +21,15 @@ "type": "string" } }, - "suspended": { - "description": "If set to true, the provider is suspended and it's not a candidate for release nor contributes dependencies to constraint calculations/CI image. Tests are excluded.", - "type:": "boolean" - }, - "removed": { - "description": "If set to true, the provider is also removed and will be soon removed from the code", - "type:": "boolean" - }, - "not-ready": { - "description": "If set to true, the provider is not included by default in release commands - for example when provider release or documentation is being prepared (not-ready providers are enabled in CI by default)", - "type:": "boolean" + "state": { + "description": "State of provider: might be not-ready, regular, suspended, removed.", + "type:": "string", + "enum": [ + "not-ready", + "ready", + "suspended", + "removed" + ] }, "dependencies": { "description": "Dependencies that should be added to the provider", @@ -462,7 +460,7 @@ "name", "package-name", "description", - "suspended", + "state", "source-date-epoch", "dependencies", "versions" diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 311a65aadc..41338f1f74 100644 --- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -55,51 +55,51 @@ If you still have doubts about building your provider, we recommend that you rea open a issue on GitHub so the community can help you. The folders are optional: example_dags, hooks, links, logs, notifications, operators, secrets, sensors, transfers, -triggers, waiters (and the list changes continuously). +triggers (and the list changes continuously). .. code-block:: bash airflow/ - ├── providers/<NEW_PROVIDER>/ - │ ├── __init__.py - │ ├── example_dags/ - │ │ ├── __init__.py - │ │ └── example_<NEW_PROVIDER>.py - │ ├── executors/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - │ ├── hooks/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - │ ├── operators/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - .... - │ ├── transfers/ - │ │ ├── __init__.py - │ │ └── <NEW_PROVIDER>.py - │ └── triggers/ - │ ├── __init__.py - │ └── <NEW_PROVIDER>.py - └── tests/providers/<NEW_PROVIDER>/ - ├── __init__.py - ├── executors/ - │ ├── __init__.py - │ └── test_<NEW_PROVIDER>.py - ├── hooks/ - │ ├── __init__.py - │ └── test_<NEW_PROVIDER>.py - ├── operators/ - │ ├── __init__.py - │ ├── test_<NEW_PROVIDER>.py - │ └── test_<NEW_PROVIDER>_system.py - ... - ├── transfers/ - │ ├── __init__.py - │ └── test_<NEW_PROVIDER>.py - └── triggers/ - ├── __init__.py - └── test_<NEW_PROVIDER>.py + ├── providers/<NEW_PROVIDER>/ + │ ├── __init__.py + │ ├── executors/ + │ │ ├── __init__.py + │ │ └── *.py + │ ├── hooks/ + │ │ ├── __init__.py + │ │ └── *.py + │ ├── operators/ + │ │ ├── __init__.py + │ │ └── *.py + │ ├── transfers/ + │ │ ├── __init__.py + │ │ └── *.py + │ └── triggers/ + │ ├── __init__.py + │ └── *.py + └── tests + ├── providers/<NEW_PROVIDER>/ + │ ├── __init__.py + │ ├── executors/ + │ │ ├── __init__.py + │ │ └── test_*.py + │ ├── hooks/ + │ │ ├── __init__.py + │ │ └── test_*>.py + │ ├── operators/ + │ │ ├── __init__.py + │ │ ├── test_*.py + │ ... + │ ├── transfers/ + │ │ ├── __init__.py + │ │ └── test_*.py + │ └── triggers/ + │ ├── __init__.py + │ └── test_*.py + └── system/providers/<NEW_PROVIDER>/ + ├── __init__.py + └── example_*.py + Considering that you have already transferred your provider's code to the above structure, it will now be necessary to create unit tests for each component you created. The example below I have already set up an environment using @@ -107,7 +107,7 @@ breeze and I'll run unit tests for my Hook. .. code-block:: bash - root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers/<NEW_PROVIDER>/hook/<NEW_PROVIDER>.py + root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers/<NEW_PROVIDER>/hook/test_*.py Adding chicken-egg providers ---------------------------- @@ -467,7 +467,7 @@ As of April 2023, we have the possibility to suspend individual providers, so th back dependencies for Airflow and other providers. The process of suspending providers is described in `description of the process <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#suspending-releases-for-providers>`_ -Technically, suspending a provider is done by setting ``suspended : true``, in the provider.yaml of the +Technically, suspending a provider is done by setting ``state: suspended``, in the provider.yaml of the provider. This should be followed by committing the change and either automatically or manually running pre-commit checks that will either update derived configuration files or ask you to update them manually. Note that you might need to run pre-commit several times until all the static checks pass, @@ -509,10 +509,10 @@ Removing providers When removing providers from Airflow code, we need to make one last release where we mark the provider as removed - in documentation and in description of the PyPI package. In order to that release manager has to -add "removed: true" flag in the provider yaml file and include the provider in the next wave of the +add "state: removed" flag in the provider yaml file and include the provider in the next wave of the providers (and then remove all the code and documentation related to the provider). -The "removed: true" flag will cause the provider to be available for the following commands (note that such +The "removed: removed" flag will cause the provider to be available for the following commands (note that such provider has to be explicitly added as selected to the package - such provider will not be included in the available list of providers or when documentation is built unless --include-removed-providers flag is used): diff --git a/airflow/providers/airbyte/provider.yaml b/airflow/providers/airbyte/provider.yaml index 127b71d5d2..c973844dd7 100644 --- a/airflow/providers/airbyte/provider.yaml +++ b/airflow/providers/airbyte/provider.yaml @@ -21,7 +21,7 @@ name: Airbyte description: | `Airbyte <https://airbyte.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288099 versions: - 3.5.1 diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml index 72e3c3b5ef..0c03ce6480 100644 --- a/airflow/providers/alibaba/provider.yaml +++ b/airflow/providers/alibaba/provider.yaml @@ -21,7 +21,7 @@ name: Alibaba description: | Alibaba Cloud integration (including `Alibaba Cloud <https://www.alibabacloud.com/>`__). -suspended: false +state: ready source-date-epoch: 1703747466 versions: - 2.7.1 diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index d99a2d2b1c..ed8a288424 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -21,7 +21,7 @@ name: Amazon description: | Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__). -suspended: false +state: ready source-date-epoch: 1703747484 versions: - 8.15.0 diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index 27fef31518..0facc9cd45 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -21,7 +21,7 @@ name: Apache Beam description: | `Apache Beam <https://beam.apache.org/>`__. -suspended: false +state: ready source-date-epoch: 1703288103 versions: - 5.5.0 diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml index 5df3785ca5..96827667f3 100644 --- a/airflow/providers/apache/cassandra/provider.yaml +++ b/airflow/providers/apache/cassandra/provider.yaml @@ -21,7 +21,7 @@ name: Apache Cassandra description: | `Apache Cassandra <https://cassandra.apache.org/>`__. -suspended: false +state: ready source-date-epoch: 1703288104 versions: - 3.4.1 diff --git a/airflow/providers/apache/drill/provider.yaml b/airflow/providers/apache/drill/provider.yaml index 7919c58455..950917e411 100644 --- a/airflow/providers/apache/drill/provider.yaml +++ b/airflow/providers/apache/drill/provider.yaml @@ -21,7 +21,7 @@ name: Apache Drill description: | `Apache Drill <https://drill.apache.org/>`__. -suspended: false +state: ready source-date-epoch: 1703288105 versions: - 2.6.0 diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index 9eebe06208..67ab94fe7a 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -21,7 +21,7 @@ name: Apache Druid description: | `Apache Druid <https://druid.apache.org/>`__. -suspended: false +state: ready source-date-epoch: 1703288106 versions: - 3.7.0 diff --git a/airflow/providers/apache/flink/provider.yaml b/airflow/providers/apache/flink/provider.yaml index 0dc4fb2d40..db6f7f224e 100644 --- a/airflow/providers/apache/flink/provider.yaml +++ b/airflow/providers/apache/flink/provider.yaml @@ -21,7 +21,7 @@ name: Apache Flink description: | `Apache Flink <https://flink.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288107 versions: - 1.3.0 diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml index 83815e3815..99155692b3 100644 --- a/airflow/providers/apache/hdfs/provider.yaml +++ b/airflow/providers/apache/hdfs/provider.yaml @@ -22,7 +22,7 @@ description: | `Hadoop Distributed File System (HDFS) <https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html>`__ and `WebHDFS <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html>`__. -suspended: false +state: ready source-date-epoch: 1703747531 versions: - 4.3.2 diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 3b471df6b8..a313d7c053 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -21,7 +21,7 @@ name: Apache Hive description: | `Apache Hive <https://hive.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288109 versions: - 6.4.0 diff --git a/airflow/providers/apache/impala/provider.yaml b/airflow/providers/apache/impala/provider.yaml index b8532f23b0..b6cc987088 100644 --- a/airflow/providers/apache/impala/provider.yaml +++ b/airflow/providers/apache/impala/provider.yaml @@ -21,7 +21,7 @@ name: Apache Impala description: | `Apache Impala <https://impala.apache.org/>`__. -suspended: false +state: ready source-date-epoch: 1703288110 versions: - 1.3.0 diff --git a/airflow/providers/apache/kafka/provider.yaml b/airflow/providers/apache/kafka/provider.yaml index c3eeda0145..5c6a98dd8b 100644 --- a/airflow/providers/apache/kafka/provider.yaml +++ b/airflow/providers/apache/kafka/provider.yaml @@ -19,7 +19,7 @@ package-name: apache-airflow-providers-apache-kafka name: Apache Kafka -suspended: false +state: ready source-date-epoch: 1703288110 description: | `Apache Kafka <https://kafka.apache.org/>`__ diff --git a/airflow/providers/apache/kylin/provider.yaml b/airflow/providers/apache/kylin/provider.yaml index c1eed26e08..6bcb1799de 100644 --- a/airflow/providers/apache/kylin/provider.yaml +++ b/airflow/providers/apache/kylin/provider.yaml @@ -21,7 +21,7 @@ name: Apache Kylin description: | `Apache Kylin <https://kylin.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703747544 versions: - 3.5.0 diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml index b2a36569d6..da48cf78cc 100644 --- a/airflow/providers/apache/livy/provider.yaml +++ b/airflow/providers/apache/livy/provider.yaml @@ -21,7 +21,7 @@ name: Apache Livy description: | `Apache Livy <https://livy.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288112 versions: - 3.7.0 diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml index 7b62ef14b3..77ed3301fa 100644 --- a/airflow/providers/apache/pig/provider.yaml +++ b/airflow/providers/apache/pig/provider.yaml @@ -21,7 +21,7 @@ name: Apache Pig description: | `Apache Pig <https://pig.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288113 versions: - 4.3.0 diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index 3972e173cd..f90e12da60 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -21,7 +21,7 @@ name: Apache Pinot description: | `Apache Pinot <https://pinot.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288114 versions: - 4.3.0 diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 1929187a71..fd5547f6a3 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -21,7 +21,7 @@ name: Apache Spark description: | `Apache Spark <https://spark.apache.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288115 versions: - 4.6.0 diff --git a/airflow/providers/apprise/provider.yaml b/airflow/providers/apprise/provider.yaml index fa55636efd..0dfb2fe6f8 100644 --- a/airflow/providers/apprise/provider.yaml +++ b/airflow/providers/apprise/provider.yaml @@ -23,7 +23,7 @@ name: Apprise description: | `Apprise <https://github.com/caronc/apprise>`__ -suspended: false +state: ready source-date-epoch: 1703288116 versions: diff --git a/airflow/providers/arangodb/provider.yaml b/airflow/providers/arangodb/provider.yaml index 28abc5cef2..bd6d75b8e5 100644 --- a/airflow/providers/arangodb/provider.yaml +++ b/airflow/providers/arangodb/provider.yaml @@ -25,7 +25,7 @@ dependencies: - apache-airflow>=2.6.0 - python-arango>=7.3.2 -suspended: false +state: ready source-date-epoch: 1703288117 versions: - 2.4.1 diff --git a/airflow/providers/asana/provider.yaml b/airflow/providers/asana/provider.yaml index c506d24211..07b764af56 100644 --- a/airflow/providers/asana/provider.yaml +++ b/airflow/providers/asana/provider.yaml @@ -21,7 +21,7 @@ name: Asana description: | `Asana <https://asana.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288118 versions: - 2.4.1 diff --git a/airflow/providers/atlassian/jira/provider.yaml b/airflow/providers/atlassian/jira/provider.yaml index c26a5cb1df..cdc2638161 100644 --- a/airflow/providers/atlassian/jira/provider.yaml +++ b/airflow/providers/atlassian/jira/provider.yaml @@ -21,7 +21,7 @@ name: Atlassian Jira description: | `Atlassian Jira <https://www.atlassian.com/software/jira/>`__ -suspended: false +state: ready source-date-epoch: 1703288118 versions: - 2.4.0 diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 5e77993925..962fe57c9d 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -21,7 +21,7 @@ name: Celery description: | `Celery <https://docs.celeryq.dev/en/stable/>`__ -suspended: false +state: ready source-date-epoch: 1703288119 versions: - 3.5.1 diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index 2496c65780..5bdd756e7a 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -21,7 +21,7 @@ name: IBM Cloudant description: | `IBM Cloudant <https://www.ibm.com/cloud/cloudant>`__ -suspended: false +state: ready source-date-epoch: 1703288120 versions: - 3.4.1 diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index 37ef5ace0f..a54edd6f3b 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -21,7 +21,7 @@ name: Kubernetes description: | `Kubernetes <https://kubernetes.io/>`__ -suspended: false +state: ready source-date-epoch: 1703747586 versions: - 7.13.0 diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml index c2775c713b..01313be518 100644 --- a/airflow/providers/cohere/provider.yaml +++ b/airflow/providers/cohere/provider.yaml @@ -23,7 +23,7 @@ name: Cohere description: | `Cohere <https://docs.cohere.com/docs>`__ -suspended: false +state: ready source-date-epoch: 1703288123 versions: diff --git a/airflow/providers/common/io/provider.yaml b/airflow/providers/common/io/provider.yaml index d0a5475bf8..bcb7418520 100644 --- a/airflow/providers/common/io/provider.yaml +++ b/airflow/providers/common/io/provider.yaml @@ -21,7 +21,7 @@ name: Common IO description: | ``Common IO Provider`` -suspended: false +state: ready source-date-epoch: 1703288123 versions: - 1.1.0 diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index f1116618c8..676a72727c 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -21,7 +21,7 @@ name: Common SQL description: | `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__ -suspended: false +state: ready source-date-epoch: 1703288124 versions: - 1.10.0 diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index 8bb5106045..de7829f1cf 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -21,7 +21,7 @@ name: Databricks description: | `Databricks <https://databricks.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288125 versions: - 6.0.0 diff --git a/airflow/providers/datadog/provider.yaml b/airflow/providers/datadog/provider.yaml index 3ece0559d1..a958527e1e 100644 --- a/airflow/providers/datadog/provider.yaml +++ b/airflow/providers/datadog/provider.yaml @@ -21,7 +21,7 @@ name: Datadog description: | `Datadog <https://www.datadoghq.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288126 versions: - 3.5.1 diff --git a/airflow/providers/dbt/cloud/provider.yaml b/airflow/providers/dbt/cloud/provider.yaml index 6c17ae8d1e..ba6237264d 100644 --- a/airflow/providers/dbt/cloud/provider.yaml +++ b/airflow/providers/dbt/cloud/provider.yaml @@ -21,7 +21,7 @@ name: dbt Cloud description: | `dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__ -suspended: false +state: ready source-date-epoch: 1703288127 versions: - 3.5.1 diff --git a/airflow/providers/dingding/provider.yaml b/airflow/providers/dingding/provider.yaml index 5ba8db7489..3647e9d4d8 100644 --- a/airflow/providers/dingding/provider.yaml +++ b/airflow/providers/dingding/provider.yaml @@ -21,7 +21,7 @@ name: Dingding description: | `DingTalk <https://www.dingtalk.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288128 versions: - 3.4.0 diff --git a/airflow/providers/discord/provider.yaml b/airflow/providers/discord/provider.yaml index 30c893a351..a98539f07c 100644 --- a/airflow/providers/discord/provider.yaml +++ b/airflow/providers/discord/provider.yaml @@ -21,7 +21,7 @@ name: Discord description: | `Discord <https://discord.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288129 versions: - 3.5.0 diff --git a/airflow/providers/docker/provider.yaml b/airflow/providers/docker/provider.yaml index 746b95e01a..4f04a065a6 100644 --- a/airflow/providers/docker/provider.yaml +++ b/airflow/providers/docker/provider.yaml @@ -21,7 +21,7 @@ name: Docker description: | `Docker <https://www.docker.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288130 versions: - 3.9.1 diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index 2cb2291c7c..208a5e57fb 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -21,7 +21,7 @@ name: Elasticsearch description: | `Elasticsearch <https://www.elastic.co/elasticsearch>`__ -suspended: false +state: ready source-date-epoch: 1703747600 versions: - 5.3.1 diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index 1714eeb64e..f47d0f7d43 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -21,7 +21,7 @@ name: Exasol description: | `Exasol <https://www.exasol.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288132 versions: - 4.4.1 diff --git a/airflow/providers/fab/provider.yaml b/airflow/providers/fab/provider.yaml index 764c917d20..80e834fcad 100644 --- a/airflow/providers/fab/provider.yaml +++ b/airflow/providers/fab/provider.yaml @@ -24,11 +24,9 @@ name: Fab description: | `Flask App Builder <https://flask-appbuilder.readthedocs.io/>`__ -suspended: false - # The provider is not yet ready to be released, we will skip it by default when preparing new release waves # For providers until we think it should be released. -not-ready: true +state: not-ready source-date-epoch: 1703288133 diff --git a/airflow/providers/facebook/provider.yaml b/airflow/providers/facebook/provider.yaml index 50089e3fa4..00fb98142a 100644 --- a/airflow/providers/facebook/provider.yaml +++ b/airflow/providers/facebook/provider.yaml @@ -21,7 +21,7 @@ name: Facebook description: | `Facebook Ads <https://www.facebook.com/about/ads>`__ -suspended: false +state: ready source-date-epoch: 1703288134 versions: - 3.4.0 diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml index a8b9d5b4ca..8dc441635b 100644 --- a/airflow/providers/ftp/provider.yaml +++ b/airflow/providers/ftp/provider.yaml @@ -21,7 +21,7 @@ name: File Transfer Protocol (FTP) description: | `File Transfer Protocol (FTP) <https://tools.ietf.org/html/rfc114>`__ -suspended: false +state: ready source-date-epoch: 1703288135 versions: - 3.7.0 diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml index 06e4bf5686..61d24d2e93 100644 --- a/airflow/providers/github/provider.yaml +++ b/airflow/providers/github/provider.yaml @@ -28,7 +28,7 @@ dependencies: # See https://github.com/PyGithub/PyGithub/issues/2436. - PyGithub!=1.58 -suspended: false +state: ready source-date-epoch: 1703288136 versions: - 2.5.1 diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 0ea04bc858..d7aca84d81 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -28,7 +28,7 @@ description: | - `Google Marketing Platform <https://marketingplatform.google.com/>`__ - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite) -suspended: false +state: ready source-date-epoch: 1703747618 versions: - 10.13.1 diff --git a/airflow/providers/grpc/provider.yaml b/airflow/providers/grpc/provider.yaml index 4cf4c68dfe..cf190109c4 100644 --- a/airflow/providers/grpc/provider.yaml +++ b/airflow/providers/grpc/provider.yaml @@ -21,7 +21,7 @@ name: gRPC description: | `gRPC <https://grpc.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288139 versions: - 3.4.1 diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml index 71627ed0cb..ce2b3846b4 100644 --- a/airflow/providers/hashicorp/provider.yaml +++ b/airflow/providers/hashicorp/provider.yaml @@ -21,7 +21,7 @@ name: Hashicorp description: | Hashicorp including `Hashicorp Vault <https://www.vaultproject.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288140 versions: - 3.6.1 diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index 07ecac0536..d193955808 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -21,7 +21,7 @@ name: Hypertext Transfer Protocol (HTTP) description: | `Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__ -suspended: false +state: ready source-date-epoch: 1703288141 versions: - 4.8.0 diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index f943a5363b..00a6fdb995 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -22,7 +22,7 @@ name: Internet Message Access Protocol (IMAP) description: | `Internet Message Access Protocol (IMAP) <https://tools.ietf.org/html/rfc3501>`__ -suspended: false +state: ready source-date-epoch: 1703288142 versions: - 3.5.0 diff --git a/airflow/providers/influxdb/provider.yaml b/airflow/providers/influxdb/provider.yaml index d39f2feda0..6820b4cfeb 100644 --- a/airflow/providers/influxdb/provider.yaml +++ b/airflow/providers/influxdb/provider.yaml @@ -28,7 +28,7 @@ dependencies: - influxdb-client>=1.19.0 - requests>=2.26.0 -suspended: false +state: ready source-date-epoch: 1703288143 versions: - 2.4.0 diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index 3d66c416d3..1acd7ee944 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -21,7 +21,7 @@ name: Java Database Connectivity (JDBC) description: | `Java Database Connectivity (JDBC) <https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/>`__ -suspended: false +state: ready source-date-epoch: 1703288143 versions: - 4.2.1 diff --git a/airflow/providers/jenkins/provider.yaml b/airflow/providers/jenkins/provider.yaml index ef90dc89a9..90124220d1 100644 --- a/airflow/providers/jenkins/provider.yaml +++ b/airflow/providers/jenkins/provider.yaml @@ -21,7 +21,7 @@ name: Jenkins description: | `Jenkins <https://jenkins.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288144 versions: - 3.5.1 diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index cdf8c0be51..fc3aefaf9c 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -19,7 +19,7 @@ package-name: apache-airflow-providers-microsoft-azure name: Microsoft Azure description: | `Microsoft Azure <https://azure.microsoft.com/>`__ -suspended: false +state: ready source-date-epoch: 1703747626 versions: - 8.5.1 diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index 40ba8c3a93..2a56bc9a2b 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -21,7 +21,7 @@ name: Microsoft SQL Server (MSSQL) description: | `Microsoft SQL Server (MSSQL) <https://www.microsoft.com/sql-server/>`__ -suspended: false +state: ready source-date-epoch: 1703288147 versions: - 3.6.0 diff --git a/airflow/providers/microsoft/psrp/provider.yaml b/airflow/providers/microsoft/psrp/provider.yaml index 8dbadf94c5..6d81aa851f 100644 --- a/airflow/providers/microsoft/psrp/provider.yaml +++ b/airflow/providers/microsoft/psrp/provider.yaml @@ -23,7 +23,7 @@ description: | `PowerShell Remoting Protocol (PSRP) <https://docs.microsoft.com/openspecs/windows_protocols/ms-psrp/>`__. -suspended: false +state: ready source-date-epoch: 1703288148 versions: - 2.5.0 diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/airflow/providers/microsoft/winrm/provider.yaml index 470dbbe035..4a4794557e 100644 --- a/airflow/providers/microsoft/winrm/provider.yaml +++ b/airflow/providers/microsoft/winrm/provider.yaml @@ -21,7 +21,7 @@ name: Windows Remote Management (WinRM) description: | `Windows Remote Management (WinRM) <https://docs.microsoft.com/windows/win32/winrm/portal>`__ -suspended: false +state: ready source-date-epoch: 1703288149 versions: - 3.4.0 diff --git a/airflow/providers/mongo/provider.yaml b/airflow/providers/mongo/provider.yaml index 0e620f1a59..9e2bea6859 100644 --- a/airflow/providers/mongo/provider.yaml +++ b/airflow/providers/mongo/provider.yaml @@ -21,7 +21,7 @@ name: MongoDB description: | `MongoDB <https://www.mongodb.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288150 versions: - 3.5.0 diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index a51034854c..9712ec8676 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -21,7 +21,7 @@ name: MySQL description: | `MySQL <https://www.mysql.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288151 versions: - 5.5.0 diff --git a/airflow/providers/neo4j/provider.yaml b/airflow/providers/neo4j/provider.yaml index 7e93afd5d2..2f0ace0688 100644 --- a/airflow/providers/neo4j/provider.yaml +++ b/airflow/providers/neo4j/provider.yaml @@ -21,7 +21,7 @@ name: Neo4j description: | `Neo4j <https://neo4j.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288152 versions: - 3.5.0 diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 2bf68a9d25..1f5c0f9fd6 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -21,7 +21,7 @@ name: ODBC description: | `ODBC <https://github.com/mkleehammer/pyodbc/wiki>`__ -suspended: false +state: ready source-date-epoch: 1703288153 versions: - 4.4.0 diff --git a/airflow/providers/openai/provider.yaml b/airflow/providers/openai/provider.yaml index 4a85040e17..5f384a3bf2 100644 --- a/airflow/providers/openai/provider.yaml +++ b/airflow/providers/openai/provider.yaml @@ -23,7 +23,7 @@ name: OpenAI description: | `OpenAI <https://platform.openai.com/docs/introduction>`__ -suspended: false +state: ready source-date-epoch: 1703288154 versions: diff --git a/airflow/providers/openfaas/provider.yaml b/airflow/providers/openfaas/provider.yaml index a359eb0311..6caf4fd0b7 100644 --- a/airflow/providers/openfaas/provider.yaml +++ b/airflow/providers/openfaas/provider.yaml @@ -21,7 +21,7 @@ name: OpenFaaS description: | `OpenFaaS <https://www.openfaas.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288154 versions: - 3.4.0 diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index 285d2547da..7c490c5a98 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -21,7 +21,7 @@ name: OpenLineage Airflow description: | `OpenLineage <https://openlineage.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288155 versions: - 1.3.1 diff --git a/airflow/providers/opensearch/provider.yaml b/airflow/providers/opensearch/provider.yaml index 44adbf527a..3fc9a21766 100644 --- a/airflow/providers/opensearch/provider.yaml +++ b/airflow/providers/opensearch/provider.yaml @@ -21,7 +21,7 @@ name: OpenSearch description: | `OpenSearch <https://opensearch.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288156 versions: - 1.1.1 diff --git a/airflow/providers/opsgenie/provider.yaml b/airflow/providers/opsgenie/provider.yaml index 31b6b5a431..892f1d1b56 100644 --- a/airflow/providers/opsgenie/provider.yaml +++ b/airflow/providers/opsgenie/provider.yaml @@ -21,7 +21,7 @@ name: Opsgenie description: | `Opsgenie <https://www.atlassian.com/software/opsgenie>`__ -suspended: false +state: ready source-date-epoch: 1703288156 versions: - 5.4.0 diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index ea56335ddf..89b6480d0e 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -21,7 +21,7 @@ name: Oracle description: | `Oracle <https://www.oracle.com/en/database/>`__ -suspended: false +state: ready source-date-epoch: 1703288157 versions: - 3.9.1 diff --git a/airflow/providers/pagerduty/provider.yaml b/airflow/providers/pagerduty/provider.yaml index a013b8b8f8..b12bb1a106 100644 --- a/airflow/providers/pagerduty/provider.yaml +++ b/airflow/providers/pagerduty/provider.yaml @@ -21,7 +21,7 @@ name: Pagerduty description: | `Pagerduty <https://www.pagerduty.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288158 versions: - 3.5.1 diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml index ccdfbf1898..1f20ecf359 100644 --- a/airflow/providers/papermill/provider.yaml +++ b/airflow/providers/papermill/provider.yaml @@ -21,7 +21,7 @@ name: Papermill description: | `Papermill <https://github.com/nteract/papermill>`__ -suspended: false +state: ready source-date-epoch: 1703288159 versions: - 3.6.0 diff --git a/airflow/providers/pgvector/provider.yaml b/airflow/providers/pgvector/provider.yaml index 852f2e2ebd..9727fb4160 100644 --- a/airflow/providers/pgvector/provider.yaml +++ b/airflow/providers/pgvector/provider.yaml @@ -23,7 +23,7 @@ name: pgvector description: | `pgvector <https://github.com/pgvector/pgvector>`__ -suspended: false +state: ready source-date-epoch: 1703288160 versions: diff --git a/airflow/providers/pinecone/provider.yaml b/airflow/providers/pinecone/provider.yaml index 3d96fa186f..80a304abd8 100644 --- a/airflow/providers/pinecone/provider.yaml +++ b/airflow/providers/pinecone/provider.yaml @@ -23,7 +23,7 @@ name: Pinecone description: | `Pinecone <https://docs.pinecone.io/docs/overview>`__ -suspended: false +state: ready source-date-epoch: 1703288160 versions: diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 9bdb618dd3..c6e4338978 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -21,7 +21,7 @@ name: PostgreSQL description: | `PostgreSQL <https://www.postgresql.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288162 versions: - 5.10.0 diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index be43f2803c..432354167a 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -21,7 +21,7 @@ name: Presto description: | `Presto <https://prestodb.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288163 versions: - 5.4.0 diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index 1640ca11e6..96fafc3c8f 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -21,7 +21,7 @@ name: Redis description: | `Redis <https://redis.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288164 versions: - 3.5.0 diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml index abcb9cdd68..c65e83e1c2 100644 --- a/airflow/providers/salesforce/provider.yaml +++ b/airflow/providers/salesforce/provider.yaml @@ -21,7 +21,7 @@ name: Salesforce description: | `Salesforce <https://www.salesforce.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288165 versions: - 5.6.1 diff --git a/airflow/providers/samba/provider.yaml b/airflow/providers/samba/provider.yaml index 36d04402a4..1277cd194f 100644 --- a/airflow/providers/samba/provider.yaml +++ b/airflow/providers/samba/provider.yaml @@ -21,7 +21,7 @@ name: Samba description: | `Samba <https://www.samba.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288166 versions: - 4.4.0 diff --git a/airflow/providers/segment/provider.yaml b/airflow/providers/segment/provider.yaml index e3a8635311..ac7174a523 100644 --- a/airflow/providers/segment/provider.yaml +++ b/airflow/providers/segment/provider.yaml @@ -21,7 +21,7 @@ name: Segment description: | `Segment <https://segment.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288167 versions: - 3.4.0 diff --git a/airflow/providers/sendgrid/provider.yaml b/airflow/providers/sendgrid/provider.yaml index 31a28220f9..ed74aa6992 100644 --- a/airflow/providers/sendgrid/provider.yaml +++ b/airflow/providers/sendgrid/provider.yaml @@ -25,7 +25,7 @@ dependencies: - apache-airflow>=2.6.0 - sendgrid>=6.0.0 -suspended: false +state: ready source-date-epoch: 1703288168 versions: - 3.4.0 diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml index 18112c648e..a91bec8791 100644 --- a/airflow/providers/sftp/provider.yaml +++ b/airflow/providers/sftp/provider.yaml @@ -21,7 +21,7 @@ name: SFTP description: | `SSH File Transfer Protocol (SFTP) <https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__ -suspended: false +state: ready source-date-epoch: 1703288169 versions: - 4.8.1 diff --git a/airflow/providers/singularity/provider.yaml b/airflow/providers/singularity/provider.yaml index d4b42a942a..a84c5ffa90 100644 --- a/airflow/providers/singularity/provider.yaml +++ b/airflow/providers/singularity/provider.yaml @@ -21,7 +21,7 @@ name: Singularity description: | `Singularity <https://sylabs.io/guides/latest/user-guide/>`__ -suspended: false +state: ready source-date-epoch: 1703288170 versions: - 3.4.0 diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index 27c11e50ff..ede4202c6b 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -24,7 +24,7 @@ description: | - `Slack API <https://api.slack.com/>`__ - `Slack Incoming Webhook <https://api.slack.com/messaging/webhooks>`__ -suspended: false +state: ready source-date-epoch: 1703288171 versions: - 8.5.1 diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml index 981d47a7bb..58619a1a5a 100644 --- a/airflow/providers/smtp/provider.yaml +++ b/airflow/providers/smtp/provider.yaml @@ -22,7 +22,7 @@ name: Simple Mail Transfer Protocol (SMTP) description: | `Simple Mail Transfer Protocol (SMTP) <https://tools.ietf.org/html/rfc5321>`__ -suspended: false +state: ready source-date-epoch: 1703288172 versions: - 1.6.0 diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index addfe8e25a..6716386632 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -21,7 +21,7 @@ name: Snowflake description: | `Snowflake <https://www.snowflake.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288173 versions: - 5.2.1 diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index 75dc931c57..753b433552 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -21,7 +21,7 @@ name: SQLite description: | `SQLite <https://www.sqlite.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288175 versions: - 3.7.0 diff --git a/airflow/providers/ssh/provider.yaml b/airflow/providers/ssh/provider.yaml index 8807a4ca37..90797ce4ac 100644 --- a/airflow/providers/ssh/provider.yaml +++ b/airflow/providers/ssh/provider.yaml @@ -21,7 +21,7 @@ name: SSH description: | `Secure Shell (SSH) <https://tools.ietf.org/html/rfc4251>`__ -suspended: false +state: ready source-date-epoch: 1703288176 versions: - 3.10.0 diff --git a/airflow/providers/tableau/provider.yaml b/airflow/providers/tableau/provider.yaml index 015faca778..4c35a77f7a 100644 --- a/airflow/providers/tableau/provider.yaml +++ b/airflow/providers/tableau/provider.yaml @@ -21,7 +21,7 @@ name: Tableau description: | `Tableau <https://www.tableau.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288177 versions: - 4.4.0 diff --git a/airflow/providers/tabular/provider.yaml b/airflow/providers/tabular/provider.yaml index 049dc9a2d3..7b161a41d2 100644 --- a/airflow/providers/tabular/provider.yaml +++ b/airflow/providers/tabular/provider.yaml @@ -21,7 +21,7 @@ name: Tabular description: | `Tabular <https://tabular.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288178 versions: - 1.4.1 diff --git a/airflow/providers/telegram/provider.yaml b/airflow/providers/telegram/provider.yaml index 43ce552966..d5f4c00d1a 100644 --- a/airflow/providers/telegram/provider.yaml +++ b/airflow/providers/telegram/provider.yaml @@ -21,7 +21,7 @@ name: Telegram description: | `Telegram <https://telegram.org/>`__ -suspended: false +state: ready source-date-epoch: 1703288178 versions: - 4.3.0 diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index 8c0b4be804..5542a0983f 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -21,7 +21,7 @@ name: Trino description: | `Trino <https://trino.io/>`__ -suspended: false +state: ready source-date-epoch: 1703288179 versions: - 5.6.0 diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index 28b3acb840..f0fb2bde70 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -21,7 +21,7 @@ name: Vertica description: | `Vertica <https://www.vertica.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288181 versions: - 3.7.0 diff --git a/airflow/providers/weaviate/provider.yaml b/airflow/providers/weaviate/provider.yaml index ad12dc8964..6559c85677 100644 --- a/airflow/providers/weaviate/provider.yaml +++ b/airflow/providers/weaviate/provider.yaml @@ -23,7 +23,7 @@ name: Weaviate description: | `Weaviate <https://weaviate.io/developers/weaviate>`__ -suspended: false +state: ready source-date-epoch: 1703747660 versions: diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index a7d71a8365..01d8a8faa1 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -20,7 +20,7 @@ package-name: apache-airflow-providers-yandex name: Yandex description: | Yandex including `Yandex.Cloud <https://cloud.yandex.com/>`__ -suspended: false +state: ready source-date-epoch: 1703288182 versions: - 3.7.1 diff --git a/airflow/providers/zendesk/provider.yaml b/airflow/providers/zendesk/provider.yaml index 25d5de690e..d5a20c7020 100644 --- a/airflow/providers/zendesk/provider.yaml +++ b/airflow/providers/zendesk/provider.yaml @@ -21,7 +21,7 @@ name: Zendesk description: | `Zendesk <https://www.zendesk.com/>`__ -suspended: false +state: ready source-date-epoch: 1703747680 versions: - 4.6.0 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 1f21b6937b..d7ae88fd13 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -451,7 +451,7 @@ def prepare_provider_documentation( except PrepareReleaseDocsUserQuitException: break else: - if provider_metadata.get("removed"): + if provider_metadata["state"] == "removed": removed_packages.append(provider_id) else: success_packages.append(provider_id) @@ -482,12 +482,12 @@ def basic_provider_checks(provider_package_id: str) -> dict[str, Any]: if not provider_metadata: get_console().print(f"[error]The package {provider_package_id} is not a provider package. Exiting[/]") sys.exit(1) - if provider_metadata.get("removed", False): + if provider_metadata["state"] == "removed": get_console().print( f"[warning]The package: {provider_package_id} is scheduled for removal, but " f"since you asked for it, it will be built [/]\n" ) - elif provider_metadata.get("suspended"): + elif provider_metadata.get("state") == "suspended": get_console().print(f"[warning]The package: {provider_package_id} is suspended " f"skipping it [/]\n") raise PackageSuspendedException() return provider_metadata diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 59fe46f2e2..49626a4668 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -156,6 +156,7 @@ def refresh_provider_metadata_with_provider_id(provider_id: str): refresh_provider_metadata_from_yaml_file(provider_yaml_path) +@lru_cache(maxsize=1) def get_provider_packages_metadata() -> dict[str, dict[str, Any]]: """ Load all data from providers files @@ -207,11 +208,7 @@ def get_provider_info_dict(provider_id: str) -> dict[str, Any]: @lru_cache def get_suspended_provider_ids() -> list[str]: - return [ - provider_id - for provider_id, provider_metadata in get_provider_packages_metadata().items() - if provider_metadata.get("suspended", False) - ] + return get_available_packages(include_suspended=True, include_regular=False) @lru_cache @@ -221,20 +218,12 @@ def get_suspended_provider_folders() -> list[str]: @lru_cache def get_removed_provider_ids() -> list[str]: - return [ - provider_id - for provider_id, provider_metadata in get_provider_packages_metadata().items() - if provider_metadata.get("removed", False) - ] + return get_available_packages(include_removed=True, include_regular=False) @lru_cache def get_not_ready_provider_ids() -> list[str]: - return [ - provider_id - for provider_id, provider_metadata in get_provider_packages_metadata().items() - if provider_metadata.get("not-ready", False) - ] + return get_available_packages(include_not_ready=True, include_regular=False) def get_provider_requirements(provider_id: str) -> list[str]: @@ -249,6 +238,7 @@ def get_available_packages( include_suspended: bool = False, include_removed: bool = False, include_not_ready: bool = False, + include_regular: bool = True, ) -> list[str]: """ Return provider ids for all packages that are available currently (not suspended). @@ -256,28 +246,33 @@ def get_available_packages( :rtype: object :param include_suspended: whether the suspended packages should be included :param include_removed: whether the removed packages should be included - :param include_not_ready: whether the not-ready ppackages should be included + :param include_not_ready: whether the not-ready packages should be included + :param include_regular: whether the regular packages should be included :param include_non_provider_doc_packages: whether the non-provider doc packages should be included (packages like apache-airflow, helm-chart, docker-stack) :param include_all_providers: whether "all-providers" should be included ni the list. """ - provider_ids: list[str] = list(json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()).keys()) - available_packages = [] - not_ready_provider_ids = get_not_ready_provider_ids() - if not include_not_ready: - provider_ids = [ - provider_id for provider_id in provider_ids if provider_id not in not_ready_provider_ids - ] + provider_dependencies = json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()) + + valid_states = set() + if include_not_ready: + valid_states.add("not-ready") + if include_regular: + valid_states.add("ready") + if include_suspended: + valid_states.add("suspended") + if include_removed: + valid_states.add("removed") + available_packages: list[str] = [ + provider_id + for provider_id, provider_dependencies in provider_dependencies.items() + if provider_dependencies["state"] in valid_states + ] if include_non_provider_doc_packages: available_packages.extend(REGULAR_DOC_PACKAGES) if include_all_providers: available_packages.append("all-providers") - available_packages.extend(provider_ids) - if include_suspended: - available_packages.extend(get_suspended_provider_ids()) - if include_removed: - available_packages.extend(get_removed_provider_ids()) return sorted(set(available_packages)) @@ -499,7 +494,7 @@ def get_provider_details(provider_id: str) -> ProviderPackageDetails: versions=provider_info["versions"], excluded_python_versions=provider_info.get("excluded-python-versions") or [], plugins=plugins, - removed=provider_info.get("removed", False), + removed=provider_info["state"] == "removed", ) diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index eb7b3a4d84..017a71e9d6 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -340,7 +340,7 @@ def test_get_provider_info_dict(): assert provider_info_dict["name"] == "Amazon" assert provider_info_dict["package-name"] == "apache-airflow-providers-amazon" assert "Amazon" in provider_info_dict["description"] - assert provider_info_dict["suspended"] is False + assert provider_info_dict["state"] == "ready" assert provider_info_dict["filesystems"] == ["airflow.providers.amazon.aws.fs.s3"] assert len(provider_info_dict["versions"]) > 45 assert len(provider_info_dict["dependencies"]) > 10 diff --git a/docs/exts/docs_build/package_filter.py b/docs/exts/docs_build/package_filter.py index d1e497e15c..26dae9ff1c 100644 --- a/docs/exts/docs_build/package_filter.py +++ b/docs/exts/docs_build/package_filter.py @@ -31,7 +31,7 @@ def get_removed_provider_ids() -> list[str]: removed_provider_ids = [] for provider_path in PROVIDERS_DIR.rglob("provider.yaml"): provider_yaml = yaml.safe_load(provider_path.read_text()) - if provider_yaml.get("removed"): + if provider_yaml["state"] == "removed": removed_provider_ids.append( provider_yaml["package-name"][len("apache-airflow-providers-") :].replace("-", ".") ) diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index 2a49cf4652..d417ba259d 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -69,7 +69,7 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: raise Exception(f"Unable to parse: {provider_yaml_path}.") - if provider["suspended"] and not include_suspended: + if provider["state"] == "suspended" and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) provider["python-module"] = _filepath_to_module(provider_yaml_dir) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 478b310198..c0131f8ca8 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -7,7 +7,8 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "alibaba": { "deps": [ @@ -17,7 +18,8 @@ "oss2>=2.14.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "amazon": { "deps": [ @@ -48,7 +50,8 @@ "salesforce", "ssh" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.beam": { "deps": [ @@ -58,7 +61,8 @@ "cross-providers-deps": [ "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.cassandra": { "deps": [ @@ -66,7 +70,8 @@ "cassandra-driver>=3.13.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.drill": { "deps": [ @@ -77,7 +82,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.druid": { "deps": [ @@ -89,7 +95,8 @@ "apache.hive", "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.flink": { "deps": [ @@ -100,7 +107,8 @@ "cross-providers-deps": [ "cncf.kubernetes" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.hdfs": { "deps": [ @@ -108,7 +116,8 @@ "hdfs[avro,dataframe,kerberos]>=2.0.4" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.hive": { "deps": [ @@ -128,7 +137,8 @@ "samba", "vertica" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.impala": { "deps": [ @@ -138,7 +148,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.kafka": { "deps": [ @@ -147,7 +158,8 @@ "confluent-kafka>=1.8.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.kylin": { "deps": [ @@ -155,7 +167,8 @@ "kylinpy>=2.6" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.livy": { "deps": [ @@ -167,14 +180,16 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.pig": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.pinot": { "deps": [ @@ -185,7 +200,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apache.spark": { "deps": [ @@ -196,7 +212,8 @@ "cross-providers-deps": [ "cncf.kubernetes" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "apprise": { "deps": [ @@ -204,7 +221,8 @@ "apprise" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "arangodb": { "deps": [ @@ -212,7 +230,8 @@ "python-arango>=7.3.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "asana": { "deps": [ @@ -220,7 +239,8 @@ "asana>=0.10,<4.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "atlassian.jira": { "deps": [ @@ -228,7 +248,8 @@ "atlassian-python-api>=1.14.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "celery": { "deps": [ @@ -240,7 +261,8 @@ "cross-providers-deps": [ "cncf.kubernetes" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "cloudant": { "deps": [ @@ -248,7 +270,8 @@ "cloudant>=2.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "cncf.kubernetes": { "deps": [ @@ -261,7 +284,8 @@ "kubernetes_asyncio>=18.20.1,<25" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "cohere": { "deps": [ @@ -269,14 +293,16 @@ "cohere>=4.27" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "common.io": { "deps": [ "apache-airflow>=2.8.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "common.sql": { "deps": [ @@ -286,7 +312,8 @@ "cross-providers-deps": [ "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "databricks": { "deps": [ @@ -299,7 +326,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "datadog": { "deps": [ @@ -307,7 +335,8 @@ "datadog>=0.14.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "dbt.cloud": { "deps": [ @@ -320,7 +349,8 @@ "http", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "dingding": { "deps": [ @@ -330,7 +360,8 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "discord": { "deps": [ @@ -340,7 +371,8 @@ "cross-providers-deps": [ "http" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "docker": { "deps": [ @@ -349,7 +381,8 @@ "python-dotenv>=0.21.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "elasticsearch": { "deps": [ @@ -360,7 +393,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "exasol": { "deps": [ @@ -372,7 +406,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "fab": { "deps": [ @@ -383,7 +418,8 @@ "google-re2>=1.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "not-ready" }, "facebook": { "deps": [ @@ -391,7 +427,8 @@ "facebook-business>=6.0.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "ftp": { "deps": [ @@ -400,7 +437,8 @@ "cross-providers-deps": [ "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "github": { "deps": [ @@ -408,7 +446,8 @@ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "google": { "deps": [ @@ -490,7 +529,8 @@ "ssh", "trino" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "grpc": { "deps": [ @@ -500,7 +540,8 @@ "grpcio>=1.15.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "hashicorp": { "deps": [ @@ -510,7 +551,8 @@ "cross-providers-deps": [ "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "http": { "deps": [ @@ -521,14 +563,16 @@ "requests_toolbelt" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "imap": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "influxdb": { "deps": [ @@ -537,7 +581,8 @@ "requests>=2.26.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "jdbc": { "deps": [ @@ -548,7 +593,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "jenkins": { "deps": [ @@ -556,7 +602,8 @@ "python-jenkins>=1.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.azure": { "deps": [ @@ -588,7 +635,8 @@ "oracle", "sftp" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.mssql": { "deps": [ @@ -599,7 +647,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.psrp": { "deps": [ @@ -607,7 +656,8 @@ "pypsrp>=0.8.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "microsoft.winrm": { "deps": [ @@ -615,7 +665,8 @@ "pywinrm>=0.4" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "mongo": { "deps": [ @@ -624,7 +675,8 @@ "pymongo>=3.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "mysql": { "deps": [ @@ -641,7 +693,8 @@ "trino", "vertica" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "neo4j": { "deps": [ @@ -649,7 +702,8 @@ "neo4j>=4.2.1" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "odbc": { "deps": [ @@ -660,7 +714,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "openai": { "deps": [ @@ -668,14 +723,16 @@ "openai[datalib]>=1.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "openfaas": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "openlineage": { "deps": [ @@ -688,7 +745,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "opensearch": { "deps": [ @@ -696,7 +754,8 @@ "opensearch-py>=2.2.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "opsgenie": { "deps": [ @@ -704,7 +763,8 @@ "opsgenie-sdk>=2.1.5" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "oracle": { "deps": [ @@ -715,7 +775,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "pagerduty": { "deps": [ @@ -723,7 +784,8 @@ "pdpyras>=4.1.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "papermill": { "deps": [ @@ -733,7 +795,8 @@ "scrapbook[all]" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "pgvector": { "deps": [ @@ -745,7 +808,8 @@ "common.sql", "postgres" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "pinecone": { "deps": [ @@ -753,7 +817,8 @@ "pinecone-client>=2.2.4" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "postgres": { "deps": [ @@ -766,7 +831,8 @@ "common.sql", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "presto": { "deps": [ @@ -779,7 +845,8 @@ "common.sql", "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "redis": { "deps": [ @@ -787,7 +854,8 @@ "redis>=4.5.2,<5.0.0,!=4.5.5" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "salesforce": { "deps": [ @@ -796,7 +864,8 @@ "simple-salesforce>=1.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "samba": { "deps": [ @@ -806,7 +875,8 @@ "cross-providers-deps": [ "google" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "segment": { "deps": [ @@ -814,7 +884,8 @@ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "sendgrid": { "deps": [ @@ -822,7 +893,8 @@ "sendgrid>=6.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "sftp": { "deps": [ @@ -834,7 +906,8 @@ "openlineage", "ssh" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "singularity": { "deps": [ @@ -842,7 +915,8 @@ "spython>=0.0.56" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "slack": { "deps": [ @@ -853,14 +927,16 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "smtp": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "snowflake": { "deps": [ @@ -873,7 +949,8 @@ "common.sql", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "sqlite": { "deps": [ @@ -883,7 +960,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "ssh": { "deps": [ @@ -892,7 +970,8 @@ "sshtunnel>=0.3.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "tableau": { "deps": [ @@ -900,14 +979,16 @@ "tableauserverclient" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "tabular": { "deps": [ "apache-airflow>=2.6.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "telegram": { "deps": [ @@ -915,7 +996,8 @@ "python-telegram-bot>=20.0.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "trino": { "deps": [ @@ -929,7 +1011,8 @@ "google", "openlineage" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "vertica": { "deps": [ @@ -940,7 +1023,8 @@ "cross-providers-deps": [ "common.sql" ], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "weaviate": { "deps": [ @@ -949,7 +1033,8 @@ "weaviate-client>=3.24.2" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "yandex": { "deps": [ @@ -957,7 +1042,8 @@ "yandexcloud>=0.228.0" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" }, "zendesk": { "deps": [ @@ -965,6 +1051,7 @@ "zenpy>=2.0.24" ], "cross-providers-deps": [], - "excluded-python-versions": [] + "excluded-python-versions": [], + "state": "ready" } } diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg index 82651b9a58..b8bc2f4b31 100644 --- a/images/breeze/output_release-management_generate-constraints.svg +++ b/images/breeze/output_release-management_generate-constraints.svg @@ -177,9 +177,9 @@ </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="312.8" textLength="97.6" clip-path="url(#breeze- [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breeze-release-mana [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r5" x="414.8" y="361.6" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">[default: constraints-source-providers]              &# [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">airflow_version >= current_version and should be installed in CI from locall [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">packages with >= current_version.dev0            &# [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">current_version and should be installed in CI from locally built packages  [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">current_version.dev0                  & [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="459.2" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-18)">(TEXT)                     [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="85.4" clip-path="url(#breeze- [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)"> diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_docs.py b/scripts/ci/pre_commit/pre_commit_check_provider_docs.py index 7814581100..0681b02287 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_docs.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_docs.py @@ -94,7 +94,7 @@ def find_all_providers(): for provider_file in AIRFLOW_PROVIDERS_DIR.rglob("provider.yaml"): provider_name = str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace(os.sep, ".") provider_info = yaml.safe_load(provider_file.read_text()) - if not provider_info["suspended"]: + if provider_info["state"] != "suspended": ALL_PROVIDERS[provider_name] = provider_info diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index a3056df3a1..5fa638505f 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -97,10 +97,9 @@ def find_all_providers_and_provider_files(): os.sep, "." ) provider_info = yaml.safe_load(provider_file.read_text()) - if not provider_info["suspended"]: - ALL_PROVIDERS[provider_name] = provider_info - else: + if provider_info["state"] == "suspended": suspended_paths.append(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR).as_posix()) + ALL_PROVIDERS[provider_name] = provider_info path = Path(root, filename) if path.is_file() and path.name.endswith(".py"): ALL_PROVIDER_FILES.append(Path(root, filename)) @@ -174,6 +173,9 @@ def check_if_different_provider_used(file_path: Path) -> None: ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider) +STATES: dict[str, str] = {} + + if __name__ == "__main__": find_all_providers_and_provider_files() num_files = len(ALL_PROVIDER_FILES) @@ -182,8 +184,8 @@ if __name__ == "__main__": for file in ALL_PROVIDER_FILES: check_if_different_provider_used(file) for provider, provider_yaml_content in ALL_PROVIDERS.items(): - if not provider_yaml_content.get("suspended"): - ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"]) + ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"]) + STATES[provider] = provider_yaml_content["state"] if warnings: console.print("[yellow]Warnings!\n") for warning in warnings: @@ -194,7 +196,7 @@ if __name__ == "__main__": for error in errors: console.print(f"[red] {error}") console.print(f"[bright_blue]Total: {len(errors)} errors.") - unique_sorted_dependencies: dict[str, dict[str, list[str]]] = defaultdict(dict) + unique_sorted_dependencies: dict[str, dict[str, list[str] | str]] = defaultdict(dict) for key in sorted(ALL_DEPENDENCIES.keys()): unique_sorted_dependencies[key]["deps"] = sorted(ALL_DEPENDENCIES[key]["deps"]) unique_sorted_dependencies[key]["cross-providers-deps"] = sorted( @@ -202,6 +204,7 @@ if __name__ == "__main__": ) excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions") unique_sorted_dependencies[key]["excluded-python-versions"] = excluded_versions or [] + unique_sorted_dependencies[key]["state"] = STATES[key] if errors: console.print() console.print("[red]Errors found during verification. Exiting!") diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index aaffa0e2f8..619ec96015 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -114,7 +114,7 @@ def _load_package_data(package_paths: Iterable[str]): jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: raise Exception(f"Unable to parse: {rel_path}.") - if not provider.get("suspended"): + if not provider["state"] == "suspended": result[rel_path] = provider else: suspended_providers.add(provider["package-name"]) @@ -680,26 +680,6 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]): return num_providers, num_errors -@run_check("Checking remove flag only set for suspended providers") -def check_removed_flag_only_set_for_suspended_providers(yaml_files: dict[str, dict]): - num_errors = 0 - num_providers = 0 - for package_info in yaml_files.values(): - num_providers += 1 - package_name = package_info["package-name"] - suspended = package_info["suspended"] - removed = package_info.get("removed", False) - if removed and not suspended: - errors.append( - f"The provider {package_name} has removed set to True in their provider.yaml file " - f"but suspended flag is set to false. You should only set removed flag in order to " - f"prepare last release for a provider that has been previously suspended. " - f"[yellow]How to fix it[/]: Please suspend the provider first before removing it." - ) - num_errors += 1 - return num_providers, num_errors - - if __name__ == "__main__": ProvidersManager().initialize_providers_configuration() architecture = Architecture.get_current() @@ -726,7 +706,6 @@ if __name__ == "__main__": check_notification_classes(all_parsed_yaml_files) check_unique_provider_name(all_parsed_yaml_files) check_providers_have_all_documentation_files(all_parsed_yaml_files) - check_removed_flag_only_set_for_suspended_providers(all_parsed_yaml_files) if all_files_loaded: # Only check those if all provider files are loaded diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 3687832fdc..aadf4e5f91 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -40,7 +40,7 @@ def get_suspended_providers_folders() -> list[str]: suspended_providers = [] for provider_path in AIRFLOW_PROVIDERS_ROOT.rglob("provider.yaml"): provider_yaml = yaml.safe_load(provider_path.read_text()) - if provider_yaml.get("suspended"): + if provider_yaml["state"] == "suspended": suspended_providers.append( provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT) .as_posix()