This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch simplify-provider-state
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7626ab12061ce33efd64f38abdd41c865de3c27c
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Dec 30 17:54:44 2023 +0100
Speed up autocompletion of Breeze by simplifying provider state
Some recent changes, adding removed and suspended state for breeze
caused significant slow-down of autocompletion retrieval - as it
turned out, because we loaded and parsed all provider yaml files
during auto-completion - in order to determine list of providers
available for some commands.
We already planned to replace the several states (suspended,
not-ready, removed) with a single state field - by doing it and
addding the field to pre-commit generated "provider_dependencies.json"
we could switch to parsing the single provider_dependencies.json
file and retrieve provider list from there following the state stored
in that json file.
This also simplifies state management following the recently
added state diagram by following the same state lifecycle:
"not-ready" -> "ready" -> "suspended" -> "removed"
---
PROVIDERS.rst | 2 +-
airflow/provider.yaml.schema.json | 22 +-
airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 90 +++----
airflow/providers/airbyte/provider.yaml | 2 +-
airflow/providers/alibaba/provider.yaml | 2 +-
airflow/providers/amazon/provider.yaml | 2 +-
airflow/providers/apache/beam/provider.yaml | 2 +-
airflow/providers/apache/cassandra/provider.yaml | 2 +-
airflow/providers/apache/drill/provider.yaml | 2 +-
airflow/providers/apache/druid/provider.yaml | 2 +-
airflow/providers/apache/flink/provider.yaml | 2 +-
airflow/providers/apache/hdfs/provider.yaml | 2 +-
airflow/providers/apache/hive/provider.yaml | 2 +-
airflow/providers/apache/impala/provider.yaml | 2 +-
airflow/providers/apache/kafka/provider.yaml | 2 +-
airflow/providers/apache/kylin/provider.yaml | 2 +-
airflow/providers/apache/livy/provider.yaml | 2 +-
airflow/providers/apache/pig/provider.yaml | 2 +-
airflow/providers/apache/pinot/provider.yaml | 2 +-
airflow/providers/apache/spark/provider.yaml | 2 +-
airflow/providers/apprise/provider.yaml | 2 +-
airflow/providers/arangodb/provider.yaml | 2 +-
airflow/providers/asana/provider.yaml | 2 +-
airflow/providers/atlassian/jira/provider.yaml | 2 +-
airflow/providers/celery/provider.yaml | 2 +-
airflow/providers/cloudant/provider.yaml | 2 +-
airflow/providers/cncf/kubernetes/provider.yaml | 2 +-
airflow/providers/cohere/provider.yaml | 2 +-
airflow/providers/common/io/provider.yaml | 2 +-
airflow/providers/common/sql/provider.yaml | 2 +-
airflow/providers/databricks/provider.yaml | 2 +-
airflow/providers/datadog/provider.yaml | 2 +-
airflow/providers/dbt/cloud/provider.yaml | 2 +-
airflow/providers/dingding/provider.yaml | 2 +-
airflow/providers/discord/provider.yaml | 2 +-
airflow/providers/docker/provider.yaml | 2 +-
airflow/providers/elasticsearch/provider.yaml | 2 +-
airflow/providers/exasol/provider.yaml | 2 +-
airflow/providers/fab/provider.yaml | 4 +-
airflow/providers/facebook/provider.yaml | 2 +-
airflow/providers/ftp/provider.yaml | 2 +-
airflow/providers/github/provider.yaml | 2 +-
airflow/providers/google/provider.yaml | 2 +-
airflow/providers/grpc/provider.yaml | 2 +-
airflow/providers/hashicorp/provider.yaml | 2 +-
airflow/providers/http/provider.yaml | 2 +-
airflow/providers/imap/provider.yaml | 2 +-
airflow/providers/influxdb/provider.yaml | 2 +-
airflow/providers/jdbc/provider.yaml | 2 +-
airflow/providers/jenkins/provider.yaml | 2 +-
airflow/providers/microsoft/azure/provider.yaml | 2 +-
airflow/providers/microsoft/mssql/provider.yaml | 2 +-
airflow/providers/microsoft/psrp/provider.yaml | 2 +-
airflow/providers/microsoft/winrm/provider.yaml | 2 +-
airflow/providers/mongo/provider.yaml | 2 +-
airflow/providers/mysql/provider.yaml | 2 +-
airflow/providers/neo4j/provider.yaml | 2 +-
airflow/providers/odbc/provider.yaml | 2 +-
airflow/providers/openai/provider.yaml | 2 +-
airflow/providers/openfaas/provider.yaml | 2 +-
airflow/providers/openlineage/provider.yaml | 2 +-
airflow/providers/opensearch/provider.yaml | 2 +-
airflow/providers/opsgenie/provider.yaml | 2 +-
airflow/providers/oracle/provider.yaml | 2 +-
airflow/providers/pagerduty/provider.yaml | 2 +-
airflow/providers/papermill/provider.yaml | 2 +-
airflow/providers/pgvector/provider.yaml | 2 +-
airflow/providers/pinecone/provider.yaml | 2 +-
airflow/providers/postgres/provider.yaml | 2 +-
airflow/providers/presto/provider.yaml | 2 +-
airflow/providers/redis/provider.yaml | 2 +-
airflow/providers/salesforce/provider.yaml | 2 +-
airflow/providers/samba/provider.yaml | 2 +-
airflow/providers/segment/provider.yaml | 2 +-
airflow/providers/sendgrid/provider.yaml | 2 +-
airflow/providers/sftp/provider.yaml | 2 +-
airflow/providers/singularity/provider.yaml | 2 +-
airflow/providers/slack/provider.yaml | 2 +-
airflow/providers/smtp/provider.yaml | 2 +-
airflow/providers/snowflake/provider.yaml | 2 +-
airflow/providers/sqlite/provider.yaml | 2 +-
airflow/providers/ssh/provider.yaml | 2 +-
airflow/providers/tableau/provider.yaml | 2 +-
airflow/providers/tabular/provider.yaml | 2 +-
airflow/providers/telegram/provider.yaml | 2 +-
airflow/providers/trino/provider.yaml | 2 +-
airflow/providers/vertica/provider.yaml | 2 +-
airflow/providers/weaviate/provider.yaml | 2 +-
airflow/providers/yandex/provider.yaml | 2 +-
airflow/providers/zendesk/provider.yaml | 2 +-
.../commands/release_management_commands.py | 6 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 53 ++---
dev/breeze/tests/test_packages.py | 2 +-
docs/exts/docs_build/package_filter.py | 2 +-
docs/exts/provider_yaml_utils.py | 2 +-
generated/provider_dependencies.json | 261 ++++++++++++++-------
...put_release-management_generate-constraints.svg | 6 +-
.../pre_commit/pre_commit_check_provider_docs.py | 2 +-
.../pre_commit_update_providers_dependencies.py | 15 +-
.../in_container/run_provider_yaml_files_check.py | 23 +-
tests/always/test_example_dags.py | 2 +-
101 files changed, 362 insertions(+), 302 deletions(-)
diff --git a/PROVIDERS.rst b/PROVIDERS.rst
index a9bdc2b7ab..d8c3360190 100644
--- a/PROVIDERS.rst
+++ b/PROVIDERS.rst
@@ -254,7 +254,7 @@ process can be suspended.
This means:
-* The provider's status is set to "suspended"
+* The provider's state in ``provider.yaml`` is set to "suspended"
* No new releases of the provider will be made until the problem with
dependencies is solved
* Sources of the provider remain in the repository for now (in the future we
might add process to remove them)
* No new changes will be accepted for the provider (other than the ones that
fix the dependencies)
diff --git a/airflow/provider.yaml.schema.json
b/airflow/provider.yaml.schema.json
index 1873efbee3..de7d6ce503 100644
--- a/airflow/provider.yaml.schema.json
+++ b/airflow/provider.yaml.schema.json
@@ -21,17 +21,15 @@
"type": "string"
}
},
- "suspended": {
- "description": "If set to true, the provider is suspended and it's
not a candidate for release nor contributes dependencies to constraint
calculations/CI image. Tests are excluded.",
- "type:": "boolean"
- },
- "removed": {
- "description": "If set to true, the provider is also removed and
will be soon removed from the code",
- "type:": "boolean"
- },
- "not-ready": {
- "description": "If set to true, the provider is not included by
default in release commands - for example when provider release or
documentation is being prepared (not-ready providers are enabled in CI by
default)",
- "type:": "boolean"
+ "state": {
+ "description": "State of provider: might be not-ready, regular,
suspended, removed.",
+ "type:": "string",
+ "enum": [
+ "not-ready",
+ "ready",
+ "suspended",
+ "removed"
+ ]
},
"dependencies": {
"description": "Dependencies that should be added to the provider",
@@ -462,7 +460,7 @@
"name",
"package-name",
"description",
- "suspended",
+ "state",
"source-date-epoch",
"dependencies",
"versions"
diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
index 311a65aadc..41338f1f74 100644
--- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
+++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
@@ -55,51 +55,51 @@ If you still have doubts about building your provider, we
recommend that you rea
open a issue on GitHub so the community can help you.
The folders are optional: example_dags, hooks, links, logs, notifications,
operators, secrets, sensors, transfers,
-triggers, waiters (and the list changes continuously).
+triggers (and the list changes continuously).
.. code-block:: bash
airflow/
- ├── providers/<NEW_PROVIDER>/
- │ ├── __init__.py
- │ ├── example_dags/
- │ │ ├── __init__.py
- │ │ └── example_<NEW_PROVIDER>.py
- │ ├── executors/
- │ │ ├── __init__.py
- │ │ └── <NEW_PROVIDER>.py
- │ ├── hooks/
- │ │ ├── __init__.py
- │ │ └── <NEW_PROVIDER>.py
- │ ├── operators/
- │ │ ├── __init__.py
- │ │ └── <NEW_PROVIDER>.py
- ....
- │ ├── transfers/
- │ │ ├── __init__.py
- │ │ └── <NEW_PROVIDER>.py
- │ └── triggers/
- │ ├── __init__.py
- │ └── <NEW_PROVIDER>.py
- └── tests/providers/<NEW_PROVIDER>/
- ├── __init__.py
- ├── executors/
- │ ├── __init__.py
- │ └── test_<NEW_PROVIDER>.py
- ├── hooks/
- │ ├── __init__.py
- │ └── test_<NEW_PROVIDER>.py
- ├── operators/
- │ ├── __init__.py
- │ ├── test_<NEW_PROVIDER>.py
- │ └── test_<NEW_PROVIDER>_system.py
- ...
- ├── transfers/
- │ ├── __init__.py
- │ └── test_<NEW_PROVIDER>.py
- └── triggers/
- ├── __init__.py
- └── test_<NEW_PROVIDER>.py
+ ├── providers/<NEW_PROVIDER>/
+ │ ├── __init__.py
+ │ ├── executors/
+ │ │ ├── __init__.py
+ │ │ └── *.py
+ │ ├── hooks/
+ │ │ ├── __init__.py
+ │ │ └── *.py
+ │ ├── operators/
+ │ │ ├── __init__.py
+ │ │ └── *.py
+ │ ├── transfers/
+ │ │ ├── __init__.py
+ │ │ └── *.py
+ │ └── triggers/
+ │ ├── __init__.py
+ │ └── *.py
+ └── tests
+ ├── providers/<NEW_PROVIDER>/
+ │ ├── __init__.py
+ │ ├── executors/
+ │ │ ├── __init__.py
+ │ │ └── test_*.py
+ │ ├── hooks/
+ │ │ ├── __init__.py
+ │ │ └── test_*>.py
+ │ ├── operators/
+ │ │ ├── __init__.py
+ │ │ ├── test_*.py
+ │ ...
+ │ ├── transfers/
+ │ │ ├── __init__.py
+ │ │ └── test_*.py
+ │ └── triggers/
+ │ ├── __init__.py
+ │ └── test_*.py
+ └── system/providers/<NEW_PROVIDER>/
+ ├── __init__.py
+ └── example_*.py
+
Considering that you have already transferred your provider's code to the
above structure, it will now be necessary
to create unit tests for each component you created. The example below I have
already set up an environment using
@@ -107,7 +107,7 @@ breeze and I'll run unit tests for my Hook.
.. code-block:: bash
- root@fafd8d630e46:/opt/airflow# python -m pytest
tests/providers/<NEW_PROVIDER>/hook/<NEW_PROVIDER>.py
+ root@fafd8d630e46:/opt/airflow# python -m pytest
tests/providers/<NEW_PROVIDER>/hook/test_*.py
Adding chicken-egg providers
----------------------------
@@ -467,7 +467,7 @@ As of April 2023, we have the possibility to suspend
individual providers, so th
back dependencies for Airflow and other providers. The process of suspending
providers is described
in `description of the process
<https://github.com/apache/airflow/blob/main/PROVIDERS.rst#suspending-releases-for-providers>`_
-Technically, suspending a provider is done by setting ``suspended : true``, in
the provider.yaml of the
+Technically, suspending a provider is done by setting ``state: suspended``, in
the provider.yaml of the
provider. This should be followed by committing the change and either
automatically or manually running
pre-commit checks that will either update derived configuration files or ask
you to update them manually.
Note that you might need to run pre-commit several times until all the static
checks pass,
@@ -509,10 +509,10 @@ Removing providers
When removing providers from Airflow code, we need to make one last release
where we mark the provider as
removed - in documentation and in description of the PyPI package. In order to
that release manager has to
-add "removed: true" flag in the provider yaml file and include the provider in
the next wave of the
+add "state: removed" flag in the provider yaml file and include the provider
in the next wave of the
providers (and then remove all the code and documentation related to the
provider).
-The "removed: true" flag will cause the provider to be available for the
following commands (note that such
+The "removed: removed" flag will cause the provider to be available for the
following commands (note that such
provider has to be explicitly added as selected to the package - such provider
will not be included in
the available list of providers or when documentation is built unless
--include-removed-providers
flag is used):
diff --git a/airflow/providers/airbyte/provider.yaml
b/airflow/providers/airbyte/provider.yaml
index 127b71d5d2..c973844dd7 100644
--- a/airflow/providers/airbyte/provider.yaml
+++ b/airflow/providers/airbyte/provider.yaml
@@ -21,7 +21,7 @@ name: Airbyte
description: |
`Airbyte <https://airbyte.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288099
versions:
- 3.5.1
diff --git a/airflow/providers/alibaba/provider.yaml
b/airflow/providers/alibaba/provider.yaml
index 72e3c3b5ef..0c03ce6480 100644
--- a/airflow/providers/alibaba/provider.yaml
+++ b/airflow/providers/alibaba/provider.yaml
@@ -21,7 +21,7 @@ name: Alibaba
description: |
Alibaba Cloud integration (including `Alibaba Cloud
<https://www.alibabacloud.com/>`__).
-suspended: false
+state: ready
source-date-epoch: 1703747466
versions:
- 2.7.1
diff --git a/airflow/providers/amazon/provider.yaml
b/airflow/providers/amazon/provider.yaml
index d99a2d2b1c..ed8a288424 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -21,7 +21,7 @@ name: Amazon
description: |
Amazon integration (including `Amazon Web Services (AWS)
<https://aws.amazon.com/>`__).
-suspended: false
+state: ready
source-date-epoch: 1703747484
versions:
- 8.15.0
diff --git a/airflow/providers/apache/beam/provider.yaml
b/airflow/providers/apache/beam/provider.yaml
index 27fef31518..0facc9cd45 100644
--- a/airflow/providers/apache/beam/provider.yaml
+++ b/airflow/providers/apache/beam/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Beam
description: |
`Apache Beam <https://beam.apache.org/>`__.
-suspended: false
+state: ready
source-date-epoch: 1703288103
versions:
- 5.5.0
diff --git a/airflow/providers/apache/cassandra/provider.yaml
b/airflow/providers/apache/cassandra/provider.yaml
index 5df3785ca5..96827667f3 100644
--- a/airflow/providers/apache/cassandra/provider.yaml
+++ b/airflow/providers/apache/cassandra/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Cassandra
description: |
`Apache Cassandra <https://cassandra.apache.org/>`__.
-suspended: false
+state: ready
source-date-epoch: 1703288104
versions:
- 3.4.1
diff --git a/airflow/providers/apache/drill/provider.yaml
b/airflow/providers/apache/drill/provider.yaml
index 7919c58455..950917e411 100644
--- a/airflow/providers/apache/drill/provider.yaml
+++ b/airflow/providers/apache/drill/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Drill
description: |
`Apache Drill <https://drill.apache.org/>`__.
-suspended: false
+state: ready
source-date-epoch: 1703288105
versions:
- 2.6.0
diff --git a/airflow/providers/apache/druid/provider.yaml
b/airflow/providers/apache/druid/provider.yaml
index 9eebe06208..67ab94fe7a 100644
--- a/airflow/providers/apache/druid/provider.yaml
+++ b/airflow/providers/apache/druid/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Druid
description: |
`Apache Druid <https://druid.apache.org/>`__.
-suspended: false
+state: ready
source-date-epoch: 1703288106
versions:
- 3.7.0
diff --git a/airflow/providers/apache/flink/provider.yaml
b/airflow/providers/apache/flink/provider.yaml
index 0dc4fb2d40..db6f7f224e 100644
--- a/airflow/providers/apache/flink/provider.yaml
+++ b/airflow/providers/apache/flink/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Flink
description: |
`Apache Flink <https://flink.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288107
versions:
- 1.3.0
diff --git a/airflow/providers/apache/hdfs/provider.yaml
b/airflow/providers/apache/hdfs/provider.yaml
index 83815e3815..99155692b3 100644
--- a/airflow/providers/apache/hdfs/provider.yaml
+++ b/airflow/providers/apache/hdfs/provider.yaml
@@ -22,7 +22,7 @@ description: |
`Hadoop Distributed File System (HDFS)
<https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html>`__
and `WebHDFS
<https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html>`__.
-suspended: false
+state: ready
source-date-epoch: 1703747531
versions:
- 4.3.2
diff --git a/airflow/providers/apache/hive/provider.yaml
b/airflow/providers/apache/hive/provider.yaml
index 3b471df6b8..a313d7c053 100644
--- a/airflow/providers/apache/hive/provider.yaml
+++ b/airflow/providers/apache/hive/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Hive
description: |
`Apache Hive <https://hive.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288109
versions:
- 6.4.0
diff --git a/airflow/providers/apache/impala/provider.yaml
b/airflow/providers/apache/impala/provider.yaml
index b8532f23b0..b6cc987088 100644
--- a/airflow/providers/apache/impala/provider.yaml
+++ b/airflow/providers/apache/impala/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Impala
description: |
`Apache Impala <https://impala.apache.org/>`__.
-suspended: false
+state: ready
source-date-epoch: 1703288110
versions:
- 1.3.0
diff --git a/airflow/providers/apache/kafka/provider.yaml
b/airflow/providers/apache/kafka/provider.yaml
index c3eeda0145..5c6a98dd8b 100644
--- a/airflow/providers/apache/kafka/provider.yaml
+++ b/airflow/providers/apache/kafka/provider.yaml
@@ -19,7 +19,7 @@
package-name: apache-airflow-providers-apache-kafka
name: Apache Kafka
-suspended: false
+state: ready
source-date-epoch: 1703288110
description: |
`Apache Kafka <https://kafka.apache.org/>`__
diff --git a/airflow/providers/apache/kylin/provider.yaml
b/airflow/providers/apache/kylin/provider.yaml
index c1eed26e08..6bcb1799de 100644
--- a/airflow/providers/apache/kylin/provider.yaml
+++ b/airflow/providers/apache/kylin/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Kylin
description: |
`Apache Kylin <https://kylin.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703747544
versions:
- 3.5.0
diff --git a/airflow/providers/apache/livy/provider.yaml
b/airflow/providers/apache/livy/provider.yaml
index b2a36569d6..da48cf78cc 100644
--- a/airflow/providers/apache/livy/provider.yaml
+++ b/airflow/providers/apache/livy/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Livy
description: |
`Apache Livy <https://livy.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288112
versions:
- 3.7.0
diff --git a/airflow/providers/apache/pig/provider.yaml
b/airflow/providers/apache/pig/provider.yaml
index 7b62ef14b3..77ed3301fa 100644
--- a/airflow/providers/apache/pig/provider.yaml
+++ b/airflow/providers/apache/pig/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Pig
description: |
`Apache Pig <https://pig.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288113
versions:
- 4.3.0
diff --git a/airflow/providers/apache/pinot/provider.yaml
b/airflow/providers/apache/pinot/provider.yaml
index 3972e173cd..f90e12da60 100644
--- a/airflow/providers/apache/pinot/provider.yaml
+++ b/airflow/providers/apache/pinot/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Pinot
description: |
`Apache Pinot <https://pinot.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288114
versions:
- 4.3.0
diff --git a/airflow/providers/apache/spark/provider.yaml
b/airflow/providers/apache/spark/provider.yaml
index 1929187a71..fd5547f6a3 100644
--- a/airflow/providers/apache/spark/provider.yaml
+++ b/airflow/providers/apache/spark/provider.yaml
@@ -21,7 +21,7 @@ name: Apache Spark
description: |
`Apache Spark <https://spark.apache.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288115
versions:
- 4.6.0
diff --git a/airflow/providers/apprise/provider.yaml
b/airflow/providers/apprise/provider.yaml
index fa55636efd..0dfb2fe6f8 100644
--- a/airflow/providers/apprise/provider.yaml
+++ b/airflow/providers/apprise/provider.yaml
@@ -23,7 +23,7 @@ name: Apprise
description: |
`Apprise <https://github.com/caronc/apprise>`__
-suspended: false
+state: ready
source-date-epoch: 1703288116
versions:
diff --git a/airflow/providers/arangodb/provider.yaml
b/airflow/providers/arangodb/provider.yaml
index 28abc5cef2..bd6d75b8e5 100644
--- a/airflow/providers/arangodb/provider.yaml
+++ b/airflow/providers/arangodb/provider.yaml
@@ -25,7 +25,7 @@ dependencies:
- apache-airflow>=2.6.0
- python-arango>=7.3.2
-suspended: false
+state: ready
source-date-epoch: 1703288117
versions:
- 2.4.1
diff --git a/airflow/providers/asana/provider.yaml
b/airflow/providers/asana/provider.yaml
index c506d24211..07b764af56 100644
--- a/airflow/providers/asana/provider.yaml
+++ b/airflow/providers/asana/provider.yaml
@@ -21,7 +21,7 @@ name: Asana
description: |
`Asana <https://asana.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288118
versions:
- 2.4.1
diff --git a/airflow/providers/atlassian/jira/provider.yaml
b/airflow/providers/atlassian/jira/provider.yaml
index c26a5cb1df..cdc2638161 100644
--- a/airflow/providers/atlassian/jira/provider.yaml
+++ b/airflow/providers/atlassian/jira/provider.yaml
@@ -21,7 +21,7 @@ name: Atlassian Jira
description: |
`Atlassian Jira <https://www.atlassian.com/software/jira/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288118
versions:
- 2.4.0
diff --git a/airflow/providers/celery/provider.yaml
b/airflow/providers/celery/provider.yaml
index 5e77993925..962fe57c9d 100644
--- a/airflow/providers/celery/provider.yaml
+++ b/airflow/providers/celery/provider.yaml
@@ -21,7 +21,7 @@ name: Celery
description: |
`Celery <https://docs.celeryq.dev/en/stable/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288119
versions:
- 3.5.1
diff --git a/airflow/providers/cloudant/provider.yaml
b/airflow/providers/cloudant/provider.yaml
index 2496c65780..5bdd756e7a 100644
--- a/airflow/providers/cloudant/provider.yaml
+++ b/airflow/providers/cloudant/provider.yaml
@@ -21,7 +21,7 @@ name: IBM Cloudant
description: |
`IBM Cloudant <https://www.ibm.com/cloud/cloudant>`__
-suspended: false
+state: ready
source-date-epoch: 1703288120
versions:
- 3.4.1
diff --git a/airflow/providers/cncf/kubernetes/provider.yaml
b/airflow/providers/cncf/kubernetes/provider.yaml
index 37ef5ace0f..a54edd6f3b 100644
--- a/airflow/providers/cncf/kubernetes/provider.yaml
+++ b/airflow/providers/cncf/kubernetes/provider.yaml
@@ -21,7 +21,7 @@ name: Kubernetes
description: |
`Kubernetes <https://kubernetes.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703747586
versions:
- 7.13.0
diff --git a/airflow/providers/cohere/provider.yaml
b/airflow/providers/cohere/provider.yaml
index c2775c713b..01313be518 100644
--- a/airflow/providers/cohere/provider.yaml
+++ b/airflow/providers/cohere/provider.yaml
@@ -23,7 +23,7 @@ name: Cohere
description: |
`Cohere <https://docs.cohere.com/docs>`__
-suspended: false
+state: ready
source-date-epoch: 1703288123
versions:
diff --git a/airflow/providers/common/io/provider.yaml
b/airflow/providers/common/io/provider.yaml
index d0a5475bf8..bcb7418520 100644
--- a/airflow/providers/common/io/provider.yaml
+++ b/airflow/providers/common/io/provider.yaml
@@ -21,7 +21,7 @@ name: Common IO
description: |
``Common IO Provider``
-suspended: false
+state: ready
source-date-epoch: 1703288123
versions:
- 1.1.0
diff --git a/airflow/providers/common/sql/provider.yaml
b/airflow/providers/common/sql/provider.yaml
index f1116618c8..676a72727c 100644
--- a/airflow/providers/common/sql/provider.yaml
+++ b/airflow/providers/common/sql/provider.yaml
@@ -21,7 +21,7 @@ name: Common SQL
description: |
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
-suspended: false
+state: ready
source-date-epoch: 1703288124
versions:
- 1.10.0
diff --git a/airflow/providers/databricks/provider.yaml
b/airflow/providers/databricks/provider.yaml
index 8bb5106045..de7829f1cf 100644
--- a/airflow/providers/databricks/provider.yaml
+++ b/airflow/providers/databricks/provider.yaml
@@ -21,7 +21,7 @@ name: Databricks
description: |
`Databricks <https://databricks.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288125
versions:
- 6.0.0
diff --git a/airflow/providers/datadog/provider.yaml
b/airflow/providers/datadog/provider.yaml
index 3ece0559d1..a958527e1e 100644
--- a/airflow/providers/datadog/provider.yaml
+++ b/airflow/providers/datadog/provider.yaml
@@ -21,7 +21,7 @@ name: Datadog
description: |
`Datadog <https://www.datadoghq.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288126
versions:
- 3.5.1
diff --git a/airflow/providers/dbt/cloud/provider.yaml
b/airflow/providers/dbt/cloud/provider.yaml
index 6c17ae8d1e..ba6237264d 100644
--- a/airflow/providers/dbt/cloud/provider.yaml
+++ b/airflow/providers/dbt/cloud/provider.yaml
@@ -21,7 +21,7 @@ name: dbt Cloud
description: |
`dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288127
versions:
- 3.5.1
diff --git a/airflow/providers/dingding/provider.yaml
b/airflow/providers/dingding/provider.yaml
index 5ba8db7489..3647e9d4d8 100644
--- a/airflow/providers/dingding/provider.yaml
+++ b/airflow/providers/dingding/provider.yaml
@@ -21,7 +21,7 @@ name: Dingding
description: |
`DingTalk <https://www.dingtalk.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288128
versions:
- 3.4.0
diff --git a/airflow/providers/discord/provider.yaml
b/airflow/providers/discord/provider.yaml
index 30c893a351..a98539f07c 100644
--- a/airflow/providers/discord/provider.yaml
+++ b/airflow/providers/discord/provider.yaml
@@ -21,7 +21,7 @@ name: Discord
description: |
`Discord <https://discord.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288129
versions:
- 3.5.0
diff --git a/airflow/providers/docker/provider.yaml
b/airflow/providers/docker/provider.yaml
index 746b95e01a..4f04a065a6 100644
--- a/airflow/providers/docker/provider.yaml
+++ b/airflow/providers/docker/provider.yaml
@@ -21,7 +21,7 @@ name: Docker
description: |
`Docker <https://www.docker.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288130
versions:
- 3.9.1
diff --git a/airflow/providers/elasticsearch/provider.yaml
b/airflow/providers/elasticsearch/provider.yaml
index 2cb2291c7c..208a5e57fb 100644
--- a/airflow/providers/elasticsearch/provider.yaml
+++ b/airflow/providers/elasticsearch/provider.yaml
@@ -21,7 +21,7 @@ name: Elasticsearch
description: |
`Elasticsearch <https://www.elastic.co/elasticsearch>`__
-suspended: false
+state: ready
source-date-epoch: 1703747600
versions:
- 5.3.1
diff --git a/airflow/providers/exasol/provider.yaml
b/airflow/providers/exasol/provider.yaml
index 1714eeb64e..f47d0f7d43 100644
--- a/airflow/providers/exasol/provider.yaml
+++ b/airflow/providers/exasol/provider.yaml
@@ -21,7 +21,7 @@ name: Exasol
description: |
`Exasol <https://www.exasol.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288132
versions:
- 4.4.1
diff --git a/airflow/providers/fab/provider.yaml
b/airflow/providers/fab/provider.yaml
index 764c917d20..80e834fcad 100644
--- a/airflow/providers/fab/provider.yaml
+++ b/airflow/providers/fab/provider.yaml
@@ -24,11 +24,9 @@ name: Fab
description: |
`Flask App Builder <https://flask-appbuilder.readthedocs.io/>`__
-suspended: false
-
# The provider is not yet ready to be released, we will skip it by default
when preparing new release waves
# For providers until we think it should be released.
-not-ready: true
+state: not-ready
source-date-epoch: 1703288133
diff --git a/airflow/providers/facebook/provider.yaml
b/airflow/providers/facebook/provider.yaml
index 50089e3fa4..00fb98142a 100644
--- a/airflow/providers/facebook/provider.yaml
+++ b/airflow/providers/facebook/provider.yaml
@@ -21,7 +21,7 @@ name: Facebook
description: |
`Facebook Ads <https://www.facebook.com/about/ads>`__
-suspended: false
+state: ready
source-date-epoch: 1703288134
versions:
- 3.4.0
diff --git a/airflow/providers/ftp/provider.yaml
b/airflow/providers/ftp/provider.yaml
index a8b9d5b4ca..8dc441635b 100644
--- a/airflow/providers/ftp/provider.yaml
+++ b/airflow/providers/ftp/provider.yaml
@@ -21,7 +21,7 @@ name: File Transfer Protocol (FTP)
description: |
`File Transfer Protocol (FTP) <https://tools.ietf.org/html/rfc114>`__
-suspended: false
+state: ready
source-date-epoch: 1703288135
versions:
- 3.7.0
diff --git a/airflow/providers/github/provider.yaml
b/airflow/providers/github/provider.yaml
index 06e4bf5686..61d24d2e93 100644
--- a/airflow/providers/github/provider.yaml
+++ b/airflow/providers/github/provider.yaml
@@ -28,7 +28,7 @@ dependencies:
# See https://github.com/PyGithub/PyGithub/issues/2436.
- PyGithub!=1.58
-suspended: false
+state: ready
source-date-epoch: 1703288136
versions:
- 2.5.1
diff --git a/airflow/providers/google/provider.yaml
b/airflow/providers/google/provider.yaml
index 0ea04bc858..d7aca84d81 100644
--- a/airflow/providers/google/provider.yaml
+++ b/airflow/providers/google/provider.yaml
@@ -28,7 +28,7 @@ description: |
- `Google Marketing Platform <https://marketingplatform.google.com/>`__
- `Google Workspace <https://workspace.google.com/>`__ (formerly Google
Suite)
-suspended: false
+state: ready
source-date-epoch: 1703747618
versions:
- 10.13.1
diff --git a/airflow/providers/grpc/provider.yaml
b/airflow/providers/grpc/provider.yaml
index 4cf4c68dfe..cf190109c4 100644
--- a/airflow/providers/grpc/provider.yaml
+++ b/airflow/providers/grpc/provider.yaml
@@ -21,7 +21,7 @@ name: gRPC
description: |
`gRPC <https://grpc.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288139
versions:
- 3.4.1
diff --git a/airflow/providers/hashicorp/provider.yaml
b/airflow/providers/hashicorp/provider.yaml
index 71627ed0cb..ce2b3846b4 100644
--- a/airflow/providers/hashicorp/provider.yaml
+++ b/airflow/providers/hashicorp/provider.yaml
@@ -21,7 +21,7 @@ name: Hashicorp
description: |
Hashicorp including `Hashicorp Vault <https://www.vaultproject.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288140
versions:
- 3.6.1
diff --git a/airflow/providers/http/provider.yaml
b/airflow/providers/http/provider.yaml
index 07ecac0536..d193955808 100644
--- a/airflow/providers/http/provider.yaml
+++ b/airflow/providers/http/provider.yaml
@@ -21,7 +21,7 @@ name: Hypertext Transfer Protocol (HTTP)
description: |
`Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288141
versions:
- 4.8.0
diff --git a/airflow/providers/imap/provider.yaml
b/airflow/providers/imap/provider.yaml
index f943a5363b..00a6fdb995 100644
--- a/airflow/providers/imap/provider.yaml
+++ b/airflow/providers/imap/provider.yaml
@@ -22,7 +22,7 @@ name: Internet Message Access Protocol (IMAP)
description: |
`Internet Message Access Protocol (IMAP)
<https://tools.ietf.org/html/rfc3501>`__
-suspended: false
+state: ready
source-date-epoch: 1703288142
versions:
- 3.5.0
diff --git a/airflow/providers/influxdb/provider.yaml
b/airflow/providers/influxdb/provider.yaml
index d39f2feda0..6820b4cfeb 100644
--- a/airflow/providers/influxdb/provider.yaml
+++ b/airflow/providers/influxdb/provider.yaml
@@ -28,7 +28,7 @@ dependencies:
- influxdb-client>=1.19.0
- requests>=2.26.0
-suspended: false
+state: ready
source-date-epoch: 1703288143
versions:
- 2.4.0
diff --git a/airflow/providers/jdbc/provider.yaml
b/airflow/providers/jdbc/provider.yaml
index 3d66c416d3..1acd7ee944 100644
--- a/airflow/providers/jdbc/provider.yaml
+++ b/airflow/providers/jdbc/provider.yaml
@@ -21,7 +21,7 @@ name: Java Database Connectivity (JDBC)
description: |
`Java Database Connectivity (JDBC)
<https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288143
versions:
- 4.2.1
diff --git a/airflow/providers/jenkins/provider.yaml
b/airflow/providers/jenkins/provider.yaml
index ef90dc89a9..90124220d1 100644
--- a/airflow/providers/jenkins/provider.yaml
+++ b/airflow/providers/jenkins/provider.yaml
@@ -21,7 +21,7 @@ name: Jenkins
description: |
`Jenkins <https://jenkins.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288144
versions:
- 3.5.1
diff --git a/airflow/providers/microsoft/azure/provider.yaml
b/airflow/providers/microsoft/azure/provider.yaml
index cdf8c0be51..fc3aefaf9c 100644
--- a/airflow/providers/microsoft/azure/provider.yaml
+++ b/airflow/providers/microsoft/azure/provider.yaml
@@ -19,7 +19,7 @@ package-name: apache-airflow-providers-microsoft-azure
name: Microsoft Azure
description: |
`Microsoft Azure <https://azure.microsoft.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703747626
versions:
- 8.5.1
diff --git a/airflow/providers/microsoft/mssql/provider.yaml
b/airflow/providers/microsoft/mssql/provider.yaml
index 40ba8c3a93..2a56bc9a2b 100644
--- a/airflow/providers/microsoft/mssql/provider.yaml
+++ b/airflow/providers/microsoft/mssql/provider.yaml
@@ -21,7 +21,7 @@ name: Microsoft SQL Server (MSSQL)
description: |
`Microsoft SQL Server (MSSQL) <https://www.microsoft.com/sql-server/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288147
versions:
- 3.6.0
diff --git a/airflow/providers/microsoft/psrp/provider.yaml
b/airflow/providers/microsoft/psrp/provider.yaml
index 8dbadf94c5..6d81aa851f 100644
--- a/airflow/providers/microsoft/psrp/provider.yaml
+++ b/airflow/providers/microsoft/psrp/provider.yaml
@@ -23,7 +23,7 @@ description: |
`PowerShell Remoting Protocol (PSRP)
<https://docs.microsoft.com/openspecs/windows_protocols/ms-psrp/>`__.
-suspended: false
+state: ready
source-date-epoch: 1703288148
versions:
- 2.5.0
diff --git a/airflow/providers/microsoft/winrm/provider.yaml
b/airflow/providers/microsoft/winrm/provider.yaml
index 470dbbe035..4a4794557e 100644
--- a/airflow/providers/microsoft/winrm/provider.yaml
+++ b/airflow/providers/microsoft/winrm/provider.yaml
@@ -21,7 +21,7 @@ name: Windows Remote Management (WinRM)
description: |
`Windows Remote Management (WinRM)
<https://docs.microsoft.com/windows/win32/winrm/portal>`__
-suspended: false
+state: ready
source-date-epoch: 1703288149
versions:
- 3.4.0
diff --git a/airflow/providers/mongo/provider.yaml
b/airflow/providers/mongo/provider.yaml
index 0e620f1a59..9e2bea6859 100644
--- a/airflow/providers/mongo/provider.yaml
+++ b/airflow/providers/mongo/provider.yaml
@@ -21,7 +21,7 @@ name: MongoDB
description: |
`MongoDB <https://www.mongodb.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288150
versions:
- 3.5.0
diff --git a/airflow/providers/mysql/provider.yaml
b/airflow/providers/mysql/provider.yaml
index a51034854c..9712ec8676 100644
--- a/airflow/providers/mysql/provider.yaml
+++ b/airflow/providers/mysql/provider.yaml
@@ -21,7 +21,7 @@ name: MySQL
description: |
`MySQL <https://www.mysql.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288151
versions:
- 5.5.0
diff --git a/airflow/providers/neo4j/provider.yaml
b/airflow/providers/neo4j/provider.yaml
index 7e93afd5d2..2f0ace0688 100644
--- a/airflow/providers/neo4j/provider.yaml
+++ b/airflow/providers/neo4j/provider.yaml
@@ -21,7 +21,7 @@ name: Neo4j
description: |
`Neo4j <https://neo4j.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288152
versions:
- 3.5.0
diff --git a/airflow/providers/odbc/provider.yaml
b/airflow/providers/odbc/provider.yaml
index 2bf68a9d25..1f5c0f9fd6 100644
--- a/airflow/providers/odbc/provider.yaml
+++ b/airflow/providers/odbc/provider.yaml
@@ -21,7 +21,7 @@ name: ODBC
description: |
`ODBC <https://github.com/mkleehammer/pyodbc/wiki>`__
-suspended: false
+state: ready
source-date-epoch: 1703288153
versions:
- 4.4.0
diff --git a/airflow/providers/openai/provider.yaml
b/airflow/providers/openai/provider.yaml
index 4a85040e17..5f384a3bf2 100644
--- a/airflow/providers/openai/provider.yaml
+++ b/airflow/providers/openai/provider.yaml
@@ -23,7 +23,7 @@ name: OpenAI
description: |
`OpenAI <https://platform.openai.com/docs/introduction>`__
-suspended: false
+state: ready
source-date-epoch: 1703288154
versions:
diff --git a/airflow/providers/openfaas/provider.yaml
b/airflow/providers/openfaas/provider.yaml
index a359eb0311..6caf4fd0b7 100644
--- a/airflow/providers/openfaas/provider.yaml
+++ b/airflow/providers/openfaas/provider.yaml
@@ -21,7 +21,7 @@ name: OpenFaaS
description: |
`OpenFaaS <https://www.openfaas.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288154
versions:
- 3.4.0
diff --git a/airflow/providers/openlineage/provider.yaml
b/airflow/providers/openlineage/provider.yaml
index 285d2547da..7c490c5a98 100644
--- a/airflow/providers/openlineage/provider.yaml
+++ b/airflow/providers/openlineage/provider.yaml
@@ -21,7 +21,7 @@ name: OpenLineage Airflow
description: |
`OpenLineage <https://openlineage.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288155
versions:
- 1.3.1
diff --git a/airflow/providers/opensearch/provider.yaml
b/airflow/providers/opensearch/provider.yaml
index 44adbf527a..3fc9a21766 100644
--- a/airflow/providers/opensearch/provider.yaml
+++ b/airflow/providers/opensearch/provider.yaml
@@ -21,7 +21,7 @@ name: OpenSearch
description: |
`OpenSearch <https://opensearch.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288156
versions:
- 1.1.1
diff --git a/airflow/providers/opsgenie/provider.yaml
b/airflow/providers/opsgenie/provider.yaml
index 31b6b5a431..892f1d1b56 100644
--- a/airflow/providers/opsgenie/provider.yaml
+++ b/airflow/providers/opsgenie/provider.yaml
@@ -21,7 +21,7 @@ name: Opsgenie
description: |
`Opsgenie <https://www.atlassian.com/software/opsgenie>`__
-suspended: false
+state: ready
source-date-epoch: 1703288156
versions:
- 5.4.0
diff --git a/airflow/providers/oracle/provider.yaml
b/airflow/providers/oracle/provider.yaml
index ea56335ddf..89b6480d0e 100644
--- a/airflow/providers/oracle/provider.yaml
+++ b/airflow/providers/oracle/provider.yaml
@@ -21,7 +21,7 @@ name: Oracle
description: |
`Oracle <https://www.oracle.com/en/database/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288157
versions:
- 3.9.1
diff --git a/airflow/providers/pagerduty/provider.yaml
b/airflow/providers/pagerduty/provider.yaml
index a013b8b8f8..b12bb1a106 100644
--- a/airflow/providers/pagerduty/provider.yaml
+++ b/airflow/providers/pagerduty/provider.yaml
@@ -21,7 +21,7 @@ name: Pagerduty
description: |
`Pagerduty <https://www.pagerduty.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288158
versions:
- 3.5.1
diff --git a/airflow/providers/papermill/provider.yaml
b/airflow/providers/papermill/provider.yaml
index ccdfbf1898..1f20ecf359 100644
--- a/airflow/providers/papermill/provider.yaml
+++ b/airflow/providers/papermill/provider.yaml
@@ -21,7 +21,7 @@ name: Papermill
description: |
`Papermill <https://github.com/nteract/papermill>`__
-suspended: false
+state: ready
source-date-epoch: 1703288159
versions:
- 3.6.0
diff --git a/airflow/providers/pgvector/provider.yaml
b/airflow/providers/pgvector/provider.yaml
index 852f2e2ebd..9727fb4160 100644
--- a/airflow/providers/pgvector/provider.yaml
+++ b/airflow/providers/pgvector/provider.yaml
@@ -23,7 +23,7 @@ name: pgvector
description: |
`pgvector <https://github.com/pgvector/pgvector>`__
-suspended: false
+state: ready
source-date-epoch: 1703288160
versions:
diff --git a/airflow/providers/pinecone/provider.yaml
b/airflow/providers/pinecone/provider.yaml
index 3d96fa186f..80a304abd8 100644
--- a/airflow/providers/pinecone/provider.yaml
+++ b/airflow/providers/pinecone/provider.yaml
@@ -23,7 +23,7 @@ name: Pinecone
description: |
`Pinecone <https://docs.pinecone.io/docs/overview>`__
-suspended: false
+state: ready
source-date-epoch: 1703288160
versions:
diff --git a/airflow/providers/postgres/provider.yaml
b/airflow/providers/postgres/provider.yaml
index 9bdb618dd3..c6e4338978 100644
--- a/airflow/providers/postgres/provider.yaml
+++ b/airflow/providers/postgres/provider.yaml
@@ -21,7 +21,7 @@ name: PostgreSQL
description: |
`PostgreSQL <https://www.postgresql.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288162
versions:
- 5.10.0
diff --git a/airflow/providers/presto/provider.yaml
b/airflow/providers/presto/provider.yaml
index be43f2803c..432354167a 100644
--- a/airflow/providers/presto/provider.yaml
+++ b/airflow/providers/presto/provider.yaml
@@ -21,7 +21,7 @@ name: Presto
description: |
`Presto <https://prestodb.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288163
versions:
- 5.4.0
diff --git a/airflow/providers/redis/provider.yaml
b/airflow/providers/redis/provider.yaml
index 1640ca11e6..96fafc3c8f 100644
--- a/airflow/providers/redis/provider.yaml
+++ b/airflow/providers/redis/provider.yaml
@@ -21,7 +21,7 @@ name: Redis
description: |
`Redis <https://redis.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288164
versions:
- 3.5.0
diff --git a/airflow/providers/salesforce/provider.yaml
b/airflow/providers/salesforce/provider.yaml
index abcb9cdd68..c65e83e1c2 100644
--- a/airflow/providers/salesforce/provider.yaml
+++ b/airflow/providers/salesforce/provider.yaml
@@ -21,7 +21,7 @@ name: Salesforce
description: |
`Salesforce <https://www.salesforce.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288165
versions:
- 5.6.1
diff --git a/airflow/providers/samba/provider.yaml
b/airflow/providers/samba/provider.yaml
index 36d04402a4..1277cd194f 100644
--- a/airflow/providers/samba/provider.yaml
+++ b/airflow/providers/samba/provider.yaml
@@ -21,7 +21,7 @@ name: Samba
description: |
`Samba <https://www.samba.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288166
versions:
- 4.4.0
diff --git a/airflow/providers/segment/provider.yaml
b/airflow/providers/segment/provider.yaml
index e3a8635311..ac7174a523 100644
--- a/airflow/providers/segment/provider.yaml
+++ b/airflow/providers/segment/provider.yaml
@@ -21,7 +21,7 @@ name: Segment
description: |
`Segment <https://segment.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288167
versions:
- 3.4.0
diff --git a/airflow/providers/sendgrid/provider.yaml
b/airflow/providers/sendgrid/provider.yaml
index 31a28220f9..ed74aa6992 100644
--- a/airflow/providers/sendgrid/provider.yaml
+++ b/airflow/providers/sendgrid/provider.yaml
@@ -25,7 +25,7 @@ dependencies:
- apache-airflow>=2.6.0
- sendgrid>=6.0.0
-suspended: false
+state: ready
source-date-epoch: 1703288168
versions:
- 3.4.0
diff --git a/airflow/providers/sftp/provider.yaml
b/airflow/providers/sftp/provider.yaml
index 18112c648e..a91bec8791 100644
--- a/airflow/providers/sftp/provider.yaml
+++ b/airflow/providers/sftp/provider.yaml
@@ -21,7 +21,7 @@ name: SFTP
description: |
`SSH File Transfer Protocol (SFTP)
<https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288169
versions:
- 4.8.1
diff --git a/airflow/providers/singularity/provider.yaml
b/airflow/providers/singularity/provider.yaml
index d4b42a942a..a84c5ffa90 100644
--- a/airflow/providers/singularity/provider.yaml
+++ b/airflow/providers/singularity/provider.yaml
@@ -21,7 +21,7 @@ name: Singularity
description: |
`Singularity <https://sylabs.io/guides/latest/user-guide/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288170
versions:
- 3.4.0
diff --git a/airflow/providers/slack/provider.yaml
b/airflow/providers/slack/provider.yaml
index 27c11e50ff..ede4202c6b 100644
--- a/airflow/providers/slack/provider.yaml
+++ b/airflow/providers/slack/provider.yaml
@@ -24,7 +24,7 @@ description: |
- `Slack API <https://api.slack.com/>`__
- `Slack Incoming Webhook <https://api.slack.com/messaging/webhooks>`__
-suspended: false
+state: ready
source-date-epoch: 1703288171
versions:
- 8.5.1
diff --git a/airflow/providers/smtp/provider.yaml
b/airflow/providers/smtp/provider.yaml
index 981d47a7bb..58619a1a5a 100644
--- a/airflow/providers/smtp/provider.yaml
+++ b/airflow/providers/smtp/provider.yaml
@@ -22,7 +22,7 @@ name: Simple Mail Transfer Protocol (SMTP)
description: |
`Simple Mail Transfer Protocol (SMTP)
<https://tools.ietf.org/html/rfc5321>`__
-suspended: false
+state: ready
source-date-epoch: 1703288172
versions:
- 1.6.0
diff --git a/airflow/providers/snowflake/provider.yaml
b/airflow/providers/snowflake/provider.yaml
index addfe8e25a..6716386632 100644
--- a/airflow/providers/snowflake/provider.yaml
+++ b/airflow/providers/snowflake/provider.yaml
@@ -21,7 +21,7 @@ name: Snowflake
description: |
`Snowflake <https://www.snowflake.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288173
versions:
- 5.2.1
diff --git a/airflow/providers/sqlite/provider.yaml
b/airflow/providers/sqlite/provider.yaml
index 75dc931c57..753b433552 100644
--- a/airflow/providers/sqlite/provider.yaml
+++ b/airflow/providers/sqlite/provider.yaml
@@ -21,7 +21,7 @@ name: SQLite
description: |
`SQLite <https://www.sqlite.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288175
versions:
- 3.7.0
diff --git a/airflow/providers/ssh/provider.yaml
b/airflow/providers/ssh/provider.yaml
index 8807a4ca37..90797ce4ac 100644
--- a/airflow/providers/ssh/provider.yaml
+++ b/airflow/providers/ssh/provider.yaml
@@ -21,7 +21,7 @@ name: SSH
description: |
`Secure Shell (SSH) <https://tools.ietf.org/html/rfc4251>`__
-suspended: false
+state: ready
source-date-epoch: 1703288176
versions:
- 3.10.0
diff --git a/airflow/providers/tableau/provider.yaml
b/airflow/providers/tableau/provider.yaml
index 015faca778..4c35a77f7a 100644
--- a/airflow/providers/tableau/provider.yaml
+++ b/airflow/providers/tableau/provider.yaml
@@ -21,7 +21,7 @@ name: Tableau
description: |
`Tableau <https://www.tableau.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288177
versions:
- 4.4.0
diff --git a/airflow/providers/tabular/provider.yaml
b/airflow/providers/tabular/provider.yaml
index 049dc9a2d3..7b161a41d2 100644
--- a/airflow/providers/tabular/provider.yaml
+++ b/airflow/providers/tabular/provider.yaml
@@ -21,7 +21,7 @@ name: Tabular
description: |
`Tabular <https://tabular.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288178
versions:
- 1.4.1
diff --git a/airflow/providers/telegram/provider.yaml
b/airflow/providers/telegram/provider.yaml
index 43ce552966..d5f4c00d1a 100644
--- a/airflow/providers/telegram/provider.yaml
+++ b/airflow/providers/telegram/provider.yaml
@@ -21,7 +21,7 @@ name: Telegram
description: |
`Telegram <https://telegram.org/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288178
versions:
- 4.3.0
diff --git a/airflow/providers/trino/provider.yaml
b/airflow/providers/trino/provider.yaml
index 8c0b4be804..5542a0983f 100644
--- a/airflow/providers/trino/provider.yaml
+++ b/airflow/providers/trino/provider.yaml
@@ -21,7 +21,7 @@ name: Trino
description: |
`Trino <https://trino.io/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288179
versions:
- 5.6.0
diff --git a/airflow/providers/vertica/provider.yaml
b/airflow/providers/vertica/provider.yaml
index 28b3acb840..f0fb2bde70 100644
--- a/airflow/providers/vertica/provider.yaml
+++ b/airflow/providers/vertica/provider.yaml
@@ -21,7 +21,7 @@ name: Vertica
description: |
`Vertica <https://www.vertica.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288181
versions:
- 3.7.0
diff --git a/airflow/providers/weaviate/provider.yaml
b/airflow/providers/weaviate/provider.yaml
index ad12dc8964..6559c85677 100644
--- a/airflow/providers/weaviate/provider.yaml
+++ b/airflow/providers/weaviate/provider.yaml
@@ -23,7 +23,7 @@ name: Weaviate
description: |
`Weaviate <https://weaviate.io/developers/weaviate>`__
-suspended: false
+state: ready
source-date-epoch: 1703747660
versions:
diff --git a/airflow/providers/yandex/provider.yaml
b/airflow/providers/yandex/provider.yaml
index a7d71a8365..01d8a8faa1 100644
--- a/airflow/providers/yandex/provider.yaml
+++ b/airflow/providers/yandex/provider.yaml
@@ -20,7 +20,7 @@ package-name: apache-airflow-providers-yandex
name: Yandex
description: |
Yandex including `Yandex.Cloud <https://cloud.yandex.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703288182
versions:
- 3.7.1
diff --git a/airflow/providers/zendesk/provider.yaml
b/airflow/providers/zendesk/provider.yaml
index 25d5de690e..d5a20c7020 100644
--- a/airflow/providers/zendesk/provider.yaml
+++ b/airflow/providers/zendesk/provider.yaml
@@ -21,7 +21,7 @@ name: Zendesk
description: |
`Zendesk <https://www.zendesk.com/>`__
-suspended: false
+state: ready
source-date-epoch: 1703747680
versions:
- 4.6.0
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 1f21b6937b..d7ae88fd13 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -451,7 +451,7 @@ def prepare_provider_documentation(
except PrepareReleaseDocsUserQuitException:
break
else:
- if provider_metadata.get("removed"):
+ if provider_metadata["state"] == "removed":
removed_packages.append(provider_id)
else:
success_packages.append(provider_id)
@@ -482,12 +482,12 @@ def basic_provider_checks(provider_package_id: str) ->
dict[str, Any]:
if not provider_metadata:
get_console().print(f"[error]The package {provider_package_id} is not
a provider package. Exiting[/]")
sys.exit(1)
- if provider_metadata.get("removed", False):
+ if provider_metadata["state"] == "removed":
get_console().print(
f"[warning]The package: {provider_package_id} is scheduled for
removal, but "
f"since you asked for it, it will be built [/]\n"
)
- elif provider_metadata.get("suspended"):
+ elif provider_metadata.get("state") == "suspended":
get_console().print(f"[warning]The package: {provider_package_id} is
suspended " f"skipping it [/]\n")
raise PackageSuspendedException()
return provider_metadata
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index 59fe46f2e2..49626a4668 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -156,6 +156,7 @@ def refresh_provider_metadata_with_provider_id(provider_id:
str):
refresh_provider_metadata_from_yaml_file(provider_yaml_path)
+@lru_cache(maxsize=1)
def get_provider_packages_metadata() -> dict[str, dict[str, Any]]:
"""
Load all data from providers files
@@ -207,11 +208,7 @@ def get_provider_info_dict(provider_id: str) -> dict[str,
Any]:
@lru_cache
def get_suspended_provider_ids() -> list[str]:
- return [
- provider_id
- for provider_id, provider_metadata in
get_provider_packages_metadata().items()
- if provider_metadata.get("suspended", False)
- ]
+ return get_available_packages(include_suspended=True,
include_regular=False)
@lru_cache
@@ -221,20 +218,12 @@ def get_suspended_provider_folders() -> list[str]:
@lru_cache
def get_removed_provider_ids() -> list[str]:
- return [
- provider_id
- for provider_id, provider_metadata in
get_provider_packages_metadata().items()
- if provider_metadata.get("removed", False)
- ]
+ return get_available_packages(include_removed=True, include_regular=False)
@lru_cache
def get_not_ready_provider_ids() -> list[str]:
- return [
- provider_id
- for provider_id, provider_metadata in
get_provider_packages_metadata().items()
- if provider_metadata.get("not-ready", False)
- ]
+ return get_available_packages(include_not_ready=True,
include_regular=False)
def get_provider_requirements(provider_id: str) -> list[str]:
@@ -249,6 +238,7 @@ def get_available_packages(
include_suspended: bool = False,
include_removed: bool = False,
include_not_ready: bool = False,
+ include_regular: bool = True,
) -> list[str]:
"""
Return provider ids for all packages that are available currently (not
suspended).
@@ -256,28 +246,33 @@ def get_available_packages(
:rtype: object
:param include_suspended: whether the suspended packages should be included
:param include_removed: whether the removed packages should be included
- :param include_not_ready: whether the not-ready ppackages should be
included
+ :param include_not_ready: whether the not-ready packages should be included
+ :param include_regular: whether the regular packages should be included
:param include_non_provider_doc_packages: whether the non-provider doc
packages should be included
(packages like apache-airflow, helm-chart, docker-stack)
:param include_all_providers: whether "all-providers" should be included
ni the list.
"""
- provider_ids: list[str] =
list(json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()).keys())
- available_packages = []
- not_ready_provider_ids = get_not_ready_provider_ids()
- if not include_not_ready:
- provider_ids = [
- provider_id for provider_id in provider_ids if provider_id not in
not_ready_provider_ids
- ]
+ provider_dependencies =
json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text())
+
+ valid_states = set()
+ if include_not_ready:
+ valid_states.add("not-ready")
+ if include_regular:
+ valid_states.add("ready")
+ if include_suspended:
+ valid_states.add("suspended")
+ if include_removed:
+ valid_states.add("removed")
+ available_packages: list[str] = [
+ provider_id
+ for provider_id, provider_dependencies in provider_dependencies.items()
+ if provider_dependencies["state"] in valid_states
+ ]
if include_non_provider_doc_packages:
available_packages.extend(REGULAR_DOC_PACKAGES)
if include_all_providers:
available_packages.append("all-providers")
- available_packages.extend(provider_ids)
- if include_suspended:
- available_packages.extend(get_suspended_provider_ids())
- if include_removed:
- available_packages.extend(get_removed_provider_ids())
return sorted(set(available_packages))
@@ -499,7 +494,7 @@ def get_provider_details(provider_id: str) ->
ProviderPackageDetails:
versions=provider_info["versions"],
excluded_python_versions=provider_info.get("excluded-python-versions")
or [],
plugins=plugins,
- removed=provider_info.get("removed", False),
+ removed=provider_info["state"] == "removed",
)
diff --git a/dev/breeze/tests/test_packages.py
b/dev/breeze/tests/test_packages.py
index eb7b3a4d84..017a71e9d6 100644
--- a/dev/breeze/tests/test_packages.py
+++ b/dev/breeze/tests/test_packages.py
@@ -340,7 +340,7 @@ def test_get_provider_info_dict():
assert provider_info_dict["name"] == "Amazon"
assert provider_info_dict["package-name"] ==
"apache-airflow-providers-amazon"
assert "Amazon" in provider_info_dict["description"]
- assert provider_info_dict["suspended"] is False
+ assert provider_info_dict["state"] == "ready"
assert provider_info_dict["filesystems"] ==
["airflow.providers.amazon.aws.fs.s3"]
assert len(provider_info_dict["versions"]) > 45
assert len(provider_info_dict["dependencies"]) > 10
diff --git a/docs/exts/docs_build/package_filter.py
b/docs/exts/docs_build/package_filter.py
index d1e497e15c..26dae9ff1c 100644
--- a/docs/exts/docs_build/package_filter.py
+++ b/docs/exts/docs_build/package_filter.py
@@ -31,7 +31,7 @@ def get_removed_provider_ids() -> list[str]:
removed_provider_ids = []
for provider_path in PROVIDERS_DIR.rglob("provider.yaml"):
provider_yaml = yaml.safe_load(provider_path.read_text())
- if provider_yaml.get("removed"):
+ if provider_yaml["state"] == "removed":
removed_provider_ids.append(
provider_yaml["package-name"][len("apache-airflow-providers-")
:].replace("-", ".")
)
diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py
index 2a49cf4652..d417ba259d 100644
--- a/docs/exts/provider_yaml_utils.py
+++ b/docs/exts/provider_yaml_utils.py
@@ -69,7 +69,7 @@ def load_package_data(include_suspended: bool = False) ->
list[dict[str, Any]]:
jsonschema.validate(provider, schema=schema)
except jsonschema.ValidationError:
raise Exception(f"Unable to parse: {provider_yaml_path}.")
- if provider["suspended"] and not include_suspended:
+ if provider["state"] == "suspended" and not include_suspended:
continue
provider_yaml_dir = os.path.dirname(provider_yaml_path)
provider["python-module"] = _filepath_to_module(provider_yaml_dir)
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 478b310198..c0131f8ca8 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -7,7 +7,8 @@
"cross-providers-deps": [
"http"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"alibaba": {
"deps": [
@@ -17,7 +18,8 @@
"oss2>=2.14.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"amazon": {
"deps": [
@@ -48,7 +50,8 @@
"salesforce",
"ssh"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.beam": {
"deps": [
@@ -58,7 +61,8 @@
"cross-providers-deps": [
"google"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.cassandra": {
"deps": [
@@ -66,7 +70,8 @@
"cassandra-driver>=3.13.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.drill": {
"deps": [
@@ -77,7 +82,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.druid": {
"deps": [
@@ -89,7 +95,8 @@
"apache.hive",
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.flink": {
"deps": [
@@ -100,7 +107,8 @@
"cross-providers-deps": [
"cncf.kubernetes"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.hdfs": {
"deps": [
@@ -108,7 +116,8 @@
"hdfs[avro,dataframe,kerberos]>=2.0.4"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.hive": {
"deps": [
@@ -128,7 +137,8 @@
"samba",
"vertica"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.impala": {
"deps": [
@@ -138,7 +148,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.kafka": {
"deps": [
@@ -147,7 +158,8 @@
"confluent-kafka>=1.8.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.kylin": {
"deps": [
@@ -155,7 +167,8 @@
"kylinpy>=2.6"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.livy": {
"deps": [
@@ -167,14 +180,16 @@
"cross-providers-deps": [
"http"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.pig": {
"deps": [
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.pinot": {
"deps": [
@@ -185,7 +200,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apache.spark": {
"deps": [
@@ -196,7 +212,8 @@
"cross-providers-deps": [
"cncf.kubernetes"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"apprise": {
"deps": [
@@ -204,7 +221,8 @@
"apprise"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"arangodb": {
"deps": [
@@ -212,7 +230,8 @@
"python-arango>=7.3.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"asana": {
"deps": [
@@ -220,7 +239,8 @@
"asana>=0.10,<4.0.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"atlassian.jira": {
"deps": [
@@ -228,7 +248,8 @@
"atlassian-python-api>=1.14.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"celery": {
"deps": [
@@ -240,7 +261,8 @@
"cross-providers-deps": [
"cncf.kubernetes"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"cloudant": {
"deps": [
@@ -248,7 +270,8 @@
"cloudant>=2.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"cncf.kubernetes": {
"deps": [
@@ -261,7 +284,8 @@
"kubernetes_asyncio>=18.20.1,<25"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"cohere": {
"deps": [
@@ -269,14 +293,16 @@
"cohere>=4.27"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"common.io": {
"deps": [
"apache-airflow>=2.8.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"common.sql": {
"deps": [
@@ -286,7 +312,8 @@
"cross-providers-deps": [
"openlineage"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"databricks": {
"deps": [
@@ -299,7 +326,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"datadog": {
"deps": [
@@ -307,7 +335,8 @@
"datadog>=0.14.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"dbt.cloud": {
"deps": [
@@ -320,7 +349,8 @@
"http",
"openlineage"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"dingding": {
"deps": [
@@ -330,7 +360,8 @@
"cross-providers-deps": [
"http"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"discord": {
"deps": [
@@ -340,7 +371,8 @@
"cross-providers-deps": [
"http"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"docker": {
"deps": [
@@ -349,7 +381,8 @@
"python-dotenv>=0.21.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"elasticsearch": {
"deps": [
@@ -360,7 +393,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"exasol": {
"deps": [
@@ -372,7 +406,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"fab": {
"deps": [
@@ -383,7 +418,8 @@
"google-re2>=1.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "not-ready"
},
"facebook": {
"deps": [
@@ -391,7 +427,8 @@
"facebook-business>=6.0.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"ftp": {
"deps": [
@@ -400,7 +437,8 @@
"cross-providers-deps": [
"openlineage"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"github": {
"deps": [
@@ -408,7 +446,8 @@
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"google": {
"deps": [
@@ -490,7 +529,8 @@
"ssh",
"trino"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"grpc": {
"deps": [
@@ -500,7 +540,8 @@
"grpcio>=1.15.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"hashicorp": {
"deps": [
@@ -510,7 +551,8 @@
"cross-providers-deps": [
"google"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"http": {
"deps": [
@@ -521,14 +563,16 @@
"requests_toolbelt"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"imap": {
"deps": [
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"influxdb": {
"deps": [
@@ -537,7 +581,8 @@
"requests>=2.26.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"jdbc": {
"deps": [
@@ -548,7 +593,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"jenkins": {
"deps": [
@@ -556,7 +602,8 @@
"python-jenkins>=1.0.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"microsoft.azure": {
"deps": [
@@ -588,7 +635,8 @@
"oracle",
"sftp"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"microsoft.mssql": {
"deps": [
@@ -599,7 +647,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"microsoft.psrp": {
"deps": [
@@ -607,7 +656,8 @@
"pypsrp>=0.8.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"microsoft.winrm": {
"deps": [
@@ -615,7 +665,8 @@
"pywinrm>=0.4"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"mongo": {
"deps": [
@@ -624,7 +675,8 @@
"pymongo>=3.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"mysql": {
"deps": [
@@ -641,7 +693,8 @@
"trino",
"vertica"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"neo4j": {
"deps": [
@@ -649,7 +702,8 @@
"neo4j>=4.2.1"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"odbc": {
"deps": [
@@ -660,7 +714,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"openai": {
"deps": [
@@ -668,14 +723,16 @@
"openai[datalib]>=1.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"openfaas": {
"deps": [
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"openlineage": {
"deps": [
@@ -688,7 +745,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"opensearch": {
"deps": [
@@ -696,7 +754,8 @@
"opensearch-py>=2.2.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"opsgenie": {
"deps": [
@@ -704,7 +763,8 @@
"opsgenie-sdk>=2.1.5"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"oracle": {
"deps": [
@@ -715,7 +775,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"pagerduty": {
"deps": [
@@ -723,7 +784,8 @@
"pdpyras>=4.1.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"papermill": {
"deps": [
@@ -733,7 +795,8 @@
"scrapbook[all]"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"pgvector": {
"deps": [
@@ -745,7 +808,8 @@
"common.sql",
"postgres"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"pinecone": {
"deps": [
@@ -753,7 +817,8 @@
"pinecone-client>=2.2.4"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"postgres": {
"deps": [
@@ -766,7 +831,8 @@
"common.sql",
"openlineage"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"presto": {
"deps": [
@@ -779,7 +845,8 @@
"common.sql",
"google"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"redis": {
"deps": [
@@ -787,7 +854,8 @@
"redis>=4.5.2,<5.0.0,!=4.5.5"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"salesforce": {
"deps": [
@@ -796,7 +864,8 @@
"simple-salesforce>=1.0.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"samba": {
"deps": [
@@ -806,7 +875,8 @@
"cross-providers-deps": [
"google"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"segment": {
"deps": [
@@ -814,7 +884,8 @@
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"sendgrid": {
"deps": [
@@ -822,7 +893,8 @@
"sendgrid>=6.0.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"sftp": {
"deps": [
@@ -834,7 +906,8 @@
"openlineage",
"ssh"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"singularity": {
"deps": [
@@ -842,7 +915,8 @@
"spython>=0.0.56"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"slack": {
"deps": [
@@ -853,14 +927,16 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"smtp": {
"deps": [
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"snowflake": {
"deps": [
@@ -873,7 +949,8 @@
"common.sql",
"openlineage"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"sqlite": {
"deps": [
@@ -883,7 +960,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"ssh": {
"deps": [
@@ -892,7 +970,8 @@
"sshtunnel>=0.3.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"tableau": {
"deps": [
@@ -900,14 +979,16 @@
"tableauserverclient"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"tabular": {
"deps": [
"apache-airflow>=2.6.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"telegram": {
"deps": [
@@ -915,7 +996,8 @@
"python-telegram-bot>=20.0.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"trino": {
"deps": [
@@ -929,7 +1011,8 @@
"google",
"openlineage"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"vertica": {
"deps": [
@@ -940,7 +1023,8 @@
"cross-providers-deps": [
"common.sql"
],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"weaviate": {
"deps": [
@@ -949,7 +1033,8 @@
"weaviate-client>=3.24.2"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"yandex": {
"deps": [
@@ -957,7 +1042,8 @@
"yandexcloud>=0.228.0"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
},
"zendesk": {
"deps": [
@@ -965,6 +1051,7 @@
"zenpy>=2.0.24"
],
"cross-providers-deps": [],
- "excluded-python-versions": []
+ "excluded-python-versions": [],
+ "state": "ready"
}
}
diff --git a/images/breeze/output_release-management_generate-constraints.svg
b/images/breeze/output_release-management_generate-constraints.svg
index 82651b9a58..b8bc2f4b31 100644
--- a/images/breeze/output_release-management_generate-constraints.svg
+++ b/images/breeze/output_release-management_generate-constraints.svg
@@ -177,9 +177,9 @@
</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="312.8" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text
class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8"
textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-12)">-</text><text
class="breeze-release-management-generate-constraints-r4" x="36.6" y="312.8"
textLength="97.6" clip-path="url(#breeze- [...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="337.2" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text
class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2"
textLength="866.2"
clip-path="url(#breeze-release-management-generate-constraints-line-13)">(constraints-source-providers | constraints | constraints-no-providers)</text><text
class="breeze-release-mana [...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text
class="breeze-release-management-generate-constraints-r5" x="414.8" y="361.6"
textLength="866.2"
clip-path="url(#breeze-release-management-generate-constraints-line-14)">[default: constraints-source-providers]              &#
[...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="386" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text
class="breeze-release-management-generate-constraints-r4" x="24.4" y="386"
textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text
class="breeze-release-management-generate-constraints-r4" x="36.6" y="386"
textLength="97.6" clip-path="url(#breeze-releas [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="410.4" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text
class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4"
textLength="1024.8"
clip-path="url(#breeze-release-management-generate-constraints-line-16)">airflow_version >= current_version and should be installed in CI from locall
[...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text
class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8"
textLength="1024.8"
clip-path="url(#breeze-release-management-generate-constraints-line-17)">packages with >= current_version.dev0            &#
[...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="386" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text
class="breeze-release-management-generate-constraints-r4" x="24.4" y="386"
textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text
class="breeze-release-management-generate-constraints-r4" x="36.6" y="386"
textLength="97.6" clip-path="url(#breeze-releas [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="410.4" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text
class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4"
textLength="1024.8"
clip-path="url(#breeze-release-management-generate-constraints-line-16)">current_version and should be installed in CI from locally built packages 
[...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text
class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8"
textLength="1024.8"
clip-path="url(#breeze-release-management-generate-constraints-line-17)">current_version.dev0                  &
[...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="459.2" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text
class="breeze-release-management-generate-constraints-r6" x="414.8" y="459.2"
textLength="1024.8"
clip-path="url(#breeze-release-management-generate-constraints-line-18)">(TEXT)                    
[...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="483.6" textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text
class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6"
textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text
class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6"
textLength="85.4" clip-path="url(#breeze- [...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0"
y="508" textLength="1464"
clip-path="url(#breeze-release-management-generate-constraints-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-generate-constraints-r1" x="1464" y="508"
textLength="12.2"
clip-path="url(#breeze-release-management-generate-constraints-line-20)">
diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_docs.py
b/scripts/ci/pre_commit/pre_commit_check_provider_docs.py
index 7814581100..0681b02287 100755
--- a/scripts/ci/pre_commit/pre_commit_check_provider_docs.py
+++ b/scripts/ci/pre_commit/pre_commit_check_provider_docs.py
@@ -94,7 +94,7 @@ def find_all_providers():
for provider_file in AIRFLOW_PROVIDERS_DIR.rglob("provider.yaml"):
provider_name =
str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace(os.sep,
".")
provider_info = yaml.safe_load(provider_file.read_text())
- if not provider_info["suspended"]:
+ if provider_info["state"] != "suspended":
ALL_PROVIDERS[provider_name] = provider_info
diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
index a3056df3a1..5fa638505f 100755
--- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
+++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
@@ -97,10 +97,9 @@ def find_all_providers_and_provider_files():
os.sep, "."
)
provider_info = yaml.safe_load(provider_file.read_text())
- if not provider_info["suspended"]:
- ALL_PROVIDERS[provider_name] = provider_info
- else:
+ if provider_info["state"] == "suspended":
suspended_paths.append(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR).as_posix())
+ ALL_PROVIDERS[provider_name] = provider_info
path = Path(root, filename)
if path.is_file() and path.name.endswith(".py"):
ALL_PROVIDER_FILES.append(Path(root, filename))
@@ -174,6 +173,9 @@ def check_if_different_provider_used(file_path: Path) ->
None:
ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider)
+STATES: dict[str, str] = {}
+
+
if __name__ == "__main__":
find_all_providers_and_provider_files()
num_files = len(ALL_PROVIDER_FILES)
@@ -182,8 +184,8 @@ if __name__ == "__main__":
for file in ALL_PROVIDER_FILES:
check_if_different_provider_used(file)
for provider, provider_yaml_content in ALL_PROVIDERS.items():
- if not provider_yaml_content.get("suspended"):
-
ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"])
+
ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"])
+ STATES[provider] = provider_yaml_content["state"]
if warnings:
console.print("[yellow]Warnings!\n")
for warning in warnings:
@@ -194,7 +196,7 @@ if __name__ == "__main__":
for error in errors:
console.print(f"[red] {error}")
console.print(f"[bright_blue]Total: {len(errors)} errors.")
- unique_sorted_dependencies: dict[str, dict[str, list[str]]] =
defaultdict(dict)
+ unique_sorted_dependencies: dict[str, dict[str, list[str] | str]] =
defaultdict(dict)
for key in sorted(ALL_DEPENDENCIES.keys()):
unique_sorted_dependencies[key]["deps"] =
sorted(ALL_DEPENDENCIES[key]["deps"])
unique_sorted_dependencies[key]["cross-providers-deps"] = sorted(
@@ -202,6 +204,7 @@ if __name__ == "__main__":
)
excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions")
unique_sorted_dependencies[key]["excluded-python-versions"] =
excluded_versions or []
+ unique_sorted_dependencies[key]["state"] = STATES[key]
if errors:
console.print()
console.print("[red]Errors found during verification. Exiting!")
diff --git a/scripts/in_container/run_provider_yaml_files_check.py
b/scripts/in_container/run_provider_yaml_files_check.py
index aaffa0e2f8..619ec96015 100755
--- a/scripts/in_container/run_provider_yaml_files_check.py
+++ b/scripts/in_container/run_provider_yaml_files_check.py
@@ -114,7 +114,7 @@ def _load_package_data(package_paths: Iterable[str]):
jsonschema.validate(provider, schema=schema)
except jsonschema.ValidationError:
raise Exception(f"Unable to parse: {rel_path}.")
- if not provider.get("suspended"):
+ if not provider["state"] == "suspended":
result[rel_path] = provider
else:
suspended_providers.add(provider["package-name"])
@@ -680,26 +680,6 @@ def
check_providers_have_all_documentation_files(yaml_files: dict[str, dict]):
return num_providers, num_errors
-@run_check("Checking remove flag only set for suspended providers")
-def check_removed_flag_only_set_for_suspended_providers(yaml_files: dict[str,
dict]):
- num_errors = 0
- num_providers = 0
- for package_info in yaml_files.values():
- num_providers += 1
- package_name = package_info["package-name"]
- suspended = package_info["suspended"]
- removed = package_info.get("removed", False)
- if removed and not suspended:
- errors.append(
- f"The provider {package_name} has removed set to True in their
provider.yaml file "
- f"but suspended flag is set to false. You should only set
removed flag in order to "
- f"prepare last release for a provider that has been previously
suspended. "
- f"[yellow]How to fix it[/]: Please suspend the provider first
before removing it."
- )
- num_errors += 1
- return num_providers, num_errors
-
-
if __name__ == "__main__":
ProvidersManager().initialize_providers_configuration()
architecture = Architecture.get_current()
@@ -726,7 +706,6 @@ if __name__ == "__main__":
check_notification_classes(all_parsed_yaml_files)
check_unique_provider_name(all_parsed_yaml_files)
check_providers_have_all_documentation_files(all_parsed_yaml_files)
- check_removed_flag_only_set_for_suspended_providers(all_parsed_yaml_files)
if all_files_loaded:
# Only check those if all provider files are loaded
diff --git a/tests/always/test_example_dags.py
b/tests/always/test_example_dags.py
index 3687832fdc..aadf4e5f91 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -40,7 +40,7 @@ def get_suspended_providers_folders() -> list[str]:
suspended_providers = []
for provider_path in AIRFLOW_PROVIDERS_ROOT.rglob("provider.yaml"):
provider_yaml = yaml.safe_load(provider_path.read_text())
- if provider_yaml.get("suspended"):
+ if provider_yaml["state"] == "suspended":
suspended_providers.append(
provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT)
.as_posix()
