This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new a45db61576b Move Apache Spark to new provider structure (#46108)
a45db61576b is described below

commit a45db61576b809d3a0d6e2d32585f19871887c0b
Author: LIU ZHE YOU <[email protected]>
AuthorDate: Mon Jan 27 17:20:24 2025 +0800

    Move Apache Spark to new provider structure (#46108)
---
 .github/boring-cyborg.yml                          |   5 +-
 .pre-commit-config.yaml                            |   8 +-
 docs/.gitignore                                    |   1 +
 .../changelog.rst                                  |  25 ---
 providers/apache/spark/README.rst                  |  83 +++++++++
 .../spark/docs}/.latest-doc-only-change.txt        |   0
 .../spark/docs/changelog.rst}                      |   0
 .../apache/spark/docs}/commits.rst                 |   0
 .../apache/spark/docs}/connections/index.rst       |   0
 .../spark/docs}/connections/spark-connect.rst      |   0
 .../apache/spark/docs}/connections/spark-sql.rst   |   0
 .../spark/docs}/connections/spark-submit.rst       |   0
 .../apache/spark/docs}/decorators/pyspark.rst      |   2 +-
 .../apache/spark/docs}/index.rst                   |   0
 .../docs}/installing-providers-from-sources.rst    |   0
 .../apache/spark/docs/integration-logos}/spark.png | Bin
 .../apache/spark/docs}/operators.rst               |   6 +-
 .../apache/spark/docs}/redirects.txt               |   0
 .../apache/spark/docs}/security.rst                |   0
 .../providers => }/apache/spark/provider.yaml      |  12 +-
 providers/apache/spark/pyproject.toml              |  89 +++++++++
 .../src/airflow/providers/apache/spark/LICENSE     | 201 +++++++++++++++++++++
 .../src/airflow/providers/apache/spark/__init__.py |   0
 .../providers/apache/spark/decorators/__init__.py  |   0
 .../providers/apache/spark/decorators/pyspark.py   |   0
 .../providers/apache/spark/get_provider_info.py    | 132 ++++++++++++++
 .../providers/apache/spark/hooks/__init__.py       |   0
 .../providers/apache/spark/hooks/spark_connect.py  |   0
 .../providers/apache/spark/hooks/spark_jdbc.py     |   0
 .../apache/spark/hooks/spark_jdbc_script.py        |   0
 .../providers/apache/spark/hooks/spark_sql.py      |   0
 .../providers/apache/spark/hooks/spark_submit.py   |   0
 .../providers/apache/spark/operators/__init__.py   |   0
 .../providers/apache/spark/operators/spark_jdbc.py |   0
 .../providers/apache/spark/operators/spark_sql.py  |   0
 .../apache/spark/operators/spark_submit.py         |   0
 .../__init__.py => apache/spark/tests/conftest.py} |  17 +-
 .../spark/tests/provider_tests}/__init__.py        |   2 +-
 .../spark/tests/provider_tests/apache}/__init__.py |   2 +-
 .../tests/provider_tests}/apache/spark/__init__.py |   0
 .../apache/spark/decorators/__init__.py            |   0
 .../apache/spark/decorators/test_pyspark.py        |   0
 .../provider_tests}/apache/spark/hooks/__init__.py |   0
 .../apache/spark/hooks/test_spark_connect.py       |   0
 .../apache/spark/hooks/test_spark_jdbc.py          |   0
 .../apache/spark/hooks/test_spark_jdbc_script.py   |   0
 .../apache/spark/hooks/test_spark_sql.py           |   0
 .../apache/spark/hooks/test_spark_submit.py        |   0
 .../apache/spark/operators/__init__.py             |   0
 .../apache/spark/operators/test_spark_jdbc.py      |   0
 .../apache/spark/operators/test_spark_sql.py       |   0
 .../apache/spark/operators/test_spark_submit.py    |   0
 .../spark}/tests/system/apache/spark/__init__.py   |   0
 .../tests/system/apache/spark/example_pyspark.py   |   0
 .../tests/system/apache/spark/example_spark_dag.py |   0
 pyproject.toml                                     |   3 +
 scripts/ci/docker-compose/remove-sources.yml       |   1 +
 scripts/ci/docker-compose/tests-sources.yml        |   1 +
 58 files changed, 539 insertions(+), 51 deletions(-)

diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index cafd932de1c..f8a21f7c921 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -106,10 +106,7 @@ labelPRBasedOnFilePath:
     - providers/tests/system/apache/pinot/**/*
 
   provider:apache-spark:
-    - providers/src/airflow/providers/apache/spark/**/*
-    - docs/apache-airflow-providers-apache-spark/**/*
-    - providers/tests/apache/spark/**/*
-    - providers/tests/system/apache/spark/**/*
+    - providers/apache/spark/**
 
   provider:apprise:
     - providers/src/airflow/providers/apprise/**/*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6ab58224b00..4e3be8011b5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -628,9 +628,9 @@ repos:
           
^providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py$|
           
^providers/src/airflow/providers/apache/hive/operators/hive_stats.py$|
           
^providers/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py$|
-          ^providers/src/airflow/providers/apache/spark/decorators/|
-          ^providers/src/airflow/providers/apache/spark/hooks/|
-          ^providers/src/airflow/providers/apache/spark/operators/|
+          
^providers/apache/spark/src/airflow/providers/apache/spark/decorators/|
+          ^providers/apache/spark/src/airflow/providers/apache/spark/hooks/|
+          
^providers/apache/spark/src/airflow/providers/apache/spark/operators/|
           ^providers/src/airflow/providers/exasol/hooks/exasol.py$|
           ^providers/src/airflow/providers/fab/auth_manager/security_manager/|
           
^providers/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
@@ -659,7 +659,7 @@ repos:
           
^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store.rst$|
           ^docs/apache-airflow-providers-apache-hdfs/connections.rst$|
           ^docs/apache-airflow-providers-apache-kafka/connections/kafka.rst$|
-          ^docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst$|
+          ^providers/apache/spark/docs/decorators/pyspark.rst$|
           
^docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst$|
           
^docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst$|
           
^docs/apache-airflow-providers-microsoft-azure/connections/azure_cosmos.rst$|
diff --git a/docs/.gitignore b/docs/.gitignore
index aea42c23b50..4e70071d3cb 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -4,6 +4,7 @@ apache-airflow-providers-airbyte
 apache-airflow-providers-alibaba
 apache-airflow-providers-apache-iceberg
 apache-airflow-providers-apache-pig
+apache-airflow-providers-apache-spark
 apache-airflow-providers-asana
 apache-airflow-providers-celery
 apache-airflow-providers-cohere
diff --git a/docs/apache-airflow-providers-apache-spark/changelog.rst 
b/docs/apache-airflow-providers-apache-spark/changelog.rst
deleted file mode 100644
index 7714d7d18e4..00000000000
--- a/docs/apache-airflow-providers-apache-spark/changelog.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
- ..   http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
-
- ..  NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-     OVERWRITTEN WHEN PREPARING PACKAGES.
-
- ..  IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
-     `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the 
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
-
-.. include:: ../../providers/src/airflow/providers/apache/spark/CHANGELOG.rst
diff --git a/providers/apache/spark/README.rst 
b/providers/apache/spark/README.rst
new file mode 100644
index 00000000000..4939d4a47d3
--- /dev/null
+++ b/providers/apache/spark/README.rst
@@ -0,0 +1,83 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE 
TEMPLATE
+    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the 
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-apache-spark``
+
+Release: ``5.0.0``
+
+
+`Apache Spark <https://spark.apache.org/>`__
+
+
+Provider package
+----------------
+
+This is a provider package for ``apache.spark`` provider. All classes for this 
provider package
+are in ``airflow.providers.apache.spark`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation 
<https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/5.0.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see 
``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-apache-spark``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+==================  ==================
+PIP package         Version required
+==================  ==================
+``apache-airflow``  ``>=2.9.0``
+``pyspark``         ``>=3.1.3``
+``grpcio-status``   ``>=1.59.0``
+==================  ==================
+
+Cross provider package dependencies
+-----------------------------------
+
+Those are dependencies that might be needed in order to use all the features 
of the package.
+You need to install the specified provider packages in order to use them.
+
+You can install such cross-provider dependencies when installing from PyPI. 
For example:
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-apache-spark[cncf.kubernetes]
+
+
+======================================================================================================================
  ===================
+Dependent package                                                              
                                         Extra
+======================================================================================================================
  ===================
+`apache-airflow-providers-cncf-kubernetes 
<https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_  
``cncf.kubernetes``
+`apache-airflow-providers-common-compat 
<https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_      
``common.compat``
+======================================================================================================================
  ===================
+
+The changelog for the provider package can be found in the
+`changelog 
<https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/5.0.0/changelog.html>`_.
diff --git 
a/providers/src/airflow/providers/apache/spark/.latest-doc-only-change.txt 
b/providers/apache/spark/docs/.latest-doc-only-change.txt
similarity index 100%
rename from 
providers/src/airflow/providers/apache/spark/.latest-doc-only-change.txt
rename to providers/apache/spark/docs/.latest-doc-only-change.txt
diff --git a/providers/src/airflow/providers/apache/spark/CHANGELOG.rst 
b/providers/apache/spark/docs/changelog.rst
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/CHANGELOG.rst
rename to providers/apache/spark/docs/changelog.rst
diff --git a/docs/apache-airflow-providers-apache-spark/commits.rst 
b/providers/apache/spark/docs/commits.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-spark/commits.rst
rename to providers/apache/spark/docs/commits.rst
diff --git a/docs/apache-airflow-providers-apache-spark/connections/index.rst 
b/providers/apache/spark/docs/connections/index.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-spark/connections/index.rst
rename to providers/apache/spark/docs/connections/index.rst
diff --git 
a/docs/apache-airflow-providers-apache-spark/connections/spark-connect.rst 
b/providers/apache/spark/docs/connections/spark-connect.rst
similarity index 100%
rename from 
docs/apache-airflow-providers-apache-spark/connections/spark-connect.rst
rename to providers/apache/spark/docs/connections/spark-connect.rst
diff --git 
a/docs/apache-airflow-providers-apache-spark/connections/spark-sql.rst 
b/providers/apache/spark/docs/connections/spark-sql.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-spark/connections/spark-sql.rst
rename to providers/apache/spark/docs/connections/spark-sql.rst
diff --git 
a/docs/apache-airflow-providers-apache-spark/connections/spark-submit.rst 
b/providers/apache/spark/docs/connections/spark-submit.rst
similarity index 100%
rename from 
docs/apache-airflow-providers-apache-spark/connections/spark-submit.rst
rename to providers/apache/spark/docs/connections/spark-submit.rst
diff --git a/docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst 
b/providers/apache/spark/docs/decorators/pyspark.rst
similarity index 96%
rename from docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst
rename to providers/apache/spark/docs/decorators/pyspark.rst
index ba61f31e9d9..7d684e947a3 100644
--- a/docs/apache-airflow-providers-apache-spark/decorators/pyspark.rst
+++ b/providers/apache/spark/docs/decorators/pyspark.rst
@@ -44,7 +44,7 @@ Example
 The following example shows how to use the ``@task.pyspark`` decorator. Note
 that the ``spark`` and ``sc`` objects are injected into the function.
 
-.. exampleinclude:: 
/../../providers/tests/system/apache/spark/example_pyspark.py
+.. exampleinclude:: 
/../../providers/apache/spark/tests/system/apache/spark/example_pyspark.py
     :language: python
     :dedent: 4
     :start-after: [START task_pyspark]
diff --git a/docs/apache-airflow-providers-apache-spark/index.rst 
b/providers/apache/spark/docs/index.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-spark/index.rst
rename to providers/apache/spark/docs/index.rst
diff --git 
a/docs/apache-airflow-providers-apache-spark/installing-providers-from-sources.rst
 b/providers/apache/spark/docs/installing-providers-from-sources.rst
similarity index 100%
rename from 
docs/apache-airflow-providers-apache-spark/installing-providers-from-sources.rst
rename to providers/apache/spark/docs/installing-providers-from-sources.rst
diff --git a/docs/integration-logos/apache/spark.png 
b/providers/apache/spark/docs/integration-logos/spark.png
similarity index 100%
rename from docs/integration-logos/apache/spark.png
rename to providers/apache/spark/docs/integration-logos/spark.png
diff --git a/docs/apache-airflow-providers-apache-spark/operators.rst 
b/providers/apache/spark/docs/operators.rst
similarity index 93%
rename from docs/apache-airflow-providers-apache-spark/operators.rst
rename to providers/apache/spark/docs/operators.rst
index 63e728b5733..1eb9629bfa9 100644
--- a/docs/apache-airflow-providers-apache-spark/operators.rst
+++ b/providers/apache/spark/docs/operators.rst
@@ -44,7 +44,7 @@ Using the operator
 
 Using ``cmd_type`` parameter, is possible to transfer data from Spark to a 
database (``spark_to_jdbc``) or from a database to Spark (``jdbc_to_spark``), 
which will write the table using the Spark command ``saveAsTable``.
 
-.. exampleinclude:: 
/../../providers/tests/system/apache/spark/example_spark_dag.py
+.. exampleinclude:: 
/../../providers/apache/spark/tests/system/apache/spark/example_spark_dag.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_spark_jdbc]
@@ -69,7 +69,7 @@ For parameter definition take a look at 
:class:`~airflow.providers.apache.spark.
 Using the operator
 """"""""""""""""""
 
-.. exampleinclude:: 
/../../providers/tests/system/apache/spark/example_spark_dag.py
+.. exampleinclude:: 
/../../providers/apache/spark/tests/system/apache/spark/example_spark_dag.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_spark_sql]
@@ -92,7 +92,7 @@ For parameter definition take a look at 
:class:`~airflow.providers.apache.spark.
 Using the operator
 """"""""""""""""""
 
-.. exampleinclude:: 
/../../providers/tests/system/apache/spark/example_spark_dag.py
+.. exampleinclude:: 
/../../providers/apache/spark/tests/system/apache/spark/example_spark_dag.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_spark_submit]
diff --git a/docs/apache-airflow-providers-apache-spark/redirects.txt 
b/providers/apache/spark/docs/redirects.txt
similarity index 100%
rename from docs/apache-airflow-providers-apache-spark/redirects.txt
rename to providers/apache/spark/docs/redirects.txt
diff --git a/docs/apache-airflow-providers-apache-spark/security.rst 
b/providers/apache/spark/docs/security.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-spark/security.rst
rename to providers/apache/spark/docs/security.rst
diff --git a/providers/src/airflow/providers/apache/spark/provider.yaml 
b/providers/apache/spark/provider.yaml
similarity index 91%
rename from providers/src/airflow/providers/apache/spark/provider.yaml
rename to providers/apache/spark/provider.yaml
index ce7ef99b818..9940adf6a31 100644
--- a/providers/src/airflow/providers/apache/spark/provider.yaml
+++ b/providers/apache/spark/provider.yaml
@@ -65,22 +65,12 @@ versions:
   - 1.0.1
   - 1.0.0
 
-dependencies:
-  - apache-airflow>=2.9.0
-  - pyspark>=3.1.3
-  - grpcio-status>=1.59.0
-
-additional-extras:
-  - name: cncf.kubernetes
-    dependencies:
-      - apache-airflow-providers-cncf-kubernetes>=7.4.0
-
 integrations:
   - integration-name: Apache Spark
     external-doc-url: https://spark.apache.org/
     how-to-guide:
       - /docs/apache-airflow-providers-apache-spark/operators.rst
-    logo: /integration-logos/apache/spark.png
+    logo: /docs/integration-logos/spark.png
     tags: [apache]
 
 operators:
diff --git a/providers/apache/spark/pyproject.toml 
b/providers/apache/spark/pyproject.toml
new file mode 100644
index 00000000000..9db7dec48c1
--- /dev/null
+++ b/providers/apache/spark/pyproject.toml
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE 
TEMPLATE
+# `pyproject_TEMPLATE.toml.jinja2` IN the 
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+[build-system]
+requires = ["flit_core==3.10.1"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "apache-airflow-providers-apache-spark"
+version = "5.0.0"
+description = "Provider package apache-airflow-providers-apache-spark for 
Apache Airflow"
+readme = "README.rst"
+authors = [
+    {name="Apache Software Foundation", email="[email protected]"},
+]
+maintainers = [
+    {name="Apache Software Foundation", email="[email protected]"},
+]
+keywords = [ "airflow-provider", "apache.spark", "airflow", "integration" ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Framework :: Apache Airflow",
+    "Framework :: Apache Airflow :: Provider",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: System :: Monitoring",
+]
+requires-python = "~=3.9"
+
+# The dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+dependencies = [
+    "apache-airflow>=2.9.0",
+    "pyspark>=3.1.3",
+    "grpcio-status>=1.59.0",
+]
+
+# The optional dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+[project.optional-dependencies]
+"cncf.kubernetes" = [
+    "apache-airflow-providers-cncf-kubernetes>=7.4.0",
+]
+"common.compat" = [
+    "apache-airflow-providers-common-compat"
+]
+
+[project.urls]
+"Documentation" = 
"https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/5.0.0";
+"Changelog" = 
"https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/5.0.0/changelog.html";
+"Bug Tracker" = "https://github.com/apache/airflow/issues";
+"Source Code" = "https://github.com/apache/airflow";
+"Slack Chat" = "https://s.apache.org/airflow-slack";
+"Twitter" = "https://x.com/ApacheAirflow";
+"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/";
+
+[project.entry-points."apache_airflow_provider"]
+provider_info = 
"airflow.providers.apache.spark.get_provider_info:get_provider_info"
+
+[tool.flit.module]
+name = "airflow.providers.apache.spark"
+
+[tool.pytest.ini_options]
+ignore = "tests/system/"
diff --git a/providers/apache/spark/src/airflow/providers/apache/spark/LICENSE 
b/providers/apache/spark/src/airflow/providers/apache/spark/LICENSE
new file mode 100644
index 00000000000..11069edd790
--- /dev/null
+++ b/providers/apache/spark/src/airflow/providers/apache/spark/LICENSE
@@ -0,0 +1,201 @@
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity authorized by
+   the copyright owner that is granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity. For the purposes of this definition,
+   "control" means (i) the power, direct or indirect, to cause the
+   direction or management of such entity, whether by contract or
+   otherwise, or (ii) ownership of fifty percent (50%) or more of the
+   outstanding shares, or (iii) beneficial ownership of such entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications,
+   including but not limited to software source code, documentation
+   source, and configuration files.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form, including but
+   not limited to compiled object code, generated documentation,
+   and conversions to other media types.
+
+   "Work" shall mean the work of authorship, whether in Source or
+   Object form, made available under the License, as indicated by a
+   copyright notice that is included in or attached to the work
+   (an example is provided in the Appendix below).
+
+   "Derivative Works" shall mean any work, whether in Source or Object
+   form, that is based on (or derived from) the Work and for which the
+   editorial revisions, annotations, elaborations, or other modifications
+   represent, as a whole, an original work of authorship. For the purposes
+   of this License, Derivative Works shall not include works that remain
+   separable from, or merely link (or bind by name) to the interfaces of,
+   the Work and Derivative Works thereof.
+
+   "Contribution" shall mean any work of authorship, including
+   the original version of the Work and any modifications or additions
+   to that Work or Derivative Works thereof, that is intentionally
+   submitted to Licensor for inclusion in the Work by the copyright owner
+   or by an individual or Legal Entity authorized to submit on behalf of
+   the copyright owner. For the purposes of this definition, "submitted"
+   means any form of electronic, verbal, or written communication sent
+   to the Licensor or its representatives, including but not limited to
+   communication on electronic mailing lists, source code control systems,
+   and issue tracking systems that are managed by, or on behalf of, the
+   Licensor for the purpose of discussing and improving the Work, but
+   excluding communication that is conspicuously marked or otherwise
+   designated in writing by the copyright owner as "Not a Contribution."
+
+   "Contributor" shall mean Licensor and any individual or Legal Entity
+   on behalf of whom a Contribution has been received by Licensor and
+   subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   copyright license to reproduce, prepare Derivative Works of,
+   publicly display, publicly perform, sublicense, and distribute the
+   Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   (except as stated in this section) patent license to make, have made,
+   use, offer to sell, sell, import, and otherwise transfer the Work,
+   where such license applies only to those patent claims licensable
+   by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s)
+   with the Work to which such Contribution(s) was submitted. If You
+   institute patent litigation against any entity (including a
+   cross-claim or counterclaim in a lawsuit) alleging that the Work
+   or a Contribution incorporated within the Work constitutes direct
+   or contributory patent infringement, then any patent licenses
+   granted to You under this License for that Work shall terminate
+   as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+   Work or Derivative Works thereof in any medium, with or without
+   modifications, and in Source or Object form, provided that You
+   meet the following conditions:
+
+   (a) You must give any other recipients of the Work or
+       Derivative Works a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices
+       stating that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works
+       that You distribute, all copyright, patent, trademark, and
+       attribution notices from the Source form of the Work,
+       excluding those notices that do not pertain to any part of
+       the Derivative Works; and
+
+   (d) If the Work includes a "NOTICE" text file as part of its
+       distribution, then any Derivative Works that You distribute must
+       include a readable copy of the attribution notices contained
+       within such NOTICE file, excluding those notices that do not
+       pertain to any part of the Derivative Works, in at least one
+       of the following places: within a NOTICE text file distributed
+       as part of the Derivative Works; within the Source form or
+       documentation, if provided along with the Derivative Works; or,
+       within a display generated by the Derivative Works, if and
+       wherever such third-party notices normally appear. The contents
+       of the NOTICE file are for informational purposes only and
+       do not modify the License. You may add Your own attribution
+       notices within Derivative Works that You distribute, alongside
+       or as an addendum to the NOTICE text from the Work, provided
+       that such additional attribution notices cannot be construed
+       as modifying the License.
+
+   You may add Your own copyright statement to Your modifications and
+   may provide additional or different license terms and conditions
+   for use, reproduction, or distribution of Your modifications, or
+   for any such Derivative Works as a whole, provided Your use,
+   reproduction, and distribution of the Work otherwise complies with
+   the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+   agreed to in writing, Licensor provides the Work (and each
+   Contributor provides its Contributions) on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied, including, without limitation, any warranties or conditions
+   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+   PARTICULAR PURPOSE. You are solely responsible for determining the
+   appropriateness of using or redistributing the Work and assume any
+   risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+   whether in tort (including negligence), contract, or otherwise,
+   unless required by applicable law (such as deliberate and grossly
+   negligent acts) or agreed to in writing, shall any Contributor be
+   liable to You for damages, including any direct, indirect, special,
+   incidental, or consequential damages of any character arising as a
+   result of this License or out of the use or inability to use the
+   Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses), even if such Contributor
+   has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+   the Work or Derivative Works thereof, You may choose to offer,
+   and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations and/or rights consistent with this
+   License. However, in accepting such obligations, You may act only
+   on Your own behalf and on Your sole responsibility, not on behalf
+   of any other Contributor, and only if You agree to indemnify,
+   defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason
+   of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+   To apply the Apache License to your work, attach the following
+   boilerplate notice, with the fields enclosed by brackets "[]"
+   replaced with your own identifying information. (Don't include
+   the brackets!)  The text should be enclosed in the appropriate
+   comment syntax for the file format. We also recommend that a
+   file or class name and description of purpose be included on the
+   same "printed page" as the copyright notice for easier
+   identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/providers/src/airflow/providers/apache/spark/__init__.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/__init__.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/__init__.py
rename to providers/apache/spark/src/airflow/providers/apache/spark/__init__.py
diff --git 
a/providers/src/airflow/providers/apache/spark/decorators/__init__.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/decorators/__init__.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/decorators/__init__.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/decorators/__init__.py
diff --git a/providers/src/airflow/providers/apache/spark/decorators/pyspark.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/decorators/pyspark.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
diff --git 
a/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py
 
b/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py
new file mode 100644
index 00000000000..bdd89fc4748
--- /dev/null
+++ 
b/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+#
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `get_provider_info_TEMPLATE.py.jinja2` IN the 
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+def get_provider_info():
+    return {
+        "package-name": "apache-airflow-providers-apache-spark",
+        "name": "Apache Spark",
+        "description": "`Apache Spark <https://spark.apache.org/>`__\n",
+        "state": "ready",
+        "source-date-epoch": 1734528589,
+        "versions": [
+            "5.0.0",
+            "4.11.3",
+            "4.11.2",
+            "4.11.1",
+            "4.11.0",
+            "4.10.0",
+            "4.9.0",
+            "4.8.2",
+            "4.8.1",
+            "4.8.0",
+            "4.7.2",
+            "4.7.1",
+            "4.7.0",
+            "4.6.0",
+            "4.5.0",
+            "4.4.0",
+            "4.3.0",
+            "4.2.0",
+            "4.1.5",
+            "4.1.4",
+            "4.1.3",
+            "4.1.2",
+            "4.1.1",
+            "4.1.0",
+            "4.0.1",
+            "4.0.0",
+            "3.0.0",
+            "2.1.3",
+            "2.1.2",
+            "2.1.1",
+            "2.1.0",
+            "2.0.3",
+            "2.0.2",
+            "2.0.1",
+            "2.0.0",
+            "1.0.3",
+            "1.0.2",
+            "1.0.1",
+            "1.0.0",
+        ],
+        "integrations": [
+            {
+                "integration-name": "Apache Spark",
+                "external-doc-url": "https://spark.apache.org/";,
+                "how-to-guide": 
["/docs/apache-airflow-providers-apache-spark/operators.rst"],
+                "logo": "/docs/integration-logos/spark.png",
+                "tags": ["apache"],
+            }
+        ],
+        "operators": [
+            {
+                "integration-name": "Apache Spark",
+                "python-modules": [
+                    "airflow.providers.apache.spark.operators.spark_jdbc",
+                    "airflow.providers.apache.spark.operators.spark_sql",
+                    "airflow.providers.apache.spark.operators.spark_submit",
+                ],
+            }
+        ],
+        "hooks": [
+            {
+                "integration-name": "Apache Spark",
+                "python-modules": [
+                    "airflow.providers.apache.spark.hooks.spark_connect",
+                    "airflow.providers.apache.spark.hooks.spark_jdbc",
+                    "airflow.providers.apache.spark.hooks.spark_jdbc_script",
+                    "airflow.providers.apache.spark.hooks.spark_sql",
+                    "airflow.providers.apache.spark.hooks.spark_submit",
+                ],
+            }
+        ],
+        "connection-types": [
+            {
+                "hook-class-name": 
"airflow.providers.apache.spark.hooks.spark_connect.SparkConnectHook",
+                "connection-type": "spark_connect",
+            },
+            {
+                "hook-class-name": 
"airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook",
+                "connection-type": "spark_jdbc",
+            },
+            {
+                "hook-class-name": 
"airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook",
+                "connection-type": "spark_sql",
+            },
+            {
+                "hook-class-name": 
"airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook",
+                "connection-type": "spark",
+            },
+        ],
+        "task-decorators": [
+            {
+                "class-name": 
"airflow.providers.apache.spark.decorators.pyspark.pyspark_task",
+                "name": "pyspark",
+            }
+        ],
+        "dependencies": ["apache-airflow>=2.9.0", "pyspark>=3.1.3", 
"grpcio-status>=1.59.0"],
+        "optional-dependencies": {
+            "cncf.kubernetes": 
["apache-airflow-providers-cncf-kubernetes>=7.4.0"],
+            "common.compat": ["apache-airflow-providers-common-compat"],
+        },
+    }
diff --git a/providers/src/airflow/providers/apache/spark/hooks/__init__.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/__init__.py
similarity index 100%
copy from providers/src/airflow/providers/apache/spark/hooks/__init__.py
copy to 
providers/apache/spark/src/airflow/providers/apache/spark/hooks/__init__.py
diff --git 
a/providers/src/airflow/providers/apache/spark/hooks/spark_connect.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_connect.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/hooks/spark_connect.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_connect.py
diff --git a/providers/src/airflow/providers/apache/spark/hooks/spark_jdbc.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_jdbc.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/hooks/spark_jdbc.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_jdbc.py
diff --git 
a/providers/src/airflow/providers/apache/spark/hooks/spark_jdbc_script.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_jdbc_script.py
similarity index 100%
rename from 
providers/src/airflow/providers/apache/spark/hooks/spark_jdbc_script.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_jdbc_script.py
diff --git a/providers/src/airflow/providers/apache/spark/hooks/spark_sql.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_sql.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/hooks/spark_sql.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_sql.py
diff --git a/providers/src/airflow/providers/apache/spark/hooks/spark_submit.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/hooks/spark_submit.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py
diff --git a/providers/src/airflow/providers/apache/spark/operators/__init__.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/operators/__init__.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/operators/__init__.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/operators/__init__.py
diff --git 
a/providers/src/airflow/providers/apache/spark/operators/spark_jdbc.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_jdbc.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/operators/spark_jdbc.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_jdbc.py
diff --git 
a/providers/src/airflow/providers/apache/spark/operators/spark_sql.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_sql.py
similarity index 100%
rename from providers/src/airflow/providers/apache/spark/operators/spark_sql.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_sql.py
diff --git 
a/providers/src/airflow/providers/apache/spark/operators/spark_submit.py 
b/providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_submit.py
similarity index 100%
rename from 
providers/src/airflow/providers/apache/spark/operators/spark_submit.py
rename to 
providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_submit.py
diff --git a/providers/src/airflow/providers/apache/spark/hooks/__init__.py 
b/providers/apache/spark/tests/conftest.py
similarity index 58%
rename from providers/src/airflow/providers/apache/spark/hooks/__init__.py
rename to providers/apache/spark/tests/conftest.py
index 217e5db9607..068fe6bbf5a 100644
--- a/providers/src/airflow/providers/apache/spark/hooks/__init__.py
+++ b/providers/apache/spark/tests/conftest.py
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,3 +14,19 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from __future__ import annotations
+
+import pathlib
+
+import pytest
+
+pytest_plugins = "tests_common.pytest_plugin"
+
+
[email protected](tryfirst=True)
+def pytest_configure(config: pytest.Config) -> None:
+    deprecations_ignore_path = 
pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml")
+    dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists() 
else []
+    config.inicfg["airflow_deprecations_ignore"] = (
+        config.inicfg.get("airflow_deprecations_ignore", []) + dep_path  # 
type: ignore[assignment,operator]
+    )
diff --git a/providers/tests/apache/spark/__init__.py 
b/providers/apache/spark/tests/provider_tests/__init__.py
similarity index 90%
copy from providers/tests/apache/spark/__init__.py
copy to providers/apache/spark/tests/provider_tests/__init__.py
index 217e5db9607..e8fd2285643 100644
--- a/providers/tests/apache/spark/__init__.py
+++ b/providers/apache/spark/tests/provider_tests/__init__.py
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,3 +14,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: 
ignore
diff --git a/providers/tests/apache/spark/__init__.py 
b/providers/apache/spark/tests/provider_tests/apache/__init__.py
similarity index 90%
copy from providers/tests/apache/spark/__init__.py
copy to providers/apache/spark/tests/provider_tests/apache/__init__.py
index 217e5db9607..e8fd2285643 100644
--- a/providers/tests/apache/spark/__init__.py
+++ b/providers/apache/spark/tests/provider_tests/apache/__init__.py
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,3 +14,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: 
ignore
diff --git a/providers/tests/apache/spark/__init__.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/__init__.py
similarity index 100%
rename from providers/tests/apache/spark/__init__.py
rename to providers/apache/spark/tests/provider_tests/apache/spark/__init__.py
diff --git a/providers/tests/apache/spark/decorators/__init__.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/decorators/__init__.py
similarity index 100%
rename from providers/tests/apache/spark/decorators/__init__.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/decorators/__init__.py
diff --git a/providers/tests/apache/spark/decorators/test_pyspark.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/decorators/test_pyspark.py
similarity index 100%
rename from providers/tests/apache/spark/decorators/test_pyspark.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/decorators/test_pyspark.py
diff --git a/providers/tests/apache/spark/hooks/__init__.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/hooks/__init__.py
similarity index 100%
rename from providers/tests/apache/spark/hooks/__init__.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/hooks/__init__.py
diff --git a/providers/tests/apache/spark/hooks/test_spark_connect.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_connect.py
similarity index 100%
rename from providers/tests/apache/spark/hooks/test_spark_connect.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_connect.py
diff --git a/providers/tests/apache/spark/hooks/test_spark_jdbc.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_jdbc.py
similarity index 100%
rename from providers/tests/apache/spark/hooks/test_spark_jdbc.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_jdbc.py
diff --git a/providers/tests/apache/spark/hooks/test_spark_jdbc_script.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_jdbc_script.py
similarity index 100%
rename from providers/tests/apache/spark/hooks/test_spark_jdbc_script.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_jdbc_script.py
diff --git a/providers/tests/apache/spark/hooks/test_spark_sql.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_sql.py
similarity index 100%
rename from providers/tests/apache/spark/hooks/test_spark_sql.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_sql.py
diff --git a/providers/tests/apache/spark/hooks/test_spark_submit.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_submit.py
similarity index 100%
rename from providers/tests/apache/spark/hooks/test_spark_submit.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/hooks/test_spark_submit.py
diff --git a/providers/tests/apache/spark/operators/__init__.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/operators/__init__.py
similarity index 100%
rename from providers/tests/apache/spark/operators/__init__.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/operators/__init__.py
diff --git a/providers/tests/apache/spark/operators/test_spark_jdbc.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/operators/test_spark_jdbc.py
similarity index 100%
rename from providers/tests/apache/spark/operators/test_spark_jdbc.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/operators/test_spark_jdbc.py
diff --git a/providers/tests/apache/spark/operators/test_spark_sql.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/operators/test_spark_sql.py
similarity index 100%
rename from providers/tests/apache/spark/operators/test_spark_sql.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/operators/test_spark_sql.py
diff --git a/providers/tests/apache/spark/operators/test_spark_submit.py 
b/providers/apache/spark/tests/provider_tests/apache/spark/operators/test_spark_submit.py
similarity index 100%
rename from providers/tests/apache/spark/operators/test_spark_submit.py
rename to 
providers/apache/spark/tests/provider_tests/apache/spark/operators/test_spark_submit.py
diff --git a/providers/tests/system/apache/spark/__init__.py 
b/providers/apache/spark/tests/system/apache/spark/__init__.py
similarity index 100%
rename from providers/tests/system/apache/spark/__init__.py
rename to providers/apache/spark/tests/system/apache/spark/__init__.py
diff --git a/providers/tests/system/apache/spark/example_pyspark.py 
b/providers/apache/spark/tests/system/apache/spark/example_pyspark.py
similarity index 100%
rename from providers/tests/system/apache/spark/example_pyspark.py
rename to providers/apache/spark/tests/system/apache/spark/example_pyspark.py
diff --git a/providers/tests/system/apache/spark/example_spark_dag.py 
b/providers/apache/spark/tests/system/apache/spark/example_spark_dag.py
similarity index 100%
rename from providers/tests/system/apache/spark/example_spark_dag.py
rename to providers/apache/spark/tests/system/apache/spark/example_spark_dag.py
diff --git a/pyproject.toml b/pyproject.toml
index b4b2e1984bb..58aec677220 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -622,6 +622,7 @@ dev = [
   "apache-airflow-providers-alibaba",
   "apache-airflow-providers-apache-iceberg",
   "apache-airflow-providers-apache-pig",
+  "apache-airflow-providers-apache-spark",
   "apache-airflow-providers-asana",
   "apache-airflow-providers-celery",
   "apache-airflow-providers-cohere",
@@ -660,6 +661,7 @@ apache-airflow-providers-airbyte = {workspace = true}
 apache-airflow-providers-alibaba = { workspace = true }
 apache-airflow-providers-apache-iceberg = {workspace = true}
 apache-airflow-providers-apache-pig = { workspace = true }
+apache-airflow-providers-apache-spark = { workspace = true }
 apache-airflow-providers-asana = { workspace = true }
 apache-airflow-providers-celery = {workspace = true}
 apache-airflow-providers-cohere = { workspace = true }
@@ -696,6 +698,7 @@ members = [
     "providers/alibaba",
     "providers/apache/iceberg",
     "providers/apache/pig",
+    "providers/apache/spark",
     "providers/asana",
     "providers/celery",
     "providers/cohere",
diff --git a/scripts/ci/docker-compose/remove-sources.yml 
b/scripts/ci/docker-compose/remove-sources.yml
index f047aa2c928..a85757455c5 100644
--- a/scripts/ci/docker-compose/remove-sources.yml
+++ b/scripts/ci/docker-compose/remove-sources.yml
@@ -36,6 +36,7 @@ services:
       - ../../../empty:/opt/airflow/providers/alibaba/src
       - ../../../empty:/opt/airflow/providers/apache/iceberg/src
       - ../../../empty:/opt/airflow/providers/apache/pig/src
+      - ../../../empty:/opt/airflow/providers/apache/spark/src
       - ../../../empty:/opt/airflow/providers/asana/src
       - ../../../empty:/opt/airflow/providers/celery/src
       - ../../../empty:/opt/airflow/providers/cohere/src
diff --git a/scripts/ci/docker-compose/tests-sources.yml 
b/scripts/ci/docker-compose/tests-sources.yml
index a0a2f979c40..3a14c527378 100644
--- a/scripts/ci/docker-compose/tests-sources.yml
+++ b/scripts/ci/docker-compose/tests-sources.yml
@@ -43,6 +43,7 @@ services:
       - ../../../providers/alibaba/tests:/opt/airflow/providers/alibaba/tests
       - 
../../../providers/apache/iceberg/tests:/opt/airflow/providers/apache/iceberg/tests
       - 
../../../providers/apache/pig/tests:/opt/airflow/providers/apache/pig/tests
+      - 
../../../providers/apache/spark/tests:/opt/airflow/providers/apache/spark/tests
       - ../../../providers/asana/tests:/opt/airflow/providers/asana/tests
       - ../../../providers/celery/tests:/opt/airflow/providers/celery/tests
       - ../../../providers/cohere/tests:/opt/airflow/providers/cohere/tests

Reply via email to