This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 08d9dc201ef Move `GitDagBundle` to providers (#47636)
08d9dc201ef is described below
commit 08d9dc201ef9e35b87b03fdc45a251efbdad2b0c
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Wed Mar 26 16:29:17 2025 +0100
Move `GitDagBundle` to providers (#47636)
* Move `GitDagBundle` to providers
This commit moves GitDagBundle into the providers package. As part of
the move, a new `GIT` provider is created and like `operators`, `transfers`
etc, we now have `bundles` package. The bundle package is where the
`GitDagBundles`
lives.
* fixup! Move `GitDagBundle` to providers
* Update documentation
* Fix static checks
* add git to compatibility tests matrix
* Update doctree
* Update with recent changes
* Fix doc, update version and add provider to prod image preinstalled
providers
* fixup! Fix doc, update version and add provider to prod image
preinstalled providers
* Fix doc
* update breeze command hash
* Fix incompatible providers regex removal syntax
* Remove providers mistakenly added in pyproject.toml
* Update providers/git/pyproject.toml
---
.../airflow_providers_bug_report.yml | 1 +
.github/workflows/test-providers.yml | 2 +-
INSTALL | 2 +-
airflow-core/docs/extra-packages-ref.rst | 2 +
.../src/airflow/config_templates/config.yml | 2 +-
.../airflow/dag_processing/bundles/provider.yaml | 14 +-
airflow-core/src/airflow/provider.yaml.schema.json | 24 +++
airflow-core/src/airflow/provider_info.schema.json | 24 +++
airflow-core/src/airflow/providers_manager.py | 3 -
airflow-core/tests/unit/api_fastapi/conftest.py | 2 +-
.../12_airflow_dependencies_and_extras.rst | 2 +-
dev/breeze/doc/images/output_build-docs.svg | 2 +-
dev/breeze/doc/images/output_build-docs.txt | 2 +-
...tput_release-management_add-back-references.svg | 2 +-
...tput_release-management_add-back-references.txt | 2 +-
.../output_release-management_publish-docs.svg | 2 +-
.../output_release-management_publish-docs.txt | 2 +-
...output_sbom_generate-providers-requirements.svg | 12 +-
...output_sbom_generate-providers-requirements.txt | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
docs/spelling_wordlist.txt | 1 +
generated/provider_dependencies.json | 11 ++
prod_image_installed_providers.txt | 1 +
providers/git/README.rst | 61 +++++++
providers/git/docs/bundles/index.rst | 40 ++++
providers/git/docs/changelog.rst | 25 +++
providers/git/docs/commits.rst | 21 +++
providers/git/docs/connections/git.rst | 53 ++++++
providers/git/docs/index.rst | 100 ++++++++++
.../git/docs/installing-providers-from-sources.rst | 18 ++
providers/git/docs/security.rst | 18 ++
.../bundles => providers/git}/provider.yaml | 18 +-
providers/git/pyproject.toml | 95 ++++++++++
.../git/src/airflow/__init__.py | 25 +--
.../git/src/airflow/providers/__init__.py | 25 +--
providers/git/src/airflow/providers/git/LICENSE | 201 +++++++++++++++++++++
.../git/src/airflow/providers/git/__init__.py | 38 ++--
.../src/airflow/providers/git/bundles/__init__.py | 25 ---
.../git/src/airflow/providers/git}/bundles/git.py | 87 +--------
.../src/airflow/providers/git/get_provider_info.py | 46 +++++
.../src/airflow/providers/git/hooks/__init__.py | 25 ---
.../git/src/airflow/providers/git/hooks/git.py | 110 +++++++++++
.../git/tests/conftest.py | 26 +--
.../git/tests/unit/__init__.py | 25 ---
.../git/tests/unit/git/__init__.py | 25 ---
.../git/tests/unit/git/bundles/__init__.py | 25 ---
.../git/tests/unit/git}/bundles/test_git.py | 191 +++-----------------
.../git/tests/unit/git/hooks/__init__.py | 25 ---
providers/git/tests/unit/git/hooks/test_git.py | 194 ++++++++++++++++++++
pyproject.toml | 4 +-
scripts/ci/docker-compose/remove-sources.yml | 1 +
scripts/ci/docker-compose/tests-sources.yml | 1 +
52 files changed, 1145 insertions(+), 524 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
index 777d0976dfc..1a017894ba7 100644
--- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
@@ -65,6 +65,7 @@ body:
- fab
- facebook
- ftp
+ - git
- github
- google
- grpc
diff --git a/.github/workflows/test-providers.yml
b/.github/workflows/test-providers.yml
index 1f0eec1b992..c503e861ffd 100644
--- a/.github/workflows/test-providers.yml
+++ b/.github/workflows/test-providers.yml
@@ -205,7 +205,7 @@ jobs:
run: |
for provider in ${REMOVE_PROVIDERS}; do
echo "Removing incompatible provider: ${provider}"
- rm -vf dist/apache_airflow_providers_${provider/./_}*
+ rm -vf dist/apache_airflow_providers_${provider/./_}-*
done
if: matrix.remove-providers != ''
- name: "Download airflow package: wheel"
diff --git a/INSTALL b/INSTALL
index 46c2411a9db..7771358124f 100644
--- a/INSTALL
+++ b/INSTALL
@@ -321,7 +321,7 @@ airbyte, alibaba, amazon, apache.beam, apache.cassandra,
apache.drill, apache.dr
apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka,
apache.kylin, apache.livy,
apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana,
atlassian.jira, celery, cloudant,
cncf.kubernetes, cohere, common.compat, common.io, common.messaging,
common.sql, databricks,
-datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol,
fab, facebook, ftp,
+datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol,
fab, facebook, ftp, git,
github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins,
microsoft.azure,
microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc,
openai, openfaas,
openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector,
pinecone, postgres,
diff --git a/airflow-core/docs/extra-packages-ref.rst
b/airflow-core/docs/extra-packages-ref.rst
index 983292f1239..3c63e5195ed 100644
--- a/airflow-core/docs/extra-packages-ref.rst
+++ b/airflow-core/docs/extra-packages-ref.rst
@@ -266,6 +266,8 @@ Some of those enable Airflow to use executors to run tasks
with them - other tha
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| fab | ``pip install 'apache-airflow[fab]'`` |
FAB auth manager |
|
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
+| git | ``pip install 'apache-airflow[git]'`` |
Git bundle and hook |
|
++---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| github | ``pip install 'apache-airflow[github]'`` |
GitHub operators and hook |
|
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| influxdb | ``pip install 'apache-airflow[influxdb]'`` |
Influxdb operators and hook |
|
diff --git a/airflow-core/src/airflow/config_templates/config.yml
b/airflow-core/src/airflow/config_templates/config.yml
index 37fd5d00714..58eb9d33d80 100644
--- a/airflow-core/src/airflow/config_templates/config.yml
+++ b/airflow-core/src/airflow/config_templates/config.yml
@@ -2478,7 +2478,7 @@ dag_processor:
[
{
"name": "my-git-repo",
- "classpath": "airflow.dag_processing.bundles.git.GitDagBundle",
+ "classpath": "airflow.providers.git.bundles.git.GitDagBundle",
"kwargs": {
"subdir": "dags",
"tracking_ref": "main",
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
index 526dc0628a8..4d209d351ed 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
@@ -16,10 +16,10 @@
# under the License.
---
-package-name: apache-airflow-providers-bundles
+package-name: apache-airflow-providers-git
name: GIT
description: |
- `GIT <https://git-scm.com/>`__
+ `Distributed version control system (GIT) <https://git-scm.com/>`__
state: not-ready
source-date-epoch: 1726861127
@@ -29,13 +29,19 @@ versions:
integrations:
- integration-name: GIT (Git)
+ external-doc-url: https://git-scm.com/
+ tags: [software]
hooks:
- integration-name: GIT
python-modules:
- - airflow.dag_processing.bundles.git
+ - airflow.providers.git.hooks.git
+bundles:
+ - integration-name: GIT
+ python-modules:
+ - airflow.providers.git.bundles.git
connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
+ - hook-class-name: airflow.providers.git.hooks.git.GitHook
connection-type: git
diff --git a/airflow-core/src/airflow/provider.yaml.schema.json
b/airflow-core/src/airflow/provider.yaml.schema.json
index 4b5e16cedc0..77c8374b7d4 100644
--- a/airflow-core/src/airflow/provider.yaml.schema.json
+++ b/airflow-core/src/airflow/provider.yaml.schema.json
@@ -313,6 +313,30 @@
]
}
},
+ "bundles": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of python modules containing the
bundles.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "python-modules"
+ ]
+ }
+ },
"connection-types": {
"type": "array",
"description": "Array of connection types mapped to hook class
names",
diff --git a/airflow-core/src/airflow/provider_info.schema.json
b/airflow-core/src/airflow/provider_info.schema.json
index 9f8fa57367c..0beeb4efe45 100644
--- a/airflow-core/src/airflow/provider_info.schema.json
+++ b/airflow-core/src/airflow/provider_info.schema.json
@@ -86,6 +86,30 @@
]
}
},
+ "bundles": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of python modules containing the
bundles.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "python-modules"
+ ]
+ }
+ },
"connection-types": {
"type": "array",
"description": "Map of connection types mapped to hook class
names.",
diff --git a/airflow-core/src/airflow/providers_manager.py
b/airflow-core/src/airflow/providers_manager.py
index 20f6e492ee2..d8452f51f27 100644
--- a/airflow-core/src/airflow/providers_manager.py
+++ b/airflow-core/src/airflow/providers_manager.py
@@ -171,9 +171,6 @@ def
_create_customized_form_field_behaviours_schema_validator():
def _check_builtin_provider_prefix(provider_package: str, class_name: str) ->
bool:
- if "bundles" in provider_package:
- # TODO: AIP-66: remove this when this package is moved to providers
directory
- return True
if provider_package.startswith("apache-airflow"):
provider_path = provider_package[len("apache-") :].replace("-", ".")
if not class_name.startswith(provider_path):
diff --git a/airflow-core/tests/unit/api_fastapi/conftest.py
b/airflow-core/tests/unit/api_fastapi/conftest.py
index 0b4310eda60..fc7c177464e 100644
--- a/airflow-core/tests/unit/api_fastapi/conftest.py
+++ b/airflow-core/tests/unit/api_fastapi/conftest.py
@@ -115,7 +115,7 @@ def configure_git_connection_for_dag_bundle(session):
(
"dag_processor",
"dag_bundle_config_list",
- ): '[{ "name": "dag_maker", "classpath":
"airflow.dag_processing.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags",
"tracking_ref": "main", "refresh_interval": 0}}, { "name":
"another_bundle_name", "classpath":
"airflow.dag_processing.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags",
"tracking_ref": "main", "refresh_interval": 0}}]'
+ ): '[{ "name": "dag_maker", "classpath":
"airflow.providers.git.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags",
"tracking_ref": "main", "refresh_interval": 0}}, { "name":
"another_bundle_name", "classpath":
"airflow.providers.git.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags",
"tracking_ref": "main", "refresh_interval": 0}}]'
}
):
yield
diff --git a/contributing-docs/12_airflow_dependencies_and_extras.rst
b/contributing-docs/12_airflow_dependencies_and_extras.rst
index 42b1536cec7..f3e1c0351e4 100644
--- a/contributing-docs/12_airflow_dependencies_and_extras.rst
+++ b/contributing-docs/12_airflow_dependencies_and_extras.rst
@@ -181,7 +181,7 @@ airbyte, alibaba, amazon, apache.beam, apache.cassandra,
apache.drill, apache.dr
apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka,
apache.kylin, apache.livy,
apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana,
atlassian.jira, celery, cloudant,
cncf.kubernetes, cohere, common.compat, common.io, common.messaging,
common.sql, databricks,
-datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol,
fab, facebook, ftp,
+datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol,
fab, facebook, ftp, git,
github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins,
microsoft.azure,
microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc,
openai, openfaas,
openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector,
pinecone, postgres,
diff --git a/dev/breeze/doc/images/output_build-docs.svg
b/dev/breeze/doc/images/output_build-docs.svg
index 25dd164a0ba..b509d6da6e9 100644
--- a/dev/breeze/doc/images/output_build-docs.svg
+++ b/dev/breeze/doc/images/output_build-docs.svg
@@ -203,7 +203,7 @@
</text><text class="breeze-build-docs-r4" x="12.2" y="142" textLength="73.2"
clip-path="url(#breeze-build-docs-line-5)">apache</text><text
class="breeze-build-docs-r1" x="85.4" y="142" textLength="12.2"
clip-path="url(#breeze-build-docs-line-5)">.</text><text
class="breeze-build-docs-r4" x="97.6" y="142" textLength="73.2"
clip-path="url(#breeze-build-docs-line-5)">impala</text><text
class="breeze-build-docs-r1" x="170.8" y="142" textLength="36.6"
clip-path="url(#breeze-build-docs-line-5) [...]
</text><text class="breeze-build-docs-r4" x="12.2" y="166.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-6)">arangodb</text><text
class="breeze-build-docs-r1" x="109.8" y="166.4" textLength="36.6"
clip-path="url(#breeze-build-docs-line-6)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="166.4" textLength="61"
clip-path="url(#breeze-build-docs-line-6)">asana</text><text
class="breeze-build-docs-r1" x="207.4" y="166.4" textLength="36.6"
clip-path="url(#bree [...]
</text><text class="breeze-build-docs-r4" x="12.2" y="190.8" textLength="73.2"
clip-path="url(#breeze-build-docs-line-7)">common</text><text
class="breeze-build-docs-r1" x="85.4" y="190.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-7)">.</text><text
class="breeze-build-docs-r4" x="97.6" y="190.8" textLength="109.8"
clip-path="url(#breeze-build-docs-line-7)">messaging</text><text
class="breeze-build-docs-r1" x="207.4" y="190.8" textLength="36.6"
clip-path="url(#breeze-build- [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="215.2"
textLength="158.6"
clip-path="url(#breeze-build-docs-line-8)">elasticsearch</text><text
class="breeze-build-docs-r1" x="170.8" y="215.2" textLength="36.6"
clip-path="url(#breeze-build-docs-line-8)"> | </text><text
class="breeze-build-docs-r4" x="207.4" y="215.2" textLength="73.2"
clip-path="url(#breeze-build-docs-line-8)">exasol</text><text
class="breeze-build-docs-r1" x="280.6" y="215.2" textLength="36.6" clip-path="
[...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="215.2"
textLength="158.6"
clip-path="url(#breeze-build-docs-line-8)">elasticsearch</text><text
class="breeze-build-docs-r1" x="170.8" y="215.2" textLength="36.6"
clip-path="url(#breeze-build-docs-line-8)"> | </text><text
class="breeze-build-docs-r4" x="207.4" y="215.2" textLength="73.2"
clip-path="url(#breeze-build-docs-line-8)">exasol</text><text
class="breeze-build-docs-r1" x="280.6" y="215.2" textLength="36.6" clip-path="
[...]
</text><text class="breeze-build-docs-r4" x="12.2" y="239.6" textLength="97.6"
clip-path="url(#breeze-build-docs-line-9)">influxdb</text><text
class="breeze-build-docs-r1" x="109.8" y="239.6" textLength="36.6"
clip-path="url(#breeze-build-docs-line-9)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="239.6" textLength="48.8"
clip-path="url(#breeze-build-docs-line-9)">jdbc</text><text
class="breeze-build-docs-r1" x="195.2" y="239.6" textLength="36.6"
clip-path="url(#bre [...]
</text><text class="breeze-build-docs-r4" x="12.2" y="264" textLength="61"
clip-path="url(#breeze-build-docs-line-10)">neo4j</text><text
class="breeze-build-docs-r1" x="73.2" y="264" textLength="36.6"
clip-path="url(#breeze-build-docs-line-10)"> | </text><text
class="breeze-build-docs-r4" x="109.8" y="264" textLength="48.8"
clip-path="url(#breeze-build-docs-line-10)">odbc</text><text
class="breeze-build-docs-r1" x="158.6" y="264" textLength="36.6"
clip-path="url(#breeze-build-d [...]
</text><text class="breeze-build-docs-r4" x="12.2" y="288.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-11)">pinecone</text><text
class="breeze-build-docs-r1" x="109.8" y="288.4" textLength="36.6"
clip-path="url(#breeze-build-docs-line-11)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="288.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-11)">postgres</text><text
class="breeze-build-docs-r1" x="244" y="288.4" textLength="36.6" clip-path="url
[...]
diff --git a/dev/breeze/doc/images/output_build-docs.txt
b/dev/breeze/doc/images/output_build-docs.txt
index a2bfaac971f..ef6d1c62e36 100644
--- a/dev/breeze/doc/images/output_build-docs.txt
+++ b/dev/breeze/doc/images/output_build-docs.txt
@@ -1 +1 @@
-8c3469cb20447510da111e0e59c97ae2
+30a1bc3b071f939cb94d52b61dbab3c6
diff --git
a/dev/breeze/doc/images/output_release-management_add-back-references.svg
b/dev/breeze/doc/images/output_release-management_add-back-references.svg
index 8e591509256..75b4db05446 100644
--- a/dev/breeze/doc/images/output_release-management_add-back-references.svg
+++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg
@@ -145,7 +145,7 @@
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="142" textLength="73.2"
clip-path="url(#breeze-release-management-add-back-references-line-5)">apache</text><text
class="breeze-release-management-add-back-references-r1" x="85.4" y="142"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-5)">.</text><text
class="breeze-release-management-add-back-references-r4" x="97.6" y="142"
textLength="73.2" clip-path="url(#breeze-relea [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="166.4" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-6)">arangodb</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="166.4"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-6)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="166.4"
textLength="61" clip-pat [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="190.8" textLength="73.2"
clip-path="url(#breeze-release-management-add-back-references-line-7)">common</text><text
class="breeze-release-management-add-back-references-r1" x="85.4" y="190.8"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-7)">.</text><text
class="breeze-release-management-add-back-references-r4" x="97.6" y="190.8"
textLength="109.8" clip-path="url(#breez [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="215.2" textLength="158.6"
clip-path="url(#breeze-release-management-add-back-references-line-8)">elasticsearch</text><text
class="breeze-release-management-add-back-references-r1" x="170.8" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-8)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="207.4" y="215.2"
textLength="73.2" [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="215.2" textLength="158.6"
clip-path="url(#breeze-release-management-add-back-references-line-8)">elasticsearch</text><text
class="breeze-release-management-add-back-references-r1" x="170.8" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-8)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="207.4" y="215.2"
textLength="73.2" [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="239.6" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-9)">influxdb</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="239.6"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-9)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="239.6"
textLength="48.8" clip-p [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="264" textLength="61"
clip-path="url(#breeze-release-management-add-back-references-line-10)">neo4j</text><text
class="breeze-release-management-add-back-references-r1" x="73.2" y="264"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-10)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="109.8" y="264"
textLength="48.8" clip-path="url(# [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="288.4" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-11)">pinecone</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="288.4"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-11)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="288.4"
textLength="97.6" clip [...]
diff --git
a/dev/breeze/doc/images/output_release-management_add-back-references.txt
b/dev/breeze/doc/images/output_release-management_add-back-references.txt
index 625dff4d6ab..1a1424c2706 100644
--- a/dev/breeze/doc/images/output_release-management_add-back-references.txt
+++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt
@@ -1 +1 @@
-28ca3febf77042587cb889cc72053872
+12a24f48d706bfdc9893951274e5574f
diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg
b/dev/breeze/doc/images/output_release-management_publish-docs.svg
index 745d0843b3e..966deeedbae 100644
--- a/dev/breeze/doc/images/output_release-management_publish-docs.svg
+++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg
@@ -202,7 +202,7 @@
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="142" textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-5)">apache</text><text
class="breeze-release-management-publish-docs-r1" x="85.4" y="142"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-5)">.</text><text
class="breeze-release-management-publish-docs-r4" x="97.6" y="142"
textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-5)" [...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="166.4" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-6)">arangodb</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="166.4"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-6)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="166.4"
textLength="61" clip-path="url(#breeze-release-management-p [...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="190.8" textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-7)">common</text><text
class="breeze-release-management-publish-docs-r1" x="85.4" y="190.8"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-7)">.</text><text
class="breeze-release-management-publish-docs-r4" x="97.6" y="190.8"
textLength="109.8" clip-path="url(#breeze-release-management-publish-docs-l
[...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="215.2" textLength="158.6"
clip-path="url(#breeze-release-management-publish-docs-line-8)">elasticsearch</text><text
class="breeze-release-management-publish-docs-r1" x="170.8" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-8)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="207.4" y="215.2"
textLength="73.2" clip-path="url(#breeze-release-mana [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="215.2" textLength="158.6"
clip-path="url(#breeze-release-management-publish-docs-line-8)">elasticsearch</text><text
class="breeze-release-management-publish-docs-r1" x="170.8" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-8)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="207.4" y="215.2"
textLength="73.2" clip-path="url(#breeze-release-mana [...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="239.6" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-9)">influxdb</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="239.6"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-9)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="239.6"
textLength="48.8" clip-path="url(#breeze-release-management [...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="264" textLength="61"
clip-path="url(#breeze-release-management-publish-docs-line-10)">neo4j</text><text
class="breeze-release-management-publish-docs-r1" x="73.2" y="264"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-10)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="109.8" y="264"
textLength="48.8" clip-path="url(#breeze-release-management-publish-d [...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="288.4" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-11)">pinecone</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="288.4"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-11)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="288.4"
textLength="97.6" clip-path="url(#breeze-release-manageme [...]
diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt
b/dev/breeze/doc/images/output_release-management_publish-docs.txt
index ff51ba37a55..8ca3f3ea8a0 100644
--- a/dev/breeze/doc/images/output_release-management_publish-docs.txt
+++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt
@@ -1 +1 @@
-34ecaf7e65f8beecd2086b093b4f4e1e
+23e9ccdf71766df7de57ea19fc4edbf3
diff --git
a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg
b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg
index d1ec17242a9..4b5b95ef093 100644
--- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg
+++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg
@@ -191,12 +191,12 @@
</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-11)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="288.4"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-11)">apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apprise | 
[...]
</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="312.8" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-12)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="312.8"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-12)">asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.co
[...]
</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="337.2" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-13)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="337.2"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-13)">common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | di
[...]
-</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="361.6"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">discord | docker | edge | elasticsearch | exasol | fab | facebook | ftp
[...]
-</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="386" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="386"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azur
[...]
-</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="410.4" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="410.4"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai&#
[...]
-</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="434.8"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector&#
[...]
-</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="459.2" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="459.2"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">postgres | presto | qdrant | redis | salesforce | samba | segment | sen
[...]
-</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="483.6" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="483.6"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">singularity | slack | smtp | snowflake | sqlite | ssh | standard | tabl
[...]
+</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="361.6"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-14)">discord | docker | edge | elasticsearch | exasol | fab | facebook | ftp
[...]
+</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="386" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="386"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-15)">google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |&
[...]
+</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="410.4" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="410.4"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-16)">microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc
[...]
+</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="434.8"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-17)">openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill&#
[...]
+</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="459.2" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="459.2"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-18)">pinecone | postgres | presto | qdrant | redis | salesforce | samba | se
[...]
+</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="483.6" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="483.6"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-19)">| singularity | slack | smtp | snowflake | sqlite | ssh | standard |
[...]
</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="508" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-20)">│</text><text
class="breeze-sbom-generate-providers-requirements-r6" x="292.8" y="508"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-20)">teradata | trino | vertica | weaviate | yandex | ydb | zendesk)   
[...]
</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="532.4" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-21)">│</text><text
class="breeze-sbom-generate-providers-requirements-r4" x="24.4" y="532.4"
textLength="219.6"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-21)">--provider-version</text><text
class="breeze-sbom-generate-providers-requirements-r1" x="292.8" y="532.4"
textLength="1146.8" clip-path="url(#b [...]
</text><text class="breeze-sbom-generate-providers-requirements-r5" x="0"
y="556.8" textLength="12.2"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-22)">│</text><text
class="breeze-sbom-generate-providers-requirements-r1" x="292.8" y="556.8"
textLength="1146.8"
clip-path="url(#breeze-sbom-generate-providers-requirements-line-22)">value to account for the most recent version of the provider     &
[...]
diff --git
a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt
b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt
index 7376906c608..f9dd9d5a6a7 100644
--- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt
+++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt
@@ -1 +1 @@
-2b7b93625b6e3adc20712223eb230494
+f1332e8ce3cc88d0ea9ece1e6ddeec76
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 9784d7f1541..421007bc964 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -605,13 +605,13 @@ PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str
| list[str]]] = [
{
"python-version": "3.9",
"airflow-version": "2.9.3",
- "remove-providers": "cloudant common.messaging fab edge",
+ "remove-providers": "cloudant common.messaging fab edge git",
"run-tests": "true",
},
{
"python-version": "3.9",
"airflow-version": "2.10.5",
- "remove-providers": "cloudant common.messaging fab",
+ "remove-providers": "cloudant common.messaging fab git",
"run-tests": "true",
},
]
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 4076f14a09d..a6db72c55ea 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -729,6 +729,7 @@ GH
GiB
gid
gif
+GitDagBundle
Github
github
gitignore
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 938474d4e09..d7970a8c116 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -647,6 +647,17 @@
"excluded-python-versions": [],
"state": "ready"
},
+ "git": {
+ "deps": [
+ "GitPython>=3.1.44",
+ "apache-airflow>=3.0.0.dev0"
+ ],
+ "devel-deps": [],
+ "plugins": [],
+ "cross-providers-deps": [],
+ "excluded-python-versions": [],
+ "state": "not-ready"
+ },
"github": {
"deps": [
"PyGithub>=2.1.1",
diff --git a/prod_image_installed_providers.txt
b/prod_image_installed_providers.txt
index 5ba30a865f4..8de9a34a203 100644
--- a/prod_image_installed_providers.txt
+++ b/prod_image_installed_providers.txt
@@ -9,6 +9,7 @@ docker
elasticsearch
fab
ftp
+git
google
grpc
hashicorp
diff --git a/providers/git/README.rst b/providers/git/README.rst
new file mode 100644
index 00000000000..96d4e6b62e4
--- /dev/null
+++ b/providers/git/README.rst
@@ -0,0 +1,61 @@
+
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+ ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the
``dev/breeze/src/airflow_breeze/templates`` DIRECTORY
+
+Package ``apache-airflow-providers-git``
+
+Release: ``0.0.1``
+
+
+`Distributed version control system (GIT) <https://git-scm.com/>`__
+
+
+Provider package
+----------------
+
+This is a provider package for ``git`` provider. All classes for this provider
package
+are in ``airflow.providers.git`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation
<https://airflow.apache.org/docs/apache-airflow-providers-git/0.0.1/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see
``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-git``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+================== ==================
+PIP package Version required
+================== ==================
+``apache-airflow`` ``>=3.0.0.dev0``
+``GitPython`` ``>=3.1.44``
+================== ==================
+
+The changelog for the provider package can be found in the
+`changelog
<https://airflow.apache.org/docs/apache-airflow-providers-git/0.0.1/changelog.html>`_.
diff --git a/providers/git/docs/bundles/index.rst
b/providers/git/docs/bundles/index.rst
new file mode 100644
index 00000000000..d31fc3cd040
--- /dev/null
+++ b/providers/git/docs/bundles/index.rst
@@ -0,0 +1,40 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Bundles
+=======
+
+Use the :class:`~airflow.providers.git.bundles.git.GitDagBundle` to configure
a Git bundle in your Airflow's
+``[dag_processor] dag_bundle_config_list``.
+
+Example of using the GitDagBundle:
+
+**JSON format example**:
+
+.. code-block:: bash
+
+ export AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST='[
+ {
+ "name": "my-git-repo",
+ "classpath": "airflow.providers.git.bundles.git.GitDagBundle",
+ "kwargs": {
+ "subdir": "dags",
+ "tracking_ref": "main",
+ "refresh_interval": 3600
+ }
+ }
+ ]'
diff --git a/providers/git/docs/changelog.rst b/providers/git/docs/changelog.rst
new file mode 100644
index 00000000000..799c667d2ad
--- /dev/null
+++ b/providers/git/docs/changelog.rst
@@ -0,0 +1,25 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+0.0.1
+.....
+
+.. note::
+ Provider is still WIP. It can be used with production but we may introduce
breaking changes without following semver until version 1.0.0
+
+Breaking changes
+~~~~~~~~~~~~~~~~
diff --git a/providers/git/docs/commits.rst b/providers/git/docs/commits.rst
new file mode 100644
index 00000000000..9eb2f4e1c9c
--- /dev/null
+++ b/providers/git/docs/commits.rst
@@ -0,0 +1,21 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+ .. THIS FILE IS UPDATED AUTOMATICALLY_AT_RELEASE_TIME
+
+Package apache-airflow-providers-git
+--------------------------------------
diff --git a/providers/git/docs/connections/git.rst
b/providers/git/docs/connections/git.rst
new file mode 100644
index 00000000000..05bbc69b68c
--- /dev/null
+++ b/providers/git/docs/connections/git.rst
@@ -0,0 +1,53 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+
+.. _howto/connection:ftp:
+
+GIT Connection
+==============
+
+The GIT connection type enables the GIT Integrations.
+
+Authenticating to GIT
+-----------------------
+
+Authenticate to FTP using `ftplib
+<https://docs.python.org/3/library/ftplib.html>`_.
+i.e. indicate ``user``, ``password``, ``host``
+
+Default Connection IDs
+----------------------
+
+Hooks, bundles related to GIT use ``git_default`` by default.
+
+Configuring the Connection
+--------------------------
+Username
+ Specify the git ``username``.
+
+Repository URL (optional)
+ Specify the repository url e.g ``[email protected]/apache/airflow.git``.
+
+Password (optional)
+ Specify the git ``password`` a.k.a ``ACCESS TOKEN`` if using https.
+
+Extra (optional)
+ Specify the extra parameters (as json dictionary) that can be used in ftp
connection.
+ You can specify the ``key_file`` path or ``private_key`` as extra
parameters. You can
+ also optionally specify the ``strict_host_key_checking`` parameter.
diff --git a/providers/git/docs/index.rst b/providers/git/docs/index.rst
new file mode 100644
index 00000000000..bc990f2907f
--- /dev/null
+++ b/providers/git/docs/index.rst
@@ -0,0 +1,100 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+``apache-airflow-providers-git``
+================================
+
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: Basics
+
+ Home <self>
+ Changelog <changelog>
+ Security <security>
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: References
+
+ Connection Types <connections/git>
+ Python API <_api/airflow/providers/git/index>
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: Bundles
+
+ Bundles <bundles/index>
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: Resources
+
+ PyPI Repository <https://pypi.org/project/apache-airflow-providers-git/>
+ Installing from sources <installing-providers-from-sources>
+ Python API <_api/airflow/providers/git/index>
+
+
+
+.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE
OVERWRITTEN AT RELEASE TIME!
+
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: Commits
+
+ Detailed list of commits <commits>
+
+
+apache-airflow-providers-git package
+------------------------------------------------------
+
+`Distributed version control system (GIT) <https://git-scm.com/>`__
+
+
+Release: 0.0.1
+
+Provider package
+----------------
+
+This package is for the ``git`` provider.
+All classes for this package are included in the ``airflow.providers.git``
python package.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation via
+``pip install apache-airflow-providers-git``.
+For the minimum Airflow version supported, see ``Requirements`` below.
+
+Requirements
+------------
+
+The minimum Apache Airflow version supported by this provider distribution is
``3.0.0.dev0``.
+
+================== ==================
+PIP package Version required
+================== ==================
+``apache-airflow`` ``>=3.0.0.dev0``
+``GitPython`` ``>=3.1.44``
+================== ==================
diff --git a/providers/git/docs/installing-providers-from-sources.rst
b/providers/git/docs/installing-providers-from-sources.rst
new file mode 100644
index 00000000000..b4e730f4ff2
--- /dev/null
+++ b/providers/git/docs/installing-providers-from-sources.rst
@@ -0,0 +1,18 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+.. include:: ../exts/includes/installing-providers-from-sources.rst
diff --git a/providers/git/docs/security.rst b/providers/git/docs/security.rst
new file mode 100644
index 00000000000..afa13dac6fc
--- /dev/null
+++ b/providers/git/docs/security.rst
@@ -0,0 +1,18 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+.. include:: ../exts/includes/security.rst
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/provider.yaml
similarity index 72%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/provider.yaml
index 526dc0628a8..ed8936e285f 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/provider.yaml
@@ -16,26 +16,32 @@
# under the License.
---
-package-name: apache-airflow-providers-bundles
+package-name: apache-airflow-providers-git
name: GIT
description: |
- `GIT <https://git-scm.com/>`__
+ `Distributed version control system (GIT) <https://git-scm.com/>`__
state: not-ready
-source-date-epoch: 1726861127
+source-date-epoch: 1742823216
# note that those versions are maintained by release manager - do not update
them manually
versions:
- - 1.0.0
+ - 0.0.1
integrations:
- integration-name: GIT (Git)
+ external-doc-url: https://git-scm.com/
+ tags: [software]
hooks:
- integration-name: GIT
python-modules:
- - airflow.dag_processing.bundles.git
+ - airflow.providers.git.hooks.git
+bundles:
+ - integration-name: GIT
+ python-modules:
+ - airflow.providers.git.bundles.git
connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
+ - hook-class-name: airflow.providers.git.hooks.git.GitHook
connection-type: git
diff --git a/providers/git/pyproject.toml b/providers/git/pyproject.toml
new file mode 100644
index 00000000000..6a895b0d6a7
--- /dev/null
+++ b/providers/git/pyproject.toml
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE
TEMPLATE
+# `pyproject_TEMPLATE.toml.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+[build-system]
+requires = ["flit_core==3.12.0"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "apache-airflow-providers-git"
+version = "0.0.1"
+description = "Provider package apache-airflow-providers-git for Apache
Airflow"
+readme = "README.rst"
+authors = [
+ {name="Apache Software Foundation", email="[email protected]"},
+]
+maintainers = [
+ {name="Apache Software Foundation", email="[email protected]"},
+]
+keywords = [ "airflow-provider", "git", "airflow", "integration" ]
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Environment :: Console",
+ "Environment :: Web Environment",
+ "Intended Audience :: Developers",
+ "Intended Audience :: System Administrators",
+ "Framework :: Apache Airflow",
+ "Framework :: Apache Airflow :: Provider",
+ "License :: OSI Approved :: Apache Software License",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Topic :: System :: Monitoring",
+]
+requires-python = "~=3.9"
+
+# The dependencies should be modified in place in the generated file.
+# Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies
--all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with
``breeze ci-image build``
+dependencies = [
+ "apache-airflow>=3.0.0.dev0",
+ "GitPython>=3.1.44",
+]
+
+[dependency-groups]
+dev = [
+ "apache-airflow",
+ "apache-airflow-task-sdk",
+ "apache-airflow-devel-common",
+ # Additional devel dependencies (do not remove this line and add extra
development dependencies)
+]
+
+[tool.uv.sources]
+# These names must match the names as defined in the pyproject.toml of the
workspace items,
+# *not* the workspace folder paths
+apache-airflow = {workspace = true}
+apache-airflow-devel-common = {workspace = true}
+apache-airflow-task-sdk = {workspace = true}
+apache-airflow-providers-common-sql = {workspace = true}
+apache-airflow-providers-fab = {workspace = true}
+apache-airflow-providers-standard = {workspace = true}
+
+[project.urls]
+"Documentation" =
"https://airflow.apache.org/docs/apache-airflow-providers-git/0.0.1";
+"Changelog" =
"https://airflow.apache.org/docs/apache-airflow-providers-git/0.0.1/changelog.html";
+"Bug Tracker" = "https://github.com/apache/airflow/issues";
+"Source Code" = "https://github.com/apache/airflow";
+"Slack Chat" = "https://s.apache.org/airflow-slack";
+"Mastodon" = "https://fosstodon.org/@airflow";
+"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/";
+
+[project.entry-points."apache_airflow_provider"]
+provider_info = "airflow.providers.git.get_provider_info:get_provider_info"
+
+[tool.flit.module]
+name = "airflow.providers.git"
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/src/airflow/__init__.py
similarity index 60%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/src/airflow/__init__.py
index 526dc0628a8..df38607688e 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/src/airflow/__init__.py
@@ -15,27 +15,4 @@
# specific language governing permissions and limitations
# under the License.
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
+__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type:
ignore
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/src/airflow/providers/__init__.py
similarity index 60%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/src/airflow/providers/__init__.py
index 526dc0628a8..df38607688e 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/src/airflow/providers/__init__.py
@@ -15,27 +15,4 @@
# specific language governing permissions and limitations
# under the License.
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
+__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type:
ignore
diff --git a/providers/git/src/airflow/providers/git/LICENSE
b/providers/git/src/airflow/providers/git/LICENSE
new file mode 100644
index 00000000000..11069edd790
--- /dev/null
+++ b/providers/git/src/airflow/providers/git/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/src/airflow/providers/git/__init__.py
similarity index 52%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/src/airflow/providers/git/__init__.py
index 526dc0628a8..f1b3284bed1 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/src/airflow/providers/git/__init__.py
@@ -14,28 +14,26 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
+#
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+#
+from __future__ import annotations
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
+import packaging.version
-integrations:
- - integration-name: GIT (Git)
+from airflow import __version__ as airflow_version
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
+__all__ = ["__version__"]
+__version__ = "0.0.1"
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
+if
packaging.version.parse(packaging.version.parse(airflow_version).base_version)
< packaging.version.parse(
+ "3.0.0.dev0"
+):
+ raise RuntimeError(
+ f"The package `apache-airflow-providers-git:{__version__}` needs
Apache Airflow 3.0.0.dev0+"
+ )
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/src/airflow/providers/git/bundles/__init__.py
similarity index 59%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/src/airflow/providers/git/bundles/__init__.py
index 526dc0628a8..13a83393a91 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/src/airflow/providers/git/bundles/__init__.py
@@ -14,28 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
diff --git a/airflow-core/src/airflow/dag_processing/bundles/git.py
b/providers/git/src/airflow/providers/git/bundles/git.py
similarity index 74%
rename from airflow-core/src/airflow/dag_processing/bundles/git.py
rename to providers/git/src/airflow/providers/git/bundles/git.py
index 3a85c5b4d5c..caee10b1fd1 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/git.py
+++ b/providers/git/src/airflow/providers/git/bundles/git.py
@@ -14,15 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
from __future__ import annotations
-import contextlib
-import json
import os
-import tempfile
from pathlib import Path
-from typing import Any
from urllib.parse import urlparse
import structlog
@@ -33,91 +28,11 @@ from airflow.dag_processing.bundles.base import (
BaseDagBundle,
)
from airflow.exceptions import AirflowException
-from airflow.hooks.base import BaseHook
+from airflow.providers.git.hooks.git import GitHook
log = structlog.get_logger()
-class GitHook(BaseHook):
- """
- Hook for git repositories.
-
- :param git_conn_id: Connection ID for SSH connection to the repository
-
- """
-
- conn_name_attr = "git_conn_id"
- default_conn_name = "git_default"
- conn_type = "git"
- hook_name = "GIT"
-
- @classmethod
- def get_ui_field_behaviour(cls) -> dict[str, Any]:
- return {
- "hidden_fields": ["schema"],
- "relabeling": {
- "login": "Username",
- "host": "Repository URL",
- "password": "Access Token (optional)",
- },
- "placeholders": {
- "extra": json.dumps(
- {
- "key_file": "optional/path/to/keyfile",
- "private_key": "optional inline private key",
- }
- )
- },
- }
-
- def __init__(
- self, git_conn_id: str = "git_default", repo_url: str | None = None,
*args, **kwargs
- ) -> None:
- super().__init__()
- connection = self.get_connection(git_conn_id)
- self.repo_url = repo_url or connection.host
- self.auth_token = connection.password
- self.private_key = connection.extra_dejson.get("private_key")
- self.key_file = connection.extra_dejson.get("key_file")
- self.strict_host_key_checking =
connection.extra_dejson.get("strict_host_key_checking", "no")
- self.env: dict[str, str] = {}
-
- if self.key_file and self.private_key:
- raise AirflowException("Both 'key_file' and 'private_key' cannot
be provided at the same time")
- self._process_git_auth_url()
-
- def _build_ssh_command(self, key_path: str) -> str:
- return (
- f"ssh -i {key_path} "
- f"-o IdentitiesOnly=yes "
- f"-o StrictHostKeyChecking={self.strict_host_key_checking}"
- )
-
- def _process_git_auth_url(self):
- if not isinstance(self.repo_url, str):
- return
- if self.auth_token and self.repo_url.startswith("https://";):
- self.repo_url = self.repo_url.replace("https://";,
f"https://{self.auth_token}@";)
- elif not self.repo_url.startswith("git@") or not
self.repo_url.startswith("https://";):
- self.repo_url = os.path.expanduser(self.repo_url)
-
- def set_git_env(self, key: str) -> None:
- self.env["GIT_SSH_COMMAND"] = self._build_ssh_command(key)
-
- @contextlib.contextmanager
- def configure_hook_env(self):
- if self.private_key:
- with tempfile.NamedTemporaryFile(mode="w", delete=True) as
tmp_keyfile:
- tmp_keyfile.write(self.private_key)
- tmp_keyfile.flush()
- os.chmod(tmp_keyfile.name, 0o600)
- self.set_git_env(tmp_keyfile.name)
- yield
- else:
- self.set_git_env(self.key_file)
- yield
-
-
class GitDagBundle(BaseDagBundle):
"""
git DAG bundle - exposes a git repository as a DAG bundle.
diff --git a/providers/git/src/airflow/providers/git/get_provider_info.py
b/providers/git/src/airflow/providers/git/get_provider_info.py
new file mode 100644
index 00000000000..bb8a7ff1c25
--- /dev/null
+++ b/providers/git/src/airflow/providers/git/get_provider_info.py
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+#
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `get_provider_info_TEMPLATE.py.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+def get_provider_info():
+ return {
+ "package-name": "apache-airflow-providers-git",
+ "name": "GIT",
+ "description": "`Distributed version control system (GIT)
<https://git-scm.com/>`__\n",
+ "state": "not-ready",
+ "source-date-epoch": 1742823216,
+ "versions": ["0.0.1"],
+ "integrations": [
+ {
+ "integration-name": "GIT (Git)",
+ "external-doc-url": "https://git-scm.com/";,
+ "tags": ["software"],
+ }
+ ],
+ "hooks": [{"integration-name": "GIT", "python-modules":
["airflow.providers.git.hooks.git"]}],
+ "bundles": [{"integration-name": "GIT", "python-modules":
["airflow.providers.git.bundles.git"]}],
+ "connection-types": [
+ {"hook-class-name": "airflow.providers.git.hooks.git.GitHook",
"connection-type": "git"}
+ ],
+ "dependencies": ["apache-airflow>=3.0.0.dev0", "GitPython>=3.1.44"],
+ "devel-dependencies": [],
+ }
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/src/airflow/providers/git/hooks/__init__.py
similarity index 59%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/src/airflow/providers/git/hooks/__init__.py
index 526dc0628a8..13a83393a91 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/src/airflow/providers/git/hooks/__init__.py
@@ -14,28 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
diff --git a/providers/git/src/airflow/providers/git/hooks/git.py
b/providers/git/src/airflow/providers/git/hooks/git.py
new file mode 100644
index 00000000000..22cfde132a5
--- /dev/null
+++ b/providers/git/src/airflow/providers/git/hooks/git.py
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+import tempfile
+from typing import Any
+
+from airflow.exceptions import AirflowException
+from airflow.hooks.base import BaseHook
+
+log = logging.getLogger(__name__)
+
+
+class GitHook(BaseHook):
+ """
+ Hook for git repositories.
+
+ :param git_conn_id: Connection ID for SSH connection to the repository
+
+ """
+
+ conn_name_attr = "git_conn_id"
+ default_conn_name = "git_default"
+ conn_type = "git"
+ hook_name = "GIT"
+
+ @classmethod
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
+ return {
+ "hidden_fields": ["schema"],
+ "relabeling": {
+ "login": "Username",
+ "host": "Repository URL",
+ "password": "Access Token (optional)",
+ },
+ "placeholders": {
+ "extra": json.dumps(
+ {
+ "key_file": "optional/path/to/keyfile",
+ "private_key": "optional inline private key",
+ }
+ )
+ },
+ }
+
+ def __init__(
+ self, git_conn_id: str = "git_default", repo_url: str | None = None,
*args, **kwargs
+ ) -> None:
+ super().__init__()
+ connection = self.get_connection(git_conn_id)
+ self.repo_url = repo_url or connection.host
+ self.auth_token = connection.password
+ self.private_key = connection.extra_dejson.get("private_key")
+ self.key_file = connection.extra_dejson.get("key_file")
+ self.strict_host_key_checking =
connection.extra_dejson.get("strict_host_key_checking", "no")
+ self.env: dict[str, str] = {}
+
+ if self.key_file and self.private_key:
+ raise AirflowException("Both 'key_file' and 'private_key' cannot
be provided at the same time")
+ self._process_git_auth_url()
+
+ def _build_ssh_command(self, key_path: str) -> str:
+ return (
+ f"ssh -i {key_path} "
+ f"-o IdentitiesOnly=yes "
+ f"-o StrictHostKeyChecking={self.strict_host_key_checking}"
+ )
+
+ def _process_git_auth_url(self):
+ if not isinstance(self.repo_url, str):
+ return
+ if self.auth_token and self.repo_url.startswith("https://";):
+ self.repo_url = self.repo_url.replace("https://";,
f"https://{self.auth_token}@";)
+ elif not self.repo_url.startswith("git@") or not
self.repo_url.startswith("https://";):
+ self.repo_url = os.path.expanduser(self.repo_url)
+
+ def set_git_env(self, key: str) -> None:
+ self.env["GIT_SSH_COMMAND"] = self._build_ssh_command(key)
+
+ @contextlib.contextmanager
+ def configure_hook_env(self):
+ if self.private_key:
+ with tempfile.NamedTemporaryFile(mode="w", delete=True) as
tmp_keyfile:
+ tmp_keyfile.write(self.private_key)
+ tmp_keyfile.flush()
+ os.chmod(tmp_keyfile.name, 0o600)
+ self.set_git_env(tmp_keyfile.name)
+ yield
+ else:
+ self.set_git_env(self.key_file)
+ yield
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/tests/conftest.py
similarity index 60%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/tests/conftest.py
index 526dc0628a8..f56ccce0a3f 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/tests/conftest.py
@@ -14,28 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
+pytest_plugins = "tests_common.pytest_plugin"
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/tests/unit/__init__.py
similarity index 59%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/tests/unit/__init__.py
index 526dc0628a8..13a83393a91 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/tests/unit/__init__.py
@@ -14,28 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/tests/unit/git/__init__.py
similarity index 59%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/tests/unit/git/__init__.py
index 526dc0628a8..13a83393a91 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/tests/unit/git/__init__.py
@@ -14,28 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/tests/unit/git/bundles/__init__.py
similarity index 59%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/tests/unit/git/bundles/__init__.py
index 526dc0628a8..13a83393a91 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/tests/unit/git/bundles/__init__.py
@@ -14,28 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
diff --git a/airflow-core/tests/unit/dag_processing/bundles/test_git.py
b/providers/git/tests/unit/git/bundles/test_git.py
similarity index 72%
rename from airflow-core/tests/unit/dag_processing/bundles/test_git.py
rename to providers/git/tests/unit/git/bundles/test_git.py
index 4c0db6bfa21..5113ffa0882 100644
--- a/airflow-core/tests/unit/dag_processing/bundles/test_git.py
+++ b/providers/git/tests/unit/git/bundles/test_git.py
@@ -27,9 +27,10 @@ from git import Repo
from git.exc import GitCommandError, NoSuchPathError
from airflow.dag_processing.bundles.base import get_bundle_storage_root_path
-from airflow.dag_processing.bundles.git import GitDagBundle, GitHook
from airflow.exceptions import AirflowException
from airflow.models import Connection
+from airflow.providers.git.bundles.git import GitDagBundle
+from airflow.providers.git.hooks.git import GitHook
from airflow.utils import db
from tests_common.test_utils.config import conf_vars
@@ -46,6 +47,12 @@ def bundle_temp_dir(tmp_path):
GIT_DEFAULT_BRANCH = "main"
+AIRFLOW_HTTPS_URL = "https://github.com/apache/airflow.git";
+ACCESS_TOKEN = "my_access_token"
+CONN_HTTPS = "my_git_conn"
+CONN_ONLY_PATH = "my_git_conn_only_path"
+CONN_NO_REPO_URL = "my_git_conn_no_repo_url"
+
@pytest.fixture
def git_repo(tmp_path_factory):
@@ -60,146 +67,6 @@ def git_repo(tmp_path_factory):
return (directory, repo)
-AIRFLOW_HTTPS_URL = "https://github.com/apache/airflow.git";
-AIRFLOW_GIT = "[email protected]:apache/airflow.git"
-ACCESS_TOKEN = "my_access_token"
-CONN_DEFAULT = "git_default"
-CONN_HTTPS = "my_git_conn"
-CONN_ONLY_PATH = "my_git_conn_only_path"
-CONN_ONLY_INLINE_KEY = "my_git_conn_only_inline_key"
-CONN_BOTH_PATH_INLINE = "my_git_conn_both_path_inline"
-CONN_NO_REPO_URL = "my_git_conn_no_repo_url"
-
-
-class TestGitHook:
- @classmethod
- def teardown_class(cls) -> None:
- clear_db_connections()
-
- @classmethod
- def setup_class(cls) -> None:
- db.merge_conn(
- Connection(
- conn_id=CONN_DEFAULT,
- host=AIRFLOW_GIT,
- conn_type="git",
- extra='{"key_file": "/files/pkey.pem"}',
- )
- )
- db.merge_conn(
- Connection(
- conn_id=CONN_HTTPS,
- host=AIRFLOW_HTTPS_URL,
- password=ACCESS_TOKEN,
- conn_type="git",
- )
- )
- db.merge_conn(
- Connection(
- conn_id=CONN_ONLY_PATH,
- host="path/to/repo",
- conn_type="git",
- )
- )
- db.merge_conn(
- Connection(
- conn_id=CONN_ONLY_INLINE_KEY,
- host="path/to/repo",
- conn_type="git",
- extra={
- "private_key": "inline_key",
- },
- )
- )
-
- @pytest.mark.parametrize(
- "conn_id, hook_kwargs, expected_repo_url",
- [
- (CONN_DEFAULT, {}, AIRFLOW_GIT),
- (CONN_HTTPS, {},
f"https://{ACCESS_TOKEN}@github.com/apache/airflow.git";),
- (
- CONN_HTTPS,
- {"repo_url": "https://github.com/apache/zzzairflow"},
- f"https://{ACCESS_TOKEN}@github.com/apache/zzzairflow";,
- ),
- (CONN_ONLY_PATH, {}, "path/to/repo"),
- ],
- )
- def test_correct_repo_urls(self, conn_id, hook_kwargs, expected_repo_url):
- hook = GitHook(git_conn_id=conn_id, **hook_kwargs)
- assert hook.repo_url == expected_repo_url
-
- def test_env_var_with_configure_hook_env(self, session):
- default_hook = GitHook(git_conn_id=CONN_DEFAULT)
- with default_hook.configure_hook_env():
- assert default_hook.env == {
- "GIT_SSH_COMMAND": "ssh -i /files/pkey.pem -o
IdentitiesOnly=yes -o StrictHostKeyChecking=no"
- }
- db.merge_conn(
- Connection(
- conn_id="my_git_conn_strict",
- host=AIRFLOW_GIT,
- conn_type="git",
- extra='{"key_file": "/files/pkey.pem",
"strict_host_key_checking": "yes"}',
- )
- )
-
- strict_default_hook = GitHook(git_conn_id="my_git_conn_strict")
- with strict_default_hook.configure_hook_env():
- assert strict_default_hook.env == {
- "GIT_SSH_COMMAND": "ssh -i /files/pkey.pem -o
IdentitiesOnly=yes -o StrictHostKeyChecking=yes"
- }
-
- def test_given_both_private_key_and_key_file(self):
- db.merge_conn(
- Connection(
- conn_id=CONN_BOTH_PATH_INLINE,
- host="path/to/repo",
- conn_type="git",
- extra={
- "key_file": "path/to/key",
- "private_key": "inline_key",
- },
- )
- )
-
- with pytest.raises(
- AirflowException, match="Both 'key_file' and 'private_key' cannot
be provided at the same time"
- ):
- GitHook(git_conn_id=CONN_BOTH_PATH_INLINE)
-
- def test_key_file_git_hook_has_env_with_configure_hook_env(self):
- hook = GitHook(git_conn_id=CONN_DEFAULT)
-
- assert hasattr(hook, "env")
- with hook.configure_hook_env():
- assert hook.env == {
- "GIT_SSH_COMMAND": "ssh -i /files/pkey.pem -o
IdentitiesOnly=yes -o StrictHostKeyChecking=no"
- }
-
- def test_private_key_lazy_env_var(self):
- hook = GitHook(git_conn_id=CONN_ONLY_INLINE_KEY)
- assert hook.env == {}
-
- hook.set_git_env("dummy_inline_key")
- assert hook.env == {
- "GIT_SSH_COMMAND": "ssh -i dummy_inline_key -o IdentitiesOnly=yes
-o StrictHostKeyChecking=no"
- }
-
- def test_configure_hook_env(self):
- hook = GitHook(git_conn_id=CONN_ONLY_INLINE_KEY)
- assert hasattr(hook, "private_key")
-
- hook.set_git_env("dummy_inline_key")
-
- with hook.configure_hook_env():
- command = hook.env.get("GIT_SSH_COMMAND")
- temp_key_path = command.split()[2]
- assert os.path.exists(temp_key_path)
-
- assert not os.path.exists(temp_key_path)
-
-
class TestGitDagBundle:
@classmethod
def teardown_class(cls) -> None:
@@ -254,7 +121,7 @@ class TestGitDagBundle:
bundle = GitDagBundle(name="test", tracking_ref=GIT_DEFAULT_BRANCH)
assert bundle.repo_url == bundle.hook.repo_url
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_get_current_version(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -264,7 +131,7 @@ class TestGitDagBundle:
assert bundle.get_current_version() == repo.head.commit.hexsha
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_get_specific_version(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -289,7 +156,7 @@ class TestGitDagBundle:
files_in_repo = {f.name for f in bundle.path.iterdir() if f.is_file()}
assert {"test_dag.py"} == files_in_repo
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_get_tag_version(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -316,7 +183,7 @@ class TestGitDagBundle:
files_in_repo = {f.name for f in bundle.path.iterdir() if f.is_file()}
assert {"test_dag.py"} == files_in_repo
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_get_latest(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -343,7 +210,7 @@ class TestGitDagBundle:
False,
],
)
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_refresh(self, mock_githook, git_repo, amend):
"""Ensure that the bundle refresh works when tracking a branch, with a
new commit and amending the commit"""
repo_path, repo = git_repo
@@ -375,7 +242,7 @@ class TestGitDagBundle:
files_in_repo = {f.name for f in bundle.path.iterdir() if f.is_file()}
assert {"test_dag.py", "new_test.py"} == files_in_repo
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_refresh_tag(self, mock_githook, git_repo):
"""Ensure that the bundle refresh works when tracking a tag"""
repo_path, repo = git_repo
@@ -406,7 +273,7 @@ class TestGitDagBundle:
files_in_repo = {f.name for f in bundle.path.iterdir() if f.is_file()}
assert {"test_dag.py", "new_test.py"} == files_in_repo
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_head(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -416,7 +283,7 @@ class TestGitDagBundle:
bundle.initialize()
assert bundle.repo.head.ref.name == "test"
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_version_not_found(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -429,7 +296,7 @@ class TestGitDagBundle:
with pytest.raises(AirflowException, match="Version not_found not
found in the repository"):
bundle.initialize()
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_subdir(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
@@ -464,8 +331,8 @@ class TestGitDagBundle:
with pytest.raises(AirflowException, match=f"Connection
{CONN_NO_REPO_URL} doesn't have a host url"):
bundle.initialize()
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
- @mock.patch("airflow.dag_processing.bundles.git.Repo")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.Repo")
def test_with_path_as_repo_url(self, mock_gitRepo, mock_githook):
bundle = GitDagBundle(
name="test",
@@ -476,7 +343,7 @@ class TestGitDagBundle:
assert mock_gitRepo.clone_from.call_count == 2
assert mock_gitRepo.return_value.git.checkout.call_count == 1
- @mock.patch("airflow.dag_processing.bundles.git.Repo")
+ @mock.patch("airflow.providers.git.bundles.git.Repo")
def test_refresh_with_git_connection(self, mock_gitRepo):
bundle = GitDagBundle(
name="test",
@@ -510,7 +377,7 @@ class TestGitDagBundle:
("file:///dev/null", None),
],
)
- @mock.patch("airflow.dag_processing.bundles.git.Repo")
+ @mock.patch("airflow.providers.git.bundles.git.Repo")
def test_view_url(self, mock_gitrepo, repo_url, expected_url, session):
session.query(Connection).delete()
conn = Connection(
@@ -529,7 +396,7 @@ class TestGitDagBundle:
assert view_url == expected_url
bundle.initialize.assert_not_called()
- @mock.patch("airflow.dag_processing.bundles.git.Repo")
+ @mock.patch("airflow.providers.git.bundles.git.Repo")
def test_view_url_returns_none_when_no_version_in_view_url(self,
mock_gitrepo):
bundle = GitDagBundle(
name="test",
@@ -538,12 +405,12 @@ class TestGitDagBundle:
view_url = bundle.view_url(None)
assert view_url is None
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_clone_bare_repo_git_command_error(self, mock_githook):
mock_githook.return_value.repo_url =
"[email protected]:apache/airflow.git"
mock_githook.return_value.env = {}
- with mock.patch("airflow.dag_processing.bundles.git.Repo.clone_from")
as mock_clone:
+ with mock.patch("airflow.providers.git.bundles.git.Repo.clone_from")
as mock_clone:
mock_clone.side_effect = GitCommandError("clone", "Simulated
error")
bundle = GitDagBundle(name="test", tracking_ref="main")
with pytest.raises(
@@ -552,12 +419,12 @@ class TestGitDagBundle:
):
bundle.initialize()
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_clone_repo_no_such_path_error(self, mock_githook):
mock_githook.return_value.repo_url =
"[email protected]:apache/airflow.git"
- with mock.patch("airflow.dag_processing.bundles.git.os.path.exists",
return_value=False):
- with
mock.patch("airflow.dag_processing.bundles.git.Repo.clone_from") as mock_clone:
+ with mock.patch("airflow.providers.git.bundles.git.os.path.exists",
return_value=False):
+ with
mock.patch("airflow.providers.git.bundles.git.Repo.clone_from") as mock_clone:
mock_clone.side_effect = NoSuchPathError("Path not found")
bundle = GitDagBundle(name="test", tracking_ref="main")
with pytest.raises(AirflowException) as exc_info:
@@ -580,11 +447,11 @@ class TestGitDagBundle:
else:
assert not hasattr(bundle, "hook")
- @mock.patch("airflow.dag_processing.bundles.git.GitHook")
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_lock_used(self, mock_githook, git_repo):
repo_path, repo = git_repo
mock_githook.return_value.repo_url = repo_path
bundle = GitDagBundle(name="test", tracking_ref=GIT_DEFAULT_BRANCH)
- with
mock.patch("airflow.dag_processing.bundles.git.GitDagBundle.lock") as mock_lock:
+ with mock.patch("airflow.providers.git.bundles.git.GitDagBundle.lock")
as mock_lock:
bundle.initialize()
assert mock_lock.call_count == 2 # both initialize and refresh
diff --git a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
b/providers/git/tests/unit/git/hooks/__init__.py
similarity index 59%
copy from airflow-core/src/airflow/dag_processing/bundles/provider.yaml
copy to providers/git/tests/unit/git/hooks/__init__.py
index 526dc0628a8..13a83393a91 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/provider.yaml
+++ b/providers/git/tests/unit/git/hooks/__init__.py
@@ -14,28 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
----
-package-name: apache-airflow-providers-bundles
-name: GIT
-description: |
- `GIT <https://git-scm.com/>`__
-
-state: not-ready
-source-date-epoch: 1726861127
-# note that those versions are maintained by release manager - do not update
them manually
-versions:
- - 1.0.0
-
-integrations:
- - integration-name: GIT (Git)
-
-hooks:
- - integration-name: GIT
- python-modules:
- - airflow.dag_processing.bundles.git
-
-
-connection-types:
- - hook-class-name: airflow.dag_processing.bundles.git.GitHook
- connection-type: git
diff --git a/providers/git/tests/unit/git/hooks/test_git.py
b/providers/git/tests/unit/git/hooks/test_git.py
new file mode 100644
index 00000000000..11302e59331
--- /dev/null
+++ b/providers/git/tests/unit/git/hooks/test_git.py
@@ -0,0 +1,194 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import os
+
+import pytest
+from git import Repo
+
+from airflow.exceptions import AirflowException
+from airflow.models import Connection
+from airflow.providers.git.hooks.git import GitHook
+from airflow.utils import db
+
+from tests_common.test_utils.config import conf_vars
+from tests_common.test_utils.db import clear_db_connections
+
+pytestmark = pytest.mark.db_test
+
+
[email protected](autouse=True)
+def bundle_temp_dir(tmp_path):
+ with conf_vars({("dag_processor", "dag_bundle_storage_path"):
str(tmp_path)}):
+ yield tmp_path
+
+
+GIT_DEFAULT_BRANCH = "main"
+
+AIRFLOW_HTTPS_URL = "https://github.com/apache/airflow.git";
+AIRFLOW_GIT = "[email protected]:apache/airflow.git"
+ACCESS_TOKEN = "my_access_token"
+CONN_DEFAULT = "git_default"
+CONN_HTTPS = "my_git_conn"
+CONN_ONLY_PATH = "my_git_conn_only_path"
+CONN_ONLY_INLINE_KEY = "my_git_conn_only_inline_key"
+CONN_BOTH_PATH_INLINE = "my_git_conn_both_path_inline"
+CONN_NO_REPO_URL = "my_git_conn_no_repo_url"
+
+
[email protected]
+def git_repo(tmp_path_factory):
+ directory = tmp_path_factory.mktemp("repo")
+ repo = Repo.init(directory)
+ repo.git.symbolic_ref("HEAD", f"refs/heads/{GIT_DEFAULT_BRANCH}")
+ file_path = directory / "test_dag.py"
+ with open(file_path, "w") as f:
+ f.write("hello world")
+ repo.index.add([file_path])
+ repo.index.commit("Initial commit")
+ return (directory, repo)
+
+
+class TestGitHook:
+ @classmethod
+ def teardown_class(cls) -> None:
+ clear_db_connections()
+
+ @classmethod
+ def setup_class(cls) -> None:
+ db.merge_conn(
+ Connection(
+ conn_id=CONN_DEFAULT,
+ host=AIRFLOW_GIT,
+ conn_type="git",
+ extra='{"key_file": "/files/pkey.pem"}',
+ )
+ )
+ db.merge_conn(
+ Connection(
+ conn_id=CONN_HTTPS,
+ host=AIRFLOW_HTTPS_URL,
+ password=ACCESS_TOKEN,
+ conn_type="git",
+ )
+ )
+ db.merge_conn(
+ Connection(
+ conn_id=CONN_ONLY_PATH,
+ host="path/to/repo",
+ conn_type="git",
+ )
+ )
+ db.merge_conn(
+ Connection(
+ conn_id=CONN_ONLY_INLINE_KEY,
+ host="path/to/repo",
+ conn_type="git",
+ extra={
+ "private_key": "inline_key",
+ },
+ )
+ )
+
+ @pytest.mark.parametrize(
+ "conn_id, hook_kwargs, expected_repo_url",
+ [
+ (CONN_DEFAULT, {}, AIRFLOW_GIT),
+ (CONN_HTTPS, {},
f"https://{ACCESS_TOKEN}@github.com/apache/airflow.git";),
+ (
+ CONN_HTTPS,
+ {"repo_url": "https://github.com/apache/zzzairflow"},
+ f"https://{ACCESS_TOKEN}@github.com/apache/zzzairflow";,
+ ),
+ (CONN_ONLY_PATH, {}, "path/to/repo"),
+ ],
+ )
+ def test_correct_repo_urls(self, conn_id, hook_kwargs, expected_repo_url):
+ hook = GitHook(git_conn_id=conn_id, **hook_kwargs)
+ assert hook.repo_url == expected_repo_url
+
+ def test_env_var_with_configure_hook_env(self, session):
+ default_hook = GitHook(git_conn_id=CONN_DEFAULT)
+ with default_hook.configure_hook_env():
+ assert default_hook.env == {
+ "GIT_SSH_COMMAND": "ssh -i /files/pkey.pem -o
IdentitiesOnly=yes -o StrictHostKeyChecking=no"
+ }
+ db.merge_conn(
+ Connection(
+ conn_id="my_git_conn_strict",
+ host=AIRFLOW_GIT,
+ conn_type="git",
+ extra='{"key_file": "/files/pkey.pem",
"strict_host_key_checking": "yes"}',
+ )
+ )
+
+ strict_default_hook = GitHook(git_conn_id="my_git_conn_strict")
+ with strict_default_hook.configure_hook_env():
+ assert strict_default_hook.env == {
+ "GIT_SSH_COMMAND": "ssh -i /files/pkey.pem -o
IdentitiesOnly=yes -o StrictHostKeyChecking=yes"
+ }
+
+ def test_given_both_private_key_and_key_file(self):
+ db.merge_conn(
+ Connection(
+ conn_id=CONN_BOTH_PATH_INLINE,
+ host="path/to/repo",
+ conn_type="git",
+ extra={
+ "key_file": "path/to/key",
+ "private_key": "inline_key",
+ },
+ )
+ )
+
+ with pytest.raises(
+ AirflowException, match="Both 'key_file' and 'private_key' cannot
be provided at the same time"
+ ):
+ GitHook(git_conn_id=CONN_BOTH_PATH_INLINE)
+
+ def test_key_file_git_hook_has_env_with_configure_hook_env(self):
+ hook = GitHook(git_conn_id=CONN_DEFAULT)
+
+ assert hasattr(hook, "env")
+ with hook.configure_hook_env():
+ assert hook.env == {
+ "GIT_SSH_COMMAND": "ssh -i /files/pkey.pem -o
IdentitiesOnly=yes -o StrictHostKeyChecking=no"
+ }
+
+ def test_private_key_lazy_env_var(self):
+ hook = GitHook(git_conn_id=CONN_ONLY_INLINE_KEY)
+ assert hook.env == {}
+
+ hook.set_git_env("dummy_inline_key")
+ assert hook.env == {
+ "GIT_SSH_COMMAND": "ssh -i dummy_inline_key -o IdentitiesOnly=yes
-o StrictHostKeyChecking=no"
+ }
+
+ def test_configure_hook_env(self):
+ hook = GitHook(git_conn_id=CONN_ONLY_INLINE_KEY)
+ assert hasattr(hook, "private_key")
+
+ hook.set_git_env("dummy_inline_key")
+
+ with hook.configure_hook_env():
+ command = hook.env.get("GIT_SSH_COMMAND")
+ temp_key_path = command.split()[2]
+ assert os.path.exists(temp_key_path)
+
+ assert not os.path.exists(temp_key_path)
diff --git a/pyproject.toml b/pyproject.toml
index b65603ce7bf..5082ddd6795 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -102,7 +102,7 @@ dynamic = ["version", "optional-dependencies",
"dependencies"]
# apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka,
apache.kylin, apache.livy,
# apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana,
atlassian.jira, celery, cloudant,
# cncf.kubernetes, cohere, common.compat, common.io, common.messaging,
common.sql, databricks,
-# datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol,
fab, facebook, ftp,
+# datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol,
fab, facebook, ftp, git,
# github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins,
microsoft.azure,
# microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc,
openai, openfaas,
# openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector,
pinecone, postgres,
@@ -845,6 +845,7 @@ apache-airflow-providers-exasol = { workspace = true }
apache-airflow-providers-fab = { workspace = true }
apache-airflow-providers-facebook = { workspace = true }
apache-airflow-providers-ftp = { workspace = true }
+apache-airflow-providers-git = { workspace = true }
apache-airflow-providers-github = { workspace = true }
apache-airflow-providers-google = { workspace = true }
apache-airflow-providers-grpc = { workspace = true }
@@ -947,6 +948,7 @@ members = [
"providers/fab",
"providers/facebook",
"providers/ftp",
+ "providers/git",
"providers/github",
"providers/google",
"providers/grpc",
diff --git a/scripts/ci/docker-compose/remove-sources.yml
b/scripts/ci/docker-compose/remove-sources.yml
index abb912a04ff..9e0ca79dffb 100644
--- a/scripts/ci/docker-compose/remove-sources.yml
+++ b/scripts/ci/docker-compose/remove-sources.yml
@@ -68,6 +68,7 @@ services:
- ../../../empty:/opt/airflow/providers/fab/src
- ../../../empty:/opt/airflow/providers/facebook/src
- ../../../empty:/opt/airflow/providers/ftp/src
+ - ../../../empty:/opt/airflow/providers/git/src
- ../../../empty:/opt/airflow/providers/github/src
- ../../../empty:/opt/airflow/providers/google/src
- ../../../empty:/opt/airflow/providers/grpc/src
diff --git a/scripts/ci/docker-compose/tests-sources.yml
b/scripts/ci/docker-compose/tests-sources.yml
index c3bf069288d..3670ac737e7 100644
--- a/scripts/ci/docker-compose/tests-sources.yml
+++ b/scripts/ci/docker-compose/tests-sources.yml
@@ -80,6 +80,7 @@ services:
- ../../../providers/fab/tests:/opt/airflow/providers/fab/tests
- ../../../providers/facebook/tests:/opt/airflow/providers/facebook/tests
- ../../../providers/ftp/tests:/opt/airflow/providers/ftp/tests
+ - ../../../providers/git/tests:/opt/airflow/providers/git/tests
- ../../../providers/github/tests:/opt/airflow/providers/github/tests
- ../../../providers/google/tests:/opt/airflow/providers/google/tests
- ../../../providers/grpc/tests:/opt/airflow/providers/grpc/tests
