This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4c3f841b411 test: add e2e opensearch remote logging test (#65374)
4c3f841b411 is described below
commit 4c3f841b4115a5c9c12c440cc645740f900a060a
Author: Owen Leung <[email protected]>
AuthorDate: Sun Apr 19 04:06:02 2026 +0800
test: add e2e opensearch remote logging test (#65374)
* test: add e2e opensearch remote logging test
* Fix failing CI
* Fix failing CI
* Add opensearch provider into prod image
---
.github/workflows/additional-prod-image-tests.yml | 16 +++++++++++
.github/workflows/airflow-e2e-tests.yml | 4 +--
.github/workflows/ci-amd-arm.yml | 2 ++
.../opensearch.yml} | 32 +++++++++++++++-------
.../tests/airflow_e2e_tests/conftest.py | 31 ++++++++++++++++++++-
.../tests/airflow_e2e_tests/constants.py | 1 +
.../e2e_test_utils/base_remote_log_search_test.py | 5 ++--
.../airflow_e2e_tests/e2e_test_utils/clients.py | 5 ----
.../test_remote_logging_elasticsearch.py | 4 ---
.../__init__.py} | 11 --------
.../test_remote_logging_opensearch.py} | 6 +---
.../images/output_testing_airflow-e2e-tests.svg | 24 +++++++++-------
.../images/output_testing_airflow-e2e-tests.txt | 2 +-
.../airflow_breeze/commands/testing_commands.py | 2 +-
.../src/airflow_breeze/utils/selective_checks.py | 12 ++++++++
dev/breeze/tests/test_selective_checks.py | 30 ++++++++++++++++++++
prod_image_installed_providers.txt | 1 +
17 files changed, 135 insertions(+), 53 deletions(-)
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index 03a94c84179..00fd2edd989 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -44,6 +44,10 @@ on: # yamllint disable-line rule:truthy
description: "Whether to run Elasticsearch remote logging e2e tests
(true/false)"
required: true
type: string
+ run-remote-logging-opensearch-e2e-tests:
+ description: "Whether to run OpenSearch remote logging e2e tests
(true/false)"
+ required: true
+ type: string
constraints-branch:
description: "Branch used to construct constraints URL from."
required: true
@@ -247,6 +251,18 @@ jobs:
e2e_test_mode: "remote_log_elasticsearch"
if: inputs.canary-run == 'true' ||
inputs.run-remote-logging-elasticsearch-e2e-tests == 'true'
+ test-e2e-integration-tests-remote-log-opensearch:
+ name: "OpenSearch remote logging tests with PROD image"
+ uses: ./.github/workflows/airflow-e2e-tests.yml
+ with:
+ workflow-name: "OpenSearch remote logging e2e test"
+ runners: ${{ inputs.runners }}
+ platform: ${{ inputs.platform }}
+ default-python-version: "${{ inputs.default-python-version }}"
+ use-uv: ${{ inputs.use-uv }}
+ e2e_test_mode: "remote_log_opensearch"
+ if: inputs.canary-run == 'true' ||
inputs.run-remote-logging-opensearch-e2e-tests == 'true'
+
test-e2e-integration-tests-xcom-object-storage:
name: "XCom object storage backend tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
diff --git a/.github/workflows/airflow-e2e-tests.yml
b/.github/workflows/airflow-e2e-tests.yml
index cee6f855dc8..49c5bbd1a5a 100644
--- a/.github/workflows/airflow-e2e-tests.yml
+++ b/.github/workflows/airflow-e2e-tests.yml
@@ -49,7 +49,7 @@ on: # yamllint disable-line rule:truthy
type: string
required: true
e2e_test_mode:
- description: "Test mode - basic, remote_log, remote_log_elasticsearch,
or xcom_object_storage"
+ description: "Test mode - basic, remote_log, remote_log_elasticsearch,
remote_log_opensearch, or xcom_object_storage" # yamllint disable-line
rule:line-length
type: string
default: "basic"
@@ -80,7 +80,7 @@ on: # yamllint disable-line rule:truthy
type: string
default: ""
e2e_test_mode:
- description: "Test mode - basic, remote_log, remote_log_elasticsearch,
or xcom_object_storage"
+ description: "Test mode - basic, remote_log, remote_log_elasticsearch,
remote_log_opensearch, or xcom_object_storage" # yamllint disable-line
rule:line-length
type: string
default: "basic"
diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml
index da98b28b4d0..edabb322483 100644
--- a/.github/workflows/ci-amd-arm.yml
+++ b/.github/workflows/ci-amd-arm.yml
@@ -115,6 +115,7 @@ jobs:
run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
run-mypy-providers: ${{
steps.selective-checks.outputs.run-mypy-providers }}
run-remote-logging-elasticsearch-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
+ run-remote-logging-opensearch-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-opensearch-e2e-tests }}
run-remote-logging-s3-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
run-task-sdk-tests: ${{
steps.selective-checks.outputs.run-task-sdk-tests }}
@@ -796,6 +797,7 @@ jobs:
run-task-sdk-integration-tests: ${{
needs.build-info.outputs.run-task-sdk-integration-tests }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
run-remote-logging-elasticsearch-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }}
+ run-remote-logging-opensearch-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-opensearch-e2e-tests }}
run-remote-logging-s3-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
b/airflow-e2e-tests/docker/opensearch.yml
similarity index 56%
copy from
airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
copy to airflow-e2e-tests/docker/opensearch.yml
index 424fa921043..f6fc0b72cc3 100644
---
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
+++ b/airflow-e2e-tests/docker/opensearch.yml
@@ -14,14 +14,26 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from __future__ import annotations
+---
+services:
+ opensearch:
+ image: opensearchproject/opensearch:2.19.0
+ environment:
+ - discovery.type=single-node
+ - DISABLE_SECURITY_PLUGIN=true
+ - DISABLE_INSTALL_DEMO_CONFIG=true
+ - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+ volumes:
+ - opensearch-db-volume:/usr/share/opensearch/data
+ ports:
+ - "9200:9200"
+ healthcheck:
+ test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health ||
exit 1"]
+ interval: 10s
+ timeout: 30s
+ start_period: 60s
+ retries: 50
+ restart: "on-failure"
-from airflow_e2e_tests.e2e_test_utils.base_remote_log_search_test import
BaseRemoteLoggingSearchTest
-from airflow_e2e_tests.e2e_test_utils.clients import get_elasticsearch_session
-
-
-class TestRemoteLoggingElasticsearch(BaseRemoteLoggingSearchTest):
- search_url = "http://localhost:9200";
-
- def _get_session(self):
- return get_elasticsearch_session()
+volumes:
+ opensearch-db-volume:
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
index 04fe9eace59..4443f7d8a28 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
@@ -36,6 +36,7 @@ from airflow_e2e_tests.constants import (
ELASTICSEARCH_PATH,
LOCALSTACK_PATH,
LOGS_FOLDER,
+ OPENSEARCH_PATH,
TEST_REPORT_FILE,
XCOM_BUCKET,
)
@@ -64,6 +65,11 @@ def _copy_elasticsearch_files(tmp_dir):
copyfile(ELASTICSEARCH_PATH, tmp_dir / "elasticsearch.yml")
+def _copy_opensearch_files(tmp_dir):
+ """Copy OpenSearch compose file into the temp directory."""
+ copyfile(OPENSEARCH_PATH, tmp_dir / "opensearch.yml")
+
+
def _setup_s3_integration(dot_env_file, tmp_dir):
_copy_localstack_files(tmp_dir)
@@ -95,6 +101,26 @@ def _setup_elasticsearch_integration(dot_env_file, tmp_dir):
os.environ["ENV_FILE_PATH"] = str(dot_env_file)
+def _setup_opensearch_integration(dot_env_file, tmp_dir):
+ _copy_opensearch_files(tmp_dir)
+
+ dot_env_file.write_text(
+ f"AIRFLOW_UID={os.getuid()}\n"
+ "AIRFLOW__LOGGING__REMOTE_LOGGING=true\n"
+ "AIRFLOW__OPENSEARCH__HOST=http://opensearch:9200\n";
+ "AIRFLOW__OPENSEARCH__PORT=9200\n"
+ "AIRFLOW__OPENSEARCH__USERNAME=admin\n"
+ "AIRFLOW__OPENSEARCH__PASSWORD=admin\n"
+ "AIRFLOW__OPENSEARCH__WRITE_STDOUT=false\n"
+ "AIRFLOW__OPENSEARCH__JSON_FORMAT=true\n"
+ "AIRFLOW__OPENSEARCH__WRITE_TO_OS=true\n"
+ "AIRFLOW__OPENSEARCH__TARGET_INDEX=airflow-e2e-logs\n"
+ "AIRFLOW__OPENSEARCH__HOST_FIELD=host\n"
+ "AIRFLOW__OPENSEARCH__OFFSET_FIELD=offset\n"
+ )
+ os.environ["ENV_FILE_PATH"] = str(dot_env_file)
+
+
def _setup_xcom_object_storage_integration(dot_env_file, tmp_dir):
_copy_localstack_files(tmp_dir)
@@ -148,6 +174,9 @@ def spin_up_airflow_environment(tmp_path_factory:
pytest.TempPathFactory):
elif E2E_TEST_MODE == "remote_log_elasticsearch":
compose_file_names.append("elasticsearch.yml")
_setup_elasticsearch_integration(dot_env_file, tmp_dir)
+ elif E2E_TEST_MODE == "remote_log_opensearch":
+ compose_file_names.append("opensearch.yml")
+ _setup_opensearch_integration(dot_env_file, tmp_dir)
elif E2E_TEST_MODE == "xcom_object_storage":
compose_file_names.append("localstack.yml")
_setup_xcom_object_storage_integration(dot_env_file, tmp_dir)
@@ -190,7 +219,7 @@ def _print_logs(compose_instance: DockerCompose):
if service:
stdout, _ = compose_instance.get_logs(service)
console.print(f"::group:: {service} Logs")
- console.print(stdout, style="red", soft_wrap=True)
+ console.print(stdout, style="red", soft_wrap=True, markup=False)
console.print("::endgroup::")
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
index 67fec2c5580..a2d99aeabab 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
@@ -41,6 +41,7 @@ LOGS_FOLDER = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "logs"
TEST_REPORT_FILE = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" /
"_e2e_test_report.json"
LOCALSTACK_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"localstack.yml"
ELASTICSEARCH_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"elasticsearch.yml"
+OPENSEARCH_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"opensearch.yml"
E2E_TEST_MODE = os.environ.get("E2E_TEST_MODE", "basic")
AWS_INIT_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "scripts" /
"init-aws.sh"
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/base_remote_log_search_test.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/base_remote_log_search_test.py
index fb03d8bb2ba..797c8324c18 100644
---
a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/base_remote_log_search_test.py
+++
b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/base_remote_log_search_test.py
@@ -21,7 +21,7 @@ from datetime import datetime, timezone
import pytest
-from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient
+from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient,
create_request_session_with_retries
class BaseRemoteLoggingSearchTest:
@@ -30,7 +30,6 @@ class BaseRemoteLoggingSearchTest:
Subclasses must set:
- ``search_url``: base URL of the search backend, e.g.
``"http://localhost:9200"``
- - ``_get_session``: zero-arg callable returning a requests.Session
with retry logic
"""
airflow_client = AirflowClient()
@@ -44,7 +43,7 @@ class BaseRemoteLoggingSearchTest:
expected_log_id_prefix = f"{dag_id}-{task_id}-"
def _get_session(self):
- raise NotImplementedError
+ return create_request_session_with_retries(status_forcelist=[429, 500,
502, 503, 504])
def _matches_expected_log(self, log_source: dict, run_id: str) -> bool:
log_id = log_source.get("log_id", "")
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
index ddbb642cf4f..b0f6573b9dc 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
@@ -43,11 +43,6 @@ def get_s3_client():
)
-def get_elasticsearch_session():
- """Return a requests session configured for talking to the local
Elasticsearch container."""
- return create_request_session_with_retries(status_forcelist=[429, 500,
502, 503, 504])
-
-
def create_request_session_with_retries(status_forcelist: list[int]):
"""Create a requests Session with retry logic for handling transient
errors."""
Retry.DEFAULT_BACKOFF_MAX = 32
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
index 424fa921043..20da667d1d0 100644
---
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
+++
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
@@ -17,11 +17,7 @@
from __future__ import annotations
from airflow_e2e_tests.e2e_test_utils.base_remote_log_search_test import
BaseRemoteLoggingSearchTest
-from airflow_e2e_tests.e2e_test_utils.clients import get_elasticsearch_session
class TestRemoteLoggingElasticsearch(BaseRemoteLoggingSearchTest):
search_url = "http://localhost:9200";
-
- def _get_session(self):
- return get_elasticsearch_session()
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/__init__.py
similarity index 66%
copy from
airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
copy to
airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/__init__.py
index 424fa921043..13a83393a91 100644
---
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
+++
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/__init__.py
@@ -14,14 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from __future__ import annotations
-
-from airflow_e2e_tests.e2e_test_utils.base_remote_log_search_test import
BaseRemoteLoggingSearchTest
-from airflow_e2e_tests.e2e_test_utils.clients import get_elasticsearch_session
-
-
-class TestRemoteLoggingElasticsearch(BaseRemoteLoggingSearchTest):
- search_url = "http://localhost:9200";
-
- def _get_session(self):
- return get_elasticsearch_session()
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/test_remote_logging_opensearch.py
similarity index 81%
copy from
airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
copy to
airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/test_remote_logging_opensearch.py
index 424fa921043..180561dd4a6 100644
---
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/test_remote_logging_elasticsearch.py
+++
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/test_remote_logging_opensearch.py
@@ -17,11 +17,7 @@
from __future__ import annotations
from airflow_e2e_tests.e2e_test_utils.base_remote_log_search_test import
BaseRemoteLoggingSearchTest
-from airflow_e2e_tests.e2e_test_utils.clients import get_elasticsearch_session
-class TestRemoteLoggingElasticsearch(BaseRemoteLoggingSearchTest):
+class TestRemoteLoggingOpenSearch(BaseRemoteLoggingSearchTest):
search_url = "http://localhost:9200";
-
- def _get_session(self):
- return get_elasticsearch_session()
diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
index e75a730eee6..1add4b96322 100644
--- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
+++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 562.4"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 586.8"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-testing-airflow-e2e-tests-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="511.4" />
+ <rect x="0" y="0" width="1463.0" height="535.8" />
</clipPath>
<clipPath id="breeze-testing-airflow-e2e-tests-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -105,9 +105,12 @@
<clipPath id="breeze-testing-airflow-e2e-tests-line-19">
<rect x="0" y="465.1" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-testing-airflow-e2e-tests-line-20">
+ <rect x="0" y="489.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="560.4" rx="8"/><text
class="breeze-testing-airflow-e2e-tests-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: testing airflow-e2e-tests</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="584.8" rx="8"/><text
class="breeze-testing-airflow-e2e-tests-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: testing airflow-e2e-tests</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -131,13 +134,14 @@
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="288.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="288.4" textLength="366"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">--github-repository           </text><text
class="breeze-testing-airflow-e2e-tests-r6" x="414.8" y="288.4"
textLength="24.4" clip-path [...]
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="312.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">│</text><text
class="breeze-testing-airflow-e2e-tests-r7" x="463.6" y="312.8"
textLength="73.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">(TEXT)</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="1451.8" y="312.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">│</text><text
class="breez [...]
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="337.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-13)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="337.2" textLength="366"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-13)">--e2e-test-mode               </text><text
class="breeze-testing-airflow-e2e-tests-r1" x="463.6" y="337.2" textLen [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="361.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">│</text><text
class="breeze-testing-airflow-e2e-tests-r7" x="463.6" y="361.6"
textLength="768.6"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">(basic|remote_log|remote_log_elasticsearch|xcom_object_storage)</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="1451.8" y="361.6"
textLength="12.2" clip-path="url(#breeze-test [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="386"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="386" textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="410.4"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">╭─</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="24.4" y="410.4"
textLength="195.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)"> Common options </text><text
class="breeze-testing-airflow-e2e-tests-r5" x="219.6" y="410.4"
textLength="1220" clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">─
[...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="434.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="434.8"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">--verbose</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="434.8"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">-v</text><text
class="br [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="459.2"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">--dry-run</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="459.2"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">-D</text><text
class="br [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="483.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="483.6"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">--help   </text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="483.6"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">-h</text> [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="508"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="508" textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="361.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">│</text><text
class="breeze-testing-airflow-e2e-tests-r7" x="463.6" y="361.6"
textLength="976"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">(basic|remote_log|remote_log_elasticsearch|remote_log_opensearch|xcom_object_sto</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="1451.8" y="361.6"
textLength="12.2" clip-path="u [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="386"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">│</text><text
class="breeze-testing-airflow-e2e-tests-r7" x="463.6" y="386" textLength="61"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">rage)</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="1451.8" y="386"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">│</text><text
class="breeze-testing [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="410.4"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="410.4"
textLength="12.2" clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="434.8"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">╭─</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="24.4" y="434.8"
textLength="195.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)"> Common options </text><text
class="breeze-testing-airflow-e2e-tests-r5" x="219.6" y="434.8"
textLength="1220" clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">─
[...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="459.2"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">--verbose</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="459.2"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">-v</text><text
class="br [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="483.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="483.6"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">--dry-run</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="483.6"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">-D</text><text
class="br [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="508"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="508" textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">--help   </text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="508"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">-h</text><text [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="532.4"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="532.4"
textLength="12.2" clip-path="url(#breeze-testing-airflow-e2e-tests-line-21)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
index b344eaf9673..4648ab65601 100644
--- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
+++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
@@ -1 +1 @@
-11e115062a47a3ee70aeaa380af9cd84
+0e0a5500aa42f7857d251cf13a1043ed
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index 3ee2e605614..eabef38ce52 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -1413,7 +1413,7 @@ option_e2e_test_mode = click.option(
show_default=True,
envvar="E2E_TEST_MODE",
type=click.Choice(
- ["basic", "remote_log", "remote_log_elasticsearch",
"xcom_object_storage"],
+ ["basic", "remote_log", "remote_log_elasticsearch",
"remote_log_opensearch", "xcom_object_storage"],
case_sensitive=False,
),
)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index ca7fbef3a3d..5b399ed3d8c 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -126,6 +126,7 @@ class FileGroupForCi(Enum):
REMOTE_LOGGING_E2E_SHARED_FILES = auto()
REMOTE_LOGGING_E2E_S3_FILES = auto()
REMOTE_LOGGING_E2E_ELASTICSEARCH_FILES = auto()
+ REMOTE_LOGGING_E2E_OPENSEARCH_FILES = auto()
ALL_PYPROJECT_TOML_FILES = auto()
ALL_PYTHON_FILES = auto()
ALL_SOURCE_FILES = auto()
@@ -198,6 +199,10 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
r"^airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/.*",
r"^providers/elasticsearch/.*",
],
+ FileGroupForCi.REMOTE_LOGGING_E2E_OPENSEARCH_FILES: [
+
r"^airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_opensearch_tests/.*",
+ r"^providers/opensearch/.*",
+ ],
FileGroupForCi.PYTHON_PRODUCTION_FILES: [
r"^airflow-core/src/airflow/.*\.py",
r"^providers/.*\.py",
@@ -950,6 +955,12 @@ class SelectiveChecks:
FileGroupForCi.REMOTE_LOGGING_E2E_ELASTICSEARCH_FILES
)
+ @cached_property
+ def run_remote_logging_opensearch_e2e_tests(self) -> bool:
+ return
self._should_be_run(FileGroupForCi.REMOTE_LOGGING_E2E_SHARED_FILES) or
self._should_be_run(
+ FileGroupForCi.REMOTE_LOGGING_E2E_OPENSEARCH_FILES
+ )
+
@cached_property
def run_amazon_tests(self) -> bool:
if self.providers_test_types_list_as_strings_in_json == "[]":
@@ -1060,6 +1071,7 @@ class SelectiveChecks:
or self.run_airflow_ctl_integration_tests
or self.run_remote_logging_s3_e2e_tests
or self.run_remote_logging_elasticsearch_e2e_tests
+ or self.run_remote_logging_opensearch_e2e_tests
or self.run_ui_e2e_tests
)
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 3838f779e4c..24c2f9befa6 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1255,6 +1255,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
{
"run-remote-logging-s3-e2e-tests": "false",
"run-remote-logging-elasticsearch-e2e-tests": "true",
+ "run-remote-logging-opensearch-e2e-tests": "false",
"ci-image-build": "true",
"prod-image-build": "true",
},
@@ -1267,18 +1268,46 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
{
"run-remote-logging-s3-e2e-tests": "false",
"run-remote-logging-elasticsearch-e2e-tests": "true",
+ "run-remote-logging-opensearch-e2e-tests": "false",
"ci-image-build": "true",
"prod-image-build": "true",
},
id="Elasticsearch helper changes enable Elasticsearch e2e",
)
),
+ (
+ pytest.param(
+
("providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "false",
+ "run-remote-logging-elasticsearch-e2e-tests": "false",
+ "run-remote-logging-opensearch-e2e-tests": "true",
+ "ci-image-build": "true",
+ "prod-image-build": "true",
+ },
+ id="OpenSearch remote logging changes enable only OpenSearch
e2e",
+ )
+ ),
+ (
+ pytest.param(
+
("providers/opensearch/src/airflow/providers/opensearch/log/os_json_formatter.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "false",
+ "run-remote-logging-elasticsearch-e2e-tests": "false",
+ "run-remote-logging-opensearch-e2e-tests": "true",
+ "ci-image-build": "true",
+ "prod-image-build": "true",
+ },
+ id="OpenSearch helper changes enable OpenSearch e2e",
+ )
+ ),
(
pytest.param(
("airflow-core/src/airflow/config_templates/airflow_local_settings.py",),
{
"run-remote-logging-s3-e2e-tests": "true",
"run-remote-logging-elasticsearch-e2e-tests": "true",
+ "run-remote-logging-opensearch-e2e-tests": "true",
"ci-image-build": "true",
"prod-image-build": "true",
},
@@ -1291,6 +1320,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
{
"run-remote-logging-s3-e2e-tests": "true",
"run-remote-logging-elasticsearch-e2e-tests": "true",
+ "run-remote-logging-opensearch-e2e-tests": "true",
"ci-image-build": "true",
"prod-image-build": "true",
},
diff --git a/prod_image_installed_providers.txt
b/prod_image_installed_providers.txt
index 376a72bd778..fc1a9c97605 100644
--- a/prod_image_installed_providers.txt
+++ b/prod_image_installed_providers.txt
@@ -19,6 +19,7 @@ microsoft.azure
mysql
odbc
openlineage
+opensearch
postgres
redis
sendgrid
