This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f816237196 Refactor string splitting (#34185)
f816237196 is described below
commit f816237196f69b70a9b628858148dcde91febf5b
Author: Miroslav Šedivý <6774676+eum...@users.noreply.github.com>
AuthorDate: Fri Oct 20 16:47:47 2023 +0000
Refactor string splitting (#34185)
---
airflow/providers/amazon/aws/hooks/s3.py | 8 ++++----
dev/breeze/src/airflow_breeze/commands/setup_commands.py | 2 +-
dev/breeze/src/airflow_breeze/params/doc_build_params.py | 10 +++++++++-
scripts/in_container/verify_providers.py | 6 +++---
tests/providers/amazon/aws/hooks/test_s3.py | 6 ++----
tests/test_utils/get_all_tests.py | 2 +-
6 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/airflow/providers/amazon/aws/hooks/s3.py
b/airflow/providers/amazon/aws/hooks/s3.py
index 2d6a78f5f6..7e81af0981 100644
--- a/airflow/providers/amazon/aws/hooks/s3.py
+++ b/airflow/providers/amazon/aws/hooks/s3.py
@@ -218,12 +218,12 @@ class S3Hook(AwsBaseHook):
elif format[0] == "https:":
temp_split = format[1].split(".")
if temp_split[0] == "s3":
- split_url = format[1].split("/")
- bucket_name = split_url[1]
- key = "/".join(split_url[2:])
+ # "https://s3.region-code.amazonaws.com/bucket-name/key-name";
+ _, bucket_name, key = format[1].split("/", 2)
elif temp_split[1] == "s3":
+ # "https://bucket-name.s3.region-code.amazonaws.com/key-name";
bucket_name = temp_split[0]
- key = "/".join(format[1].split("/")[1:])
+ key = format[1].partition("/")[-1]
else:
raise S3HookUriParseFailure(
"Please provide a bucket name using a valid virtually
hosted format which should "
diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands.py
b/dev/breeze/src/airflow_breeze/commands/setup_commands.py
index 4ee18045d2..24a3d2e8a2 100644
--- a/dev/breeze/src/airflow_breeze/commands/setup_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/setup_commands.py
@@ -428,7 +428,7 @@ def get_commands() -> list[str]:
for line in content.splitlines():
strip_line = line.strip()
if strip_line and not strip_line.startswith("#"):
- results.append(":".join(strip_line.split(":")[:-1]))
+ results.append(strip_line.rpartition(":")[0])
return results
diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py
b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
index 267f84ee54..8b6749a3cd 100644
--- a/dev/breeze/src/airflow_breeze/params/doc_build_params.py
+++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
@@ -20,7 +20,15 @@ import os
from dataclasses import dataclass
from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
-from airflow_breeze.utils.general_utils import
get_provider_name_from_short_hand
+
+providers_prefix = "apache-airflow-providers-"
+
+
+def get_provider_name_from_short_hand(short_form_providers: tuple[str]):
+ return tuple(
+ providers_prefix + short_form_provider.replace(".", "-")
+ for short_form_provider in short_form_providers
+ )
@dataclass
diff --git a/scripts/in_container/verify_providers.py
b/scripts/in_container/verify_providers.py
index bb0e989fe0..d4704d7cc9 100755
--- a/scripts/in_container/verify_providers.py
+++ b/scripts/in_container/verify_providers.py
@@ -192,7 +192,7 @@ def import_all_classes(
console.print(f"Skipping module: {modinfo.name}")
continue
if print_imports:
- package_to_print = ".".join(modinfo.name.split(".")[:-1])
+ package_to_print = modinfo.name.rpartition(".")[0]
if package_to_print not in printed_packages:
printed_packages.add(package_to_print)
console.print(f"Importing package: {package_to_print}")
@@ -247,7 +247,7 @@ def is_imported_from_same_module(the_class: str,
imported_name: str) -> bool:
:param imported_name: name of the imported class
:return: true if the class was imported from another module
"""
- return ".".join(imported_name.split(".")[:-1]) == the_class.__module__
+ return imported_name.rpartition(":")[0] == the_class.__module__
def is_example_dag(imported_name: str) -> bool:
@@ -360,7 +360,7 @@ def convert_class_name_to_url(base_url: str, class_name) ->
str:
:param class_name: name of the class
:return: URL to the class
"""
- return base_url + os.path.sep.join(class_name.split(".")[:-1]) + ".py"
+ return base_url + class_name.rpartition(".")[0].replace(".", "/") + ".py"
def get_class_code_link(base_package: str, class_name: str, git_tag: str) ->
str:
diff --git a/tests/providers/amazon/aws/hooks/test_s3.py
b/tests/providers/amazon/aws/hooks/test_s3.py
index 869f4ca2a3..032d95c6e1 100644
--- a/tests/providers/amazon/aws/hooks/test_s3.py
+++ b/tests/providers/amazon/aws/hooks/test_s3.py
@@ -24,6 +24,7 @@ import re
import unittest
from unittest import mock, mock as async_mock
from unittest.mock import MagicMock, Mock, patch
+from urllib.parse import parse_qs
import boto3
import pytest
@@ -1052,10 +1053,7 @@ class TestAwsS3Hook:
presigned_url = hook.generate_presigned_url(
client_method="get_object", params={"Bucket": s3_bucket, "Key":
"my_key"}
)
-
- url = presigned_url.split("?")[1]
- params = {x[0]: x[1] for x in [x.split("=") for x in
url[0:].split("&")]}
-
+ params = parse_qs(presigned_url.partition("?")[-1])
assert {"AWSAccessKeyId", "Signature",
"Expires"}.issubset(set(params.keys()))
def test_should_throw_error_if_extra_args_is_not_dict(self):
diff --git a/tests/test_utils/get_all_tests.py
b/tests/test_utils/get_all_tests.py
index 1ed04ed64d..10993ce000 100644
--- a/tests/test_utils/get_all_tests.py
+++ b/tests/test_utils/get_all_tests.py
@@ -56,7 +56,7 @@ def print_all_cases(xunit_test_file_path):
modules = set()
for test_case in test_cases:
- the_module = ".".join(test_case.get("classname").split(".")[:-1])
+ the_module = test_case["classname"].rpartition(".")[0]
the_class = last_replace(test_case.get("classname"), ".", ":", 1)
test_method = test_case.get("name")
modules.add(the_module)
