This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 90953d1e2f Allow non-str value for path concat (#35290)
90953d1e2f is described below
commit 90953d1e2fb8d23665e7e4427fdb689a5c02a6a6
Author: Bolke de Bruin <bo...@xs4all.nl>
AuthorDate: Tue Oct 31 21:17:54 2023 +0100
Allow non-str value for path concat (#35290)
It was not possible to do path / int which is
a typical thing when using date partioned files.
---
airflow/io/store/path.py | 9 ++++++---
tests/io/store/test_store.py | 6 ++++++
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/airflow/io/store/path.py b/airflow/io/store/path.py
index 7ee0971855..01f30fd446 100644
--- a/airflow/io/store/path.py
+++ b/airflow/io/store/path.py
@@ -88,7 +88,7 @@ class ObjectStoragePath(os.PathLike):
protocol = ""
key = ""
- path = stringify_path(path)
+ path = str(stringify_path(path))
i = path.find("://")
if i > 0:
@@ -162,13 +162,16 @@ class ObjectStoragePath(os.PathLike):
def __truediv__(self, other) -> ObjectStoragePath:
o_protocol, o_bucket, o_key = self.split_path(other)
- if not isinstance(other, str) and o_bucket and self._bucket !=
o_bucket:
+ if isinstance(other, ObjectStoragePath) and o_bucket and self._bucket
!= o_bucket:
raise ValueError("Cannot combine paths from different buckets /
containers")
if o_protocol and self._protocol != o_protocol:
raise ValueError("Cannot combine paths from different protocols")
- path =
f"{stringify_path(self).rstrip(self.sep)}/{stringify_path(other).lstrip(self.sep)}"
+ self_path = str(stringify_path(self))
+ other_path = str(stringify_path(other))
+
+ path = f"{self_path.rstrip(self.sep)}/{other_path.lstrip(self.sep)}"
return ObjectStoragePath(path, conn_id=self._conn_id)
def _unsupported(self, method_name):
diff --git a/tests/io/store/test_store.py b/tests/io/store/test_store.py
index db1609f283..7a7018e7ec 100644
--- a/tests/io/store/test_store.py
+++ b/tests/io/store/test_store.py
@@ -66,6 +66,12 @@ class TestFs:
assert path2.key == "key/part1/part2/part3"
assert path2._protocol == "file"
+ # check if we can append a non string to the path
+ path3 = ObjectStoragePath(path2 / 2023)
+ assert path3.bucket == "bucket"
+ assert path3.key == "key/part1/part2/part3/2023"
+ assert path3._protocol == "file"
+
def test_read_write(self):
o = ObjectStoragePath(f"file:///tmp/{str(uuid.uuid4())}")
