mccormickt12 commented on code in PR #2291:
URL: https://github.com/apache/iceberg-python/pull/2291#discussion_r2302429908
##########
pyiceberg/io/pyarrow.py:
##########
@@ -387,15 +387,29 @@ def __init__(self, properties: Properties = EMPTY_DICT):
super().__init__(properties=properties)
@staticmethod
- def parse_location(location: str) -> Tuple[str, str, str]:
- """Return the path without the scheme."""
+ def parse_location(location: str, properties: Properties = EMPTY_DICT) ->
Tuple[str, str, str]:
+ """Return (scheme, netloc, path) for the given location.
+
+ Uses DEFAULT_SCHEME and DEFAULT_NETLOC if scheme/netloc are missing.
+ """
uri = urlparse(location)
+
+ default_scheme = properties.get("DEFAULT_SCHEME", "file")
+ default_netloc = properties.get("DEFAULT_NETLOC", "")
+
if not uri.scheme:
- return "file", uri.netloc, os.path.abspath(location)
- elif uri.scheme in ("hdfs", "viewfs"):
- return uri.scheme, uri.netloc, uri.path
+ scheme = default_scheme
+ if not uri.netloc:
+ netloc = default_netloc
+
+ if scheme in ("hdfs", "viewfs"):
+ return scheme, netloc, uri.path
else:
- return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"
+ # For non-HDFS URIs, include netloc in the path if present
+ path = uri.path if uri.scheme else os.path.abspath(location)
+ if netloc and not path.startswith(netloc):
+ path = f"{netloc}{path}"
+ return scheme, netloc, path
Review Comment:
this assumes the default scheme will only ever be hdfs, I think its better
to leave this generic and before to allow other schemes to be default and still
work
##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL:
"pyiceberg.DoesNotExist"})
with pytest.warns(UserWarning, match="Could not initialize S3 retry
strategy: pyiceberg.DoesNotExist"):
io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+ """Test that parse_location uses environment variables for defaults."""
Review Comment:
good catch
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
