Fokko commented on code in PR #5588:
URL: https://github.com/apache/iceberg/pull/5588#discussion_r950890054
##########
python/pyiceberg/io/__init__.py:
##########
@@ -218,11 +234,53 @@ def delete(self, location: Union[str, InputFile,
OutputFile]) -> None:
"""
-def load_file_io(_: Properties) -> FileIO:
- # To be implemented in a different PR.
- # - If py-file-io is present, load the right Python class
- # - When the property is missing, map from Java's filo-io to an
appropriate FileIO
- # - Extend the FileIO structure with a initialize that pass in properties
(could also be the constructor?)
+ARROW_FILE_IO = "pyiceberg.io.pyarrow.PyArrowFileIO"
+
+# Mappings from the Java FileIO impl to a Python one. The list is ordered by
preference.
+# If a implementation isn't installed, it will fall back to the next one.
+JAVA_FILE_IO_MAPPINGS: Dict[str, List[str]] = {
+ "org.apache.iceberg.dell.ecs.EcsFileIO": [ARROW_FILE_IO],
+ "org.apache.iceberg.gcp.gcs.GCSFileIO": [ARROW_FILE_IO],
+ "org.apache.iceberg.hadoop.HadoopFileIO": [ARROW_FILE_IO],
+ "org.apache.iceberg.aliyun.oss.OSSFileIO": [ARROW_FILE_IO],
+ "org.apache.iceberg.io.ResolvingFileIO": [ARROW_FILE_IO],
+ "org.apache.iceberg.aws.s3.S3FileIO": [ARROW_FILE_IO],
+}
+
+
+def _import_file_io(io_impl: str, properties: Properties) -> Optional[FileIO]:
+ try:
+ path_parts = io_impl.split(".")
+ if len(path_parts) < 2:
+ raise ValueError(f"py-io-impl should be full path
(module.CustomFileIO), got: {io_impl}")
+ module_name, class_name = ".".join(path_parts[:-1]), path_parts[-1]
+ module = importlib.import_module(module_name)
+ class_ = getattr(module, class_name)
+ return class_(properties)
+ except ImportError:
+ logger.exception("Could not initialize FileIO: %s", io_impl)
+ return None
+
+
+PY_IO_IMPL = "py-io-impl"
+IO_IMPL = "io-impl"
+
+
+def load_file_io(properties: Properties) -> FileIO:
Review Comment:
Ah got it. I thought you wanted to expand the properties. It would then look
like this:
```python
def load_file_io(py-io-impl: Optional[str], **properties: str) -> FileIO:
```
The dashes are not allowed.
You're right, I was confused with the namespaces where we annotate the
properties with the location and comment. Updated the code!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
