JonasJ-ap commented on code in PR #6997:
URL: https://github.com/apache/iceberg/pull/6997#discussion_r1146934576
##########
python/tests/test_integration.py:
##########
@@ -118,3 +123,22 @@ def test_ray_all_types(table_test_all_types: Table) ->
None:
pandas_dataframe = table_test_all_types.scan().to_pandas()
assert ray_dataset.count() == pandas_dataframe.shape[0]
assert pandas_dataframe.equals(ray_dataset.to_pandas())
+
+
+@pytest.mark.integration
+def test_pyarrow_to_iceberg_all_types(table_test_all_types: Table) -> None:
+ fs = S3FileSystem(
+ **{
+ "endpoint_override": "http://localhost:9000";,
+ "access_key": "admin",
+ "secret_key": "password",
+ }
+ )
+ data_file_paths = [task.file.file_path for task in
table_test_all_types.scan().plan_files()]
+ for data_file_path in data_file_paths:
+ uri = urlparse(data_file_path)
+ with fs.open_input_file(f"{uri.netloc}{uri.path}") as fout:
+ parquet_schema = pq.read_schema(fout)
+ stored_iceberg_schema =
Schema.parse_raw(parquet_schema.metadata.get(b"iceberg.schema"))
+ converted_iceberg_schema = pyarrow_to_schema(parquet_schema)
+ assert converted_iceberg_schema == stored_iceberg_schema
Review Comment:
Rebase to the master branch. Since python integration tests are in, we can
now test pyarrow to iceberg schema visitor on real parquet file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
