jorisvandenbossche commented on code in PR #39825:
URL: https://github.com/apache/arrow/pull/39825#discussion_r1507552877
##########
python/pyarrow/tests/parquet/conftest.py:
##########
@@ -81,6 +81,7 @@ def s3_example_fs(s3_server):
host, port, access_key, secret_key = s3_server['connection']
uri = (
"s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
+ "&allow_bucket_creation=True"
Review Comment:
Do you remember why this was needed?
##########
python/pyarrow/tests/parquet/test_dataset.py:
##########
@@ -569,6 +548,9 @@ def _generate_partition_directories(fs, base_dir,
partition_spec, df):
# partition_spec : list of lists, e.g. [['foo', [0, 1, 2],
# ['bar', ['a', 'b', 'c']]
# part_table : a pyarrow.Table to write to each partition
+ if (not isinstance(fs, FileSystem)):
Review Comment:
```suggestion
if not isinstance(fs, FileSystem):
```
##########
python/pyarrow/tests/parquet/test_dataset.py:
##########
@@ -1009,15 +994,22 @@ def _test_write_to_dataset_no_partitions(base_path,
output_table = pa.Table.from_pandas(output_df)
if filesystem is None:
- filesystem = LocalFileSystem._get_instance()
+ filesystem = LocalFileSystem()
+ elif (not isinstance(filesystem, FileSystem)):
Review Comment:
```suggestion
elif not isinstance(filesystem, FileSystem):
```
##########
python/pyarrow/tests/parquet/test_dataset.py:
##########
@@ -1009,15 +994,22 @@ def _test_write_to_dataset_no_partitions(base_path,
output_table = pa.Table.from_pandas(output_df)
if filesystem is None:
- filesystem = LocalFileSystem._get_instance()
+ filesystem = LocalFileSystem()
+ elif (not isinstance(filesystem, FileSystem)):
+ filesystem = PyFileSystem(FSSpecHandler(filesystem))
# Without partitions, append files to root_path
n = 5
for i in range(n):
pq.write_to_dataset(output_table, base_path,
filesystem=filesystem)
- output_files = [file for file in filesystem.ls(str(base_path))
- if file.endswith(".parquet")]
+
+ selector = FileSelector(str(base_path), allow_not_found=False,
+ recursive=True)
+ assert selector.base_dir == str(base_path)
Review Comment:
```suggestion
```
That line is probably unnecessary?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
