jorisvandenbossche commented on a change in pull request #8367:
URL: https://github.com/apache/arrow/pull/8367#discussion_r501100412
##########
File path: python/pyarrow/_dataset.pyx
##########
@@ -1403,34 +1412,42 @@ cdef class HivePartitioning(Partitioning):
self.hive_partitioning = <CHivePartitioning*> sp.get()
@staticmethod
- def discover(object max_partition_dictionary_size=0):
+ def discover(infer_dictionary=False, max_partition_dictionary_size=0):
"""
Discover a HivePartitioning.
- Params
- ------
- max_partition_dictionary_size : int or None, default 0
- The maximum number of unique values to consider for dictionary
- encoding. By default no field will be inferred as dictionary
- encoded. If -1 is provided dictionary encoding will be used for
- every string field.
+ Parameters
+ ----------
+ infer_dictionary : bool, default False
+ When inferring a schema for partition fields, yield dictionary
+ encoded types instead of plain. This can be more efficient when
+ materializing virtual columns, and Expressions parsed by the
+ finished Partitioning will include dictionaries of all unique
+ inspected values for each field.
+ max_partition_dictionary_size : int, default 0
+ Synonymous with infer_dictionary for backwards compatibility with
+ 1.0: setting this to -1 or None is equivalent to passing
+ infer_dictionary=True.
Returns
-------
PartitioningFactory
To be used in the FileSystemFactoryOptions.
"""
cdef:
- CPartitioningFactoryOptions options
+ CPartitioningFactoryOptions c_options
- if max_partition_dictionary_size is None:
- max_partition_dictionary_size = -1
+ if max_partition_dictionary_size in {-1, None}:
+ infer_dictionary = True
+ else if max_partition_dictionary_size != 0:
Review comment:
```suggestion
elif max_partition_dictionary_size != 0:
```
##########
File path: python/pyarrow/_dataset.pyx
##########
@@ -1321,38 +1321,47 @@ cdef class DirectoryPartitioning(Partitioning):
self.directory_partitioning = <CDirectoryPartitioning*> sp.get()
@staticmethod
- def discover(field_names, object max_partition_dictionary_size=0):
+ def discover(field_names, infer_dictionary=False,
+ max_partition_dictionary_size=0):
"""
Discover a DirectoryPartitioning.
Parameters
----------
field_names : list of str
The names to associate with the values from the subdirectory names.
- max_partition_dictionary_size : int or None, default 0
- The maximum number of unique values to consider for dictionary
- encoding. By default no field will be inferred as dictionary
- encoded. If None is provided dictionary encoding will be used for
- every string field.
+ infer_dictionary : bool, default False
+ When inferring a schema for partition fields, yield dictionary
+ encoded types instead of plain types. This can be more efficient
+ when materializing virtual columns, and Expressions parsed by the
+ finished Partitioning will include dictionaries of all unique
+ inspected values for each field.
+ max_partition_dictionary_size : int, default 0
+ Synonymous with infer_dictionary for backwards compatibility with
+ 1.0: setting this to -1 or None is equivalent to passing
+ infer_dictionary=True.
Returns
-------
DirectoryPartitioningFactory
To be used in the FileSystemFactoryOptions.
"""
cdef:
- CPartitioningFactoryOptions options
+ CPartitioningFactoryOptions c_options
vector[c_string] c_field_names
- if max_partition_dictionary_size is None:
- max_partition_dictionary_size = -1
+ if max_partition_dictionary_size in {-1, None}:
+ infer_dictionary = True
+ else if max_partition_dictionary_size != 0:
Review comment:
```suggestion
elif max_partition_dictionary_size != 0:
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
