This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new d235f69 ARROW-5436: [Python] parquet.read_table add filters keyword d235f69 is described below commit d235f69bc7ebfbbd03f031b2191c69244861cf4f Author: Joris Van den Bossche <jorisvandenboss...@gmail.com> AuthorDate: Thu Jun 6 11:35:34 2019 -0400 ARROW-5436: [Python] parquet.read_table add filters keyword https://issues.apache.org/jira/browse/ARROW-5436 I suppose the fact that `parquet.read_table` dispatched to FileSystem.read_parquet was for historical reasons (that function was added before ParquetDataset was added), but directly calling ParquetDataset there looks cleaner instead of going through FileSystem.read_parquet. So therefore I also changed that. In addition, I made sure the `memory_map` keyword was actually passed through, I think an oversight of https://github.com/apache/arrow/pull/2954. (those two changes should be useful anyway, regardless of adding `filters` keyword or not) Author: Joris Van den Bossche <jorisvandenboss...@gmail.com> Closes #4409 from jorisvandenbossche/ARROW-5436-parquet-read_table and squashes the following commits: 85e5b0e1 <Joris Van den Bossche> lint 0ae1488d <Joris Van den Bossche> add test with nested list 9baf420b <Joris Van den Bossche> add filters to read_pandas 0df8c881 <Joris Van den Bossche> Merge remote-tracking branch 'upstream/master' into ARROW-5436-parquet-read_table 4ea7b77d <Joris Van den Bossche> fix test 4eb2ea7f <Joris Van den Bossche> add filters keyword 9c10f700 <Joris Van den Bossche> fix passing of memory_map (leftover from ARROW-2807) 896abb2a <Joris Van den Bossche> simplify read_table (use ParquetDataset directly) --- python/pyarrow/parquet.py | 20 ++++++++++++-------- python/pyarrow/tests/test_parquet.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index d44deee..34a9c42 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -1181,6 +1181,11 @@ memory_map : boolean, default True If the source is a file path, use a memory map to read file, which can improve performance in some environments {1} +filters : List[Tuple] or List[List[Tuple]] or None (default) + List of filters to apply, like ``[[('x', '=', 0), ...], ...]``. This + implements partition-level (hive) filtering only, i.e., to prevent the + loading of some files of the dataset if `source` is a directory. + See the docstring of ParquetDataset for more details. Returns ------- @@ -1190,14 +1195,12 @@ Returns def read_table(source, columns=None, use_threads=True, metadata=None, use_pandas_metadata=False, memory_map=True, - filesystem=None): + filesystem=None, filters=None): if _is_path_like(source): - fs, path = _get_filesystem_and_path(filesystem, source) - return fs.read_parquet(path, columns=columns, - use_threads=use_threads, metadata=metadata, - use_pandas_metadata=use_pandas_metadata) - - pf = ParquetFile(source, metadata=metadata) + pf = ParquetDataset(source, metadata=metadata, memory_map=memory_map, + filesystem=filesystem, filters=filters) + else: + pf = ParquetFile(source, metadata=metadata, memory_map=memory_map) return pf.read(columns=columns, use_threads=use_threads, use_pandas_metadata=use_pandas_metadata) @@ -1212,10 +1215,11 @@ read_table.__doc__ = _read_table_docstring.format( def read_pandas(source, columns=None, use_threads=True, memory_map=True, - metadata=None): + metadata=None, filters=None): return read_table(source, columns=columns, use_threads=use_threads, metadata=metadata, memory_map=True, + filters=filters, use_pandas_metadata=True) diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 76ec864..a97e885 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1523,6 +1523,38 @@ def test_invalid_pred_op(tempdir): ]) +@pytest.mark.pandas +def test_filters_read_table(tempdir): + # test that filters keyword is passed through in read_table + fs = LocalFileSystem.get_instance() + base_path = tempdir + + integer_keys = [0, 1, 2, 3, 4] + partition_spec = [ + ['integers', integer_keys], + ] + N = 5 + + df = pd.DataFrame({ + 'index': np.arange(N), + 'integers': np.array(integer_keys, dtype='i4'), + }, columns=['index', 'integers']) + + _generate_partition_directories(fs, base_path, partition_spec, df) + + table = pq.read_table( + base_path, filesystem=fs, filters=[('integers', '<', 3)]) + assert table.num_rows == 3 + + table = pq.read_table( + base_path, filesystem=fs, filters=[[('integers', '<', 3)]]) + assert table.num_rows == 3 + + table = pq.read_pandas( + base_path, filters=[('integers', '<', 3)]) + assert table.num_rows == 3 + + @pytest.yield_fixture def s3_example(): access_key = os.environ['PYARROW_TEST_S3_ACCESS_KEY']