[arrow-datafusion] branch main updated: Unnecessary to list all files during partition pruning (#7395)

[alamb]

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 980557e2ef Unnecessary to list all files during partition pruning 
(#7395)
980557e2ef is described below

commit 980557e2ef0720a1e32afa07a20edb2d4bacd718
Author: jokercurry <982458...@qq.com>
AuthorDate: Thu Aug 24 21:38:33 2023 +0800

    Unnecessary to list all files during partition pruning (#7395)
    
    * Unnecessary to list all files during partition pruning
    
    * nit
    
    * nit
    
    ---------
    
    Co-authored-by: zhongjingxiong <zhongjingxi...@bytedance.com>
---
 datafusion/core/src/datasource/listing/helpers.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/datafusion/core/src/datasource/listing/helpers.rs 
b/datafusion/core/src/datasource/listing/helpers.rs
index efafd7d2c3..476c58b698 100644
--- a/datafusion/core/src/datasource/listing/helpers.rs
+++ b/datafusion/core/src/datasource/listing/helpers.rs
@@ -321,11 +321,13 @@ pub async fn pruned_partition_list<'a>(
     file_extension: &'a str,
     partition_cols: &'a [(String, DataType)],
 ) -> Result<BoxStream<'a, Result<PartitionedFile>>> {
-    let list = table_path.list_all_files(store, file_extension);
-
     // if no partition col => simply list all the files
     if partition_cols.is_empty() {
-        return Ok(Box::pin(list.map_ok(|object_meta| object_meta.into())));
+        return Ok(Box::pin(
+            table_path
+                .list_all_files(store, file_extension)
+                .map_ok(|object_meta| object_meta.into()),
+        ));
     }
 
     let partitions = list_partitions(store, table_path, 
partition_cols.len()).await?;