Copilot commented on code in PR #55665:
URL: https://github.com/apache/arrow/pull/55665#discussion_r4608935922


##########
python/pyarrow/_dataset.pyx:
##########
@@ -1097,6 +1109,51 @@ cdef class UnionDataset(Dataset):
         return [Dataset.wrap(children[i]) for i in range(children.size())]
 
 
+cdef class FilteredDataset(Dataset):
+    """
+    A Dataset viewing only the rows of a child dataset that match a filter.
+
+    Unlike :meth:`Dataset.filter`, the filter is part of the dataset itself, so
+    it is preserved when this dataset is unioned with datasets that have
+    different filters. Rows for which the filter evaluates to null are dropped.
+
+    Parameters
+    ----------
+    child : Dataset
+        The dataset to filter. Any filter already applied to it with
+        :meth:`Dataset.filter` is combined with ``filter``.
+    filter : Expression
+        A boolean expression referencing fields of the child's schema.
+    """
+
+    cdef:
+        CFilteredDataset* filtered_dataset
+
+    def __init__(self, Dataset child not None, Expression filter not None):
+        current_filter = child._scan_options.get("filter")
+        if current_filter is not None:
+            filter = current_filter & filter
+        self.init(<shared_ptr[CDataset]> GetResultValue(
+            CFilteredDataset.Make(child.wrapped, (<Expression> 
filter).unwrap())))
+
+    cdef void init(self, const shared_ptr[CDataset]& sp):
+        Dataset.init(self, sp)
+        self.filtered_dataset = <CFilteredDataset*> sp.get()
+
+    def __reduce__(self):
+        return FilteredDataset, (self.child, self.filter)
+
+    @property
+    def child(self):
+        """The unfiltered child Dataset."""
+        return Dataset.wrap(self.filtered_dataset.child())
+
+    @property
+    def filter(self):
+        """The filter applied to the child Dataset."""
+        return Expression.wrap(self.filtered_dataset.filter())

Review Comment:
   This property shadows the inherited `Dataset.filter(...)` method, so a 
`FilteredDataset` cannot be filtered again through the normal Dataset API 
(`filtered.filter(expr)` resolves to an Expression and raises instead of 
returning a new dataset). Please expose the stored predicate under a different 
name (for example `filter_expression`) and preserve the inherited `filter` 
method, updating the public test/docs accordingly.



##########
cpp/src/arrow/dataset/dataset.h:
##########
@@ -485,6 +485,67 @@ class ARROW_DS_EXPORT UnionDataset : public Dataset {
   friend class UnionDatasetFactory;
 };
 
+/// \brief A Fragment which wraps another Fragment and applies a filter.
+class ARROW_DS_EXPORT FilteredFragment : public Fragment {
+ public:
+  FilteredFragment(std::shared_ptr<Fragment> fragment, compute::Expression 
filter);
+
+  Result<RecordBatchGenerator> ScanBatchesAsync(
+      const std::shared_ptr<ScanOptions>& options) override;
+
+  Future<std::optional<int64_t>> CountRows(
+      compute::Expression predicate,
+      const std::shared_ptr<ScanOptions>& options) override;
+
+  Status ClearCachedMetadata() override;
+
+  std::string type_name() const override { return "filtered"; }
+
+  std::string ToString() const override;
+
+  const std::shared_ptr<Fragment>& fragment() const { return fragment_; }
+
+  const compute::Expression& filter() const { return filter_; }
+
+ protected:
+  Result<std::shared_ptr<Schema>> ReadPhysicalSchemaImpl() override;
+
+  std::shared_ptr<Fragment> fragment_;
+  compute::Expression filter_;
+};
+
+/// \brief A Dataset which wraps another Dataset and applies a filter.
+///
+/// Unlike a filter in ScanOptions, the filter is part of the Dataset itself, 
so it
+/// is preserved when the Dataset is combined with others, e.g. as a child of a
+/// UnionDataset whose other children have different filters.
+class ARROW_DS_EXPORT FilteredDataset : public Dataset {

Review Comment:
   `FilteredDataset` (and the new exported `FilteredFragment`) are public C++ 
types, but neither is declared in `arrow/dataset/type_fwd.h`. Consumers that 
use the forward-declaration API cannot name these new types without including 
the full definitions, unlike `UnionDataset` and `InMemoryDataset`; please add 
the corresponding forward declarations there.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to