Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/21230#discussion_r185986217
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownOperatorsToDataSource.scala
---
@@ -23,49 +23,46 @@ import
org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project
import org.apache.spark.sql.catalyst.rules.Rule
object PushDownOperatorsToDataSource extends Rule[LogicalPlan] {
- override def apply(
- plan: LogicalPlan): LogicalPlan = plan transformUp {
- // PhysicalOperation guarantees that filters are deterministic; no
need to check
- case PhysicalOperation(project, newFilters, relation :
DataSourceV2Relation) =>
- // merge the filters
- val filters = relation.filters match {
- case Some(existing) =>
- existing ++ newFilters
- case _ =>
- newFilters
- }
+ override def apply(plan: LogicalPlan): LogicalPlan = {
+ var pushed = false
+ plan transformDown {
+ // PhysicalOperation guarantees that filters are deterministic; no
need to check
+ case PhysicalOperation(project, filters, relation:
DataSourceV2Relation) if !pushed =>
--- End diff --
Is that possible one plan has multiple `PhysicalOperation`?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
