pan3793 commented on code in PR #58702:
URL: https://github.com/apache/spark/pull/58702#discussion_r4047887121
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryPartitionPredicateDeleteTable.scala:
##########
@@ -107,6 +113,72 @@ class InMemoryPartitionPredicateDeleteTable(
}
}
+ /**
+ * Row-level scans push V2 predicates iteratively, so a group-based
operation receives a
+ * second-pass [[PartitionPredicate]] the same way a metadata-only DELETE
does. Only partition
+ * predicates prune, by partition key; a data predicate is always returned
since the scan
+ * cannot filter rows.
+ */
+ override protected def newRowLevelScanBuilder(
+ options: CaseInsensitiveStringMap)(
+ onBuild: BatchScanBaseClass => Unit): ScanBuilder = {
+ new PartitionPredicateRowLevelScanBuilder(onBuild)
+ }
+
+ class PartitionPredicateRowLevelScanBuilder(onBuild: BatchScanBaseClass =>
Unit)
+ extends ScanBuilder with SupportsPushDownV2Filters with
SupportsPushDownRequiredColumns {
+
+ private var readSchema: StructType = schema
+ private val pushed = ArrayBuffer.empty[Predicate]
+
+ override def supportsIterativePushdown(): Boolean = true
+
+ override def pushPredicates(predicates: Array[Predicate]):
Array[Predicate] = {
+ val (accepted, returned) = predicates.partition {
+ case _: PartitionPredicate => acceptPartitionPredicates
+ case p => refsOnlyPartCols(p) &&
InMemoryTableWithV2Filter.supportsPredicates(Array(p))
Review Comment:
Good catch, fixed in f46e5503. `partPaths` now comes from identity
transforms only, so on `(dep, bucket(4, pk))` a filter on `pk` is a data filter
and the metadata-only path declines it instead of comparing `pk` with a bucket
value. Test added: `DELETE ... WHERE pk = 5`, which previously matched nothing
since `bucket(4, 5)` is 1.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
