[spark] branch master updated: [SPARK-44402][SQL][TESTS] Add tests for schema pruning in delta-based DELETEs

[dongjoon]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new af27400bd1f [SPARK-44402][SQL][TESTS] Add tests for schema pruning in 
delta-based DELETEs
af27400bd1f is described below

commit af27400bd1f674c59564eb66570209a783859f6e
Author: aokolnychyi <aokolnyc...@apple.com>
AuthorDate: Thu Jul 13 10:18:51 2023 -0700

    [SPARK-44402][SQL][TESTS] Add tests for schema pruning in delta-based 
DELETEs
    
    ### What changes were proposed in this pull request?
    
    This PR adds tests for schema pruning for delta-based DELETEs, similar to 
the ones we have for MERGE and UPDATE.
    
    ### Why are the changes needed?
    
    These changes are needed to verify schema pruning works as expected.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    This PR comes with tests.
    
    Closes #41975 from aokolnychyi/spark-44402.
    
    Authored-by: aokolnychyi <aokolnyc...@apple.com>
    Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
 .../connector/DeltaBasedDeleteFromTableSuite.scala  | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeltaBasedDeleteFromTableSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeltaBasedDeleteFromTableSuite.scala
index 4da85a5ce05..7336b3a6e92 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeltaBasedDeleteFromTableSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeltaBasedDeleteFromTableSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector
 
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, Row}
 
 class DeltaBasedDeleteFromTableSuite extends DeleteFromTableSuiteBase {
 
@@ -59,4 +59,23 @@ class DeltaBasedDeleteFromTableSuite extends 
DeleteFromTableSuiteBase {
     }
     assert(exception.message.contains("Row ID attributes cannot be nullable"))
   }
+
+  test("delete with schema pruning") {
+    createAndInitTable("pk INT NOT NULL, id INT, country STRING, dep STRING",
+      """{ "pk": 1, "id": 1, "country": "uk", "dep": "hr" }
+        |{ "pk": 2, "id": 2, "country": "us", "dep": "software" }
+        |{ "pk": 3, "id": 3, "country": "canada", "dep": "hr" }
+        |""".stripMargin)
+
+    executeAndCheckScan(
+      s"DELETE FROM $tableNameAsString WHERE id <= 1",
+      // `pk` is used to encode deletes
+      // `id` is used in the condition
+      // `_partition` is used in the requested write distribution
+      expectedScanSchema = "pk INT, id INT, _partition STRING")
+
+    checkAnswer(
+      sql(s"SELECT * FROM $tableNameAsString"),
+      Row(2, 2, "us", "software") :: Row(3, 3, "canada", "hr") :: Nil)
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org