viirya commented on a change in pull request #28556:
URL: https://github.com/apache/spark/pull/28556#discussion_r429048929
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
##########
@@ -338,6 +349,93 @@ abstract class SchemaPruningSuite
}
}
+ testSchemaPruning("select one deep nested complex field after repartition") {
+ val query = sql("select * from contacts")
+ .repartition(100)
+ .where("employer.company.address is not null")
+ .selectExpr("employer.id as employer_id")
+ checkScan(query,
+ "struct<employer:struct<id:int,company:struct<address:string>>>")
+ checkAnswer(query, Row(0) :: Nil)
+ }
+
+ testSchemaPruning("select one deep nested complex field after repartition by
expression") {
+ val query1 = sql("select * from contacts")
+ .repartition(100, col("id"))
+ .where("employer.company.address is not null")
+ .selectExpr("employer.id as employer_id")
+ checkScan(query1,
+ "struct<id:int,employer:struct<id:int,company:struct<address:string>>>")
+ checkAnswer(query1, Row(0) :: Nil)
+
+ val query2 = sql("select * from contacts")
+ .repartition(100, col("employer"))
+ .where("employer.company.address is not null")
+ .selectExpr("employer.id as employer_id")
+ checkScan(query2,
+
"struct<employer:struct<id:int,company:struct<name:string,address:string>>>")
+ checkAnswer(query2, Row(0) :: Nil)
+
+ val query3 = sql("select * from contacts")
+ .repartition(100, col("employer.company"))
+ .where("employer.company.address is not null")
+ .selectExpr("employer.company as employer_company")
+ checkScan(query3,
+ "struct<employer:struct<company:struct<name:string,address:string>>>")
+ checkAnswer(query3, Row(Row("abc", "123 Business Street")) :: Nil)
+
+ val query4 = sql("select * from contacts")
+ .repartition(100, col("employer.company.address"))
+ .where("employer.company.address is not null")
+ .selectExpr("employer.company.address as employer_company_addr")
+ checkScan(query4,
+ "struct<employer:struct<company:struct<address:string>>>")
+ checkAnswer(query4, Row("123 Business Street") :: Nil)
+ }
+
+ testSchemaPruning("select one deep nested complex field after join") {
+ val query1 = sql("select contacts.name.middle from contacts, departments
where " +
+ "contacts.id = departments.contactId")
+ checkScan(query1,
+ "struct<id:int,name:struct<middle:string>>",
+ "struct<contactId:int>")
+ checkAnswer(query1, Row("X.") :: Row("Y.") :: Nil)
+
+ val query2 = sql("select contacts.name.middle from contacts, departments
where " +
Review comment:
Seems all tests in this test suite are using lowercases. Changing all
tests seems too bothering... :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
