Re: [PR] chore: [native scans] Ignore Spark SQL test for string predicate pushdown [datafusion-comet]
andygrove merged PR #1768: URL: https://github.com/apache/datafusion-comet/pull/1768 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] - To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
Re: [PR] chore: [native scans] Ignore Spark SQL test for string predicate pushdown [datafusion-comet]
andygrove commented on PR #1768: URL: https://github.com/apache/datafusion-comet/pull/1768#issuecomment-2919781336 Could I get a review @parthchandra @mbutrovich -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] - To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
Re: [PR] chore: [native scans] Ignore Spark SQL test for string predicate pushdown [datafusion-comet]
andygrove commented on code in PR #1768:
URL: https://github.com/apache/datafusion-comet/pull/1768#discussion_r2114221607
##
spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala:
##
@@ -114,92 +95,91 @@ class CometStringExpressionSuite extends CometTestBase {
}
test("InitCap") {
-Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
-val table = "names"
-withTable(table) {
- sql(s"create table $table(id int, name varchar(20)) using parquet")
- sql(
-s"insert into $table values(1, 'james smith'), (2, 'michael
rose'), " +
- "(3, 'robert williams'), (4, 'rames rose'), (5, 'james smith'),
" +
- "(6, 'robert rose-smith'), (7, 'james ähtäri')")
- if (CometConf.COMET_EXEC_INITCAP_ENABLED.get()) {
-// TODO: remove this if clause
https://github.com/apache/datafusion-comet/issues/1052
-checkSparkAnswerAndOperator(s"SELECT initcap(name) FROM $table")
- } else {
-checkSparkAnswer(s"SELECT initcap(name) FROM $table")
- }
-}
+val table = "names"
+withTable(table) {
+ sql(s"create table $table(id int, name varchar(20)) using parquet")
+ sql(
+s"insert into $table values(1, 'james smith'), (2, 'michael rose'), " +
+ "(3, 'robert williams'), (4, 'rames rose'), (5, 'james smith'), " +
+ "(6, 'robert rose-smith'), (7, 'james ähtäri')")
+ if (CometConf.COMET_EXEC_INITCAP_ENABLED.get()) {
+// TODO: remove this if clause
https://github.com/apache/datafusion-comet/issues/1052
+checkSparkAnswerAndOperator(s"SELECT initcap(name) FROM $table")
+ } else {
+checkSparkAnswer(s"SELECT initcap(name) FROM $table")
}
}
}
test("trim") {
-Seq(false, true).foreach { dictionary =>
- withSQLConf(
-"parquet.enable.dictionary" -> dictionary.toString,
-CometConf.COMET_CASE_CONVERSION_ENABLED.key -> "true") {
-val table = "test"
-withTable(table) {
- sql(s"create table $table(col varchar(20)) using parquet")
- sql(s"insert into $table values('SparkSQL '), ('SSparkSQLS')")
+withSQLConf(CometConf.COMET_CASE_CONVERSION_ENABLED.key -> "true") {
+ val table = "test"
+ withTable(table) {
+sql(s"create table $table(col varchar(20)) using parquet")
+sql(s"insert into $table values('SparkSQL '), ('SSparkSQLS')")
- checkSparkAnswerAndOperator(s"SELECT upper(trim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT trim('SL', col) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT upper(trim(col)) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT trim('SL', col) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT upper(btrim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT btrim('SL', col) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT upper(btrim(col)) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT btrim('SL', col) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT upper(ltrim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT ltrim('SL', col) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT upper(ltrim(col)) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT ltrim('SL', col) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT upper(rtrim(col)) FROM $table")
- checkSparkAnswerAndOperator(s"SELECT rtrim('SL', col) FROM $table")
-}
+checkSparkAnswerAndOperator(s"SELECT upper(rtrim(col)) FROM $table")
+checkSparkAnswerAndOperator(s"SELECT rtrim('SL', col) FROM $table")
}
}
}
test("string concat_ws") {
-Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
-val table = "names"
-withTable(table) {
- sql(
-s"create table $table(id int, first_name varchar(20),
middle_initial char(1), last_name varchar(20)) using parquet")
- sql(
-s"insert into $table values(1, 'James', 'B', 'Taylor'), (2,
'Smith', 'C', 'Davis')," +
- " (3, NULL, NULL, NULL), (4, 'Smith', 'C', 'Davis')")
- checkSparkAnswerAndOperator(
-s"SELECT concat_ws(' ', first_name, middle_initial, last_name)
FROM $table")
-}
- }
+val table = "names"
+withTable(table) {
+ sql(
+s"create table $table(id int, first_name varchar(20), middle_initial
char(1), last_name varchar(20)) using parquet")
+ sql(
+s"insert into $table values(1, 'James', 'B', 'Taylor'), (2, 'Smith',
'C', 'Davis')," +
+ " (3, NULL, NULL, NULL), (4, 'Smith', 'C', 'Davis')")
+ checkSparkAnswerAndOperator(
+s"SELECT concat_ws(' ', first_name, middle_initial, l
Re: [PR] chore: [native scans] Ignore Spark SQL test for string predicate pushdown [datafusion-comet]
andygrove commented on code in PR #1768:
URL: https://github.com/apache/datafusion-comet/pull/1768#discussion_r2114220564
##
spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala:
##
@@ -20,87 +20,68 @@
package org.apache.comet
import org.apache.spark.sql.CometTestBase
+import org.apache.spark.sql.internal.SQLConf
class CometStringExpressionSuite extends CometTestBase {
test("Various String scalar functions") {
-Seq(false, true).foreach { dictionary =>
- withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
Review Comment:
This is an unrelated change but none of the tests in this suite create data
that would use dictionary encoding (they just insert a few rows and there are
no duplicates) so it is pointless running all of these tests with and without
dictionary encoding enabled.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
-
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
Re: [PR] chore: [native scans] Ignore Spark SQL test for string predicate pushdown [datafusion-comet]
parthchandra commented on code in PR #1768:
URL: https://github.com/apache/datafusion-comet/pull/1768#discussion_r2112650217
##
spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala:
##
@@ -1330,6 +1330,25 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
}
+ // based on Spark SQL ParquetFilterSuite test "filter pushdown -
StringPredicate"
+ test("string predicate filter") {
Review Comment:
> Shouldn't this be in Comet itself?
Uh, nvm.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
-
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
Re: [PR] chore: [native scans] Ignore Spark SQL test for string predicate pushdown [datafusion-comet]
parthchandra commented on code in PR #1768:
URL: https://github.com/apache/datafusion-comet/pull/1768#discussion_r2105360116
##
spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala:
##
@@ -1330,6 +1330,25 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
}
+ // based on Spark SQL ParquetFilterSuite test "filter pushdown -
StringPredicate"
+ test("string predicate filter") {
Review Comment:
Shouldn't this be in Comet itself?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
-
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
