kazuyukitanimura commented on code in PR #3236:
URL: https://github.com/apache/datafusion-comet/pull/3236#discussion_r2719196124
##########
spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala:
##########
@@ -201,38 +202,19 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
test("uint data type support") {
- Seq(true, false).foreach { dictionaryEnabled =>
- // TODO: Once the question of what to get back from uint_8, uint_16
types is resolved,
- // we can also update this test to check for
COMET_SCAN_ALLOW_INCOMPATIBLE=true
- Seq(false).foreach { allowIncompatible =>
- {
- withSQLConf(CometConf.COMET_SCAN_ALLOW_INCOMPATIBLE.key ->
allowIncompatible.toString) {
- withTempDir { dir =>
- val path = new Path(dir.toURI.toString, "testuint.parquet")
- makeParquetFileAllPrimitiveTypes(
- path,
- dictionaryEnabled = dictionaryEnabled,
- Byte.MinValue,
- Byte.MaxValue)
- withParquetTable(path.toString, "tbl") {
- val qry = "select _9 from tbl order by _11"
- if (usingDataSourceExec(conf)) {
- if (!allowIncompatible) {
- checkSparkAnswerAndOperator(qry)
- } else {
- // need to convert the values to unsigned values
- val expected = (Byte.MinValue to Byte.MaxValue)
- .map(v => {
- if (v < 0) Byte.MaxValue.toShort - v else v
- })
- .toDF("a")
- checkAnswer(sql(qry), expected)
- }
- } else {
- checkSparkAnswerAndOperator(qry)
- }
- }
- }
+ // this test requires native_comet scan due to unsigned u8/u16 issue
+ withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key ->
CometConf.SCAN_NATIVE_COMET) {
+ Seq(true, false).foreach { dictionaryEnabled =>
+ withTempDir { dir =>
+ val path = new Path(dir.toURI.toString, "testuint.parquet")
+ makeParquetFileAllPrimitiveTypes(
+ path,
+ dictionaryEnabled = dictionaryEnabled,
+ Byte.MinValue,
+ Byte.MaxValue)
+ withParquetTable(path.toString, "tbl") {
+ val qry = "select _9 from tbl order by _11"
Review Comment:
Is `_9 ` uint 32?
##########
spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala:
##########
@@ -187,11 +187,12 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
test("basic data type support") {
- Seq(true, false).foreach { dictionaryEnabled =>
- withTempDir { dir =>
- val path = new Path(dir.toURI.toString, "test.parquet")
- makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled =
dictionaryEnabled, 10000)
- withSQLConf(CometConf.COMET_SCAN_ALLOW_INCOMPATIBLE.key -> "false") {
+ // this test requires native_comet scan due to unsigned u8/u16 issue
Review Comment:
Should we divert the scan type to native_comet only for u8/u16? So that the
rest of the types can be tested for iceberg_compat?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
