cloud-fan commented on a change in pull request #31319: URL: https://github.com/apache/spark/pull/31319#discussion_r564704944
########## File path: sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala ########## @@ -3868,6 +3869,48 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark assert(unions.size == 1) } + + test("SPARK-34212 Parquet should read decimals correctly") { + // a is int-decimal (4 bytes), b is long-decimal (8 bytes), c is binary-decimal (16 bytes) + val df = sql("SELECT 1.0 a, CAST(1.23 AS DECIMAL(17, 2)) b, CAST(1.23 AS DECIMAL(36, 2)) c") + + withTempPath { path => + df.write.parquet(path.toString) + + withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") { + val schema = "a DECIMAL(3, 2), b DECIMAL(18, 3)" + checkAnswer(spark.read.schema(schema).parquet(path.toString), df.select("a", "b")) + } + + withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") { + val e1 = intercept[SparkException] { + spark.read.schema("a DECIMAL(3, 2)").parquet(path.toString).collect() + }.getCause.getCause + assert(e1.isInstanceOf[SchemaColumnConvertNotSupportedException]) + + val e2 = intercept[SparkException] { + spark.read.schema("b DECIMAL(18, 1)").parquet(path.toString).collect() + }.getCause.getCause + assert(e2.isInstanceOf[SchemaColumnConvertNotSupportedException]) + + val e3 = intercept[SparkException] { + spark.read.schema("c DECIMAL(37, 1)").parquet(path.toString).collect() + }.getCause.getCause + assert(e3.isInstanceOf[SchemaColumnConvertNotSupportedException]) + } + } + + withTempPath { path => + sql("SELECT 1 a").write.parquet(path.toString) Review comment: can we test row-based reader here too? This can trigger the NPE before your recent fix. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org