viirya commented on code in PR #59542:
URL: https://github.com/apache/spark/pull/59542#discussion_r4318670400
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala:
##########
@@ -2569,6 +2581,26 @@ class ParquetIOSuite extends ParquetTest with
SharedSparkSession {
val e = readCorruptedRleSection("id >= 0 as b", writerVersion =
"PARQUET_2_0")(setMaxLength)
assertInvalidLengthReportsColumn(e, "b")
}
+
+ test("SPARK-59854: a ZSTD page that cannot be decompressed reports the
column") {
+ Seq(PageType.DATA_PAGE -> false, PageType.DICTIONARY_PAGE -> true).foreach
{
+ case (pageType, dictionary) =>
+ val options = Map(
+ ParquetOutputFormat.ENABLE_DICTIONARY -> dictionary.toString,
+ "compression" -> "zstd")
+ val e = readCorruptedFile("cast(id % 10 as int) as a", options) {
bytes =>
+ // The first page of the only column chunk starts right after the
magic number.
+ val in = new ByteArrayInputStream(bytes,
ParquetFileWriter.MAGIC.length, bytes.length)
Review Comment:
Done, it now passes `bytes.length - start`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
