bersprockets commented on a change in pull request #30221:
URL: https://github.com/apache/spark/pull/30221#discussion_r516771531
##########
File path:
external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
##########
@@ -2005,3 +2027,61 @@ class AvroV2Suite extends AvroSuite with
ExplainSuiteHelper {
}
}
}
+
+class AvroRowReaderSuite
+ extends QueryTest
+ with SharedSparkSession {
+
+ import testImplicits._
+
+ override protected def sparkConf: SparkConf =
+ super
+ .sparkConf
+ .set(SQLConf.USE_V1_SOURCE_LIST, "") // need this for BatchScanExec
+
+ test("SPARK-33314: hasNextRow and nextRow properly handle consecutive
calls") {
+ withTempPath { dir =>
+ Seq((1), (2), (3))
+ .toDF("value")
+ .coalesce(1)
+ .write
+ .format("avro")
+ .save(dir.getCanonicalPath)
+
+ val df = spark.read.format("avro").load(dir.getCanonicalPath)
+ val fileScan = df.queryExecution.executedPlan collectFirst {
+ case BatchScanExec(_, f: AvroScan) => f
+ }
+ val filePath = fileScan.get.fileIndex.inputFiles(0)
+ val fileSize = new File(new URI(filePath)).length
+ val in = new FsInput(new Path(new URI(filePath)), new Configuration())
+ val reader = DataFileReader.openReader(in, new
GenericDatumReader[GenericRecord]())
+
+ val it = new Iterator[InternalRow] with AvroUtils.RowReader {
+ override val fileReader = reader
+ override val deserializer = new AvroDeserializer(
+ reader.getSchema,
+ StructType(new StructField("value", IntegerType, true) :: Nil),
+ CORRECTED,
+ new NoopFilters)
+ override val stopPosition = 0 + fileSize
Review comment:
OK, that's kind of dumb. I will fix it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
