Github user mgaido91 commented on a diff in the pull request:
https://github.com/apache/spark/pull/22621#discussion_r222320450
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala ---
@@ -250,7 +250,9 @@ abstract class SparkPlan extends QueryPlan[SparkPlan]
with Logging with Serializ
val codec = CompressionCodec.createCodec(SparkEnv.get.conf)
val bos = new ByteArrayOutputStream()
val out = new DataOutputStream(codec.compressedOutputStream(bos))
- while (iter.hasNext && (n < 0 || count < n)) {
+ // `iter.hasNext` may produce one row and buffer it, we should only
call it when the limit is
+ // not hit.
+ while ((n < 0 || count < n) && iter.hasNext) {
--- End diff --
nice catch this one!
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
