Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/20292#discussion_r162054765
--- Diff:
core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
---
@@ -547,9 +550,9 @@ class ExternalAppendOnlyMap[K, V, C](
private def cleanup() {
batchIndex = batchOffsets.length // Prevent reading any other batch
val ds = deserializeStream
- if (ds != null) {
- ds.close()
- deserializeStream = null
+ if (ds != null && ds.isEmpty) {
--- End diff --
This check is wrong
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
