Github user dbtsai commented on a diff in the pull request:
https://github.com/apache/spark/pull/21322#discussion_r188035443
--- Diff:
core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala ---
@@ -526,4 +526,84 @@ class MemoryStoreSuite
}
}
}
+
+ test("[SPARK-24225]: remove should close AutoCloseable object") {
+
+ val (store, _) = makeMemoryStore(12000)
+
+ val id = BroadcastBlockId(0)
+ val tracker = new CloseTracker()
+ store.putIteratorAsValues(id, Iterator(tracker), ClassTag.Any)
+ assert(store.contains(id))
+ store.remove(id)
+ assert(tracker.getClosed())
+ }
+
+ test("[SPARK-24225]: remove should close AutoCloseable objects even if
they throw exceptions") {
+
+ val (store, _) = makeMemoryStore(12000)
+
+ val id = BroadcastBlockId(0)
+ val tracker = new CloseTracker(true)
+ store.putIteratorAsValues(id, Iterator(tracker), ClassTag.Any)
+ assert(store.contains(id))
+ store.remove(id)
+ assert(tracker.getClosed())
+ }
+
+ test("[SPARK-24225]: clear should close AutoCloseable objects") {
+
+ val (store, _) = makeMemoryStore(12000)
+
+ val id = BroadcastBlockId(0)
+ val tracker = new CloseTracker
+ store.putIteratorAsValues(id, Iterator(tracker), ClassTag.Any)
+ assert(store.contains(id))
+ store.clear()
+ assert(tracker.getClosed())
+ }
+
+ test("[SPARK-24225]: clear should close all AutoCloseable objects put
together in an iterator") {
--- End diff --
Can you check if you have multiple autocloseable objects in an iterator,
and only one of them is removed, the rests of the objects will not be closed?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
