ivoson commented on code in PR #39459: URL: https://github.com/apache/spark/pull/39459#discussion_r1118028518
########## core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala: ########## @@ -2266,6 +2270,160 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with PrivateMethodTe } } + test("SPARK-41497: getOrElseUpdateRDDBlock do compute based on cache visibility statue") { + val store = makeBlockManager(8000, "executor1") + val blockId = RDDBlockId(rddId = 1, splitIndex = 1) + var computed: Boolean = false + val data = Seq(1, 2, 3) + val makeIterator = () => { + computed = true + data.iterator + } + + // Cache doesn't exist and is not visible. + assert(store.getStatus(blockId).isEmpty && !store.isRDDBlockVisible(blockId)) + val res1 = store.getOrElseUpdateRDDBlock( + 1, blockId, StorageLevel.MEMORY_ONLY, classTag[Int], makeIterator) + // Put cache successfully and reported block task info. + assert(res1.isLeft && computed) + verify(master, times(1)).updateRDDBlockTaskInfo(blockId, 1) + + // Cache exists but not visible. + computed = false + assert(store.getStatus(blockId).nonEmpty && !store.isRDDBlockVisible(blockId)) + val res2 = store.getOrElseUpdateRDDBlock( + 1, blockId, StorageLevel.MEMORY_ONLY, classTag[Int], makeIterator) + // Load cache successfully and reported block task info. + assert(res2.isLeft && computed) + verify(master, times(2)).updateRDDBlockTaskInfo(blockId, 1) Review Comment: HI @mridulm actually this is to make sure that task2blockId information will be reported only when the block is invisble as this is part of the work flow design. If this doesn't make sense, I can remove `verify` statement in a new iteration. Thanks. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org