Ngone51 commented on code in PR #36512: URL: https://github.com/apache/spark/pull/36512#discussion_r892317333
########## core/src/main/scala/org/apache/spark/storage/BlockManager.scala: ########## @@ -933,46 +935,56 @@ private[spark] class BlockManager( }) Some(new BlockResult(ci, DataReadMethod.Memory, info.size)) } else if (level.useDisk && diskStore.contains(blockId)) { + var diskData: BlockData = null try { - val diskData = diskStore.getBytes(blockId) - val iterToReturn: Iterator[Any] = { - if (level.deserialized) { - val diskValues = serializerManager.dataDeserializeStream( - blockId, - diskData.toInputStream())(info.classTag) - maybeCacheDiskValuesInMemory(info, blockId, level, diskValues) - } else { - val stream = maybeCacheDiskBytesInMemory(info, blockId, level, diskData) - .map { _.toInputStream(dispose = false) } - .getOrElse { diskData.toInputStream() } - serializerManager.dataDeserializeStream(blockId, stream)(info.classTag) - } + diskData = diskStore.getBytes(blockId) + val iterToReturn = if (level.deserialized) { + val diskValues = serializerManager.dataDeserializeStream( + blockId, + diskData.toInputStream())(info.classTag) + maybeCacheDiskValuesInMemory(info, blockId, level, diskValues) + } else { + val stream = maybeCacheDiskBytesInMemory(info, blockId, level, diskData) + .map { _.toInputStream(dispose = false) } + .getOrElse { diskData.toInputStream() } + serializerManager.dataDeserializeStream(blockId, stream)(info.classTag) } val ci = CompletionIterator[Any, Iterator[Any]](iterToReturn, { releaseLockAndDispose(blockId, diskData, taskContext) }) Some(new BlockResult(ci, DataReadMethod.Disk, info.size)) } catch { - case ex: KryoException if ex.getCause.isInstanceOf[IOException] => - // We need to have detailed log message to catch environmental problems easily. - // Further details: https://issues.apache.org/jira/browse/SPARK-37710 - processKryoException(ex, blockId) - throw ex + case t: Throwable => + if (diskData != null) { + diskData.dispose() Review Comment: Shouldn't we only dispose the block data when we need to remove the block in the case of IO error? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org