attilapiros commented on code in PR #36512: URL: https://github.com/apache/spark/pull/36512#discussion_r880644194
########## core/src/main/scala/org/apache/spark/storage/BlockManager.scala: ########## @@ -933,46 +935,56 @@ private[spark] class BlockManager( }) Some(new BlockResult(ci, DataReadMethod.Memory, info.size)) } else if (level.useDisk && diskStore.contains(blockId)) { + var diskData: BlockData = null try { - val diskData = diskStore.getBytes(blockId) - val iterToReturn: Iterator[Any] = { - if (level.deserialized) { - val diskValues = serializerManager.dataDeserializeStream( - blockId, - diskData.toInputStream())(info.classTag) - maybeCacheDiskValuesInMemory(info, blockId, level, diskValues) - } else { - val stream = maybeCacheDiskBytesInMemory(info, blockId, level, diskData) - .map { _.toInputStream(dispose = false) } - .getOrElse { diskData.toInputStream() } - serializerManager.dataDeserializeStream(blockId, stream)(info.classTag) - } + diskData = diskStore.getBytes(blockId) + val iterToReturn = if (level.deserialized) { + val diskValues = serializerManager.dataDeserializeStream( + blockId, + diskData.toInputStream())(info.classTag) + maybeCacheDiskValuesInMemory(info, blockId, level, diskValues) + } else { + val stream = maybeCacheDiskBytesInMemory(info, blockId, level, diskData) + .map { _.toInputStream(dispose = false) } + .getOrElse { diskData.toInputStream() } + serializerManager.dataDeserializeStream(blockId, stream)(info.classTag) } val ci = CompletionIterator[Any, Iterator[Any]](iterToReturn, { releaseLockAndDispose(blockId, diskData, taskContext) }) Some(new BlockResult(ci, DataReadMethod.Disk, info.size)) } catch { - case ex: KryoException if ex.getCause.isInstanceOf[IOException] => - // We need to have detailed log message to catch environmental problems easily. - // Further details: https://issues.apache.org/jira/browse/SPARK-37710 - processKryoException(ex, blockId) - throw ex + case t: Throwable => + if (diskData != null) { + diskData.dispose() + diskData = null + } + releaseLock(blockId, taskContext) + if (isIORelatedException(t)) { + logInfo(extendMessageWithBlockDetails(t.getMessage, blockId)) + // Remove the block so that its unavailability is reported to the driver + removeBlock(blockId) Review Comment: @Ngone51 is there anything else? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org