joelrobin18 commented on code in PR #58907:
URL: https://github.com/apache/spark/pull/58907#discussion_r4068737692
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala:
##########
@@ -582,8 +593,24 @@ class CacheManager extends Logging with
AdaptiveSparkPlanHelper {
* `HadoopFsRelation` node(s) as part of its logical plan.
*/
def recacheByPath(spark: SparkSession, resourcePath: Path, fs: FileSystem):
Unit = {
+ recacheByPath(spark, resourcePath, fs, includeTimeTravel = true)
+ }
+
+ /**
+ * Tries to re-cache matching entries whose logical plans contain a
file-backed relation under
+ * `resourcePath`. If `includeTimeTravel` is false, immutable time-travel
relations are ignored
+ * when deciding whether an entry must be re-cached; another matching live
relation in the same
+ * plan can still trigger re-caching.
+ */
+ def recacheByPath(
+ spark: SparkSession,
+ resourcePath: Path,
+ fs: FileSystem,
+ includeTimeTravel: Boolean): Unit = {
val qualifiedPath = fs.makeQualified(resourcePath)
- recacheByCondition(spark, _.plan.exists(lookupAndRefresh(_, fs,
qualifiedPath)))
+ recacheByCondition(
+ spark,
+ _.plan.exists(lookupAndRefresh(_, fs, qualifiedPath, includeTimeTravel)))
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
