HyukjinKwon commented on code in PR #46570:
URL: https://github.com/apache/spark/pull/46570#discussion_r1600949983
##########
python/pyspark/sql/tests/connect/test_connect_basic.py:
##########
@@ -1358,6 +1359,37 @@ def test_verify_col_name(self):
self.assertTrue(verify_col_name("`m```.`s.s`.v", cdf.schema))
self.assertTrue(verify_col_name("`m```.`s.s`.`v`", cdf.schema))
+ def test_garbage_collection_checkpoint(self):
+ """
+ SPARK-48258: Make sure garbage-collecting DataFrame remove the paired
state
+ in Spark Connect server
+ """
+ df = self.connect.range(10).localCheckpoint()
+ self.assertIsNotNone(df._cached_remote_relation_id)
+ cached_remote_relation_id = df._cached_remote_relation_id
+
+ jvm = self.spark._jvm
+ session_holder = getattr(
+ getattr(
+ jvm.org.apache.spark.sql.connect.service,
+ "SparkConnectService$",
+ ),
+ "MODULE$",
+ ).getOrCreateIsolatedSession(self.connect.client._user_id,
self.connect.client._session_id)
+
+ # Check the state exists.
+ self.assertIsNotNone(
+
session_holder.dataFrameCache().getOrDefault(cached_remote_relation_id, None)
+ )
+
+ del df
+
+ time.sleep(3) # Make sure removing is triggered, and executed in the
server.
Review Comment:
This is fine - test-only. It waits for client to send to remove request to
the server (and the server to remove the state).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
