mathewjacob1002 commented on code in PR #42118:
URL: https://github.com/apache/spark/pull/42118#discussion_r1274228543
##########
python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py:
##########
@@ -163,6 +168,93 @@ def test_create_torchrun_command_distributed(self) -> None:
)
self.assertEqual(distributed_cmd_args_expected,
distributed_command_with_args)
+def _create_basic_function():
+ def pythagoras(leg1: float, leg2: float):
+ import deepspeed
+ return (leg1 * leg1 + leg2 * leg2)**0.5
+ return pythagoras
+
+from pyspark.ml.torch.tests.test_distributor import get_distributed_mode_conf
+
+class DeepspeedTorchDistributorDistributedEndToEnd(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) =
set_up_test_dirs()
+ print(f"Distributed e2e fname: {cls.gpu_discovery_script_file_name}")
+ conf = SparkConf(loadDefaults=False)
+ print("BEFORE setting anything: ", conf.getAll(), " |||||||||||\n")
+ for k, v in get_distributed_mode_conf().items():
+ if k is "spark.driver.resource.gpu.discoveryScript":
+ raise RuntimeError("Found trying to set the driver discovery
script")
+ conf = conf.set(k, v)
+ conf = conf.set(
+ "spark.worker.resource.gpu.discoveryScript",
cls.gpu_discovery_script_file_name
+ )
+ print("Distributed babby:", conf.getAll())
+ sc = SparkContext("local-cluster[2,2,512]",cls.__name__,conf=conf)
Review Comment:
This is the same as the TorchDistributor tests
(https://github.com/apache/spark/blob/master/python/pyspark/ml/torch/tests/test_distributor.py).
It's just to set up the env so that Spark doesn't complain.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
