wForget opened a new issue, #2581: URL: https://github.com/apache/uniffle/issues/2581
### Code of Conduct - [x] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) ### Search before asking - [x] I have searched in the [issues](https://github.com/apache/incubator-uniffle/issues?q=is%3Aissue) and found no similar issues. ### Describe the bug When creating a SparkContext fails, new SparkContexts are created repeatedly. ``` 25/08/13 15:01:31 INFO SparkContext: Successfully stopped SparkContext 25/08/13 15:01:31 ERROR Utils: Uncaught exception in thread Driver java.lang.IllegalArgumentException: requirement failed: initial executor number 1000 must between min executor number 0 and max executor number 500 at scala.Predef$.require(Predef.scala:281) at org.apache.spark.scheduler.cluster.SchedulerBackendUtils$.getInitialTargetExecutorNumber(SchedulerBackendUtils.scala:39) at org.apache.spark.scheduler.cluster.YarnClusterSchedulerBackend.start(YarnClusterSchedulerBackend.scala:34) at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:235) at org.apache.spark.SparkContext.<init>(SparkContext.scala:604) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2888) at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:1099) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:1093) at org.apache.kyuubi.engine.spark.SparkSQLEngine$.createSpark(SparkSQLEngine.scala:303) at org.apache.kyuubi.engine.spark.SparkSQLEngine$.main(SparkSQLEngine.scala:377) at org.apache.kyuubi.engine.spark.SparkSQLEngine.main(SparkSQLEngine.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:738) 25/08/13 15:01:31 INFO SparkContext: Successfully stopped SparkContext 25/08/13 15:01:31 ERROR Utils: Uncaught exception in thread Driver java.lang.IllegalArgumentException: requirement failed: initial executor number 1000 must between min executor number 0 and max executor number 500 at scala.Predef$.require(Predef.scala:281) at org.apache.spark.scheduler.cluster.SchedulerBackendUtils$.getInitialTargetExecutorNumber(SchedulerBackendUtils.scala:39) at org.apache.spark.scheduler.cluster.YarnClusterSchedulerBackend.start(YarnClusterSchedulerBackend.scala:34) at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:235) at org.apache.spark.SparkContext.<init>(SparkContext.scala:604) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext.<init>(SparkContext.scala:131) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2910) at org.apache.spark.SparkContext.getOrCreate(SparkContext.scala) at org.apache.spark.shuffle.RssSparkShuffleUtils.getActiveSparkContext(RssSparkShuffleUtils.java:271) at org.apache.uniffle.shuffle.manager.RssShuffleManagerBase.stop(RssShuffleManagerBase.java:1215) at org.apache.spark.shuffle.QiyiRssShuffleManager.stop(QiyiRssShuffleManager.java:352) at org.apache.spark.SparkEnv.stop(SparkEnv.scala:93) at org.apache.spark.SparkContext.$anonfun$stop$25(SparkContext.scala:2310) at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1406) at org.apache.spark.SparkContext.stop(SparkContext.scala:2310) at org.apache.spark.SparkContext.stop(SparkContext.scala:2216) at org.apache.spark.SparkContext.<init>(SparkContext.scala:711) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2888) at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:1099) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:1093) at org.apache.kyuubi.engine.spark.SparkSQLEngine$.createSpark(SparkSQLEngine.scala:303) at org.apache.kyuubi.engine.spark.SparkSQLEngine$.main(SparkSQLEngine.scala:377) at org.apache.kyuubi.engine.spark.SparkSQLEngine.main(SparkSQLEngine.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:738) ``` ### Affects Version(s) master ### Uniffle Server Log Output ```logtalk ``` ### Uniffle Engine Log Output ```logtalk ``` ### Uniffle Server Configurations ```yaml ``` ### Uniffle Engine Configurations ```yaml ``` ### Additional context _No response_ ### Are you willing to submit PR? - [x] Yes I am willing to submit a PR! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
