[ https://issues.apache.org/jira/browse/SPARK-41555?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
jiahong.li updated SPARK-41555: ------------------------------- Description: In spark , if we create multi sparkSession in the program, we will get multi-SQLTab in UI, At the same time, we will get muti-SQLAppStatusListener object, it is waste of memory. code like this: {code:java} // code placeholder def main(args: Array[String]): Unit = { val sparkConf = new SparkConf() .setAppName("demo") .setMaster("local[*]") val spark = SparkSession.builder() .config(sparkConf) .getOrCreate() setDefaultSession(null) setActiveSession(null) val spark2 = SparkSession.builder() .config(sparkConf) .getOrCreate() import spark.implicits._ val testData = spark.sparkContext .parallelize((1 to 3).map(i => TestData(i, i.toString))).toDF() testData.createOrReplaceTempView("testTable") val testData2 = spark.sparkContext.parallelize( TestData2(1, "1") :: TestData2(1, "2") :: TestData2(2, "1") :: TestData2(2, "2") :: TestData2(3, "1") :: TestData2(3, "2") :: Nil, 2).toDF() testData2.createOrReplaceTempView("testTable2") val query = "select ind2,count(*) from ( select * from testTable2 join testTable on testTable.ind = testTable2.ind2 where testTable.name <> '1') group by ind2" spark.sql(query).collect() Thread.sleep(500000) spark.stop() } {code} was: In spark , if we create multi sparkSession in the program, we will get multi-SQLTab in UI, At the same time, we will get muti-SQLAppStatusListener object, it is waste of memory. code like this: def main(args: Array[String]): Unit = { val sparkConf = new SparkConf() .setAppName("demo") .setMaster("local[*]") val spark = SparkSession.builder() .config(sparkConf) .getOrCreate() setDefaultSession(null) setActiveSession(null) val spark2 = SparkSession.builder() .config(sparkConf) .getOrCreate() import spark.implicits._ val testData = spark.sparkContext .parallelize((1 to 3).map(i => TestData(i, i.toString))).toDF() testData.createOrReplaceTempView("testTable") val testData2 = spark.sparkContext.parallelize( TestData2(1, "1") :: TestData2(1, "2") :: TestData2(2, "1") :: TestData2(2, "2") :: TestData2(3, "1") :: TestData2(3, "2") :: Nil, 2).toDF() testData2.createOrReplaceTempView("testTable2") val query = "select ind2,count(*) from ( select * from testTable2 join testTable on testTable.ind = testTable2.ind2 where testTable.name <> '1') group by ind2" spark.sql(query).collect() Thread.sleep(500000) spark.stop() } > Multi sparkSession should share single SQLAppStatusStore > -------------------------------------------------------- > > Key: SPARK-41555 > URL: https://issues.apache.org/jira/browse/SPARK-41555 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 3.1.1, 3.2.1, 3.3.0 > Reporter: jiahong.li > Priority: Major > Attachments: muti-SQLStore.png, muti-sqltab.png > > > In spark , if we create multi sparkSession in the program, we will get > multi-SQLTab in UI, > At the same time, we will get muti-SQLAppStatusListener object, it is waste > of memory. > code like this: > > {code:java} > // code placeholder > def main(args: Array[String]): Unit = { > val sparkConf = new SparkConf() > .setAppName("demo") > .setMaster("local[*]") > val spark = SparkSession.builder() > .config(sparkConf) > .getOrCreate() > setDefaultSession(null) > setActiveSession(null) > val spark2 = SparkSession.builder() > .config(sparkConf) > .getOrCreate() > import spark.implicits._ > val testData = spark.sparkContext > .parallelize((1 to 3).map(i => TestData(i, i.toString))).toDF() > testData.createOrReplaceTempView("testTable") > val testData2 = spark.sparkContext.parallelize( > TestData2(1, "1") :: > TestData2(1, "2") :: > TestData2(2, "1") :: > TestData2(2, "2") :: > TestData2(3, "1") :: > TestData2(3, "2") :: > Nil, 2).toDF() > testData2.createOrReplaceTempView("testTable2") > val query = "select ind2,count(*) from ( select * from testTable2 join > testTable on testTable.ind = testTable2.ind2 where testTable.name <> '1') > group by ind2" > spark.sql(query).collect() > Thread.sleep(500000) > spark.stop() > } > {code} -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org