This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-19926-e8a076d08d5875c404736941ae88c9bbc7449b82
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 498e5c26b152c119ee0381b923f402407d962c31
Author: voonhous <[email protected]>
AuthorDate: Thu Nov 19 08:09:08 2026 +0000

    test(spark): shared SparkContext for the SQL suites, opt-in (#19926)
    
    * test(spark): shared SparkContext for the SQL suites, opt-in
    
    Add hudi.spark.test.sharedSession (default false), forwarded by the
    scalatest plugin. When set, every suite on HoodieSparkSqlTestBase
    shares one never-stopped SparkContext and works in its own
    SparkSession.newSession() child: own SQL conf, temp views and
    HoodieCatalog; shared external catalog and warehouse. A suite's
    conf deltas (extraConf, sparkConf overrides) are validated first
    and then applied to the child session; spark.hadoop.* keys go to
    the shared Hadoop conf and are restored in afterAll, any other
    context-level or static key is rejected before anything mutates.
    The child session is pinned as the active session on the suite
    thread, with a check that its HoodieCatalog is bound to it. The
    per-test cleanup drops only the suite's own tables.
    
    With the property off nothing changes. It is switched on for the
    three Java 17 Scala shards whose packages hold nothing but
    HoodieSparkSqlTestBase suites (dml-1, dml-2, other-1); other-2
    mixes in suites that create their own SparkContext.
    
    TestHoodieDataUtils no longer closes its JavaSparkContext in
    afterAll, which stopped the context.
    
    Supersedes the proof of concept in #19921, where this ran green
    serially on the dml packages at master's serial time.
    
    * fix: read parquet Avro records on a conf copy
    
    ParquetUtils.readAvroRecords set the Avro read schema on the
    caller's Hadoop Configuration, which is usually the SparkContext
    wide one, and the schema-less overload built its reader on that
    same object. Whatever read schema an earlier caller left behind
    was silently applied to later schema-less reads.
    
    With the SQL suites sharing one SparkContext, the hdfs parquet
    import procedure (HDFSParquetImporterUtils sets the schema on
    jsc.hadoopConfiguration after Job.getInstance has already copied
    it) leaked a schema without Hudi meta fields into the JVM, and
    repair_deduplicate then failed with "Avro field
    '_hoodie_commit_time' not found".
    
    Both overloads now read on a private copy of the configuration.
    The schema-less one also drops any read schema or projection the
    copy carried, so it reads with the file's own schema.
    
    * test(spark): clean up listeners and global views
    
    Review follow-ups for the shared SparkContext mode.
    
    - TestInsertTable4 removes its StageParallelismListener in finally;
      the shared context outlives the suite, so the listener did too.
    - dropSuiteTables also drops the suite's global temp views in shared
      mode. They live in the shared state, outside listDatabases, so
      TestCopyToTempViewProcedure's views outlived the suite. Matched on
      the generateTableName prefix only, which stays safe once suites
      run concurrently.
    - TestParquetUtils covers readAvroRecords leaving the caller's
      Configuration untouched and ignoring a stale read schema or
      projection on a schema-less read. Both halves fail on the previous
      implementation.
    
    * fix: drop a stale projection on schema reads too
    
    readAvroRecords with an explicit schema kept any requested projection
    a previous caller left on the configuration, so stored fields outside
    that projection came back as their defaults. Both overloads now build
    their private copy through one helper that drops the read schema and
    the projection. TestParquetUtils plants a narrow projection and checks
    that a later explicit-schema read still returns the stored values.
---
 .github/workflows/bot.yml                          |  15 +-
 .../org/apache/hudi/common/util/ParquetUtils.java  |  31 +++-
 .../apache/hudi/common/util/TestParquetUtils.java  |  44 ++++++
 .../sql/hudi/common/HoodieSparkSqlTestBase.scala   | 175 ++++++++++++++++++---
 .../sql/hudi/common/TestHoodieDataUtils.scala      |   7 -
 .../sql/hudi/dml/insert/TestInsertTable4.scala     |  56 ++++---
 pom.xml                                            |   3 +
 7 files changed, 270 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 36ab7f8ea13c..a05bb99d3b6e 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1120,6 +1120,7 @@ jobs:
           flags: spark-java-tests
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  # sharedSession: this shard holds only HoodieSparkSqlTestBase suites, which 
share one SparkContext
   test-spark-java17-scala-dml-1:
     if: *route
     needs: changes
@@ -1164,14 +1165,14 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-Dtest=skipJavaTests $SCALA_TEST_DML1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false
+          mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-Dtest=skipJavaTests $SCALA_TEST_DML1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS 
-Djacoco.skip=false -Dhudi.spark.test.sharedSession=true
       - name: Scala FT - Spark (dml.others)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false
+          mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false 
-Dhudi.spark.test.sharedSession=true
       - name: Test timing summary
         if: always()
         run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job 
}}"
@@ -1185,6 +1186,7 @@ jobs:
           flags: spark-scala-tests
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  # sharedSession: this shard holds only HoodieSparkSqlTestBase suites, which 
share one SparkContext
   test-spark-java17-scala-dml-2:
     if: *route
     needs: changes
@@ -1229,14 +1231,14 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-Dtest=skipJavaTests $SCALA_TEST_DML2_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false
+          mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-Dtest=skipJavaTests $SCALA_TEST_DML2_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS 
-Djacoco.skip=false -Dhudi.spark.test.sharedSession=true
       - name: Scala FT - Spark (dml.insert, dml.schema)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML2_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false
+          mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML2_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false 
-Dhudi.spark.test.sharedSession=true
       - name: Test timing summary
         if: always()
         run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job 
}}"
@@ -1286,6 +1288,7 @@ jobs:
           if-no-files-found: ignore
 
 
+  # sharedSession: this shard holds only HoodieSparkSqlTestBase suites, which 
share one SparkContext
   test-spark-java17-scala-other-1:
     if: *route
     needs: changes
@@ -1330,14 +1333,14 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-Dtest=skipJavaTests $SCALA_TEST_OTHERS1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false
+          mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" 
-Dtest=skipJavaTests $SCALA_TEST_OTHERS1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS 
-Djacoco.skip=false -Dhudi.spark.test.sharedSession=true
       - name: Scala FT - Spark (procedure, ddl)
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_MODULES: ${{ matrix.sparkModules }}
         run:
-          mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_OTHERS1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false
+          mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" 
-D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_OTHERS1_FILTER 
-Dsurefire.failIfNoSpecifiedTests=false -pl 
"$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false 
-Dhudi.spark.test.sharedSession=true
       - name: Test timing summary
         if: always()
         run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job 
}}"
diff --git 
a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
 
b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
index 90f0dc64952f..585e9c204d1c 100644
--- 
a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
+++ 
b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
@@ -452,10 +452,33 @@ public class ParquetUtils extends FileFormatUtils {
    */
   @Override
   public List<GenericRecord> readAvroRecords(HoodieStorage storage, 
StoragePath filePath) {
+    // A schema-less read takes the schema from the file.
+    return readAvroRecords(readConf(storage), filePath);
+  }
+
+  @Override
+  public List<GenericRecord> readAvroRecords(HoodieStorage storage, 
StoragePath filePath, HoodieSchema schema) {
+    Configuration conf = readConf(storage);
+    AvroReadSupport.setAvroReadSchema(conf, schema.toAvroSchema());
+    return readAvroRecords(conf, filePath);
+  }
+
+  /**
+   * A private copy of the storage configuration, so a read never changes the 
caller's configuration,
+   * without any read schema or projection a previous caller left on it, so 
neither leaks into this read.
+   */
+  private static Configuration readConf(HoodieStorage storage) {
+    Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+    conf.unset("parquet.avro.read.schema");
+    conf.unset(AvroReadSupport.AVRO_REQUESTED_PROJECTION);
+    return conf;
+  }
+
+  private static List<GenericRecord> readAvroRecords(Configuration conf, 
StoragePath filePath) {
     List<GenericRecord> records = new ArrayList<>();
     try (ParquetReader reader = AvroParquetReader.builder(new 
Path(filePath.toUri()))
         .withDataModel(GenericData.get())
-        .withConf(storage.getConf().unwrapAs(Configuration.class)).build()) {
+        .withConf(conf).build()) {
       Object obj = reader.read();
       while (obj != null) {
         if (obj instanceof GenericRecord) {
@@ -470,12 +493,6 @@ public class ParquetUtils extends FileFormatUtils {
     return records;
   }
 
-  @Override
-  public List<GenericRecord> readAvroRecords(HoodieStorage storage, 
StoragePath filePath, HoodieSchema schema) {
-    
AvroReadSupport.setAvroReadSchema(storage.getConf().unwrapAs(Configuration.class),
 schema.toAvroSchema());
-    return readAvroRecords(storage, filePath);
-  }
-
   /**
    * Returns the number of records in the parquet file.
    *
diff --git 
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
 
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
index d1e477694fdf..234561054f3f 100644
--- 
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
+++ 
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
@@ -36,11 +36,14 @@ import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.keygen.BaseKeyGenerator;
 import org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata;
+import org.apache.hudi.storage.HoodieStorage;
 import org.apache.hudi.storage.StoragePath;
 
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.parquet.avro.AvroReadSupport;
 import org.apache.parquet.avro.AvroSchemaConverter;
 import org.apache.parquet.hadoop.ParquetWriter;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -69,6 +72,7 @@ import static 
org.apache.hudi.common.schema.HoodieSchemaUtils.METADATA_FIELD_SCH
 import static org.apache.hudi.metadata.HoodieIndexVersion.V1;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -77,6 +81,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
  */
 public class TestParquetUtils extends HoodieCommonTestHarness {
 
+  // AvroReadSupport.AVRO_READ_SCHEMA is private in the parquet-avro versions 
Hudi builds against.
+  private static final String AVRO_READ_SCHEMA_KEY = 
"parquet.avro.read.schema";
+
   private ParquetUtils parquetUtils;
 
   public static List<Arguments> bloomFilterTypeCodes() {
@@ -250,6 +257,43 @@ public class TestParquetUtils extends 
HoodieCommonTestHarness {
         HoodieTestUtils.getStorage(filePath), new StoragePath(filePath)));
   }
 
+  @Test
+  public void testReadAvroRecordsLeavesCallerConfUntouched() throws Exception {
+    String filePath = Paths.get(basePath, "test.parquet").toUri().toString();
+    HoodieSchema fileSchema = getSchemaWithFields(
+        Arrays.asList(HoodieRecord.RECORD_KEY_METADATA_FIELD, 
HoodieRecord.PARTITION_PATH_METADATA_FIELD));
+    writeParquetFile(BloomFilterTypeCode.SIMPLE.name(), filePath, 
Arrays.asList("k1", "k2"), fileSchema, true, "p1");
+    HoodieSchema evolvedSchema = getSchemaWithFields(
+        Arrays.asList(HoodieRecord.RECORD_KEY_METADATA_FIELD, 
HoodieRecord.PARTITION_PATH_METADATA_FIELD, "extra_field"));
+    HoodieSchema keyOnlySchema = 
getSchemaWithFields(Collections.singletonList(HoodieRecord.RECORD_KEY_METADATA_FIELD));
+
+    // One storage, so every read sees the same Configuration, like readers 
sharing a SparkContext's.
+    HoodieStorage storage = HoodieTestUtils.getStorage(filePath);
+    StoragePath storagePath = new StoragePath(filePath);
+    Configuration conf = storage.getConf().unwrapAs(Configuration.class);
+
+    // A read with a schema applies it to its own records only.
+    List<GenericRecord> evolved = parquetUtils.readAvroRecords(storage, 
storagePath, evolvedSchema);
+    assertEquals(2, evolved.size());
+    assertEquals(3, evolved.get(0).getSchema().getFields().size());
+    assertNull(conf.get(AVRO_READ_SCHEMA_KEY));
+
+    // A schema-less read uses the file schema even if another caller left a 
read schema or projection behind.
+    AvroReadSupport.setAvroReadSchema(conf, keyOnlySchema.toAvroSchema());
+    AvroReadSupport.setRequestedProjection(conf, keyOnlySchema.toAvroSchema());
+    List<GenericRecord> full = parquetUtils.readAvroRecords(storage, 
storagePath);
+    assertEquals(2, full.size());
+    assertEquals(2, full.get(0).getSchema().getFields().size());
+    assertEquals("p1", 
String.valueOf(full.get(0).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD)));
+    assertEquals(keyOnlySchema.toAvroSchema().toString(), 
conf.get(AVRO_READ_SCHEMA_KEY));
+
+    // A read with a schema ignores the stale projection too, so stored fields 
are read rather than defaulted.
+    List<GenericRecord> evolvedAfterStale = 
parquetUtils.readAvroRecords(storage, storagePath, evolvedSchema);
+    assertEquals(2, evolvedAfterStale.size());
+    assertEquals(3, evolvedAfterStale.get(0).getSchema().getFields().size());
+    assertEquals("p1", 
String.valueOf(evolvedAfterStale.get(0).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD)));
+  }
+
   @Test
   public void testReadColumnStatsFromMetadata() throws Exception {
     List<Pair<Pair<String, String>, Boolean>> valueList = new ArrayList<>();
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
index f197be325d8e..ace64841b78b 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
@@ -39,7 +39,10 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import 
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.checkMessageContains
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.hudi.catalog.HoodieCatalog
+import 
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.{checkMessageContains, 
sharedSessionEnabled}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructField
 import org.apache.spark.util.Utils
 import org.joda.time.DateTimeZone
@@ -58,7 +61,9 @@ class HoodieSparkSqlTestBase extends FunSuite with 
BeforeAndAfterAll {
   org.apache.log4j.Logger.getRootLogger.setLevel(org.apache.log4j.Level.WARN)
   private val LOG = LoggerFactory.getLogger(getClass)
 
-  private lazy val sparkWareHouse = {
+  private lazy val sparkWareHouse: File = if (sharedSessionEnabled) {
+    HoodieSparkSqlTestBase.sharedWarehouse
+  } else {
     val dir = Utils.createTempDir()
     Utils.deleteRecursively(dir)
     dir
@@ -71,25 +76,56 @@ class HoodieSparkSqlTestBase extends FunSuite with 
BeforeAndAfterAll {
   //       is consistent with the fixtures
   DateTimeZone.setDefault(DateTimeZone.UTC)
   TimeZone.setDefault(DateTimeUtils.getTimeZone("UTC"))
-  protected lazy val spark: SparkSession = SparkSession.builder()
-    .config("spark.sql.warehouse.dir", sparkWareHouse.getCanonicalPath)
-    .config("spark.sql.session.timeZone", "UTC")
-    .config("hoodie.insert.shuffle.parallelism", "4")
-    .config("hoodie.upsert.shuffle.parallelism", "4")
-    .config("hoodie.delete.shuffle.parallelism", "4")
-    .config(sparkConf())
-    .getOrCreate()
+  protected lazy val spark: SparkSession = if (sharedSessionEnabled) {
+    val session = HoodieSparkSqlTestBase.sharedBaseSession().newSession()
+    SparkSession.setActiveSession(session)
+    applySuiteConfToSharedSession(session)
+    session
+  } else {
+    HoodieSparkSqlTestBase.sessionBuilder(sparkWareHouse, 
sparkConf()).getOrCreate()
+  }
 
   private var tableId = new AtomicInteger(0)
 
   private var extraConf = Map[String, String]()
 
+  // Shared mode: spark.hadoop.* keys this suite set on the shared Hadoop 
conf, with the value they replaced.
+  private var hadoopConfOverrides: Seq[(String, String)] = Seq.empty
+
   def sparkConf(): SparkConf = {
     val conf = getSparkConfForTest("Hoodie SQL Test")
     conf.setAll(extraConf)
     conf
   }
 
+  /**
+   * Shared mode: the context-level SparkConf is fixed, so the deltas a suite 
adds through extraConf or a
+   * sparkConf() override go to its session conf (hoodie.* and spark.sql.* 
keys), except spark.hadoop.*
+   * keys, which the write client reads from sparkContext.hadoopConfiguration 
and which are restored in
+   * afterAll. Any other spark.* key, and any static SQL conf, is a setting a 
child session cannot change,
+   * so it is rejected before anything is mutated: a partial apply would leave 
the shared Hadoop conf
+   * changed for every later suite in the JVM.
+   */
+  private def applySuiteConfToSharedSession(session: SparkSession): Unit = {
+    val defaults = getSparkConfForTest("Hoodie SQL Test").getAll.toMap
+    val deltas = sparkConf().getAll.filterNot { case (k, v) => 
defaults.get(k).contains(v) }
+    val (hadoopKeys, sessionKeys) = deltas.partition { case (k, _) => 
k.startsWith("spark.hadoop.") }
+    val rejected = sessionKeys.collect {
+      case (k, _) if (k.startsWith("spark.") && !k.startsWith("spark.sql.")) 
|| SQLConf.isStaticConfigKey(k) => k
+    }
+    if (rejected.nonEmpty) {
+      throw new IllegalArgumentException(
+        s"${rejected.mkString(", ")}: SparkContext-level or static settings; 
shared session mode cannot apply them per suite")
+    }
+    val hadoopConf = session.sparkContext.hadoopConfiguration
+    hadoopKeys.foreach { case (k, v) =>
+      val key = k.stripPrefix("spark.hadoop.")
+      hadoopConfOverrides :+= (key -> hadoopConf.get(key))
+      hadoopConf.set(key, v)
+    }
+    sessionKeys.foreach { case (k, v) => session.conf.set(k, v) }
+  }
+
   protected def initQueryIndexConf(): Unit = {
     extraConf = extraConf ++ Map(
       DataSkippingFailureMode.configName -> 
DataSkippingFailureMode.Strict.value
@@ -110,6 +146,9 @@ class HoodieSparkSqlTestBase extends FunSuite with 
BeforeAndAfterAll {
   override protected def test(testName: String, testTags: Tag*)(testFun: => 
Any /* Assertion */)(implicit pos: source.Position): Unit = {
     super.test(testName, testTags: _*)(
       try {
+        if (sharedSessionEnabled) {
+          bindSuiteSession()
+        }
         testFun
       } finally {
         // The INMEMORY index keeps a JVM-static record-location map; reset it 
after every test so
@@ -118,23 +157,74 @@ class HoodieSparkSqlTestBase extends FunSuite with 
BeforeAndAfterAll {
         // it, so a throwing or non-withRecordType INMEMORY test would 
otherwise leak state here.
         // Runs before the catalog cleanup so it holds even if a drop throws.
         HoodieInMemoryHashIndex.clear()
-        val catalog = spark.sessionState.catalog
-        catalog.listDatabases().foreach { db =>
-          catalog.listTables(db).foreach { table =>
-            catalog.dropTable(table, true, true)
-          }
-        }
+        dropSuiteTables()
       }
     )
   }
 
+  /**
+   * Shared mode: scalatest runs each suite on its own thread and Hudi reads 
SparkSession.active in a few
+   * places (HoodieCatalog captures it when the session's catalog is first 
built, BaseProcedure per CALL),
+   * so pin this suite's child session to the test thread and fail fast if the 
session's HoodieCatalog was
+   * built against another session.
+   */
+  private def bindSuiteSession(): Unit = {
+    SparkSession.setActiveSession(spark)
+    
spark.sessionState.catalogManager.catalog(CatalogManager.SESSION_CATALOG_NAME) 
match {
+      case hoodieCatalog: HoodieCatalog =>
+        assert(hoodieCatalog.spark eq spark,
+          s"HoodieCatalog of ${getClass.getSimpleName} is bound to another 
SparkSession")
+      case _ =>
+    }
+  }
+
+  /**
+   * Drops the tables a test left behind. Per-suite mode owns the whole 
catalog. Shared mode shares the
+   * external catalog with every other suite in the JVM, so it drops only this 
suite's generateTableName
+   * tables plus any table whose name is not of that form (fixed names and 
temp views); under serial
+   * execution those can only come from the test that just ran. Fixed names 
are renamed in a later step.
+   */
+  private def dropSuiteTables(): Unit = {
+    val catalog = spark.sessionState.catalog
+    catalog.listDatabases().foreach { db =>
+      val tables = catalog.listTables(db)
+      val toDrop = if (sharedSessionEnabled) tables.filter(table => 
ownsTable(table.table)) else tables
+      toDrop.foreach(table => catalog.dropTable(table, true, true))
+    }
+    if (sharedSessionEnabled) {
+      // Global temp views live in the shared state, outside listDatabases, 
and would otherwise outlive the suite.
+      catalog.globalTempViewManager.listViewNames("*")
+        .filter(name => name.startsWith(tableNamePrefix))
+        .foreach(name => catalog.dropGlobalTempView(name))
+    }
+  }
+
+  /** Shared mode: a table is this suite's if generateTableName produced it, 
or if no suite's generateTableName could have. */
+  private def ownsTable(name: String): Boolean = {
+    name.startsWith(tableNamePrefix) || 
!HoodieSparkSqlTestBase.GENERATED_TABLE_NAME.matcher(name).matches()
+  }
+
+  private lazy val tableNamePrefix: String = 
s"h${getClass.getSimpleName.toLowerCase}_"
+
   protected def generateTableName: String = {
-    s"h${getClass.getSimpleName.toLowerCase}_${tableId.incrementAndGet()}"
+    s"$tableNamePrefix${tableId.incrementAndGet()}"
   }
 
   override protected def afterAll(): Unit = {
-    Utils.deleteRecursively(sparkWareHouse)
-    spark.stop()
+    if (sharedSessionEnabled) {
+      // The context and warehouse outlive the suite; only undo this suite's 
Hadoop conf overrides.
+      if (hadoopConfOverrides.nonEmpty) {
+        val hadoopConf = 
HoodieSparkSqlTestBase.sharedBaseSession().sparkContext.hadoopConfiguration
+        hadoopConfOverrides.reverse.foreach {
+          case (key, null) => hadoopConf.unset(key)
+          case (key, previous) => hadoopConf.set(key, previous)
+        }
+        hadoopConfOverrides = Seq.empty
+      }
+    } else {
+      Utils.deleteRecursively(sparkWareHouse)
+      spark.stop()
+    }
   }
 
   protected def checkAnswer(sql: String)(expects: Seq[Any]*): Unit = {
@@ -413,6 +503,53 @@ class HoodieSparkSqlTestBase extends FunSuite with 
BeforeAndAfterAll {
 
 object HoodieSparkSqlTestBase {
 
+  private val LOG = LoggerFactory.getLogger(classOf[HoodieSparkSqlTestBase])
+
+  /**
+   * System property forwarded by the scalatest plugin from the pom property 
of the same name
+   * (`-Dhudi.spark.test.sharedSession=true`). When set, every suite in the 
JVM shares one
+   * SparkContext and works in its own `newSession()` child; the context is 
never stopped by a suite.
+   * Default off: each suite builds and stops its own session, as before.
+   *
+   * Only for runs whose suites all extend HoodieSparkSqlTestBase: a suite 
that builds its own
+   * SparkContext (for example TestHoodieInternalRowUtils) fails with "Only 
one SparkContext
+   * should be running in this JVM" and aborts the run, so CI enables the 
property only on
+   * shards whose packages hold nothing else.
+   */
+  val SHARED_SESSION_PROPERTY = "hudi.spark.test.sharedSession"
+  val sharedSessionEnabled: Boolean = 
java.lang.Boolean.getBoolean(SHARED_SESSION_PROPERTY)
+
+  // Table names produced by generateTableName (without a database prefix).
+  private[common] val GENERATED_TABLE_NAME: Pattern = 
Pattern.compile("h[a-z0-9]+_[0-9]+")
+
+  private[common] lazy val sharedWarehouse: File = {
+    val dir = Utils.createTempDir()
+    Utils.deleteRecursively(dir)
+    dir
+  }
+
+  @volatile private var sharedBase: SparkSession = _
+
+  /** The JVM-wide base session; rebuilt if something stopped its context. 
Only its children are handed to suites. */
+  private[common] def sharedBaseSession(): SparkSession = synchronized {
+    if (sharedBase == null || sharedBase.sparkContext.isStopped) {
+      LOG.warn("Shared session mode on ({}=true): one SparkContext for this 
JVM, a child session per suite",
+        SHARED_SESSION_PROPERTY)
+      sharedBase = sessionBuilder(sharedWarehouse, getSparkConfForTest("Hoodie 
SQL Test")).getOrCreate()
+    }
+    sharedBase
+  }
+
+  private[common] def sessionBuilder(warehouse: File, conf: SparkConf): 
SparkSession.Builder = {
+    SparkSession.builder()
+      .config("spark.sql.warehouse.dir", warehouse.getCanonicalPath)
+      .config("spark.sql.session.timeZone", "UTC")
+      .config("hoodie.insert.shuffle.parallelism", "4")
+      .config("hoodie.upsert.shuffle.parallelism", "4")
+      .config("hoodie.delete.shuffle.parallelism", "4")
+      .config(conf)
+  }
+
   // the naming format of 0.x version
   final val NAME_FORMAT_0_X: Pattern = 
Pattern.compile("^(\\d+)(\\.\\w+)(\\.\\D+)?$")
 
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieDataUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieDataUtils.scala
index 1c3e1a16cffd..d52c68b409ee 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieDataUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieDataUtils.scala
@@ -44,13 +44,6 @@ class TestHoodieDataUtils extends HoodieSparkSqlTestBase {
     context = new HoodieSparkEngineContext(jsc)
   }
 
-  override def afterAll(): Unit = {
-    if (jsc != null) {
-      jsc.close()
-    }
-    super.afterAll()
-  }
-
   // Helper method to create a Pair from key and value
   private def pair(key: String, value: String): Pair[String, String] = {
     Pair.of(key, value)
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/insert/TestInsertTable4.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/insert/TestInsertTable4.scala
index 8d786fde8c8c..eebc34fa9625 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/insert/TestInsertTable4.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/insert/TestInsertTable4.scala
@@ -338,19 +338,25 @@ class TestInsertTable4 extends HoodieSparkSqlTestBase {
            |select '1' as id, 'aa' as name, 123 as dt, '2023-10-12' as `day`, 
12 as `hour`
            |""".stripMargin)
       val stageClassName = classOf[HoodieSparkEngineContext].getSimpleName
-      spark.sparkContext.addSparkListener(new 
StageParallelismListener(stageName = stageClassName))
-      val df = spark.sql(
-        s"""
-           |select * from ${targetTable} where day='2023-10-12' and hour=11
-           |""".stripMargin)
-      var rddHead = df.rdd
-      while (rddHead.dependencies.size > 0) {
+      // The SparkContext outlives the suite in shared session mode, so the 
listener must not.
+      val listener = new StageParallelismListener(stageName = stageClassName)
+      spark.sparkContext.addSparkListener(listener)
+      try {
+        val df = spark.sql(
+          s"""
+             |select * from ${targetTable} where day='2023-10-12' and hour=11
+             |""".stripMargin)
+        var rddHead = df.rdd
+        while (rddHead.dependencies.size > 0) {
+          assertResult(1)(rddHead.partitions.size)
+          rddHead = rddHead.firstParent
+        }
         assertResult(1)(rddHead.partitions.size)
-        rddHead = rddHead.firstParent
+        countDownLatch.await(1, TimeUnit.MINUTES)
+        assert(listenerCallCount >= 1)
+      } finally {
+        spark.sparkContext.removeSparkListener(listener)
       }
-      assertResult(1)(rddHead.partitions.size)
-      countDownLatch.await(1, TimeUnit.MINUTES)
-      assert(listenerCallCount >= 1)
     }
   }
 
@@ -391,19 +397,25 @@ class TestInsertTable4 extends HoodieSparkSqlTestBase {
            |select '1' as id, 'aa' as name, 123 as dt, '2023-10-12' as `day`, 
12 as `hour`
            |""".stripMargin)
       val stageClassName = classOf[HoodieSparkEngineContext].getSimpleName
-      spark.sparkContext.addSparkListener(new 
StageParallelismListener(stageName = stageClassName))
-      val df = spark.sql(
-        s"""
-           |select * from ${targetTable} where day='2023-10-12' and hour=11
-           |""".stripMargin)
-      var rddHead = df.rdd
-      while (rddHead.dependencies.size > 0) {
+      // The SparkContext outlives the suite in shared session mode, so the 
listener must not.
+      val listener = new StageParallelismListener(stageName = stageClassName)
+      spark.sparkContext.addSparkListener(listener)
+      try {
+        val df = spark.sql(
+          s"""
+             |select * from ${targetTable} where day='2023-10-12' and hour=11
+             |""".stripMargin)
+        var rddHead = df.rdd
+        while (rddHead.dependencies.size > 0) {
+          assertResult(1)(rddHead.partitions.size)
+          rddHead = rddHead.firstParent
+        }
         assertResult(1)(rddHead.partitions.size)
-        rddHead = rddHead.firstParent
+        countDownLatch.await(1, TimeUnit.MINUTES)
+        assert(listenerCallCount >= 1)
+      } finally {
+        spark.sparkContext.removeSparkListener(listener)
       }
-      assertResult(1)(rddHead.partitions.size)
-      countDownLatch.await(1, TimeUnit.MINUTES)
-      assert(listenerCallCount >= 1)
     }
   }
 
diff --git a/pom.xml b/pom.xml
index 44b6c3aee980..1972638fa6cf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -276,6 +276,8 @@
     <lance.spark.connector.version>0.4.0</lance.spark.connector.version>
     
<lance.spark.artifact>lance-spark-3.5_${scala.binary.version}</lance.spark.artifact>
     <lance.skip.tests>false</lance.skip.tests>
+    <!-- Scala SQL suites: one shared SparkContext per JVM with a child 
session per suite (see HoodieSparkSqlTestBase) -->
+    <hudi.spark.test.sharedSession>false</hudi.spark.test.sharedSession>
     <vortex.version>0.76.0</vortex.version>
     
<vortex.spark.artifact>vortex-spark_${scala.binary.version}</vortex.spark.artifact>
     <vortex.skip.tests>false</vortex.skip.tests>
@@ -624,6 +626,7 @@
               
<log4j.configurationFile>${surefire-log4j.file}</log4j.configurationFile>
               <lance.skip.tests>${lance.skip.tests}</lance.skip.tests>
               <vortex.skip.tests>${vortex.skip.tests}</vortex.skip.tests>
+              
<hudi.spark.test.sharedSession>${hudi.spark.test.sharedSession}</hudi.spark.test.sharedSession>
             </systemProperties>
           </configuration>
           <executions>

Reply via email to