This is an automated email from the ASF dual-hosted git repository.

philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new a9102e68d2 [GLUTEN-13180][CORE] Fall back the scan when the 
ignoreMissingFiles read option differs from the conf (#13207)
a9102e68d2 is described below

commit a9102e68d2075f7c1c696ef1570d10bc3fe46de3
Author: rdtr <[email protected]>
AuthorDate: Sat Nov 7 17:22:47 2026 -0800

    [GLUTEN-13180][CORE] Fall back the scan when the ignoreMissingFiles read 
option differs from the conf (#13207)
---
 .../apache/gluten/execution/VeloxScanSuite.scala   | 54 +++++++++++++++++++++-
 .../execution/BasicScanExecTransformer.scala       | 15 ++++++
 .../execution/BatchScanExecTransformer.scala       | 12 +++++
 .../execution/FileSourceScanExecTransformer.scala  |  2 +
 4 files changed, 82 insertions(+), 1 deletion(-)

diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
index 2a1fd3bf2b..117bea4333 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
@@ -22,12 +22,14 @@ import org.apache.gluten.config.{GlutenConfig, VeloxConfig}
 import org.apache.gluten.utils.VeloxFileSystemValidationJniWrapper
 
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.catalyst.expressions.GreaterThan
 import org.apache.spark.sql.execution.ScalarSubquery
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
+import org.apache.hadoop.fs.{FileSystem, Path}
+
 import scala.reflect.ClassTag
 
 class VeloxScanSuite extends VeloxWholeStageTransformerSuite {
@@ -330,4 +332,54 @@ class VeloxScanSuite extends 
VeloxWholeStageTransformerSuite {
       }
     }
   }
+
+  test("ignoreMissingFiles read option overrides the session conf") {
+    // Reads two parquet dirs, then deletes the files of the first dir before 
running the query.
+    def readWithMissingFile(basePath: String, options: Map[String, String]): 
DataFrame = {
+      val path1 = new Path(basePath, "first")
+      val path2 = new Path(basePath, "second")
+      spark.range(1).toDF("a").write.parquet(path1.toString)
+      spark.range(1, 2).toDF("a").write.parquet(path2.toString)
+      val df = spark.read.options(options).parquet(path1.toString, 
path2.toString)
+      val fs = FileSystem.get(spark.sessionState.newHadoopConf())
+      fs.listStatus(path1)
+        .filter(f => f.isFile && !f.getPath.getName.startsWith("_"))
+        .foreach(f => fs.delete(f.getPath, false))
+      df
+    }
+
+    def isOffloaded(df: DataFrame): Boolean =
+      df.queryExecution.executedPlan.collect { case s: 
BasicScanExecTransformer => s }.nonEmpty
+
+    // (spark.sql.files.ignoreMissingFiles, read options, expected to ignore 
the missing file)
+    val cases = Seq(
+      ("true", Map.empty[String, String], true),
+      ("false", Map("ignoreMissingFiles" -> "true"), true),
+      ("false", Map.empty[String, String], false),
+      ("true", Map("ignoreMissingFiles" -> "false"), false)
+    )
+    // "parquet" uses the V1 scan, "" uses the V2 scan.
+    Seq("parquet", "").foreach {
+      v1Sources =>
+        cases.foreach {
+          case (conf, options, ignored) =>
+            withTempDir {
+              dir =>
+                withSQLConf(
+                  SQLConf.USE_V1_SOURCE_LIST.key -> v1Sources,
+                  SQLConf.IGNORE_MISSING_FILES.key -> conf) {
+                  val df = readWithMissingFile(dir.getCanonicalPath, options)
+                  if (ignored) {
+                    checkAnswer(df, Seq(Row(1)))
+                  } else {
+                    intercept[Exception](df.collect())
+                  }
+                  // The native scan only reads the session conf, so it must 
fall back when
+                  // the read option is set to a different value.
+                  assert(isOffloaded(df) == options.isEmpty)
+                }
+            }
+        }
+    }
+  }
 }
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
index c65fe7d135..4b0a83cd6c 100644
--- 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
@@ -27,8 +27,10 @@ import org.apache.gluten.substrait.rel.{ReadRelNode, 
RelBuilder, SplitInfo}
 import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
 
 import org.apache.spark.Partition
+import org.apache.spark.sql.catalyst.FileSourceOptions
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution.adaptive.InputStats
+import org.apache.spark.sql.internal.SQLConf
 
 import com.google.protobuf.StringValue
 import io.substrait.proto.NamedStruct
@@ -114,6 +116,9 @@ trait BasicScanExecTransformer extends LeafTransformSupport 
with BaseDataSource
   /** Returns the file format properties. */
   def getProperties: Map[String, String] = Map.empty
 
+  /** Returns the options given to this read, e.g. `spark.read.option(...)`. */
+  def getReadOptions: Map[String, String] = Map.empty
+
   def getInputStats: Option[InputStats] = Option.empty
 
   protected def neededInputFileRelatedMetadataKeys: Seq[String] = {
@@ -170,6 +175,16 @@ trait BasicScanExecTransformer extends 
LeafTransformSupport with BaseDataSource
           "resolution: native scan cannot produce both simultaneously")
     }
 
+    // The native scan only gets spark.sql.files.ignoreMissingFiles from the 
session conf.
+    // Spark also accepts ignoreMissingFiles as a read option, and the option 
wins over the conf.
+    // When the two differ, the native scan would use the wrong value, so fall 
back.
+    val ignoreMissingFiles = new 
FileSourceOptions(getReadOptions).ignoreMissingFiles
+    if (ignoreMissingFiles != SQLConf.get.ignoreMissingFiles) {
+      return ValidationResult.failed(
+        s"The ignoreMissingFiles read option ($ignoreMissingFiles) is 
different from " +
+          s"${SQLConf.IGNORE_MISSING_FILES.key} 
(${SQLConf.get.ignoreMissingFiles}).")
+    }
+
     val validationResult = BackendsApiManager.getSettings
       .validateScanExec(
         fileFormat,
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
index c95237c7ed..de61390d03 100644
--- 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
@@ -32,11 +32,15 @@ import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.connector.catalog.Table
 import org.apache.spark.sql.connector.read.Scan
 import org.apache.spark.sql.execution.datasources.v2.{BatchScanExecShim, 
FileScan}
+import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
+import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.types.StructType
 
 import com.google.common.base.Objects
 
+import scala.collection.JavaConverters._
+
 /** Columnar Based BatchScanExec. */
 case class BatchScanExecTransformer(
     override val output: Seq[AttributeReference],
@@ -145,6 +149,14 @@ abstract class BatchScanExecTransformerBase(
     case _ => new StructType()
   }
 
+  override def getReadOptions: Map[String, String] = {
+    scan match {
+      case parquetScan: ParquetScan => 
parquetScan.options.asCaseSensitiveMap.asScala.toMap
+      case orcScan: OrcScan => orcScan.options.asCaseSensitiveMap.asScala.toMap
+      case _ => Map.empty
+    }
+  }
+
   override def getRootPathsInternal: Seq[String] = {
     scan match {
       case fileScan: FileScan =>
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
index 4e587d3528..2fb4e5c303 100644
--- 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala
@@ -242,6 +242,8 @@ abstract class FileSourceScanExecTransformerBase(
     s"${getClass.getSimpleName} $relation 
${tableIdentifier.map(_.unquotedString).getOrElse("")}"
   }
 
+  override def getReadOptions: Map[String, String] = relation.options
+
   override def getProperties: Map[String, String] = {
     this.fileFormat match {
       case ReadFileFormat.TextReadFormat =>


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to