This is an automated email from the ASF dual-hosted git repository.

jinchengchenghh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 66aec06951 [VL][Iceberg] Support SQL MERGE INTO for parquet(#12964)
66aec06951 is described below

commit 66aec06951daa5638995deed1724a763e74d4736
Author: inf / Hazmi <[email protected]>
AuthorDate: Fri Nov 13 10:02:02 2026 +0000

    [VL][Iceberg] Support SQL MERGE INTO for parquet(#12964)
---
 .../util/iceberg-upstream/flaky-tests.csv          |   5 +-
 .../gluten/execution/VeloxIcebergSuite.scala       |  49 ++++++
 .../gluten/execution/IcebergScanTransformer.scala  | 164 +++++++++++++--------
 .../spark/source/GlutenIcebergSourceUtil.scala     |  39 +++--
 .../execution/BasicScanExecTransformer.scala       |  41 ++++--
 .../execution/BatchScanExecTransformer.scala       |   5 +-
 .../parquet/GlutenParquetRowIndexSuite.scala       |  40 ++++-
 .../parquet/GlutenParquetRowIndexSuite.scala       |  45 +++++-
 .../parquet/GlutenParquetRowIndexSuite.scala       |  45 +++++-
 .../parquet/GlutenParquetRowIndexSuite.scala       |  45 +++++-
 10 files changed, 378 insertions(+), 100 deletions(-)

diff --git a/.github/workflows/util/iceberg-upstream/flaky-tests.csv 
b/.github/workflows/util/iceberg-upstream/flaky-tests.csv
index 1dceb3bd3f..b34131c8f9 100644
--- a/.github/workflows/util/iceberg-upstream/flaky-tests.csv
+++ b/.github/workflows/util/iceberg-upstream/flaky-tests.csv
@@ -16,6 +16,7 @@
 # All invocations of these methods run and are reported, but do not affect the 
gate.
 # Use exact class and method names; omit method arguments and invocation 
indices.
 class,method
-# Example
-# 
org.apache.iceberg.spark.extensions.TestAlterTablePartitionFields,testDropPartitionAndSourceColumnLong
 
org.apache.iceberg.spark.extensions.TestViews,createViewWithCTEReferencingTempView
+org.apache.iceberg.spark.extensions.TestCopyOnWriteDelete,testDeleteWithVectorizedReads
+org.apache.iceberg.spark.extensions.TestCopyOnWriteMerge,testMergeWithVectorizedReads
+org.apache.iceberg.spark.extensions.TestCopyOnWriteUpdate,testUpdateWithVectorizedReads
diff --git 
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
 
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
index 6ff43d256c..eaef2f10c3 100644
--- 
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
+++ 
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
@@ -20,6 +20,7 @@ import org.apache.gluten.config.{GlutenConfig, VeloxConfig}
 import org.apache.gluten.extension.columnar.FallbackTags
 import org.apache.gluten.utils.VeloxFileSystemValidationJniWrapper
 
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -36,6 +37,54 @@ import org.apache.iceberg.types.{Type, Types}
 import scala.collection.JavaConverters._
 
 class VeloxIcebergSuite extends IcebergSuite {
+  override protected def sparkConf: SparkConf =
+    super.sparkConf.set("spark.sql.ansi.enabled", "false")
+
+  test("iceberg metadata row positions are file-relative across splits and 
filters") {
+    withTable("iceberg_metadata_split_tb") {
+      withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+        spark.sql("""
+                    |CREATE TABLE iceberg_metadata_split_tb (id BIGINT, data 
STRING)
+                    |USING iceberg
+                    |TBLPROPERTIES (
+                    |  'write.parquet.row-group-size-bytes' = '1024',
+                    |  'write.parquet.compression-codec' = 'uncompressed',
+                    |  'write.distribution-mode' = 'none',
+                    |  'read.split.target-size' = '1024',
+                    |  'read.split.open-file-cost' = '0')
+                    |""".stripMargin)
+        spark.range(0, 2000, 1, 1)
+          .selectExpr("id", "sha2(cast(id as string), 256) AS data")
+          .writeTo("iceberg_metadata_split_tb")
+          .append()
+      }
+
+      Seq(
+        "" -> (0L until 2000L),
+        "WHERE id >= 1000" -> (1000L until 2000L),
+        "WHERE _pos >= 1000 AND _pos < 1500" -> (1000L until 1500L),
+        "WHERE id % 3 = 0" -> (0L until 2000L by 3)
+      ).foreach {
+        case (filter, expectedIds) =>
+          runQueryAndCompare(
+            s"SELECT id, _file, _pos FROM iceberg_metadata_split_tb $filter") {
+            df =>
+              checkGlutenPlanCount[IcebergScanTransformer](df, 1)
+              val scans = getExecutedPlan(df).collect { case scan: 
IcebergScanTransformer => scan }
+              assert(scans.size == 1)
+              assert(
+                scans.head.inputRDD.partitions.length > 1,
+                "The single data file must be read using multiple splits")
+              val rows = df.collect()
+              assert(rows.map(_.getString(1)).distinct.length == 1)
+              assert(rows.map(_.getLong(0)).sorted.toSeq == expectedIds)
+              assert(rows.forall(row => !row.isNullAt(2)))
+              assert(rows.forall(row => row.getLong(0) == row.getLong(2)))
+          }
+      }
+    }
+  }
+
   private def withCredentialTable(catalogName: String)(
       f: (String, ResolvingFileIO) => Unit): Unit = {
     withTempDir {
diff --git 
a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
 
b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
index a96706bf4b..cbaaaeedc7 100644
--- 
a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
+++ 
b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
@@ -22,30 +22,34 @@ import org.apache.gluten.exception.GlutenNotSupportException
 import 
org.apache.gluten.execution.IcebergScanTransformer.{containsMetadataColumn, 
containsUuidOrFixedType}
 import org.apache.gluten.expression.ConverterUtils
 import org.apache.gluten.sql.shims.SparkShimLoader
-import org.apache.gluten.substrait.rel.{LocalFilesNode, SplitInfo}
+import org.apache.gluten.substrait.SubstraitContext
+import org.apache.gluten.substrait.expression.{ExpressionBuilder, 
ExpressionNode}
+import org.apache.gluten.substrait.rel.{LocalFilesNode, RelBuilder, SplitInfo}
 import org.apache.gluten.substrait.rel.LocalFilesNode.ColumnMappingMode
 import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
 
 import org.apache.spark.Partition
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, 
DynamicPruningExpression, Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, 
AttributeReference, DynamicPruningExpression, Expression, Literal}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.connector.catalog.Table
 import org.apache.spark.sql.connector.read.Scan
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.sql.types.{ArrayType, DataType, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType, 
StringType, StructType}
 
 import org.apache.iceberg.{BaseTable, MetadataColumns, SnapshotSummary, 
TableProperties}
 import org.apache.iceberg.avro.AvroSchemaUtil
-import org.apache.iceberg.spark.source.{GlutenIcebergSourceUtil, SparkTable}
+import org.apache.iceberg.spark.source.GlutenIcebergSourceUtil
 import org.apache.iceberg.spark.source.metrics.NumSplits
-import org.apache.iceberg.types.{Type, Types}
+import org.apache.iceberg.types.{Type, Types, TypeUtil}
 import org.apache.iceberg.types.Type.TypeID
 import org.apache.iceberg.types.Types.{ListType, MapType, NestedField}
 
 import java.util.{HashMap => JHashMap}
 
+import scala.collection.JavaConverters._
+
 case class IcebergScanTransformer(
     override val output: Seq[AttributeReference],
     @transient override val scan: Scan,
@@ -92,58 +96,57 @@ case class IcebergScanTransformer(
         "Iceberg scan uses catalog-vended credentials, which native readers do 
not support")
     }
 
+    if (GlutenIcebergSourceUtil.hasOrcScanTask(scan)) {
+      if 
(output.exists(_.name.equalsIgnoreCase(MetadataColumns.ROW_POSITION.name()))) {
+        return ValidationResult.failed(
+          "Iceberg row position reads with ORC files are not supported")
+      }
+      val hasTimestampWithZone = 
GlutenIcebergSourceUtil.getTable(scan).schemas().values().stream()
+        .flatMap(schema => 
TypeUtil.indexById(schema.asStruct()).values().stream())
+        .anyMatch(
+          field =>
+            field.`type`() match {
+              case timestamp: Types.TimestampType => 
timestamp.shouldAdjustToUTC()
+              case _ => false
+            })
+      if (hasTimestampWithZone) {
+        return ValidationResult.failed("Iceberg ORC timestamp-with-zone reads 
are not supported")
+      }
+    }
+
     val validationResult = super.doValidateInternal()
     if (!validationResult.ok()) {
       return validationResult
     }
 
     if (!BackendsApiManager.getSettings.supportIcebergEqualityDeleteRead()) {
-      val notSupport = table match {
-        case t: SparkTable =>
-          t.table() match {
-            case t: BaseTable =>
-              t.operations()
-                .current()
-                .schema()
-                .columns()
-                .stream
-                .anyMatch(c => containsUuidOrFixedType(c.`type`()) || 
containsMetadataColumn(c))
-            case _ => false
-          }
+      val notSupport = GlutenIcebergSourceUtil.getTable(scan) match {
+        case t: BaseTable =>
+          t.operations()
+            .current()
+            .schema()
+            .columns()
+            .stream
+            .anyMatch(c => containsUuidOrFixedType(c.`type`()) || 
containsMetadataColumn(c))
         case _ => false
       }
       if (notSupport) {
         return ValidationResult.failed("Contains not supported data type or 
metadata column")
       }
-      // Allow input_file_name() and related metadata functions
-      val allowedMetadataColumns =
-        IcebergScanTransformer.InputFileRelatedMetadataColumnNames
       val hasUnsupportedMetadata = scan.readSchema().fieldNames.exists {
         f =>
           MetadataColumns.isMetadataColumn(f) &&
-          !allowedMetadataColumns.contains(ConverterUtils.normalizeColName(f))
+          !IcebergScanTransformer.NativeSupportedMetadataColumnNames
+            .contains(ConverterUtils.normalizeColName(f))
       }
       if (hasUnsupportedMetadata) {
         return ValidationResult.failed("Read unsupported metadata column")
       }
-      val containsEqualityDelete = table match {
-        case t: SparkTable =>
-          t.table() match {
-            case t: BaseTable =>
-              val snapshot = t
-                .operations()
-                .current()
-                .currentSnapshot()
-              if (snapshot == null) {
-                false
-              } else {
-                snapshot
-                  .summary()
-                  .getOrDefault(SnapshotSummary.TOTAL_EQ_DELETES_PROP, "0")
-                  .toInt > 0
-              }
-            case _ => false
-          }
+      val containsEqualityDelete = GlutenIcebergSourceUtil.getTable(scan) 
match {
+        case t: BaseTable =>
+          val snapshot = t.operations().current().currentSnapshot()
+          snapshot != null &&
+          
snapshot.summary().getOrDefault(SnapshotSummary.TOTAL_EQ_DELETES_PROP, 
"0").toInt > 0
         case _ => false
       }
       if (containsEqualityDelete) {
@@ -156,12 +159,8 @@ case class IcebergScanTransformer(
       }
     }
 
-    val baseTable = table match {
-      case t: SparkTable =>
-        t.table() match {
-          case t: BaseTable => t
-          case _ => null
-        }
+    val baseTable = GlutenIcebergSourceUtil.getTable(scan) match {
+      case t: BaseTable => t
       case _ => null
     }
     if (baseTable == null) {
@@ -217,17 +216,58 @@ case class IcebergScanTransformer(
   }
 
   override def getMetadataColumns(): Seq[AttributeReference] = {
-    val extraMetadataColumns = inputFileRelatedMetadataColumns.filterNot {
-      metadataAttr =>
-        metadataColumns.exists {
-          existingAttr =>
-            ConverterUtils.normalizeColName(existingAttr.name) ==
-              ConverterUtils.normalizeColName(metadataAttr.name)
-        }
-    }
+    val extraMetadataColumns =
+      (inputFileRelatedMetadataColumns ++ output.filter(
+        _.name.equalsIgnoreCase(MetadataColumns.FILE_PATH.name()))).filterNot {
+        metadataAttr =>
+          metadataColumns.exists {
+            existingAttr =>
+              ConverterUtils.normalizeColName(existingAttr.name) ==
+                ConverterUtils.normalizeColName(metadataAttr.name)
+          }
+      }
     metadataColumns ++ extraMetadataColumns
   }
 
+  override protected def isRowIndexMetadataColumn(attr: Attribute): Boolean = {
+    attr.name.equalsIgnoreCase(MetadataColumns.ROW_POSITION.name()) ||
+    super.isRowIndexMetadataColumn(attr)
+  }
+
+  // Velox's Iceberg reader does not fill row-index columns, so read `_pos` 
from the
+  // `$target_table_row_id` column it synthesizes and project out its 
`row_position` field.
+  // TODO: Remove once https://github.com/facebookincubator/velox/pull/20324 
is merged.
+  override protected def doTransform(context: SubstraitContext): 
TransformContext = {
+    val rowPositionIndex =
+      
output.indexWhere(_.name.equalsIgnoreCase(MetadataColumns.ROW_POSITION.name()))
+    if (rowPositionIndex < 0) {
+      return super.doTransform(context)
+    }
+    val targetTableRowId = AttributeReference(
+      "$target_table_row_id",
+      new StructType()
+        .add("file_path", StringType)
+        .add("row_position", LongType)
+        .add("spec_id", IntegerType)
+        .add("partition_data", StringType)
+    )()
+    val readOutput: Seq[Attribute] = output.updated(rowPositionIndex, 
targetTableRowId)
+    val operatorId = context.nextOperatorId(this.nodeName)
+    val readRel = makeReadRel(context, readOutput, operatorId)
+    val projections: Seq[ExpressionNode] = output.indices.map {
+      case `rowPositionIndex` => 
ExpressionBuilder.makeSelection(rowPositionIndex, 1)
+      case i => ExpressionBuilder.makeSelection(i)
+    }
+    val projectRel = RelBuilder.makeProjectRel(
+      readRel,
+      projections.asJava,
+      RelBuilder.createExtensionNode(readOutput.asJava),
+      context,
+      operatorId,
+      output.size)
+    TransformContext(output, projectRel)
+  }
+
   override lazy val fileFormat: ReadFileFormat = fileFormatAndRootPaths._1
 
   override def getSplitInfosFromPartitions(
@@ -274,6 +314,8 @@ case class IcebergScanTransformer(
       runtimeFilters = QueryPlan.normalizePredicates(
         runtimeFilters.filterNot(_ == 
DynamicPruningExpression(Literal.TrueLiteral)),
         output),
+      keyGroupedPartitioning = keyGroupedPartitioning.map(
+        _.map(QueryPlan.normalizeExpressions(_, output))),
       pushDownFilters = pushDownFilters.map(QueryPlan.normalizePredicates(_, 
output))
     )
   }
@@ -285,12 +327,8 @@ case class IcebergScanTransformer(
   private[execution] def getKeyGroupPartitioning: Option[Seq[Expression]] = 
keyGroupedPartitioning
 
   private def hasRenamedColumn: Boolean = {
-    val icebergTable = table match {
-      case t: SparkTable =>
-        t.table() match {
-          case t: BaseTable => t
-          case _ => null
-        }
+    val icebergTable = GlutenIcebergSourceUtil.getTable(scan) match {
+      case t: BaseTable => t
       case _ => null
     }
     if (icebergTable == null) {
@@ -301,11 +339,13 @@ case class IcebergScanTransformer(
     // but may have different id in history schemas
     val ops = icebergTable.operations().current()
     val currentSchema = ops.schema()
+    val dataReadSchema = StructType(
+      scan.readSchema().fields.filterNot(f => 
MetadataColumns.isMetadataColumn(f.name)))
     val oldSchemas = icebergTable.operations().current().schemas()
     oldSchemas
       .stream()
       .filter(s => s.schemaId() != ops.currentSchemaId())
-      .anyMatch(s => !typesMatch(s.asStruct(), currentSchema.asStruct(), 
scan.readSchema()))
+      .anyMatch(s => !typesMatch(s.asStruct(), currentSchema.asStruct(), 
dataReadSchema))
   }
 
   private def typesMatch(icebergType: Type, currentType: Type, sparkType: 
DataType): Boolean = {
@@ -366,6 +406,10 @@ object IcebergScanTransformer {
   private val InputFileRelatedMetadataColumnNames =
     Set("input_file_name", "input_file_block_start", "input_file_block_length")
 
+  private val NativeSupportedMetadataColumnNames = 
InputFileRelatedMetadataColumnNames ++
+    Set(MetadataColumns.FILE_PATH.name(), MetadataColumns.ROW_POSITION.name())
+      .map(ConverterUtils.normalizeColName)
+
   def apply(batchScan: BatchScanExec): IcebergScanTransformer = {
     val transformer = new IcebergScanTransformer(
       batchScan.output.map(a => 
a.withName(AvroSchemaUtil.makeCompatibleName(a.name))),
diff --git 
a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
 
b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
index d460f674df..b4f3aa0166 100644
--- 
a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
+++ 
b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
@@ -46,12 +46,23 @@ object GlutenIcebergSourceUtil {
   private val InputFileBlockLengthCol = "input_file_block_length"
 
   def isSupportedScan(sparkScan: Scan): Boolean = sparkScan match {
-    case _: SparkBatchQueryScan | _: SparkStagedScan => true
+    case _: SparkBatchQueryScan | _: SparkStagedScan | _: SparkCopyOnWriteScan 
=> true
     case _ => false
   }
 
   def isSparkStagedScan(sparkScan: Scan): Boolean = 
sparkScan.isInstanceOf[SparkStagedScan]
 
+  def hasOrcScanTask(sparkScan: Scan): Boolean =
+    getScanTasks(sparkScan).exists {
+      case task if task.isFileScanTask =>
+        task.asFileScanTask().file().format() == FileFormat.ORC
+      case task: CombinedScanTask =>
+        task.tasks().asScala.exists(_.file().format() == FileFormat.ORC)
+      case _ =>
+        throw new UnsupportedOperationException(
+          "Only support iceberg CombinedScanTask and FileScanTask.")
+    }
+
   def hasVendedCredentials(sparkScan: Scan): Boolean =
     isSupportedScan(sparkScan) && 
hasVendedCredentials(getTable(sparkScan).io())
 
@@ -182,6 +193,8 @@ object GlutenIcebergSourceUtil {
           case InputFileNameCol => metadataColumns.put(name, filePath)
           case InputFileBlockStartCol => metadataColumns.put(name, 
start.toString)
           case InputFileBlockLengthCol => metadataColumns.put(name, 
length.toString)
+          case filePathCol if filePathCol == MetadataColumns.FILE_PATH.name() 
=>
+            metadataColumns.put(name, filePath)
           case _ =>
         }
     }
@@ -233,14 +246,14 @@ object GlutenIcebergSourceUtil {
       }
     }
 
-    asFileScanTask(getScanTasks(sparkScan)).foreach {
+    asFileScanTask(getScanTasks(sparkScan).toList).foreach {
       task =>
+        val currentFileFormat = convertFileFormat(task.file().format())
         if (fileFormat == ReadFileFormat.UnknownFormat) {
-          task.file().format() match {
-            case FileFormat.PARQUET => fileFormat = 
ReadFileFormat.ParquetReadFormat
-            case FileFormat.ORC => fileFormat = ReadFileFormat.OrcReadFormat
-            case _ =>
-          }
+          fileFormat = currentFileFormat
+        } else if (fileFormat != currentFileFormat) {
+          throw new GlutenNotSupportException(
+            "Iceberg scans with mixed file formats are not supported")
         }
         if (collectRootPaths) {
           recordPath(task.file().path().toString)
@@ -255,7 +268,7 @@ object GlutenIcebergSourceUtil {
   }
 
   def getReadPartitionSchema(sparkScan: Scan): StructType = {
-    asFileScanTask(getScanTasks(sparkScan)).foreach {
+    asFileScanTask(getScanTasks(sparkScan).toList).foreach {
       task =>
         val spec = task.spec()
         if (spec.isPartitioned) {
@@ -293,18 +306,20 @@ object GlutenIcebergSourceUtil {
     throw new UnsupportedOperationException("Failed to get partition schema 
from iceberg scan.")
   }
 
-  private def getTable(sparkScan: Scan): Table = sparkScan match {
+  def getTable(sparkScan: Scan): Table = sparkScan match {
     case scan: SparkBatchQueryScan => scan.table()
+    case scan: SparkCopyOnWriteScan => scan.table()
     case scan: SparkStagedScan => scan.table()
     case _ =>
       throw new GlutenNotSupportException(
         s"Unsupported Iceberg scan: ${sparkScan.getClass.getName}.")
   }
 
-  private def getScanTasks(sparkScan: Scan): List[ScanTask] = sparkScan match {
-    case scan: SparkBatchQueryScan => scan.tasks().asScala.toList
+  private def getScanTasks(sparkScan: Scan): Iterator[ScanTask] = sparkScan 
match {
+    case scan: SparkBatchQueryScan => scan.tasks().asScala.iterator
+    case scan: SparkCopyOnWriteScan => scan.tasks().asScala.iterator
     case scan: SparkStagedScan =>
-      scan.taskGroups().asScala.flatMap(_.tasks().asScala).toList
+      scan.taskGroups().asScala.iterator.flatMap(_.tasks().asScala.iterator)
     case _ =>
       throw new GlutenNotSupportException(
         s"Unsupported Iceberg scan: ${sparkScan.getClass.getName}.")
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
index 4b0a83cd6c..e552f1953f 100644
--- 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
@@ -23,7 +23,7 @@ import 
org.apache.gluten.extension.columnar.PushDownInputFileExpression
 import org.apache.gluten.substrait.`type`.ColumnTypeNode
 import org.apache.gluten.substrait.SubstraitContext
 import org.apache.gluten.substrait.extensions.ExtensionBuilder
-import org.apache.gluten.substrait.rel.{ReadRelNode, RelBuilder, SplitInfo}
+import org.apache.gluten.substrait.rel.{ReadRelNode, RelBuilder, RelNode, 
SplitInfo}
 import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
 
 import org.apache.spark.Partition
@@ -46,24 +46,26 @@ trait BasicScanExecTransformer extends LeafTransformSupport 
with BaseDataSource
       val (_, scanFiltersNotInPushDownFilters) =
         scanFilters.partition(pushDownFilters.get.contains(_))
       // For filters that only exists in scan, we need to check if they are 
supported.
-      val unsupportedFilters = scanFiltersNotInPushDownFilters.filter(
-        
!BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(_, this))
+      val unsupportedFilters = 
scanFiltersNotInPushDownFilters.filterNot(isSupportedScanFilter)
       if (unsupportedFilters.nonEmpty) {
         throw new UnsupportedOperationException(
           "Found unsupported filter in scan " + unsupportedFilters.mkString(", 
"))
       }
-      val supportedPushDownFilters = pushDownFilters.get
-        
.filter(BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(_, 
this))
+      val supportedPushDownFilters = 
pushDownFilters.get.filter(isSupportedScanFilter)
       FilterHandler.combineFilters(supportedPushDownFilters, 
scanFiltersNotInPushDownFilters)
     } else {
       // todo: When PushDownFilterToScan is not performed, find a way to throw 
an
       //  exception to trigger scan fallback when encountering unsupported 
scan filters,
       //  instead of simply filtering them out.
-      scanFilters.filter(
-        
BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(_, this))
+      scanFilters.filter(isSupportedScanFilter)
     }
   }
 
+  private def isSupportedScanFilter(filter: Expression): Boolean = {
+    !filter.references.exists(isRowIndexMetadataColumn) &&
+    
BackendsApiManager.getSparkPlanExecApiInstance.isSupportedScanFilter(filter, 
this)
+  }
+
   /** Returns the filters that already exists in scan. */
   def scanFilters: Seq[Expression]
 
@@ -205,10 +207,14 @@ trait BasicScanExecTransformer extends 
LeafTransformSupport with BaseDataSource
     doNativeValidation(substraitContext, relNode)
   }
 
+  protected def isRowIndexMetadataColumn(attr: Attribute): Boolean = {
+    
BackendsApiManager.getSparkPlanExecApiInstance.isRowIndexMetadataColumn(attr.name)
+  }
+
   private def makeColumnTypeNode(attr: Attribute): ColumnTypeNode = {
     if (getPartitionSchema.exists(_.name.equals(attr.name))) {
       new ColumnTypeNode(NamedStruct.ColumnType.PARTITION_COL)
-    } else if 
(BackendsApiManager.getSparkPlanExecApiInstance.isRowIndexMetadataColumn(attr.name))
 {
+    } else if (isRowIndexMetadataColumn(attr)) {
       new ColumnTypeNode(NamedStruct.ColumnType.ROWINDEX_COL)
     } else if (attr.isMetadataCol) {
       new ColumnTypeNode(NamedStruct.ColumnType.METADATA_COL)
@@ -218,13 +224,20 @@ trait BasicScanExecTransformer extends 
LeafTransformSupport with BaseDataSource
   }
 
   override protected def doTransform(context: SubstraitContext): 
TransformContext = {
-    val typeNodes = ConverterUtils.collectAttributeTypeNodes(output)
-    val nameList = ConverterUtils.collectAttributeNamesWithoutExprId(output)
-    val columnTypeNodes = output.map(makeColumnTypeNode).asJava
+    TransformContext(output, makeReadRel(context, output, 
context.nextOperatorId(this.nodeName)))
+  }
+
+  protected def makeReadRel(
+      context: SubstraitContext,
+      attributes: Seq[Attribute],
+      operatorId: java.lang.Long): RelNode = {
+    val typeNodes = ConverterUtils.collectAttributeTypeNodes(attributes)
+    val nameList = 
ConverterUtils.collectAttributeNamesWithoutExprId(attributes)
+    val columnTypeNodes = attributes.map(makeColumnTypeNode).asJava
     // Will put all filter expressions into an AND expression
     val exprNode = FilterHandler
       
.combineConjuncts(filterExprs().map(ExpressionConverter.replaceAttributeReference))
-      .map(ExpressionConverter.replaceWithExpressionTransformer(_, output))
+      .map(ExpressionConverter.replaceWithExpressionTransformer(_, attributes))
       .map(_.doTransform(context))
       .orNull
 
@@ -247,12 +260,12 @@ trait BasicScanExecTransformer extends 
LeafTransformSupport with BaseDataSource
       columnTypeNodes,
       extensionNode,
       context,
-      context.nextOperatorId(this.nodeName))
+      operatorId)
     getInputStats.foreach(
       inputStats => {
         logInfo(s"hit scan inputStats with $inputStats")
         readNode.asInstanceOf[ReadRelNode].setInputStats(inputStats)
       })
-    TransformContext(output, readNode)
+    readNode
   }
 }
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
index de61390d03..645b3fc6a0 100644
--- 
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala
@@ -176,8 +176,11 @@ abstract class BatchScanExecTransformerBase(
       return ValidationResult.failed(s"Unsupported aggregation push down for 
$scan.")
     }
 
+    val hasRowIndexColumn =
+      SparkShimLoader.getSparkShims.findRowIndexColumnIndexInSchema(schema) >= 
0 ||
+        output.exists(isRowIndexMetadataColumn)
     if (
-      SparkShimLoader.getSparkShims.findRowIndexColumnIndexInSchema(schema) > 
0 &&
+      hasRowIndexColumn &&
       !BackendsApiManager.getSettings.supportNativeRowIndexColumn()
     ) {
       return ValidationResult.failed("Unsupported row index column scan in 
native.")
diff --git 
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
 
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 5cf41b7a9e..9c3f9073e1 100644
--- 
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++ 
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
 import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.execution.{BatchScanExecTransformer, 
FileSourceScanExecTransformer}
 
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import 
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
 import org.apache.spark.sql.functions.{col, max, min}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
 
+import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.column.ParquetProperties._
 import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
 class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with 
GlutenSQLTestsBaseTrait {
   import testImplicits._
 
+  for (rowIndexType <- Seq(LongType, StringType)) {
+    testGluten(s"datasource v2 row index at schema index zero - 
${rowIndexType.typeName}") {
+      withSQLConf(
+        SQLConf.ANSI_ENABLED.key -> "false",
+        SQLConf.USE_V1_SOURCE_LIST.key -> "",
+        GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+        withTempPath {
+          path =>
+            spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+            val rowIndexName = 
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+            val readSchema = new StructType()
+              .add(rowIndexName, rowIndexType)
+              .add("id", LongType)
+            val df = spark.read
+              .format(classOf[ParquetDataSourceV2].getCanonicalName)
+              .schema(readSchema)
+              .load(path.getAbsolutePath)
+              .select(rowIndexName, "id")
+
+            if (rowIndexType == LongType) {
+              checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+              val scans = getExecutedPlan(df).collect {
+                case scan: BatchScanExecTransformer => scan
+              }
+              assert(scans.size == 1)
+              assert(scans.head.schema.fieldNames.head == rowIndexName)
+            } else {
+              val exception = intercept[Exception](df.collect())
+              
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+            }
+        }
+      }
+    }
+  }
+
   private def readRowGroupRowCounts(path: String): Seq[Long] = {
     ParquetFooterReader
       .readFooter(
diff --git 
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
 
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 570b6d5e0c..f85a3cf5e6 100644
--- 
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++ 
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
 import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.execution.{BatchScanExecTransformer, 
FileSourceScanExecTransformer}
 
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import 
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
 import org.apache.spark.sql.functions.{col, max, min}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
 
+import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.column.ParquetProperties._
 import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
 class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with 
GlutenSQLTestsBaseTrait {
   import testImplicits._
 
+  for (rowIndexType <- Seq(LongType, StringType)) {
+    testGluten(s"datasource v2 row index at schema index zero - 
${rowIndexType.typeName}") {
+      withSQLConf(
+        SQLConf.ANSI_ENABLED.key -> "false",
+        SQLConf.USE_V1_SOURCE_LIST.key -> "",
+        GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+        withTempPath {
+          path =>
+            spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+            val rowIndexName = 
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+            val readSchema = new StructType()
+              .add(rowIndexName, rowIndexType)
+              .add("id", LongType)
+            val df = spark.read
+              .format(classOf[ParquetDataSourceV2].getCanonicalName)
+              .schema(readSchema)
+              .load(path.getAbsolutePath)
+              .select(rowIndexName, "id")
+
+            if (rowIndexType == LongType) {
+              checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+              val scans = getExecutedPlan(df).collect {
+                case scan: BatchScanExecTransformer => scan
+              }
+              assert(scans.size == 1)
+              assert(scans.head.schema.fieldNames.head == rowIndexName)
+            } else {
+              val exception = intercept[Exception](df.collect())
+              
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+            }
+        }
+      }
+    }
+  }
+
   private def readRowGroupRowCounts(path: String): Seq[Long] = {
     ParquetFooterReaderShim
       .readFooter(
@@ -353,7 +389,10 @@ class GlutenParquetRowIndexSuite extends 
ParquetRowIndexSuite with GlutenSQLTest
               .load(path.getAbsolutePath)
 
             val exception = intercept[Exception](dfRead.collect())
-            
assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
+            assert(
+              ExceptionUtils
+                .getRootCauseMessage(exception)
+                .contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
         }
       }
     }
diff --git 
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
 
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 570b6d5e0c..f85a3cf5e6 100644
--- 
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++ 
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
 import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.execution.{BatchScanExecTransformer, 
FileSourceScanExecTransformer}
 
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import 
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
 import org.apache.spark.sql.functions.{col, max, min}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
 
+import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.column.ParquetProperties._
 import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
 class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with 
GlutenSQLTestsBaseTrait {
   import testImplicits._
 
+  for (rowIndexType <- Seq(LongType, StringType)) {
+    testGluten(s"datasource v2 row index at schema index zero - 
${rowIndexType.typeName}") {
+      withSQLConf(
+        SQLConf.ANSI_ENABLED.key -> "false",
+        SQLConf.USE_V1_SOURCE_LIST.key -> "",
+        GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+        withTempPath {
+          path =>
+            spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+            val rowIndexName = 
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+            val readSchema = new StructType()
+              .add(rowIndexName, rowIndexType)
+              .add("id", LongType)
+            val df = spark.read
+              .format(classOf[ParquetDataSourceV2].getCanonicalName)
+              .schema(readSchema)
+              .load(path.getAbsolutePath)
+              .select(rowIndexName, "id")
+
+            if (rowIndexType == LongType) {
+              checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+              val scans = getExecutedPlan(df).collect {
+                case scan: BatchScanExecTransformer => scan
+              }
+              assert(scans.size == 1)
+              assert(scans.head.schema.fieldNames.head == rowIndexName)
+            } else {
+              val exception = intercept[Exception](df.collect())
+              
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+            }
+        }
+      }
+    }
+  }
+
   private def readRowGroupRowCounts(path: String): Seq[Long] = {
     ParquetFooterReaderShim
       .readFooter(
@@ -353,7 +389,10 @@ class GlutenParquetRowIndexSuite extends 
ParquetRowIndexSuite with GlutenSQLTest
               .load(path.getAbsolutePath)
 
             val exception = intercept[Exception](dfRead.collect())
-            
assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
+            assert(
+              ExceptionUtils
+                .getRootCauseMessage(exception)
+                .contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
         }
       }
     }
diff --git 
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
 
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
index 570b6d5e0c..f85a3cf5e6 100644
--- 
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
+++ 
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.datasources.parquet
 import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.execution.{BatchScanExecTransformer, 
FileSourceScanExecTransformer}
 
-import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row}
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.datasources.FileFormat
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import 
org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2
 import org.apache.spark.sql.functions.{col, max, min}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{LongType, StringType}
+import org.apache.spark.sql.types.{LongType, StringType, StructType}
 
+import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.column.ParquetProperties._
 import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -41,6 +42,41 @@ import scala.collection.JavaConverters._
 class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with 
GlutenSQLTestsBaseTrait {
   import testImplicits._
 
+  for (rowIndexType <- Seq(LongType, StringType)) {
+    testGluten(s"datasource v2 row index at schema index zero - 
${rowIndexType.typeName}") {
+      withSQLConf(
+        SQLConf.ANSI_ENABLED.key -> "false",
+        SQLConf.USE_V1_SOURCE_LIST.key -> "",
+        GlutenConfig.NATIVE_WRITER_ENABLED.key -> "false") {
+        withTempPath {
+          path =>
+            spark.range(0, 10, 1, 1).write.parquet(path.getAbsolutePath)
+            val rowIndexName = 
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME
+            val readSchema = new StructType()
+              .add(rowIndexName, rowIndexType)
+              .add("id", LongType)
+            val df = spark.read
+              .format(classOf[ParquetDataSourceV2].getCanonicalName)
+              .schema(readSchema)
+              .load(path.getAbsolutePath)
+              .select(rowIndexName, "id")
+
+            if (rowIndexType == LongType) {
+              checkAnswer(df, (0L until 10L).map(id => Row(id, id)))
+              val scans = getExecutedPlan(df).collect {
+                case scan: BatchScanExecTransformer => scan
+              }
+              assert(scans.size == 1)
+              assert(scans.head.schema.fieldNames.head == rowIndexName)
+            } else {
+              val exception = intercept[Exception](df.collect())
+              
assert(ExceptionUtils.getRootCauseMessage(exception).contains(rowIndexName))
+            }
+        }
+      }
+    }
+  }
+
   private def readRowGroupRowCounts(path: String): Seq[Long] = {
     ParquetFooterReaderShim
       .readFooter(
@@ -353,7 +389,10 @@ class GlutenParquetRowIndexSuite extends 
ParquetRowIndexSuite with GlutenSQLTest
               .load(path.getAbsolutePath)
 
             val exception = intercept[Exception](dfRead.collect())
-            
assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
+            assert(
+              ExceptionUtils
+                .getRootCauseMessage(exception)
+                .contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME))
         }
       }
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to