This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b22a3d7af37a test(spark): add extended SQL parser coverage for index
DDL and Hudi column types (#19218)
b22a3d7af37a is described below
commit b22a3d7af37a5c87665477b5e5775307d0f3137d
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Jul 9 10:51:13 2026 -0700
test(spark): add extended SQL parser coverage for index DDL and Hudi column
types (#19218)
* test(spark): add extended SQL parser coverage for index DDL and Hudi
column types
* test(spark): strengthen parser coverage assertions per review
- BLOB/VECTOR tests now assert the hudi_type field metadata (BLOB, canonical
VECTOR(3)) and the mapped dataType (BlobType(), ArrayType(FloatType,
false))
instead of only checking the column name exists, so they fail if the
custom
type maps incorrectly.
- Drop the unused create-table setup from the four index-DDL tests;
parsePlan
is purely syntactic, so the parse is identical whether or not the table
exists (noted on the parse helper).
* test(spark): fold parser coverage into TestIndexSyntax per review
TestExtendedSqlParserCoverage duplicated existing coverage: BLOB/VECTOR
create-table + hudi_type assertions live in TestCreateTable and
TestBlobDataType, and CREATE/DROP/SHOW INDEX parse+resolution is covered
by TestIndexSyntax. The only genuine gap was REFRESH INDEX, which
TestIndexSyntax's 'Test Create/Drop/Show/Refresh Index' promised in its
title but never exercised. Add the refresh-index parse+analyze
assertions there, mirroring the sibling statements, and drop the
redundant suite.
* test(spark): assert catalog-stored schema retains hudi_type for
BLOB/VECTOR
The dropped parser-coverage suite read the schema back via
catalog.getTableMetadata rather than spark.table; that was the one
assertion without an existing home: the CatalogTable persisted by
CreateHoodieTableCommand retains the hudi_type field metadata. Pin it in
TestCreateTable's BLOB and VECTOR create-table tests.
---------
Co-authored-by: voon <[email protected]>
---
.../scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala | 10 ++++++++++
.../apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala | 7 ++++++-
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
index 5c1019f4d21c..b770c17b462c 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
@@ -2065,6 +2065,11 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
// Verify structure matches blob schema
assertTrue(videoField.dataType.isInstanceOf[StructType])
assertEquals(BlobType(), videoField.dataType)
+
+ // The catalog-stored copy of the schema must retain the hudi_type
metadata too
+ val catalogField =
spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
+ .schema.find(_.name == "video").get
+ assertEquals(HoodieSchemaType.BLOB.name(),
catalogField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
}
}
@@ -2184,6 +2189,11 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
assertEquals("VECTOR(128)",
embeddingField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
assertEquals("document embedding",
embeddingField.metadata.getString("comment"))
assertEquals(ArrayType(FloatType, containsNull = false),
embeddingField.dataType)
+
+ // The catalog-stored copy of the schema must retain the hudi_type
metadata too
+ val catalogField =
spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
+ .schema.find(_.name == "embedding").get
+ assertEquals("VECTOR(128)",
catalogField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
index 8fb3fd8648c0..2e186ae5347b 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
@@ -26,7 +26,7 @@ import org.apache.hudi.metadata.HoodieTableMetadataUtil
import org.apache.spark.sql.catalyst.analysis.Analyzer
import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.hudi.command.{CreateIndexCommand,
DropIndexCommand, ShowIndexesCommand}
+import org.apache.spark.sql.hudi.command.{CreateIndexCommand,
DropIndexCommand, RefreshIndexCommand, ShowIndexesCommand}
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue}
@@ -90,6 +90,11 @@ class TestIndexSyntax extends HoodieSparkSqlTestBase {
assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].table,
databaseName, tableName)
assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].indexName)
assertResult(true)(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].ignoreIfNotExists)
+
+ logicalPlan = sqlParser.parsePlan(s"refresh index idx_name on
$tableName")
+ resolvedLogicalPlan = analyzer.execute(logicalPlan)
+
assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].table,
databaseName, tableName)
+
assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].indexName)
}
}
}
