This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit dc5c845de2235ff6e9c69ccda3e4c41784de5285 Author: Y Ethan Guo <[email protected]> AuthorDate: Thu Jul 9 10:51:13 2026 -0700 test(spark): add extended SQL parser coverage for index DDL and Hudi column types (#19218) * test(spark): add extended SQL parser coverage for index DDL and Hudi column types * test(spark): strengthen parser coverage assertions per review - BLOB/VECTOR tests now assert the hudi_type field metadata (BLOB, canonical VECTOR(3)) and the mapped dataType (BlobType(), ArrayType(FloatType, false)) instead of only checking the column name exists, so they fail if the custom type maps incorrectly. - Drop the unused create-table setup from the four index-DDL tests; parsePlan is purely syntactic, so the parse is identical whether or not the table exists (noted on the parse helper). * test(spark): fold parser coverage into TestIndexSyntax per review TestExtendedSqlParserCoverage duplicated existing coverage: BLOB/VECTOR create-table + hudi_type assertions live in TestCreateTable and TestBlobDataType, and CREATE/DROP/SHOW INDEX parse+resolution is covered by TestIndexSyntax. The only genuine gap was REFRESH INDEX, which TestIndexSyntax's 'Test Create/Drop/Show/Refresh Index' promised in its title but never exercised. Add the refresh-index parse+analyze assertions there, mirroring the sibling statements, and drop the redundant suite. * test(spark): assert catalog-stored schema retains hudi_type for BLOB/VECTOR The dropped parser-coverage suite read the schema back via catalog.getTableMetadata rather than spark.table; that was the one assertion without an existing home: the CatalogTable persisted by CreateHoodieTableCommand retains the hudi_type field metadata. Pin it in TestCreateTable's BLOB and VECTOR create-table tests. --------- Co-authored-by: voon <[email protected]> (cherry picked from commit b22a3d7af37a5c87665477b5e5775307d0f3137d) --- .../scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala | 10 ++++++++++ .../apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala index 88ec4b7fb4c3..ab70edb18b4e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala @@ -2064,6 +2064,11 @@ class TestCreateTable extends HoodieSparkSqlTestBase { // Verify structure matches blob schema assertTrue(videoField.dataType.isInstanceOf[StructType]) assertEquals(BlobType(), videoField.dataType) + + // The catalog-stored copy of the schema must retain the hudi_type metadata too + val catalogField = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + .schema.find(_.name == "video").get + assertEquals(HoodieSchemaType.BLOB.name(), catalogField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD)) } } @@ -2183,6 +2188,11 @@ class TestCreateTable extends HoodieSparkSqlTestBase { assertEquals("VECTOR(128)", embeddingField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD)) assertEquals("document embedding", embeddingField.metadata.getString("comment")) assertEquals(ArrayType(FloatType, containsNull = false), embeddingField.dataType) + + // The catalog-stored copy of the schema must retain the hudi_type metadata too + val catalogField = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + .schema.find(_.name == "embedding").get + assertEquals("VECTOR(128)", catalogField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD)) } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala index 8fb3fd8648c0..2e186ae5347b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala @@ -26,7 +26,7 @@ import org.apache.hudi.metadata.HoodieTableMetadataUtil import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.parser.ParserInterface -import org.apache.spark.sql.hudi.command.{CreateIndexCommand, DropIndexCommand, ShowIndexesCommand} +import org.apache.spark.sql.hudi.command.{CreateIndexCommand, DropIndexCommand, RefreshIndexCommand, ShowIndexesCommand} import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue} @@ -90,6 +90,11 @@ class TestIndexSyntax extends HoodieSparkSqlTestBase { assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].table, databaseName, tableName) assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].indexName) assertResult(true)(resolvedLogicalPlan.asInstanceOf[DropIndexCommand].ignoreIfNotExists) + + logicalPlan = sqlParser.parsePlan(s"refresh index idx_name on $tableName") + resolvedLogicalPlan = analyzer.execute(logicalPlan) + assertTableIdentifier(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].table, databaseName, tableName) + assertResult("idx_name")(resolvedLogicalPlan.asInstanceOf[RefreshIndexCommand].indexName) } } }
