(hudi) branch master updated: fix(gcp): skip "null" type fields when syncing the BigQuery schema (#20356)

[yihua]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new c82957df5f75 fix(gcp): skip "null" type fields when syncing the 
BigQuery schema (#20356)
c82957df5f75 is described below

commit c82957df5f757c2cc3faaf8b971fb554251100c1
Author: Prashant Wason <pwa...@uber.com>
AuthorDate: Thu Oct 15 22:49:27 2026 -0700

    fix(gcp): skip "null" type fields when syncing the BigQuery schema (#20356)
---
 .../hudi/gcp/bigquery/BigQuerySchemaResolver.java  | 14 ++++++---
 .../gcp/bigquery/TestBigQuerySchemaResolver.java   | 36 ++++++++++++++++++++++
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git 
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
 
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
index a26affe33ab2..24d7011e34c9 100644
--- 
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
+++ 
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
@@ -172,11 +172,15 @@ public class BigQuerySchemaResolver {
   }
 
   private List<Field> getFields(HoodieSchema schema) {
-    return schema.getFields().stream().map(field -> {
-      final HoodieSchema fieldSchema = field.getNonNullSchema();
-      final boolean nullable = field.isNullable();
-      return getField(fieldSchema, field.name(), nullable);
-    }).collect(Collectors.toList());
+    return schema.getFields().stream()
+        // Fields of the "null" type always hold null and are not written to 
the data files, so they are skipped
+        .filter(field -> !field.schema().isSchemaNull())
+        .map(field -> {
+          final HoodieSchema fieldSchema = field.getNonNullSchema();
+          final boolean nullable = field.isNullable();
+          return getField(fieldSchema, field.name(), nullable);
+        })
+        .collect(Collectors.toList());
   }
 
   private void validateUnion(List<HoodieSchema> subTypes) {
diff --git 
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
 
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
index c47d96c05869..2c9994f80313 100644
--- 
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
+++ 
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
@@ -243,6 +243,42 @@ public class TestBigQuerySchemaResolver {
     Assertions.assertEquals(expected, SCHEMA_RESOLVER.convertSchema(input));
   }
 
+  @Test
+  void convertSchema_nullTypeField() {
+    HoodieSchema input = HoodieSchema.parse("{\"type\": \"record\", \"name\": 
\"testRecord\", \"fields\": ["
+        + "{\"name\": \"name\", \"type\": \"string\"},"
+        + "{\"name\": \"clientpromotion\", \"type\": \"null\", \"default\": 
null},"
+        + "{\"name\": \"age\", \"type\": [\"null\", \"int\"], \"default\": 
null}]}");
+
+    Schema expected = Schema.of(
+        Field.newBuilder("name", 
StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(),
+        Field.newBuilder("age", 
StandardSQLTypeName.INT64).setMode(Field.Mode.NULLABLE).build());
+
+    Schema result = SCHEMA_RESOLVER.convertSchema(input);
+    Assertions.assertEquals(expected, result);
+    Assertions.assertEquals("`name` STRING NOT NULL, `age` INT64", 
schemaToSqlString(result));
+  }
+
+  @Test
+  void convertSchema_nestedNullTypeField() {
+    HoodieSchema input = HoodieSchema.parse("{\"type\": \"record\", \"name\": 
\"testRecord\", \"fields\": ["
+        + "{\"name\": \"name\", \"type\": \"string\"},"
+        + "{\"name\": \"address\", \"type\": {\"type\": \"record\", \"name\": 
\"address\", \"fields\": ["
+        + "{\"name\": \"city\", \"type\": \"string\"},"
+        + "{\"name\": \"nestedpromotion\", \"type\": \"null\", \"default\": 
null}]}}]}");
+
+    Schema expected = Schema.of(
+        Field.newBuilder("name", 
StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(),
+        Field.newBuilder("address", StandardSQLTypeName.STRUCT,
+                Field.newBuilder("city", 
StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build())
+            .setMode(Field.Mode.REQUIRED).build());
+
+    Schema result = SCHEMA_RESOLVER.convertSchema(input);
+    Assertions.assertEquals(expected, result);
+    Assertions.assertEquals("`name` STRING NOT NULL, `address` STRUCT<`city` 
STRING NOT NULL> NOT NULL",
+        schemaToSqlString(result));
+  }
+
   @Test
   void getTableSchema_withPartitionFields() throws Exception {
     HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);