This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6ef0c903540b test(writer): cover legacy populate.meta.fields
persistence on table version 6 (#19695)
6ef0c903540b is described below
commit 6ef0c903540ba26c92cef36536e8c4681ecf9637
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Aug 20 19:07:47 2026 -0700
test(writer): cover legacy populate.meta.fields persistence on table
version 6 (#19695)
* test(writer): cover legacy populate.meta.fields persistence on table
version 6
The existing v6 assertion resolved through the mode property and could not
fail when the boolean was not persisted. Assert the raw persisted key at
both persistence sites (table builder and HoodieTableConfig.create), and
add a Spark datasource end-to-end case writing a table version 6 table for
every meta fields mode.
Claude-Session: https://claude.ai/code/session_01XZmMF237YSc3Mbfvx8Cz2D
* Parameterize the persistence tests across table versions 6, 9, and 10
Claude-Session: https://claude.ai/code/session_01XZmMF237YSc3Mbfvx8Cz2D
---
.../hudi/common/table/TestHoodieTableConfig.java | 35 ++++++++++++++++++
.../common/table/TestHoodieTableMetaClient.java | 28 ++++++++------
.../hudi/functional/TestMetaFieldsModeE2E.java | 43 ++++++++++++++++++++++
3 files changed, 95 insertions(+), 11 deletions(-)
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
index a6dd7912318a..cdc023602c66 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
@@ -423,6 +423,41 @@ class TestHoodieTableConfig extends
HoodieCommonTestHarness {
"the mode must survive on a v6 table -- Uber-style deployments set it
there via hudi-cli");
}
+ private static Stream<Arguments> tableVersionsAndMetaFieldsModes() {
+ return Stream.of(HoodieTableVersion.SIX, HoodieTableVersion.NINE,
HoodieTableVersion.TEN)
+ .flatMap(version -> Arrays.stream(MetaFieldsMode.values())
+ .map(mode -> arguments(version, mode)));
+ }
+
+ /**
+ * {@link HoodieTableConfig#create} must derive and persist the legacy
populate boolean below
+ * table version 10 on its own: callers such as the repair-overwrite-props
procedure invoke it
+ * directly with user-supplied properties, without going through the table
builder that also
+ * derives the boolean. From version 10 the mode alone is recorded.
+ */
+ @ParameterizedTest
+ @MethodSource("tableVersionsAndMetaFieldsModes")
+ void testCreatePersistsLegacyBooleanPerTableVersion(HoodieTableVersion
version, MetaFieldsMode mode)
+ throws IOException {
+ StoragePath versionedMetaPath = new StoragePath(basePath,
+ "v" + version.versionCode() + "-" + mode.name() + "/" +
HoodieTableMetaClient.METAFOLDER_NAME);
+ Properties props = new Properties();
+ props.setProperty(HoodieTableConfig.NAME.key(), "test-table");
+ props.setProperty(HoodieTableConfig.VERSION.key(),
String.valueOf(version.versionCode()));
+ props.setProperty(HoodieTableConfig.META_FIELDS_MODE.key(), mode.name());
+ HoodieTableConfig.create(storage, versionedMetaPath, props);
+
+ HoodieTableConfig persisted = new HoodieTableConfig(storage,
versionedMetaPath);
+ if (version.lesserThan(HoodieTableVersion.TEN)) {
+ assertEquals(Boolean.toString(mode.toLegacyPopulateMetaFields()),
+
persisted.getProps().getProperty(HoodieTableConfig.POPULATE_META_FIELDS.key()),
+ "create() must persist the derived boolean below v10 for unpatched
readers, " + mode);
+ } else {
+
assertFalse(persisted.getProps().containsKey(HoodieTableConfig.POPULATE_META_FIELDS.key()),
+ "create() must record the mode alone from v10, " + mode);
+ }
+ }
+
@Test
void testDropInvalidConfigs() {
// test invalid configs are dropped
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
index 3cc87601cb57..eda38d12c7f9 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -229,17 +230,22 @@ class TestHoodieTableMetaClient extends
HoodieCommonTestHarness {
assertFalse(newTable.getProps().containsKey(HoodieTableConfig.POPULATE_META_FIELDS.key()),
"a table created at v10 records the mode alone for " + mode);
- final String oldPath = tempDir.toAbsolutePath() + Path.SEPARATOR +
"mfm-v6-" + mode.name();
- HoodieTableConfig oldTable = HoodieTableMetaClient.newTableBuilder()
- .setTableType(HoodieTableType.COPY_ON_WRITE.name())
- .setTableName("mfm-v6-" + mode.name())
- .setTableVersion(HoodieTableVersion.SIX.versionCode())
- .setMetaFieldsMode(mode)
- .initTable(this.metaClient.getStorageConf(), oldPath)
- .getTableConfig();
- assertEquals(mode, oldTable.getMetaFieldsMode());
- assertEquals(mode.toLegacyPopulateMetaFields(),
oldTable.populateMetaFields(),
- "a table below v10 must still carry the derived boolean for
unpatched readers, " + mode);
+ for (HoodieTableVersion oldVersion :
Arrays.asList(HoodieTableVersion.SIX, HoodieTableVersion.NINE)) {
+ final String oldPath = tempDir.toAbsolutePath() + Path.SEPARATOR
+ + "mfm-v" + oldVersion.versionCode() + "-" + mode.name();
+ HoodieTableConfig oldTable = HoodieTableMetaClient.newTableBuilder()
+ .setTableType(HoodieTableType.COPY_ON_WRITE.name())
+ .setTableName("mfm-v" + oldVersion.versionCode() + "-" +
mode.name())
+ .setTableVersion(oldVersion.versionCode())
+ .setMetaFieldsMode(mode)
+ .initTable(this.metaClient.getStorageConf(), oldPath)
+ .getTableConfig();
+ assertEquals(mode, oldTable.getMetaFieldsMode());
+ assertEquals(Boolean.toString(mode.toLegacyPopulateMetaFields()),
+
oldTable.getProps().getProperty(HoodieTableConfig.POPULATE_META_FIELDS.key()),
+ "a table at version " + oldVersion.versionCode()
+ + " must persist the derived boolean for unpatched readers, "
+ mode);
+ }
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsModeE2E.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsModeE2E.java
index 58886e72776d..76cfbe687ed2 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsModeE2E.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsModeE2E.java
@@ -27,7 +27,9 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.MetaFieldsMode;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
import org.apache.spark.api.java.function.VoidFunction2;
@@ -42,7 +44,9 @@ import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
import java.nio.file.Path;
import java.util.Arrays;
@@ -52,6 +56,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -281,6 +286,44 @@ class TestMetaFieldsModeE2E extends
SparkClientFunctionalTestHarness {
"ALL must persist populate.meta.fields=true for pre-1.3.0 readers");
}
+ private static Stream<Arguments> tableVersionsAndMetaFieldsModes() {
+ return Stream.of(HoodieTableVersion.SIX, HoodieTableVersion.NINE,
HoodieTableVersion.TEN)
+ .flatMap(version -> Arrays.stream(MetaFieldsMode.values())
+ .map(mode -> Arguments.of(version, mode)));
+ }
+
+ /**
+ * A write at any supported table version must record the mode verbatim, and
below version 10 also
+ * the derived legacy boolean as a raw property; from version 10 the mode
alone is recorded. The
+ * boolean is asserted on the raw persisted key rather than through {@code
populateMetaFields()}
+ * (which resolves through the mode and answers the same either way) because
an unpatched
+ * pre-1.3.0 reader sees only the raw key. Meta-column population is
verified for every case.
+ */
+ @ParameterizedTest
+ @MethodSource("tableVersionsAndMetaFieldsModes")
+ void writePersistsModeAndLegacyBooleanPerTableVersion(HoodieTableVersion
version, MetaFieldsMode mode) {
+ Map<String, String> options = baseOptions();
+ options.put(HoodieWriteConfig.WRITE_TABLE_VERSION.key(),
String.valueOf(version.versionCode()));
+ options.put(HoodieTableConfig.META_FIELDS_MODE.key(), mode.name());
+ options.put(DataSourceWriteOptions.OPERATION().key(),
DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL());
+
+ HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath());
+
+ assertEquals(version, tc.getTableVersion());
+ assertEquals(mode, tc.getMetaFieldsMode());
+ assertEquals(mode.name(), tc.getString(HoodieTableConfig.META_FIELDS_MODE),
+ "the mode must be persisted at table version " +
version.versionCode());
+ if (version.lesserThan(HoodieTableVersion.TEN)) {
+ assertEquals(Boolean.toString(mode.toLegacyPopulateMetaFields()),
+
tc.getProps().getProperty(HoodieTableConfig.POPULATE_META_FIELDS.key()),
+ "a table below v10 must persist the derived legacy boolean for
unpatched readers");
+ } else {
+
assertFalse(tc.getProps().containsKey(HoodieTableConfig.POPULATE_META_FIELDS.key()),
+ "a table at v10 must record the mode alone");
+ }
+ assertMetaColumnPopulation(basePath(), mode);
+ }
+
@Test
void unknownModeValueIsRejected() {
Map<String, String> options = baseOptions();
