This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7888bf909694 fix(common): reload index definitions before changing
them (#20245)
7888bf909694 is described below
commit 7888bf909694f3377c2ea7dc860b78e16509623f
Author: Vinish Reddy <[email protected]>
AuthorDate: Sun Nov 8 01:29:09 2026 +0000
fix(common): reload index definitions before changing them (#20245)
---
.../hudi/common/table/HoodieTableMetaClient.java | 20 ++++++++++++
.../common/table/TestHoodieTableMetaClient.java | 37 ++++++++++++++++++++++
2 files changed, 57 insertions(+)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 87218cae4f40..dd55bacac555 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -252,6 +252,7 @@ public class HoodieTableMetaClient implements Serializable {
* @return true if index definition is updated.
*/
public boolean buildIndexDefinition(HoodieIndexDefinition indexDefinition) {
+ reloadIndexMetadata();
String indexName = indexDefinition.getIndexName();
boolean updateIndexDefn = true;
if (indexMetadataOpt.isPresent()) {
@@ -292,6 +293,7 @@ public class HoodieTableMetaClient implements Serializable {
* @param indexName Name of the index
*/
public void deleteIndexDefinition(String indexName) {
+ reloadIndexMetadata();
checkState(indexMetadataOpt.isPresent(), "Index metadata is not present");
indexMetadataOpt.get().getIndexDefinitions().remove(indexName);
writeIndexMetadataToStorage();
@@ -300,6 +302,24 @@ public class HoodieTableMetaClient implements Serializable
{
}
}
+ /**
+ * Reloads the index definitions from storage before they are changed. The
definitions are written as one file, so a
+ * change applied to the definitions this meta client loaded earlier would
drop the definitions that another meta
+ * client of the table wrote since, for example the secondary index
definition that the metadata writer registers
+ * during a commit before the commit updates the column stats index
definition.
+ */
+ private void reloadIndexMetadata() {
+ StoragePath indexDefinitionPath = new
StoragePath(getIndexDefinitionPath());
+ try {
+ Option<byte[]> bytesOpt = FileIOUtils.readDataFromPath(storage,
indexDefinitionPath, true);
+ if (bytesOpt.isPresent()) {
+ indexMetadataOpt =
Option.of(HoodieIndexMetadata.fromJson(fromUTF8Bytes(bytesOpt.get())));
+ }
+ } catch (IOException e) {
+ throw new HoodieIOException("Could not load index definition at path: "
+ indexDefinitionPath, e);
+ }
+ }
+
/**
* Writes the current index metadata to storage.
*/
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
index ccf8335f6e53..6f300a6b5a04 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
@@ -593,6 +593,43 @@ class TestHoodieTableMetaClient extends
HoodieCommonTestHarness {
assertTrue(indexMetadata.getIndexDefinitions().isEmpty());
}
+ @Test
+ void testIndexDefinitionChangesKeepDefinitionsOfOtherMetaClients() throws
IOException {
+ final String basePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "t8";
+ HoodieTableMetaClient firstMetaClient =
HoodieTableMetaClient.newTableBuilder()
+ .setTableType(HoodieTableType.COPY_ON_WRITE.name())
+ .setTableName("table")
+ .initTable(this.metaClient.getStorageConf(), basePath);
+ // both meta clients load the index definitions before either of them
changes them
+ HoodieTableMetaClient secondMetaClient =
HoodieTableMetaClient.reload(firstMetaClient);
+ String secondaryIndexName =
MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "c1";
+ String columnStatsIndexName =
MetadataPartitionType.COLUMN_STATS.getPartitionPath();
+
+ firstMetaClient.buildIndexDefinition(HoodieIndexDefinition.newBuilder()
+ .withIndexName(secondaryIndexName)
+
.withIndexType(MetadataPartitionType.SECONDARY_INDEX.getPartitionPath())
+
.withVersion(HoodieIndexVersion.getCurrentVersion(HoodieTableVersion.current(),
secondaryIndexName))
+ .withSourceFields(Collections.singletonList("c1"))
+ .build());
+ // the second meta client did not load the secondary index definition, and
must not drop it
+ secondMetaClient.buildIndexDefinition(HoodieIndexDefinition.newBuilder()
+ .withIndexName(columnStatsIndexName)
+ .withIndexType(columnStatsIndexName)
+ .withIndexFunction(columnStatsIndexName)
+
.withVersion(HoodieIndexVersion.getCurrentVersion(HoodieTableVersion.current(),
columnStatsIndexName))
+ .withSourceFields(Arrays.asList("c1", "c2"))
+ .build());
+ HoodieTableMetaClient reloadedMetaClient =
HoodieTableMetaClient.reload(firstMetaClient);
+
assertTrue(reloadedMetaClient.getIndexForMetadataPartition(secondaryIndexName).isPresent());
+
assertTrue(reloadedMetaClient.getIndexForMetadataPartition(columnStatsIndexName).isPresent());
+
+ // a delete through the first meta client must keep the definition the
second meta client wrote
+ firstMetaClient.deleteIndexDefinition(secondaryIndexName);
+ reloadedMetaClient = HoodieTableMetaClient.reload(firstMetaClient);
+
assertFalse(reloadedMetaClient.getIndexForMetadataPartition(secondaryIndexName).isPresent());
+
assertTrue(reloadedMetaClient.getIndexForMetadataPartition(columnStatsIndexName).isPresent());
+ }
+
@Test
void testReadIndexDefFromStorage() throws Exception {
final String basePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "t8";