This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 1fc97453dcfea11bbf9439719e8c1a2034c1812c Author: Sivabalan Narayanan <[email protected]> AuthorDate: Thu Jun 25 19:58:24 2026 -0700 [MINOR] Guard detailed metadata size metrics with a config (#18803) * feat(metadata): guard detailed metadata size metrics behind a config Detailed size metrics in the metadata table cause FSV to be built for MDT in the driver. Add hoodie.metadata.enable.detailed.metrics (default false) to gate emission of these detailed size metrics, reducing driver memory pressure at scale. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]> --------- Co-authored-by: Claude Opus 4.7 (1M context) <[email protected]> (cherry picked from commit c6d707c6a8bda4a18edab98247ebd03ff1530111) --- .../metadata/HoodieBackedTableMetadataWriter.java | 6 +++++- .../table/action/index/RunIndexActionExecutor.java | 2 +- .../FlinkHoodieBackedTableMetadataWriter.java | 2 +- .../JavaHoodieBackedTableMetadataWriter.java | 2 +- .../hudi/testutils/TestHoodieMetadataBase.java | 1 + .../SparkHoodieBackedTableMetadataWriter.java | 2 +- ...ieBackedTableMetadataWriterTableVersionSix.java | 2 +- .../client/functional/TestHoodieMetadataBase.java | 1 + .../hudi/common/config/HoodieMetadataConfig.java | 18 ++++++++++++++++ .../apache/hudi/metadata/BaseTableMetadata.java | 3 ++- .../hudi/metadata/HoodieMetadataMetrics.java | 8 +++++++- .../common/config/TestHoodieMetadataConfig.java | 24 ++++++++++++++++++++++ 12 files changed, 63 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 4789accc8caa..df61ed996b1f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -1984,7 +1984,11 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab metadataMetaClient.reloadActiveTimeline(); // Update total size of the metadata and count of base/log files - metrics.ifPresent(m -> m.updateSizeMetrics(metadataMetaClient, metadata, dataMetaClient.getTableConfig().getMetadataPartitions())); + metrics.ifPresent(m -> { + if (m.isDetailedMetricsEnabled()) { + m.updateSizeMetrics(metadataMetaClient, metadata, dataMetaClient.getTableConfig().getMetadataPartitions()); + } + }); } protected abstract void bulkInsertAndCommit(BaseHoodieWriteClient<?, I, ?, O> writeClient, String instantTime, I preppedRecordInputs, Option<BulkInsertPartitioner> bulkInsertPartitioner); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java index 6bb15573c829..c0a29695bf45 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java @@ -98,7 +98,7 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, super(context, config, table, instantTime); this.txnManager = new TransactionManager(config, table.getStorage()); if (config.getMetadataConfig().isMetricsEnabled()) { - this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage(), config.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java index 268ae942b92e..8cc44f068df5 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java @@ -98,7 +98,7 @@ public class FlinkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad protected void initRegistry() { if (metadataWriteConfig.isMetricsOn()) { // should support executor metrics - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java index 2a035f560d5c..b5d9d2822f19 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java @@ -82,7 +82,7 @@ public class JavaHoodieBackedTableMetadataWriter extends HoodieBackedTableMetada @Override protected void initRegistry() { if (metadataWriteConfig.isMetricsOn()) { - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java index b555b7696461..792f742ed92a 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java @@ -308,6 +308,7 @@ public class TestHoodieMetadataBase extends HoodieJavaClientTestHarness { .enable(useFileListingMetadata) .withMetadataIndexColumnStats(false) .enableMetrics(enableMetrics) + .enableDetailedMetadataMetrics(enableMetrics) .ignoreSpuriousDeletes(validateMetadataPayloadConsistency) .withMetadataIndexColumnStats(false) // HUDI-8774 .withEngineType(EngineType.JAVA) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java index dfb295d5d16a..7ea05926768d 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java @@ -138,7 +138,7 @@ public class SparkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad } else { registry = Registry.getRegistry("HoodieMetadata"); } - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java index 617b4b4e820f..facd02965749 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java @@ -106,7 +106,7 @@ public class SparkHoodieBackedTableMetadataWriterTableVersionSix extends HoodieB } else { registry = Registry.getRegistry("HoodieMetadata"); } - this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage())); + this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java index f28de3266c7f..4a210227bb74 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java @@ -344,6 +344,7 @@ public class TestHoodieMetadataBase extends HoodieSparkClientTestHarness { .withMetadataConfig(HoodieMetadataConfig.newBuilder() .enable(useFileListingMetadata) .enableMetrics(enableMetrics) + .enableDetailedMetadataMetrics(enableMetrics) .ignoreSpuriousDeletes(validateMetadataPayloadConsistency) .build()) .withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index cd4d03ee4ee7..7ce1e447e65f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -683,6 +683,15 @@ public final class HoodieMetadataConfig extends HoodieConfig { + "with the actual record count stored in the metadata table. This validation runs in a distributed manner " + "using the compute engine. Disabled by default as it adds overhead to the initialization process."); + public static final ConfigProperty<Boolean> ENABLE_DETAILED_METRICS = ConfigProperty + .key(METADATA_PREFIX + ".enable.detailed.metrics") + .defaultValue(false) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Enables detailed metadata table metrics — per-metadata-partition file size and base/log " + + "file counts. Emitting these requires building a HoodieTableFileSystemView for the metadata table on " + + "the driver, which adds memory pressure at scale; leave disabled unless you need the breakdown."); + public long getMaxLogFileSize() { return getLong(MAX_LOG_FILE_SIZE_BYTES_PROP); } @@ -1020,6 +1029,10 @@ public final class HoodieMetadataConfig extends HoodieConfig { return subIndexNameToDrop.contains(indexName); } + public boolean isDetailedMetricsEnabled() { + return getBoolean(ENABLE_DETAILED_METRICS); + } + public static class Builder { private EngineType engineType = EngineType.SPARK; @@ -1349,6 +1362,11 @@ public final class HoodieMetadataConfig extends HoodieConfig { return this; } + public Builder enableDetailedMetadataMetrics(boolean enable) { + metadataConfig.setValue(ENABLE_DETAILED_METRICS, String.valueOf(enable)); + return this; + } + public HoodieMetadataConfig build() { metadataConfig.setDefaultValue(ENABLE, getDefaultMetadataEnable(engineType)); metadataConfig.setDefaultValue(ENABLE_METADATA_INDEX_COLUMN_STATS, getDefaultColStatsEnable(engineType)); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java index f59a257dd777..c849262d2d94 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java @@ -94,7 +94,8 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata { if (metadataConfig.isMetricsEnabled()) { this.metrics = Option.of(new HoodieMetadataMetrics(HoodieMetricsConfig.newBuilder() - .fromProperties(metadataConfig.getProps()).withPath(dataBasePath).build(), dataMetaClient.getStorage())); + .fromProperties(metadataConfig.getProps()).withPath(dataBasePath).build(), dataMetaClient.getStorage(), + metadataConfig.isDetailedMetricsEnabled())); } else { this.metrics = Option.empty(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java index c7244a30d094..32f7eca343f3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java @@ -85,10 +85,12 @@ public class HoodieMetadataMetrics implements Serializable { private final transient MetricRegistry metricsRegistry; private final transient Metrics metrics; + private final boolean detailedMetricsEnabled; - public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage) { + public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage, boolean detailedMetricsEnabled) { this.metrics = Metrics.getInstance(metricsConfig, storage); this.metricsRegistry = metrics.getRegistry(); + this.detailedMetricsEnabled = detailedMetricsEnabled; } public Map<String, String> getStats(boolean detailed, HoodieTableMetaClient metaClient, HoodieTableMetadata metadata, Set<String> metadataPartitions) { @@ -101,6 +103,10 @@ public class HoodieMetadataMetrics implements Serializable { } } + public boolean isDetailedMetricsEnabled() { + return detailedMetricsEnabled; + } + private Map<String, String> getStats(HoodieTableFileSystemView fsView, boolean detailed, HoodieTableMetadata tableMetadata, Set<String> metadataPartitions) throws IOException { Map<String, String> stats = new HashMap<>(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java index b66b432abc24..f3091ed58fe3 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java @@ -269,4 +269,28 @@ class TestHoodieMetadataConfig { .withTableServiceManagerEnabled(true) .build()); } + + @Test + void testMetricsConfig() { + // Test default value + HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().build(); + assertFalse(config.isMetricsEnabled()); + + Properties props = new Properties(); + props.put(HoodieMetadataConfig.METRICS_ENABLE.key(), true); + config = HoodieMetadataConfig.newBuilder() + .fromProperties(props) + .build(); + assertTrue(config.isMetricsEnabled()); + assertFalse(config.isDetailedMetricsEnabled()); + + props = new Properties(); + props.put(HoodieMetadataConfig.METRICS_ENABLE.key(), true); + props.put(HoodieMetadataConfig.ENABLE_DETAILED_METRICS.key(), true); + config = HoodieMetadataConfig.newBuilder() + .fromProperties(props) + .build(); + assertTrue(config.isMetricsEnabled()); + assertTrue(config.isDetailedMetricsEnabled()); + } }
