This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit aca664d5abc59787e686bedc43ba9e350df81c0b Author: Prashant Wason <[email protected]> AuthorDate: Fri Jun 12 16:14:30 2026 -0700 feat: Add HUDI version and engine properties to commit metadata (#18183) Add hudi.version and engine (SPARK/FLINK/JAVA) to every commit's extraMetadata so post-hoc debugging can answer "which writer wrote this commit?". Engine- specific properties (spark.application.id, java.version, os.name, etc.) and a configurable allowlist of HoodieWriteConfig values are also embeddable under new opt-in configs: - hoodie.commit.metadata.engine.properties.embed.enable (default false) - hoodie.write.config.keys.to.serialize.to.commit.metadata (default: small allowlist of operationally useful keys) Enrichment is centralized in CommitMetadataProperties.enrich() and invoked once via BaseHoodieClient.updateExtraMetadata() from the write-client commit path and the table-service schedule path (after early-return checks). Also enrich the hoodie.properties file comment line with hostname and hudi version. Fires only on properties (re)writes — table creation, schema updates, upgrades — not on every commit. Hostname is resolved once per JVM with a "unknown" fallback for restricted environments. HoodieVersion.get() memoizes the manifest read. No storage-format changes; no public-API breakage (HoodieEngineContext .getEngineProperties() ships with a default empty-map implementation). Co-Authored-By: Claude Opus 4.7 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]> Co-authored-by: sivabalan <[email protected]> (cherry picked from commit cde86ffd64a4aadb55ec049940713d199d92eb03) --- .../org/apache/hudi/client/BaseHoodieClient.java | 4 + .../hudi/client/BaseHoodieTableServiceClient.java | 2 + .../apache/hudi/client/BaseHoodieWriteClient.java | 1 + .../hudi/client/CommitMetadataProperties.java | 145 ++++++++++++++++++ .../hudi/client/TestCommitMetadataProperties.java | 170 +++++++++++++++++++++ .../client/common/HoodieJavaEngineContext.java | 25 +++ .../client/common/HoodieSparkEngineContext.java | 14 ++ .../main/java/org/apache/hudi/HoodieVersion.java | 43 ++++-- .../hudi/common/engine/HoodieEngineContext.java | 16 ++ .../hudi/common/table/HoodieTableConfig.java | 34 ++++- .../streamer/TestHoodieIncrSourceE2E.java | 6 +- .../TestHoodieIncrSourceE2EAutoUpgrade.java | 6 +- 12 files changed, 452 insertions(+), 14 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java index 002ac1e7454f..c04454bf0857 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java @@ -458,4 +458,8 @@ public abstract class BaseHoodieClient implements Serializable, AutoCloseable { return foundRollingMetadata; } + + protected Option<Map<String, String>> updateExtraMetadata(Option<Map<String, String>> extraMetadata) { + return CommitMetadataProperties.enrich(extraMetadata, config, context); + } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java index 17106d8d940e..01105c8bd8d7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java @@ -726,6 +726,8 @@ public abstract class BaseHoodieTableServiceClient<I, T, O> extends BaseHoodieCl // so it is handled differently to avoid locking for planning. return scheduleCleaning(createTable(config, storageConf), providedInstantTime); } + // Only enrich metadata after early-return checks, when we're actually going to use it + extraMetadata = updateExtraMetadata(extraMetadata); Option<HoodieInstant> lastCompletedInstant = lastCompletedTxnAndMetadata.isPresent() ? Option.of(lastCompletedTxnAndMetadata.get().getLeft()) : Option.empty(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index 76ffe51a256d..8284f61baa15 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -254,6 +254,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient if (!config.allowEmptyCommit() && tableWriteStats.isEmptyDataTableWriteStats()) { return true; } + extraMetadata = updateExtraMetadata(extraMetadata); log.info("Committing {} action {}", instantTime, commitActionType); // Create a Hoodie table which encapsulated the commits and files visible HoodieTable table = hoodieTableOpt.orElse(createTable(config)); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java new file mode 100644 index 000000000000..3a7b79812c8e --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.HoodieVersion; +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Enriches the {@code extraMetadata} map persisted with every commit, with version, engine, and + * (optionally) engine-specific properties and a configurable subset of {@link HoodieWriteConfig} + * values. + * + * <p>Key namespacing: + * <ul> + * <li>{@code hudi.version} — writer version. Always emitted.</li> + * <li>{@code engine} — engine type (SPARK/FLINK/JAVA). Always emitted.</li> + * <li>Engine-supplied keys (Spark: {@code spark.*}, Java: {@code java.*}/{@code os.*}, etc.) + * — gated by {@link #EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA}.</li> + * <li>{@code config.<key>} — values of {@link HoodieWriteConfig} entries whose keys are listed + * in {@link #WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA}.</li> + * </ul> + */ +public class CommitMetadataProperties { + + static final String HUDI_VERSION_KEY = "hudi.version"; + static final String ENGINE_KEY = "engine"; + static final String CONFIG_KEY_PREFIX = "config."; + + /** + * Default allowlist of write-config keys serialized into commit metadata. These are values that + * change across jobs/runs but aren't already captured in {@code hoodie.properties}, so they're + * useful for after-the-fact debugging. Intentionally excludes immutable table identity + * (already in {@code hoodie.properties}) and per-record/sensitive values. + */ + private static final String DEFAULT_WRITE_CONFIG_KEYS = String.join(",", + Arrays.asList( + "hoodie.datasource.write.operation", + "hoodie.insert.shuffle.parallelism", + "hoodie.upsert.shuffle.parallelism", + "hoodie.bulkinsert.shuffle.parallelism", + "hoodie.delete.shuffle.parallelism", + "hoodie.write.concurrency.mode", + "hoodie.metadata.enable")); + + /** + * When enabled, engine-specific properties supplied by + * {@link HoodieEngineContext#getEngineProperties()} are embedded into commit metadata for + * debugging (e.g. {@code spark.application.id}, {@code spark.user}). {@code hudi.version} and + * {@code engine} are always embedded regardless of this flag. + * + * <p>Default is {@code false} since these add per-commit growth to the timeline. Long-running + * ingestion workloads writing many commits should leave this off unless debugging. + */ + public static final ConfigProperty<Boolean> EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA = + ConfigProperty + .key("hoodie.commit.metadata.engine.properties.embed.enable") + .defaultValue(false) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("When enabled, engine-specific properties (e.g. spark.application.id, " + + "spark.user, java.version) are embedded into commit metadata for debugging. " + + "hudi.version and engine name are always embedded regardless of this flag."); + + /** + * Comma-separated list of {@link HoodieWriteConfig} keys whose values should be serialized into + * commit metadata under the {@code config.<key>} prefix. Use with care: every key listed here + * adds an entry to every commit, which lives forever in the active and archived timeline. + * + * <p>Empty value disables config-key serialization entirely (only {@code hudi.version} and + * {@code engine} are emitted). + */ + public static final ConfigProperty<String> WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA = + ConfigProperty + .key("hoodie.write.config.keys.to.serialize.to.commit.metadata") + .defaultValue(DEFAULT_WRITE_CONFIG_KEYS) + .markAdvanced() + .sinceVersion("1.3.0") + .withDocumentation("Comma-separated list of write-config keys whose values are " + + "serialized into the extraMetadata map of every commit (under the 'config.' " + + "prefix). Set to empty to skip config-key serialization entirely. Avoid adding " + + "keys whose values may contain credentials or large payloads, since commit " + + "metadata is persisted in the timeline."); + + public static Option<Map<String, String>> enrich(Option<Map<String, String>> extraMetadata, + HoodieWriteConfig config, + HoodieEngineContext context) { + Map<String, String> newMetadata = new HashMap<>(); + if (extraMetadata.isPresent()) { + newMetadata.putAll(extraMetadata.get()); + } + + newMetadata.put(HUDI_VERSION_KEY, HoodieVersion.get()); + newMetadata.put(ENGINE_KEY, config.getEngineType().name()); + + if (config.getBoolean(EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA)) { + newMetadata.putAll(context.getEngineProperties()); + } + + for (String key : parseConfigKeys(config.getString(WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA))) { + String value = config.getString(key); + if (!StringUtils.isNullOrEmpty(value)) { + newMetadata.put(CONFIG_KEY_PREFIX + key, value); + } + } + + return Option.of(newMetadata); + } + + private static List<String> parseConfigKeys(String csv) { + if (StringUtils.isNullOrEmpty(csv)) { + return Collections.emptyList(); + } + return Arrays.stream(csv.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java new file mode 100644 index 000000000000..acd83424ff3e --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client; + +import org.apache.hudi.common.engine.EngineType; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.apache.hudi.client.CommitMetadataProperties.CONFIG_KEY_PREFIX; +import static org.apache.hudi.client.CommitMetadataProperties.EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA; +import static org.apache.hudi.client.CommitMetadataProperties.ENGINE_KEY; +import static org.apache.hudi.client.CommitMetadataProperties.HUDI_VERSION_KEY; +import static org.apache.hudi.client.CommitMetadataProperties.WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestCommitMetadataProperties { + + /** Always-emitted keys (hudi.version, engine) are present even when input is empty. */ + @Test + void enrich_emptyInput_emitsVersionAndEngine() { + HoodieWriteConfig config = newConfig(new Properties()); + HoodieEngineContext context = newContext(Collections.emptyMap()); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertNotNull(result.get(HUDI_VERSION_KEY)); + assertEquals(EngineType.SPARK.name(), result.get(ENGINE_KEY)); + } + + /** Passing an immutable map must not throw — yihua's defensive-copy fix. */ + @Test + void enrich_immutableInputMap_doesNotThrow() { + HoodieWriteConfig config = newConfig(new Properties()); + HoodieEngineContext context = newContext(Collections.emptyMap()); + Map<String, String> input = Collections.unmodifiableMap( + Collections.singletonMap("caller.key", "caller.value")); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.of(input), config, context).get(); + + assertEquals("caller.value", result.get("caller.key")); + assertTrue(input.equals(Collections.singletonMap("caller.key", "caller.value")), + "Input map must not be mutated"); + } + + /** Default (opt-in flag = false) suppresses engine-supplied keys. */ + @Test + void enrich_engineEmbedFlagOff_omitsEngineSuppliedKeys() { + HoodieWriteConfig config = newConfig(new Properties()); + Map<String, String> engineProps = new HashMap<>(); + engineProps.put("spark.application.id", "app-123"); + HoodieEngineContext context = newContext(engineProps); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertFalse(result.containsKey("spark.application.id"), + "Engine-supplied keys must be omitted when embed flag is off"); + assertNotNull(result.get(HUDI_VERSION_KEY)); + assertEquals(EngineType.SPARK.name(), result.get(ENGINE_KEY)); + } + + /** Opt-in flag = true emits engine-supplied keys. */ + @Test + void enrich_engineEmbedFlagOn_emitsEngineSuppliedKeys() { + Properties props = new Properties(); + props.put(EMBED_ENGINE_PROPERTIES_IN_COMMIT_METADATA.key(), "true"); + HoodieWriteConfig config = newConfig(props); + Map<String, String> engineProps = new HashMap<>(); + engineProps.put("spark.application.id", "app-123"); + engineProps.put("spark.user", "sivabalan"); + HoodieEngineContext context = newContext(engineProps); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertEquals("app-123", result.get("spark.application.id")); + assertEquals("sivabalan", result.get("spark.user")); + } + + /** Config-key allowlist serializes present values; skips truly absent keys (no default). */ + @Test + void enrich_writeConfigKeyAllowlist_emitsPresentNonEmptyValues() { + Properties props = new Properties(); + props.put(WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA.key(), + "hoodie.metadata.enable,hoodie.does.not.exist"); + props.put("hoodie.metadata.enable", "true"); + HoodieWriteConfig config = newConfig(props); + HoodieEngineContext context = newContext(Collections.emptyMap()); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + assertEquals("true", result.get(CONFIG_KEY_PREFIX + "hoodie.metadata.enable")); + assertFalse(result.containsKey(CONFIG_KEY_PREFIX + "hoodie.does.not.exist"), + "Keys absent from the config must not be serialized"); + } + + /** Empty allowlist disables config-key serialization entirely. */ + @Test + void enrich_emptyConfigKeyList_emitsNoConfigKeys() { + Properties props = new Properties(); + props.put(WRITE_CONFIG_KEYS_TO_SERIALIZE_TO_COMMIT_METADATA.key(), ""); + props.put("hoodie.metadata.enable", "true"); + HoodieWriteConfig config = newConfig(props); + HoodieEngineContext context = newContext(Collections.emptyMap()); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.empty(), config, context).get(); + + long configKeyCount = result.keySet().stream().filter(k -> k.startsWith(CONFIG_KEY_PREFIX)).count(); + assertEquals(0L, configKeyCount, "No config.* keys when allowlist is empty"); + // hudi.version and engine still always emitted + assertNotNull(result.get(HUDI_VERSION_KEY)); + assertEquals(EngineType.SPARK.name(), result.get(ENGINE_KEY)); + } + + /** Caller-provided extra metadata is preserved alongside enrichment. */ + @Test + void enrich_preservesCallerProvidedKeys() { + HoodieWriteConfig config = newConfig(new Properties()); + HoodieEngineContext context = newContext(Collections.emptyMap()); + Map<String, String> input = new HashMap<>(); + input.put("caller.key1", "caller.value1"); + input.put("caller.key2", "caller.value2"); + + Map<String, String> result = CommitMetadataProperties.enrich(Option.of(input), config, context).get(); + + assertEquals("caller.value1", result.get("caller.key1")); + assertEquals("caller.value2", result.get("caller.key2")); + assertNotNull(result.get(HUDI_VERSION_KEY)); + } + + private static HoodieWriteConfig newConfig(Properties overrides) { + Properties props = new Properties(); + props.put(HoodieWriteConfig.BASE_PATH.key(), "/tmp/test-commit-metadata-properties"); + props.putAll(overrides); + return HoodieWriteConfig.newBuilder().withProperties(props).build(); + } + + private static HoodieEngineContext newContext(Map<String, String> engineProperties) { + HoodieEngineContext context = mock(HoodieEngineContext.class); + when(context.getEngineProperties()).thenReturn(engineProperties); + return context; + } +} diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java index 8da9e2aca0e1..a24d6fb4c25d 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/common/HoodieJavaEngineContext.java @@ -49,6 +49,7 @@ import org.apache.avro.generic.IndexedRecord; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -192,6 +193,30 @@ public class HoodieJavaEngineContext extends HoodieEngineContext { // no operation for now } + // Allowlist of safe system properties to include in commit metadata. Avoid wildcarding system + // properties since callers may pass credentials via -D flags (e.g. -Ddb.password=...). + private static final String[] SAFE_SYSTEM_PROPERTIES = { + "java.version", + "java.vendor", + "java.vm.name", + "java.vm.version", + "os.name", + "os.version", + "os.arch" + }; + + @Override + public Map<String, String> getEngineProperties() { + Map<String, String> info = new HashMap<>(); + for (String property : SAFE_SYSTEM_PROPERTIES) { + String value = System.getProperty(property); + if (value != null) { + info.put(property, value); + } + } + return info; + } + @Override public <I, O> O aggregate(HoodieData<I> data, O zeroValue, Functions.Function2<O, I, O> seqOp, Functions.Function2<O, O, O> combOp) { return data.collectAsList().stream().reduce(zeroValue, seqOp::apply, combOp::apply); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java index d764dcd3cd54..0e4fae4f70e6 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java @@ -408,4 +408,18 @@ public class HoodieSparkEngineContext extends HoodieEngineContext { new ConditionalRangePartitioner.CompositeKeyComparator<>()) .mapToPair(e -> e._1)); } + + @Override + public Map<String, String> getEngineProperties() { + Map<String, String> info = new HashMap<>(); + info.put("spark.application.id", javaSparkContext.sc().applicationId()); + info.put("spark.user", javaSparkContext.sparkUser()); + info.put("spark.master", javaSparkContext.master()); + info.put("spark.application", javaSparkContext.appName()); + info.put("spark.version", javaSparkContext.version()); + info.put("spark.defaultParallelism", String.valueOf(javaSparkContext.defaultParallelism())); + info.put("spark.defaultMinPartitions", String.valueOf(javaSparkContext.defaultMinPartitions())); + info.put("spark.executor.instances", javaSparkContext.getConf().get("spark.executor.instances", "")); + return info; + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java b/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java index dd88836468c9..23cee1015841 100644 --- a/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/HoodieVersion.java @@ -33,23 +33,46 @@ public final class HoodieVersion { public static final String HOODIE_WRITER_VERSION = "hudi_writer_version"; + // Cached result of reading version from the manifest. Null means "not loaded yet". An empty + // string means "manifest absent or unreadable" — fall back to HOODIE_DEFAULT_VERSION so tests + // that swap the default via setVersionOverride continue to work. + private static volatile String cachedManifestVersion = null; + /** * Returns the complete version of HUDI code * Example: 0.12.2 or 0.12.3-snapshot */ public static String get() { - String hudiPropertiesFilePath = "META-INF/maven/org.apache.hudi/hudi-common/pom.properties"; - try (InputStream inputStream = HoodieVersion.class.getClassLoader().getResourceAsStream(hudiPropertiesFilePath)) { - Properties properties = new Properties(); - if (inputStream != null) { - properties.load(inputStream); - // Access properties - return properties.getProperty("version"); + String fromManifest = loadManifestVersion(); + return fromManifest.isEmpty() ? HOODIE_DEFAULT_VERSION : fromManifest; + } + + private static String loadManifestVersion() { + String local = cachedManifestVersion; + if (local != null) { + return local; + } + synchronized (HoodieVersion.class) { + if (cachedManifestVersion != null) { + return cachedManifestVersion; + } + String hudiPropertiesFilePath = "META-INF/maven/org.apache.hudi/hudi-common/pom.properties"; + String resolved = ""; + try (InputStream inputStream = HoodieVersion.class.getClassLoader().getResourceAsStream(hudiPropertiesFilePath)) { + if (inputStream != null) { + Properties properties = new Properties(); + properties.load(inputStream); + String version = properties.getProperty("version"); + if (version != null) { + resolved = version; + } + } + } catch (Exception ignored) { + // Ignoring the exception as there is a fallback to default version } - } catch (Exception ignored) { - // Ignoring the exception as there is as fallback to default version + cachedManifestVersion = resolved; + return resolved; } - return HOODIE_DEFAULT_VERSION; } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java index b4663509002d..eea766b56dda 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java @@ -45,6 +45,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import java.io.IOException; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -117,6 +118,21 @@ public abstract class HoodieEngineContext { public abstract void cancelAllJobs(); + /** + * Returns engine-specific properties to be included in commit metadata for debugging. + * <p>Contract: + * <ul> + * <li>Implementations must only return safe, non-sensitive values (no credentials, no PII).</li> + * <li>This is invoked on the driver, on every commit. It must be cheap and free of side effects.</li> + * <li>Must not reach into checkpoint / runtime state (e.g. for streaming engines like Flink, + * per-checkpoint metadata is set up via the coordinator, not here).</li> + * </ul> + * Default returns an empty map so external subclasses are not forced to implement this. + */ + public Map<String, String> getEngineProperties() { + return Collections.emptyMap(); + } + /** * Returns the application id of the engine (e.g. Spark application id). * Used to populate lock metadata so lock holders can be identified. diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 73703d70d85a..aef84c9f8adc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -18,6 +18,7 @@ package org.apache.hudi.common.table; +import org.apache.hudi.HoodieVersion; import org.apache.hudi.common.HoodieTableFormat; import org.apache.hudi.common.NativeTableFormat; import org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex; @@ -51,6 +52,7 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.util.BinaryUtil; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.HoodieTableConfigUtils; +import org.apache.hudi.common.util.NetworkUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; @@ -126,6 +128,9 @@ import static org.apache.hudi.common.util.ValidationUtils.checkArgument; + " initializing a path as hoodie base path and never changes during the lifetime of a hoodie table.") public class HoodieTableConfig extends HoodieConfig { + // Cached hostname to avoid repeated synchronized network calls + private static volatile String cachedHostname = null; + public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties"; public static final String HOODIE_PROPERTIES_FILE_BACKUP = "hoodie.properties.backup"; public static final String HOODIE_WRITE_TABLE_NAME_KEY = "hoodie.datasource.write.table.name"; @@ -500,10 +505,10 @@ public class HoodieTableConfig extends HoodieConfig { final String checksum; if (isValidChecksum(props)) { checksum = props.getProperty(TABLE_CHECKSUM.key()); - props.store(outputStream, "Updated at " + Instant.now()); + props.store(outputStream, getFileComment()); } else { Properties propsWithChecksum = getOrderedPropertiesWithTableChecksum(props); - propsWithChecksum.store(outputStream, "Properties saved on " + Instant.now()); + propsWithChecksum.store(outputStream, getFileComment()); checksum = propsWithChecksum.getProperty(TABLE_CHECKSUM.key()); props.setProperty(TABLE_CHECKSUM.key(), checksum); } @@ -1385,6 +1390,31 @@ public class HoodieTableConfig extends HoodieConfig { .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue()))); } + /** + * Returns the cached hostname, fetching it lazily on first call. + * Falls back to "unknown" if network resolution fails. + */ + private static String getHostnameSafe() { + if (cachedHostname == null) { + synchronized (HoodieTableConfig.class) { + if (cachedHostname == null) { + try { + cachedHostname = NetworkUtils.getHostname(); + } catch (Exception e) { + log.warn("Failed to resolve hostname, using 'unknown'", e); + cachedHostname = "unknown"; + } + } + } + } + return cachedHostname; + } + + public static String getFileComment() { + return String.format("Updated at %s, host=%s, hudi_version=%s", + Instant.now(), getHostnameSafe(), HoodieVersion.get()); + } + /** * @deprecated Use {@link #BASE_FILE_FORMAT} and its methods. */ diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java index 1b010efab7cb..64c432474953 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java @@ -108,7 +108,11 @@ public class TestHoodieIncrSourceE2E extends S3EventsHoodieIncrSourceHarness { Option<HoodieCommitMetadata> metadata = HoodieClientTestUtils.getCommitMetadataForInstant( metaClient, metaClient.getActiveTimeline().lastInstant().get()); assertFalse(metadata.isEmpty()); - assertEquals(metadata.get().getExtraMetadata(), expectedMetadata); + // Assert expected entries are a subset of the actual extra metadata. CommitMetadataProperties + // also enriches commit metadata with hudi.version, engine, and config.* entries on every write, + // so the actual map is a superset. + Map<String, String> actual = metadata.get().getExtraMetadata(); + expectedMetadata.forEach((k, v) -> assertEquals(v, actual.get(k), "extraMetadata[" + k + "]")); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java index 3fd2068c9100..52a6734ec331 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2EAutoUpgrade.java @@ -137,7 +137,11 @@ public class TestHoodieIncrSourceE2EAutoUpgrade extends S3EventsHoodieIncrSource Option<HoodieCommitMetadata> metadata = HoodieClientTestUtils.getCommitMetadataForInstant( metaClient, metaClient.getActiveTimeline().lastInstant().get()); assertFalse(metadata.isEmpty()); - assertEquals(expectedMetadata, metadata.get().getExtraMetadata()); + // Assert expected entries are a subset of the actual extra metadata. CommitMetadataProperties + // also enriches commit metadata with hudi.version, engine, and config.* entries on every write, + // so the actual map is a superset. + Map<String, String> actual = metadata.get().getExtraMetadata(); + expectedMetadata.forEach((k, v) -> assertEquals(v, actual.get(k), "extraMetadata[" + k + "]")); } /**
