This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new fff794d4b384 fix(flink): keep reporting hive sync metrics after a 
coordinator restart (#20499)
fff794d4b384 is described below

commit fff794d4b384bf45fb8f5d396434be010fdf4833
Author: Shihuan Liu <[email protected]>
AuthorDate: Wed Nov 4 18:32:19 2026 -0800

    fix(flink): keep reporting hive sync metrics after a coordinator restart 
(#20499)
    
    * fix(flink): keep reporting hive sync metrics after a coordinator restart
    
    * fix(flink): hold shared hive sync metrics weakly so finished jobs release 
them
---
 .../apache/hudi/metrics/FlinkHiveSyncMetrics.java  |  71 ++++++++++++---
 .../hudi/sink/StreamWriteOperatorCoordinator.java  |   3 +-
 .../hudi/metrics/TestFlinkHiveSyncMetrics.java     |  59 ++++++++++++
 .../sink/TestStreamWriteOperatorCoordinator.java   |  19 ++++
 .../hudi/utils/SharedCoordinatorMetricGroup.java   | 100 +++++++++++++++++++++
 5 files changed, 239 insertions(+), 13 deletions(-)

diff --git 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
index dd1a6d99127d..9ae6fc074bb0 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
@@ -21,11 +21,16 @@ package org.apache.hudi.metrics;
 import org.apache.hudi.hive.HoodieHiveSyncStats;
 
 import com.codahale.metrics.SlidingWindowReservoir;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import org.apache.flink.dropwizard.metrics.DropwizardHistogramWrapper;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Histogram;
 import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.metrics.ThreadSafeSimpleCounter;
+import org.apache.flink.runtime.metrics.scope.ScopeFormat;
+
+import java.util.Map;
 
 /**
  * Metrics for the Hive sync that {@link 
org.apache.hudi.sink.StreamWriteOperatorCoordinator} runs
@@ -37,10 +42,27 @@ import org.apache.flink.metrics.ThreadSafeSimpleCounter;
  *
  * <p>The breakdown of each sync, from the init duration to the schema evolved 
count, is only
  * registered when {@code 
hoodie.datasource.hive_sync.detailed_metrics.enabled} is set.
+ *
+ * <p>Obtain an instance with {@link #register}. Flink keeps one coordinator 
metric group per operator
+ * for the life of the job, but creates a new coordinator whenever it rebuilds 
the execution graph,
+ * as the adaptive scheduler does on every restart and rescale. That group 
ignores a metric registered
+ * under a name it already holds, so metrics owned by each coordinator would 
stay frozen at the first
+ * coordinator's values. Every coordinator of the same operator in the same 
job therefore shares one
+ * instance.
  */
 public class FlinkHiveSyncMetrics extends HoodieFlinkMetrics {
   private static final int HISTOGRAM_WINDOW_SIZE = 100;
 
+  /**
+   * The instance shared by the coordinators of each operator, keyed by job 
and operator ID.
+   *
+   * <p>Values are weak: between coordinators, the only strong reference to an 
instance is the
+   * {@link #HIVE_SYNC_LAST_SUCCESS_TIME_MS} gauge registered on the 
operator's group, so an instance
+   * lives as long as that group reports it and its entry goes once the job's 
group is closed. That
+   * gauge must keep referencing the instance, and the cache must not be 
bounded by size.
+   */
+  private static final Cache<String, FlinkHiveSyncMetrics> SHARED_METRICS = 
Caffeine.newBuilder().weakValues().build();
+
   static final String HIVE_SYNC_SUCCESS_COUNT = "hiveSyncSuccessCount";
   static final String HIVE_SYNC_FAILURE_COUNT = "hiveSyncFailureCount";
   static final String HIVE_SYNC_DURATION_MS = "hiveSyncDurationMs";
@@ -114,7 +136,7 @@ public class FlinkHiveSyncMetrics extends 
HoodieFlinkMetrics {
    * @param detailedMetricsEnabled whether to register and update the 
breakdown of each sync, which the sync records
    *     only when its detailed metrics are enabled
    */
-  public FlinkHiveSyncMetrics(MetricGroup metricGroup, boolean 
detailedMetricsEnabled) {
+  FlinkHiveSyncMetrics(MetricGroup metricGroup, boolean 
detailedMetricsEnabled) {
     super(metricGroup);
     this.detailedMetricsEnabled = detailedMetricsEnabled;
     // Detailed histograms are only accessed when detailed metrics are enabled.
@@ -124,19 +146,46 @@ public class FlinkHiveSyncMetrics extends 
HoodieFlinkMetrics {
     this.remainingDurationMs = detailedMetricsEnabled ? newHistogram() : null;
   }
 
+  /**
+   * Returns the metrics of the coordinator that owns {@code metricGroup}, 
registered on it.
+   *
+   * <p>A coordinator that replaces an earlier one of the same operator and 
job gets the earlier
+   * one's instance, so its syncs update the metrics the group already 
reports, and the counts carry
+   * on across the restart. The instance is still registered on {@code 
metricGroup}, which covers a
+   * group that is not the one it was first registered on, such as the group 
of a JobMaster
+   * recreated for the same job.
+   *
+   * <p>A group that does not identify its job and operator, as in tests, gets 
an instance of its own.
+   */
+  public static FlinkHiveSyncMetrics register(MetricGroup metricGroup, boolean 
detailedMetricsEnabled) {
+    Map<String, String> variables = metricGroup.getAllVariables();
+    String jobId = variables.get(ScopeFormat.SCOPE_JOB_ID);
+    String operatorId = variables.get(ScopeFormat.SCOPE_OPERATOR_ID);
+    FlinkHiveSyncMetrics metrics = jobId == null || operatorId == null
+        ? new FlinkHiveSyncMetrics(metricGroup, detailedMetricsEnabled)
+        : SHARED_METRICS.get(jobId + "/" + operatorId,
+            key -> new FlinkHiveSyncMetrics(metricGroup, 
detailedMetricsEnabled));
+    metrics.registerMetrics(metricGroup);
+    return metrics;
+  }
+
   @Override
   public void registerMetrics() {
-    metricGroup.counter(HIVE_SYNC_SUCCESS_COUNT, syncSuccessCount);
-    metricGroup.counter(HIVE_SYNC_FAILURE_COUNT, syncFailureCount);
-    metricGroup.histogram(HIVE_SYNC_DURATION_MS, syncDurationMs);
-    metricGroup.gauge(HIVE_SYNC_LAST_SUCCESS_TIME_MS, () -> 
lastSyncSuccessTimeMs);
+    registerMetrics(metricGroup);
+  }
+
+  private void registerMetrics(MetricGroup group) {
+    group.counter(HIVE_SYNC_SUCCESS_COUNT, syncSuccessCount);
+    group.counter(HIVE_SYNC_FAILURE_COUNT, syncFailureCount);
+    group.histogram(HIVE_SYNC_DURATION_MS, syncDurationMs);
+    group.gauge(HIVE_SYNC_LAST_SUCCESS_TIME_MS, () -> lastSyncSuccessTimeMs);
     if (detailedMetricsEnabled) {
-      metricGroup.histogram(HIVE_SYNC_INIT_DURATION_MS, initDurationMs);
-      metricGroup.histogram(HIVE_SYNC_SCHEMA_READ_DURATION_MS, 
schemaReadDurationMs);
-      metricGroup.histogram(HIVE_SYNC_PARTITION_SCAN_DURATION_MS, 
partitionScanDurationMs);
-      metricGroup.histogram(HIVE_SYNC_REMAINING_DURATION_MS, 
remainingDurationMs);
-      metricGroup.counter(HIVE_SYNC_PARTITIONS_ADDED_COUNT, 
partitionsAddedCount);
-      metricGroup.counter(HIVE_SYNC_SCHEMA_EVOLVED_COUNT, schemaEvolvedCount);
+      group.histogram(HIVE_SYNC_INIT_DURATION_MS, initDurationMs);
+      group.histogram(HIVE_SYNC_SCHEMA_READ_DURATION_MS, schemaReadDurationMs);
+      group.histogram(HIVE_SYNC_PARTITION_SCAN_DURATION_MS, 
partitionScanDurationMs);
+      group.histogram(HIVE_SYNC_REMAINING_DURATION_MS, remainingDurationMs);
+      group.counter(HIVE_SYNC_PARTITIONS_ADDED_COUNT, partitionsAddedCount);
+      group.counter(HIVE_SYNC_SCHEMA_EVOLVED_COUNT, schemaEvolvedCount);
     }
   }
 
diff --git 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
index 595578c05d7f..87c944780d3a 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
@@ -475,8 +475,7 @@ public class StreamWriteOperatorCoordinator
         .threadFactory(getThreadFactory("hive-sync"))
         .waitForTasksFinish(true).build();
     this.hiveSyncContext = HiveSyncContext.create(conf, this.storageConf);
-    this.hiveSyncMetrics = new FlinkHiveSyncMetrics(context.metricGroup(), 
hiveSyncContext.isDetailedMetricsEnabled());
-    this.hiveSyncMetrics.registerMetrics();
+    this.hiveSyncMetrics = 
FlinkHiveSyncMetrics.register(context.metricGroup(), 
hiveSyncContext.isDetailedMetricsEnabled());
   }
 
   private void syncHiveAsync() {
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
index 9026ea0d2219..149053e5b5e9 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
@@ -20,11 +20,14 @@ package org.apache.hudi.metrics;
 
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.hive.HoodieHiveSyncStats;
+import org.apache.hudi.utils.SharedCoordinatorMetricGroup;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.Histogram;
 import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -48,7 +51,9 @@ import static 
org.apache.hudi.metrics.FlinkHiveSyncMetrics.HIVE_SYNC_SCHEMA_READ
 import static 
org.apache.hudi.metrics.FlinkHiveSyncMetrics.HIVE_SYNC_SUCCESS_COUNT;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -235,6 +240,60 @@ class TestFlinkHiveSyncMetrics {
     assertEquals(2L * syncsPerThread, duration().getCount());
   }
 
+  @Test
+  void testRestartedCoordinatorUpdatesRegisteredMetrics() {
+    SharedCoordinatorMetricGroup group = new SharedCoordinatorMetricGroup(new 
JobID().toString(), new OperatorID().toString());
+    FlinkHiveSyncMetrics first = FlinkHiveSyncMetrics.register(group, true);
+    first.markSyncSucceeded(10L);
+
+    FlinkHiveSyncMetrics second = 
FlinkHiveSyncMetrics.register(group.newView(), true);
+    second.markSyncSucceeded(20L);
+    second.markSyncFailed(30L);
+    second.updateSyncMetrics(stats(Option.of(1L), Option.of(2L), 
Option.of(3L), 5, true));
+
+    assertSame(first, second);
+    assertEquals(2, group.getCounter(HIVE_SYNC_SUCCESS_COUNT).getCount());
+    assertEquals(1, group.getCounter(HIVE_SYNC_FAILURE_COUNT).getCount());
+    assertEquals(3, group.getHistogram(HIVE_SYNC_DURATION_MS).getCount());
+    assertTrue((Long) 
group.getGauge(HIVE_SYNC_LAST_SUCCESS_TIME_MS).getValue() > 0L);
+    assertEquals(5, 
group.getCounter(HIVE_SYNC_PARTITIONS_ADDED_COUNT).getCount());
+    assertEquals(1, 
group.getCounter(HIVE_SYNC_SCHEMA_EVOLVED_COUNT).getCount());
+  }
+
+  @Test
+  void testSharedMetricsRegisteredOnNewGroupOfSameOperator() {
+    String jobId = new JobID().toString();
+    String operatorId = new OperatorID().toString();
+    FlinkHiveSyncMetrics first = FlinkHiveSyncMetrics.register(new 
SharedCoordinatorMetricGroup(jobId, operatorId), true);
+    first.markSyncSucceeded(10L);
+
+    SharedCoordinatorMetricGroup newGroup = new 
SharedCoordinatorMetricGroup(jobId, operatorId);
+    FlinkHiveSyncMetrics second = FlinkHiveSyncMetrics.register(newGroup, 
true);
+
+    assertSame(first, second);
+    assertEquals(1, newGroup.getCounter(HIVE_SYNC_SUCCESS_COUNT).getCount());
+    assertNotNull(newGroup.getHistogram(HIVE_SYNC_INIT_DURATION_MS));
+  }
+
+  @Test
+  void testMetricsNotSharedAcrossOperatorsOrJobs() {
+    String jobId = new JobID().toString();
+    String operatorId = new OperatorID().toString();
+    FlinkHiveSyncMetrics metrics = FlinkHiveSyncMetrics.register(new 
SharedCoordinatorMetricGroup(jobId, operatorId), true);
+
+    assertNotSame(metrics,
+        FlinkHiveSyncMetrics.register(new SharedCoordinatorMetricGroup(jobId, 
new OperatorID().toString()), true));
+    assertNotSame(metrics,
+        FlinkHiveSyncMetrics.register(new SharedCoordinatorMetricGroup(new 
JobID().toString(), operatorId), true));
+  }
+
+  @Test
+  void testMetricsNotSharedWhenGroupDoesNotIdentifyOperator() {
+    CapturingMetricGroup group = new CapturingMetricGroup();
+
+    assertNotSame(FlinkHiveSyncMetrics.register(group, true), 
FlinkHiveSyncMetrics.register(group, true));
+  }
+
   private long successCount() {
     return metricGroup.counters.get(HIVE_SYNC_SUCCESS_COUNT).getCount();
   }
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
index 5664dcbb9c60..7adfa975ef59 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
@@ -65,9 +65,11 @@ import org.apache.hudi.storage.StoragePath;
 import org.apache.hudi.storage.StoragePathInfo;
 import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
 import org.apache.hudi.util.StreamerUtil;
+import org.apache.hudi.utils.SharedCoordinatorMetricGroup;
 import org.apache.hudi.utils.TestConfigurations;
 import org.apache.hudi.utils.TestUtils;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.metrics.Gauge;
@@ -776,6 +778,23 @@ public class TestStreamWriteOperatorCoordinator {
         "A sync that never ran records no remaining time");
   }
 
+  @Test
+  void testHiveSyncMetricsReportSyncsAfterCoordinatorRestart() throws 
Exception {
+    coordinator.close();
+    Configuration conf = getUnreachableMetastoreConf();
+    SharedCoordinatorMetricGroup metricGroup = new 
SharedCoordinatorMetricGroup(new JobID().toString(), new 
OperatorID().toString());
+    coordinator = startCoordinator(conf, metricGroup);
+    assertThrows(HoodieHiveSyncException.class, () -> 
coordinator.doSyncHive());
+    coordinator.close();
+
+    // a restart that rebuilds the execution graph creates a new coordinator 
on the same metric group
+    coordinator = startCoordinator(conf, metricGroup.newView());
+    assertThrows(HoodieHiveSyncException.class, () -> 
coordinator.doSyncHive());
+
+    assertEquals(2, metricGroup.getCounter("hiveSyncFailureCount").getCount());
+    assertEquals(2, metricGroup.getHistogram("hiveSyncDurationMs").getCount());
+  }
+
   private Configuration getUnreachableMetastoreConf() {
     Configuration conf = 
TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
     conf.set(FlinkOptions.HIVE_SYNC_ENABLED, true);
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/SharedCoordinatorMetricGroup.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/SharedCoordinatorMetricGroup.java
new file mode 100644
index 000000000000..4c4ae782b23d
--- /dev/null
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/SharedCoordinatorMetricGroup.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.groups.OperatorCoordinatorMetricGroup;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.metrics.scope.ScopeFormat;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A coordinator metric group that behaves like the one Flink gives each 
coordinator of an operator.
+ *
+ * <p>It identifies its job and operator, and {@link #newView()} returns 
another group over the same
+ * registrations, as Flink gives every coordinator it creates for the operator 
during the job. Like
+ * Flink's, it keeps the first metric registered under a name and ignores 
later ones.
+ */
+public class SharedCoordinatorMetricGroup extends UnregisteredMetricsGroup
+    implements OperatorCoordinatorMetricGroup {
+  private final Map<String, Metric> registered;
+  private final Map<String, String> variables;
+
+  public SharedCoordinatorMetricGroup(String jobId, String operatorId) {
+    this(new ConcurrentHashMap<>(), variables(jobId, operatorId));
+  }
+
+  private SharedCoordinatorMetricGroup(Map<String, Metric> registered, 
Map<String, String> variables) {
+    this.registered = registered;
+    this.variables = variables;
+  }
+
+  /** Returns another group over the same registrations, as a coordinator 
created by a restart gets. */
+  public SharedCoordinatorMetricGroup newView() {
+    return new SharedCoordinatorMetricGroup(registered, variables);
+  }
+
+  @Override
+  public <C extends Counter> C counter(String name, C counter) {
+    registered.putIfAbsent(name, counter);
+    return counter;
+  }
+
+  @Override
+  public <H extends Histogram> H histogram(String name, H histogram) {
+    registered.putIfAbsent(name, histogram);
+    return histogram;
+  }
+
+  @Override
+  public <T, G extends Gauge<T>> G gauge(String name, G gauge) {
+    registered.putIfAbsent(name, gauge);
+    return gauge;
+  }
+
+  @Override
+  public Map<String, String> getAllVariables() {
+    return variables;
+  }
+
+  public Counter getCounter(String name) {
+    return (Counter) registered.get(name);
+  }
+
+  public Histogram getHistogram(String name) {
+    return (Histogram) registered.get(name);
+  }
+
+  public Gauge<?> getGauge(String name) {
+    return (Gauge<?>) registered.get(name);
+  }
+
+  private static Map<String, String> variables(String jobId, String 
operatorId) {
+    Map<String, String> variables = new HashMap<>();
+    variables.put(ScopeFormat.SCOPE_JOB_ID, jobId);
+    variables.put(ScopeFormat.SCOPE_OPERATOR_ID, operatorId);
+    return variables;
+  }
+}

Reply via email to