voonhous commented on code in PR #19575: URL: https://github.com/apache/hudi/pull/19575#discussion_r3863066845
########## hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/ExecutorMetricRegistry.java: ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.function.Predicate; + +/** + * Every class of executor-collected metric, and the only thing a new one is added to. The driver must + * declare it up front because an {@code AccumulatorV2} must be registered with the {@code SparkContext} + * before a task can contribute; the bundle sent to executors and the commit drain both iterate this. + */ +public enum ExecutorMetricRegistry implements ExecutorMetricGroup { + + RECORD_INDEX_LOOKUP( + "HoodieRecordIndexLookup", + "hoodie.rli.lookup.", + "rli", + "lookup", + HoodieWriteConfig::isRecordIndexLookupMetricsEnabled); + + private final String registryName; + private final String commitMetadataPrefix; + private final String metricAction; + private final String metricQualifier; + private final Predicate<HoodieWriteConfig> enabled; + + ExecutorMetricRegistry(String registryName, String commitMetadataPrefix, String metricAction, + String metricQualifier, Predicate<HoodieWriteConfig> enabled) { + this.registryName = registryName; + this.commitMetadataPrefix = commitMetadataPrefix; + this.metricAction = metricAction; + this.metricQualifier = metricQualifier; + this.enabled = enabled; + } + + /** The bare name emitting code passes to {@link Registry#getRegistry(String)}. */ + @Override + public String registryName() { + return registryName; + } + + @Override + public String commitMetadataPrefix() { + return commitMetadataPrefix; + } + + @Override + public String metricAction() { + return metricAction; + } + + @Override + public String metricQualifier() { + return metricQualifier; + } + + /** Gating here, rather than in the drain, is what lets a new class of metric need no new config. */ + @Override + public boolean isEnabled(HoodieWriteConfig config) { + return enabled.test(config); + } + + /** + * Driver-side {@code REGISTRY_MAP} key. Table name alone is not an identity: two tables can share one + * and would then share a registry. Executors use the bare {@link #registryName()}. + */ + @Override + public String scopedName(String basePath) { Review Comment: Spelled out: `Metrics.shutdown()` calls `registerHoodieCommonMetrics()`, which runs `Registry.getAllMetrics(true, true)` over every entry in `REGISTRY_MAP`, registers what it finds as gauges, then clears. The common prefix is applied only to registry names without a `.`, and this one's name is `<table>.HoodieRecordIndexLookup.<12hex>`, so any counters still in the registry at shutdown (the failed-commit case in `TestRliLookupMetricsAcrossFailedCommit`, or a straggler) are emitted as `<prefix>.<table>.HoodieRecordIndexLookup.<12hex>.lookup_record_index_key_count`, one series per base path. `rliCountersFromLatestCommit` filters on `.rli.lookup.` so the test cannot see it; the `report()` dump after the failed commit should. It goes away with an instance-scoped registry, so folding it into that follow-up is fine -- could the PR body's "dropped rather than double-counted" sentence say they are published under that name instead, since that is what a pushgateway user will see? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
