This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 571214daf1 Metric for count of tables configured with various tier
backends (#12940)
571214daf1 is described below
commit 571214daf14780cb4b832e142b2c65a395d86a0a
Author: Shounak kulkarni <[email protected]>
AuthorDate: Fri Apr 26 11:44:13 2024 +0500
Metric for count of tables configured with various tier backends (#12940)
* Metric for count of tables using various tier backends
* avoid multi tiered double counting
* remove unused import
* Handle metrics deletion
* minor
* Metric for count of tables using various tier backends
* avoid multi tiered double counting
* remove unused import
* Handle metrics deletion
* minor
* reformat metric naming
* Revert "Merge branch 'tier-backend-metric' of
https://github.com/shounakmk219/pinot into tier-backend-metric"
This reverts commit 9646c83517db2afd1a0a95805ea0c73f668b2a8e, reversing
changes made to a3f3d10eb2e8f42ecef56eff9f345fd34292e61c.
---
.../configs/controller.yml | 3 +++
.../pinot/common/metrics/AbstractMetrics.java | 4 ++++
.../pinot/common/metrics/ControllerGauge.java | 1 +
.../controller/helix/SegmentStatusChecker.java | 28 ++++++++++++++++++++++
4 files changed, 36 insertions(+)
diff --git
a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
index a441f714e4..a036a13053 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
@@ -85,6 +85,9 @@ rules:
- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ControllerMetrics\",
name=\"pinot\\.controller\\.offlineTableCount\"><>(\\w+)"
name: "pinot_controller_offlineTableCount_$1"
cache: true
+- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ControllerMetrics\",
name=\"pinot\\.controller\\.tierBackendTableCount\\.(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_tierBackendTableCount_$1_$2"
+ cache: true
- pattern:
"\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ValidationMetrics\",
name=\"pinot\\.controller\\.(([^.]+)\\.)?([^.]*)\\.(\\w+)\"><>(\\w+)"
name: "pinot_controller_validateion_$4_$5"
cache: true
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
index ee13493e15..dfdc2abb0f 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
@@ -766,6 +766,10 @@ public abstract class AbstractMetrics<QP extends
AbstractMetrics.QueryPhase, M e
return gauge.getGaugeName() + "." + getTableName(tableName) + "." + key;
}
+ public String composePluginGaugeName(String pluginName, Gauge gauge) {
+ return gauge.getGaugeName() + "." + pluginName;
+ }
+
/**
* Remove gauge from Pinot metrics.
* @param gaugeName gauge name
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index 938242ef78..82c4e666e1 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -65,6 +65,7 @@ public enum ControllerGauge implements AbstractMetrics.Gauge {
NUM_MINION_SUBTASKS_ERROR("NumMinionSubtasksError", true),
PERCENT_MINION_SUBTASKS_IN_QUEUE("PercentMinionSubtasksInQueue", true),
PERCENT_MINION_SUBTASKS_IN_ERROR("PercentMinionSubtasksInError", true),
+ TIER_BACKEND_TABLE_COUNT("TierBackendTableCount", true),
// Pinot controller leader
PINOT_CONTROLLER_LEADER("PinotControllerLeader", true),
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
index fa265c4f52..4bcdc2f516 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
@@ -19,6 +19,7 @@
package org.apache.pinot.controller.helix;
import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -50,6 +51,7 @@ import
org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentMa
import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.config.table.TierConfig;
import org.apache.pinot.spi.stream.StreamConfig;
import org.apache.pinot.spi.utils.IngestionConfigUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
@@ -78,6 +80,7 @@ public class SegmentStatusChecker extends
ControllerPeriodicTask<SegmentStatusCh
private long _lastDisabledTableLogTimestamp = 0;
private TableSizeReader _tableSizeReader;
+ private Set<String> _tierBackendGauges = new HashSet<>();
/**
* Constructs the segment status checker.
@@ -135,6 +138,17 @@ public class SegmentStatusChecker extends
ControllerPeriodicTask<SegmentStatusCh
_controllerMetrics.setValueOfGlobalGauge(ControllerGauge.UPSERT_TABLE_COUNT,
context._upsertTableCount);
_controllerMetrics.setValueOfGlobalGauge(ControllerGauge.DISABLED_TABLE_COUNT,
context._disabledTables.size());
+ _tierBackendGauges.forEach(_controllerMetrics::removeGauge);
+ // metric for total number of tables using a particular tier backend
+ context._tierBackendTableCountMap.forEach((tier, count) -> {
+ String gaugeName = _controllerMetrics.composePluginGaugeName(tier,
ControllerGauge.TIER_BACKEND_TABLE_COUNT);
+ _tierBackendGauges.add(gaugeName);
+ _controllerMetrics.setOrUpdateGauge(gaugeName, count);
+ });
+ // metric for total number of tables having tier backend configured
+
_controllerMetrics.setOrUpdateGauge(ControllerGauge.TIER_BACKEND_TABLE_COUNT.getGaugeName(),
+ context._tierBackendConfiguredTableCount);
+
//emit a 0 for tables that are not paused/disabled. This makes alert
expressions simpler as we don't have to deal
// with missing metrics
context._processedTables.forEach(tableNameWithType -> {
@@ -171,6 +185,18 @@ public class SegmentStatusChecker extends
ControllerPeriodicTask<SegmentStatusCh
if (tableConfig.isUpsertEnabled()) {
context._upsertTableCount++;
}
+ List<TierConfig> tierConfigList = tableConfig.getTierConfigsList();
+ if (tierConfigList != null && !tierConfigList.isEmpty()) {
+ Set<String> tierBackendSet = new HashSet<>(tierConfigList.size());
+ for (TierConfig config : tierConfigList) {
+ if (config.getTierBackend() != null) {
+ tierBackendSet.add(config.getTierBackend());
+ }
+ }
+ tierBackendSet.forEach(tierBackend ->
context._tierBackendTableCountMap.put(tierBackend,
+ context._tierBackendTableCountMap.getOrDefault(tierBackend, 0) +
1));
+ context._tierBackendConfiguredTableCount += tierBackendSet.isEmpty() ? 0
: 1;
+ }
int replication = tableConfig.getReplication();
_controllerMetrics.setValueOfTableGauge(tableNameWithType,
ControllerGauge.REPLICATION_FROM_CONFIG, replication);
}
@@ -391,6 +417,8 @@ public class SegmentStatusChecker extends
ControllerPeriodicTask<SegmentStatusCh
private int _realTimeTableCount;
private int _offlineTableCount;
private int _upsertTableCount;
+ private int _tierBackendConfiguredTableCount;
+ private Map<String, Integer> _tierBackendTableCountMap = new HashMap<>();
private Set<String> _processedTables = new HashSet<>();
private Set<String> _disabledTables = new HashSet<>();
private Set<String> _pausedTables = new HashSet<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
