This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 02c4998 Add mergeRollupTask delay metrics (#7368)
02c4998 is described below
commit 02c4998c9e1cac35e7102def23fc17d6645e8232
Author: Seunghyun Lee <[email protected]>
AuthorDate: Mon Oct 4 15:07:30 2021 -0700
Add mergeRollupTask delay metrics (#7368)
* Add mergeRollupTask delay metrics
When operating merge/roll-up tasks in production, we should have
a metrics that indicates the progress or delay of merge/roll-up
task. mergeRollupTaskDelayInNumTimeBuckets is computed as follows:
(current_time - watermark - bufferTime) / bucketTime
* Address the non-leader case
---
.../pinot/common/metrics/AbstractMetrics.java | 12 +-
.../helix/core/minion/ClusterInfoAccessor.java | 27 ++-
.../helix/core/minion/PinotTaskManager.java | 4 +-
.../MergeRollupMinionClusterIntegrationTest.java | 43 +++--
.../mergerollup/MergeRollupTaskGenerator.java | 186 +++++++++++++++++++--
5 files changed, 241 insertions(+), 31 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
index 533bcf2..9cf33de 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java
@@ -494,7 +494,7 @@ public abstract class AbstractMetrics<QP extends
AbstractMetrics.QueryPhase, M e
* Remove gauge from Pinot metrics.
* @param gaugeName gauge name
*/
- private void removeGauge(final String gaugeName) {
+ public void removeGauge(final String gaugeName) {
if (_gaugeValues.remove(gaugeName) != null) {
removeCallbackGauge(gaugeName);
}
@@ -518,4 +518,14 @@ public abstract class AbstractMetrics<QP extends
AbstractMetrics.QueryPhase, M e
protected String getTableName(String tableName) {
return _isTableLevelMetricsEnabled || _allowedTables.contains(tableName) ?
tableName : "allTables";
}
+
+ /**
+ * Check if the metric name appears in the gauge value map.
+ * @param metricName metric name
+ * @return True if the metric name appears on the gauge value map. False
otherwise.
+ */
+ @VisibleForTesting
+ public boolean containsGauge(String metricName) {
+ return _gaugeValues.containsKey(metricName);
+ }
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
index e22376b..7ea31fb 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/ClusterInfoAccessor.java
@@ -30,10 +30,12 @@ import org.apache.pinot.common.lineage.SegmentLineage;
import org.apache.pinot.common.lineage.SegmentLineageAccessHelper;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.minion.MergeRollupTaskMetadata;
import org.apache.pinot.common.minion.MinionTaskMetadataUtils;
import org.apache.pinot.common.minion.RealtimeToOfflineSegmentsTaskMetadata;
import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.core.common.MinionConstants;
import org.apache.pinot.core.minion.PinotTaskConfig;
@@ -49,12 +51,17 @@ public class ClusterInfoAccessor {
private final PinotHelixResourceManager _pinotHelixResourceManager;
private final PinotHelixTaskResourceManager _pinotHelixTaskResourceManager;
private final ControllerConf _controllerConf;
+ private final ControllerMetrics _controllerMetrics;
+ private final LeadControllerManager _leadControllerManager;
public ClusterInfoAccessor(PinotHelixResourceManager
pinotHelixResourceManager,
- PinotHelixTaskResourceManager pinotHelixTaskResourceManager,
ControllerConf controllerConf) {
+ PinotHelixTaskResourceManager pinotHelixTaskResourceManager,
ControllerConf controllerConf,
+ ControllerMetrics controllerMetrics, LeadControllerManager
leadControllerManager) {
_pinotHelixResourceManager = pinotHelixResourceManager;
_pinotHelixTaskResourceManager = pinotHelixTaskResourceManager;
_controllerConf = controllerConf;
+ _controllerMetrics = controllerMetrics;
+ _leadControllerManager = leadControllerManager;
}
/**
@@ -181,4 +188,22 @@ public class ClusterInfoAccessor {
_pinotHelixResourceManager.getHelixAdmin().getConfig(helixConfigScope,
Collections.singletonList(configName));
return configMap != null ? configMap.get(configName) : null;
}
+
+ /**
+ * Get the controller metrics.
+ *
+ * @return controller metrics
+ */
+ public ControllerMetrics getControllerMetrics() {
+ return _controllerMetrics;
+ }
+
+ /**
+ * Get the leader controller manager.
+ *
+ * @return leader controller manager
+ */
+ public LeadControllerManager getLeaderControllerManager() {
+ return _leadControllerManager;
+ }
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
index 016a8cd..5bafa0a 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotTaskManager.java
@@ -95,7 +95,9 @@ public class PinotTaskManager extends
ControllerPeriodicTask<Void> {
controllerConf.getPinotTaskManagerInitialDelaySeconds(),
helixResourceManager, leadControllerManager,
controllerMetrics);
_helixTaskResourceManager = helixTaskResourceManager;
- _clusterInfoAccessor = new ClusterInfoAccessor(helixResourceManager,
helixTaskResourceManager, controllerConf);
+ _clusterInfoAccessor =
+ new ClusterInfoAccessor(helixResourceManager,
helixTaskResourceManager, controllerConf, controllerMetrics,
+ leadControllerManager);
_taskGeneratorRegistry = new TaskGeneratorRegistry(_clusterInfoAccessor);
if (controllerConf.isPinotTaskManagerSchedulerEnabled()) {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
index 3708ad0..d98dc7e 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MergeRollupMinionClusterIntegrationTest.java
@@ -50,13 +50,13 @@ import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.util.TestUtils;
-import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
@@ -259,7 +259,7 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
// {merged_100days_T4_1_16400_16404_1, myTable1_16405_16435_2}
// -> {merged_100days_T5_0_myTable1_16400_16435_0}
- String sqlQuery = "SELECT count(*) FROM mytable1"; // 115545 rows for the
test table
+ String sqlQuery = "SELECT count(*) FROM myTable1"; // 115545 rows for the
test table
JsonNode expectedJson = postSqlQuery(sqlQuery, _brokerBaseApiUrl);
int[] expectedNumSubTasks = {1, 2, 2, 2, 1};
int[] expectedNumSegmentsQueried = {13, 12, 13, 13, 12};
@@ -269,11 +269,11 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
for (String tasks =
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE);
tasks != null; tasks =
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE),
numTasks++) {
-
Assert.assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(),
expectedNumSubTasks[numTasks]);
- Assert.assertTrue(_helixTaskResourceManager.getTaskQueues()
+ assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(),
expectedNumSubTasks[numTasks]);
+ assertTrue(_helixTaskResourceManager.getTaskQueues()
.contains(PinotHelixTaskResourceManager.getHelixJobQueueName(MinionConstants.MergeRollupTask.TASK_TYPE)));
// Will not schedule task if there's incomplete task
- Assert.assertNull(
+ assertNull(
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.RealtimeToOfflineSegmentsTask.TASK_TYPE));
waitForTaskToComplete();
@@ -281,7 +281,7 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
MergeRollupTaskMetadata minionTaskMetadata = MergeRollupTaskMetadata
.fromZNRecord(_taskManager.getClusterInfoAccessor().getMinionMergeRollupTaskZNRecord(offlineTableName));
assertNotNull(minionTaskMetadata);
- Assert.assertEquals((long)
minionTaskMetadata.getWatermarkMap().get("100days"), expectedWatermark);
+ assertEquals((long) minionTaskMetadata.getWatermarkMap().get("100days"),
expectedWatermark);
expectedWatermark += 100 * 86_400_000L;
// Check metadata of merged segments
@@ -306,6 +306,9 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
// Check total tasks
assertEquals(numTasks, 5);
+ assertTrue(_controllerStarter.getControllerMetrics()
+
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable1_OFFLINE.100days"));
+
// Drop the table
dropOfflineTable(SINGLE_LEVEL_CONCAT_TEST_TABLE);
@@ -361,11 +364,11 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
for (String tasks =
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE);
tasks != null; tasks =
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE),
numTasks++) {
-
Assert.assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 1);
- Assert.assertTrue(_helixTaskResourceManager.getTaskQueues()
+ assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(), 1);
+ assertTrue(_helixTaskResourceManager.getTaskQueues()
.contains(PinotHelixTaskResourceManager.getHelixJobQueueName(MinionConstants.MergeRollupTask.TASK_TYPE)));
// Will not schedule task if there's incomplete task
- Assert.assertNull(
+ assertNull(
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.RealtimeToOfflineSegmentsTask.TASK_TYPE));
waitForTaskToComplete();
@@ -373,7 +376,7 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
MergeRollupTaskMetadata minionTaskMetadata = MergeRollupTaskMetadata
.fromZNRecord(_taskManager.getClusterInfoAccessor().getMinionMergeRollupTaskZNRecord(offlineTableName));
assertNotNull(minionTaskMetadata);
- Assert.assertEquals((long)
minionTaskMetadata.getWatermarkMap().get("150days"), expectedWatermark);
+ assertEquals((long) minionTaskMetadata.getWatermarkMap().get("150days"),
expectedWatermark);
expectedWatermark += 150 * 86_400_000L;
// Check metadata of merged segments
@@ -407,10 +410,13 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
postSqlQuery("SELECT count(*), DaysSinceEpoch FROM myTable2 GROUP BY
DaysSinceEpoch ORDER BY DaysSinceEpoch");
for (int i = 0; i < responseJson.get("resultTable").get("rows").size();
i++) {
int daysSinceEpoch =
responseJson.get("resultTable").get("rows").get(i).get(1).asInt();
- Assert.assertTrue(daysSinceEpoch % 7 == 0);
+ assertTrue(daysSinceEpoch % 7 == 0);
}
// Check total tasks
assertEquals(numTasks, 3);
+
+ assertTrue(_controllerStarter.getControllerMetrics()
+
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable2_OFFLINE.150days"));
}
/**
@@ -497,11 +503,11 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
for (String tasks =
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE);
tasks != null; tasks =
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.MergeRollupTask.TASK_TYPE),
numTasks++) {
-
Assert.assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(),
expectedNumSubTasks[numTasks]);
- Assert.assertTrue(_helixTaskResourceManager.getTaskQueues()
+ assertEquals(_helixTaskResourceManager.getTaskConfigs(tasks).size(),
expectedNumSubTasks[numTasks]);
+ assertTrue(_helixTaskResourceManager.getTaskQueues()
.contains(PinotHelixTaskResourceManager.getHelixJobQueueName(MinionConstants.MergeRollupTask.TASK_TYPE)));
// Will not schedule task if there's incomplete task
- Assert.assertNull(
+ assertNull(
_taskManager.scheduleTasks(offlineTableName).get(MinionConstants.RealtimeToOfflineSegmentsTask.TASK_TYPE));
waitForTaskToComplete();
@@ -509,8 +515,8 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
MergeRollupTaskMetadata minionTaskMetadata = MergeRollupTaskMetadata
.fromZNRecord(_taskManager.getClusterInfoAccessor().getMinionMergeRollupTaskZNRecord(offlineTableName));
assertNotNull(minionTaskMetadata);
- Assert.assertEquals(minionTaskMetadata.getWatermarkMap().get("45days"),
expectedWatermarks45Days[numTasks]);
- Assert.assertEquals(minionTaskMetadata.getWatermarkMap().get("90days"),
expectedWatermarks90Days[numTasks]);
+ assertEquals(minionTaskMetadata.getWatermarkMap().get("45days"),
expectedWatermarks45Days[numTasks]);
+ assertEquals(minionTaskMetadata.getWatermarkMap().get("90days"),
expectedWatermarks90Days[numTasks]);
// Check metadata of merged segments
for (SegmentZKMetadata metadata :
_pinotHelixResourceManager.getSegmentsZKMetadata(offlineTableName)) {
@@ -539,6 +545,11 @@ public class MergeRollupMinionClusterIntegrationTest
extends BaseClusterIntegrat
}
// Check total tasks
assertEquals(numTasks, 8);
+
+ assertTrue(_controllerStarter.getControllerMetrics()
+
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable3_OFFLINE.45days"));
+ assertTrue(_controllerStarter.getControllerMetrics()
+
.containsGauge("mergeRollupTaskDelayInNumBuckets.myTable3_OFFLINE.90days"));
}
protected void verifyTableDelete(String tableNameWithType) {
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
index 031585f..5c8ec8f 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupTaskGenerator.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import org.I0Itec.zkclient.exception.ZkException;
import org.apache.commons.lang3.StringUtils;
import org.apache.helix.ZNRecord;
@@ -35,6 +36,7 @@ import org.apache.pinot.common.lineage.SegmentLineage;
import org.apache.pinot.common.lineage.SegmentLineageUtils;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.minion.MergeRollupTaskMetadata;
import org.apache.pinot.controller.helix.core.minion.ClusterInfoAccessor;
import
org.apache.pinot.controller.helix.core.minion.generator.PinotTaskGenerator;
@@ -90,16 +92,31 @@ import org.slf4j.LoggerFactory;
*/
@TaskGenerator
public class MergeRollupTaskGenerator implements PinotTaskGenerator {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(MergeRollupTaskGenerator.class);
+
public static final int END_REPLACE_SEGMENTS_SOCKET_TIMEOUT_MS = 30 * 60 *
1000; // 30 mins
private static final int DEFAULT_MAX_NUM_RECORDS_PER_TASK = 50_000_000;
private static final String REFRESH = "REFRESH";
- private static final Logger LOGGER =
LoggerFactory.getLogger(MergeRollupTaskGenerator.class);
+ // This is the metric that keeps track of the task delay in the number of
time buckets. For example, if we see this
+ // number to be 7 and merge task is configured with "bucketTimePeriod = 1d",
this means that we have 7 days of
+ // delay. When operating merge/roll-up task in production, we should set the
alert on this metrics to find out the
+ // delay. Setting the alert on 7 time buckets of delay would be a good
starting point.
+ //
+ // NOTE: Based on the current scheduler logic, we are bumping up the
watermark with some delay. (the current round
+ // will bump up the watermark for the window that got processed from the
previous round). Due to this, we will
+ // correctly report the delay with one edge case. When we processed all
available time windows, the watermark
+ // will not get bumped up until we schedule some task for the table. Due to
this, we will always see the delay >= 1.
+ private static final String MERGE_ROLLUP_TASK_DELAY_IN_NUM_BUCKETS =
"mergeRollupTaskDelayInNumBuckets";
+
+ // tableNameWithType -> mergeLevel -> watermarkMs
+ private Map<String, Map<String, Long>> _mergeRollupWatermarks;
private ClusterInfoAccessor _clusterInfoAccessor;
@Override
public void init(ClusterInfoAccessor clusterInfoAccessor) {
_clusterInfoAccessor = clusterInfoAccessor;
+ _mergeRollupWatermarks = new HashMap<>();
}
@Override
@@ -111,7 +128,6 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
public List<PinotTaskConfig> generateTasks(List<TableConfig> tableConfigs) {
String taskType = MergeRollupTask.TASK_TYPE;
List<PinotTaskConfig> pinotTaskConfigs = new ArrayList<>();
-
for (TableConfig tableConfig : tableConfigs) {
if (!validate(tableConfig, taskType)) {
continue;
@@ -138,8 +154,11 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
}
if (preSelectedSegments.isEmpty()) {
+ // Reset the watermark time if no segment found. This covers the case
where the table is newly created or
+ // all segments for the existing table got deleted.
+ resetDelayMetrics(offlineTableName);
LOGGER
- .info("Skip generating task: {} for table: {}, no segment is found
to merge.", taskType, offlineTableName);
+ .info("Skip generating task: {} for table: {}, no segment is
found.", taskType, offlineTableName);
continue;
}
@@ -197,6 +216,11 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
// Get the bucket size and buffer
long bucketMs =
TimeUtils.convertPeriodToMillis(mergeConfigs.get(MinionConstants.MergeTask.BUCKET_TIME_PERIOD_KEY));
+ if (bucketMs <= 0) {
+ LOGGER.error("Bucket time period (table : {}, mergeLevel : {}) must
be larger than 0", offlineTableName,
+ mergeLevel);
+ continue;
+ }
long bufferMs =
TimeUtils.convertPeriodToMillis(mergeConfigs.get(MinionConstants.MergeTask.BUFFER_TIME_PERIOD_KEY));
@@ -208,10 +232,8 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
long windowEndMs = windowStartMs + bucketMs;
if (!isValidMergeWindowEndTime(windowEndMs, bufferMs, lowerMergeLevel,
mergeRollupTaskMetadata)) {
- LOGGER.info(
- "Window with start: {} and end: {} of mergeLevel: {} is not a
valid merge window, Skipping task "
- + "generation: {}",
- windowStartMs, windowEndMs, mergeLevel, taskType);
+ LOGGER.info("Window with start: {} and end: {} of mergeLevel: {} is
not a valid merge window, Skipping task "
+ + "generation: {}", windowStartMs, windowEndMs, mergeLevel,
taskType);
continue;
}
@@ -264,10 +286,8 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
}
if (!isValidMergeWindow) {
- LOGGER.info(
- "Window with start: {} and end: {} of mergeLevel: {} is not a
valid merge window, Skipping task "
- + "generation: {}",
- windowStartMs, windowEndMs, mergeLevel, taskType);
+ LOGGER.info("Window with start: {} and end: {} of mergeLevel: {} is
not a valid merge window, Skipping task "
+ + "generation: {}", windowStartMs, windowEndMs, mergeLevel,
taskType);
continue;
}
@@ -279,7 +299,10 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
Long prevWatermarkMs =
mergeRollupTaskMetadata.getWatermarkMap().put(mergeLevel, windowStartMs);
LOGGER.info("Update watermark of mergeLevel: {} for table: {} from: {}
to: {}", mergeLevel, offlineTableName,
- prevWatermarkMs, waterMarkMs);
+ prevWatermarkMs, windowStartMs);
+
+ // Update the delay metrics
+ updateDelayMetrics(offlineTableName, mergeLevel, windowStartMs,
bufferMs, bucketMs);
// Create task configs
int maxNumRecordsPerTask =
mergeConfigs.get(MergeRollupTask.MAX_NUM_RECORDS_PER_TASK_KEY) != null ? Integer
@@ -340,6 +363,10 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
.info("Finished generating task configs for table: {} for task: {},
numTasks: {}", offlineTableName, taskType,
pinotTaskConfigsForTable.size());
}
+
+ // Clean up metrics
+ cleanUpDelayMetrics(tableConfigs);
+
return pinotTaskConfigs;
}
@@ -466,4 +493,139 @@ public class MergeRollupTaskGenerator implements
PinotTaskGenerator {
return pinotTaskConfigs;
}
+
+ private long getMergeRollupTaskDelayInNumTimeBuckets(long watermarkMs, long
bufferTimeMs, long bucketTimeMs) {
+ if (watermarkMs == -1) {
+ return 0;
+ }
+ return (System.currentTimeMillis() - watermarkMs - bufferTimeMs) /
bucketTimeMs;
+ }
+
+ /**
+ * Update the delay metrics for the given table and merge level. We create
the new gauge metric if the metric is not
+ * available.
+ *
+ * @param tableNameWithType table name with type
+ * @param mergeLevel merge level
+ * @param watermarkMs current watermark value
+ * @param bufferTimeMs buffer time
+ * @param bucketTimeMs bucket time
+ */
+ private void updateDelayMetrics(String tableNameWithType, String mergeLevel,
long watermarkMs, long bufferTimeMs,
+ long bucketTimeMs) {
+ ControllerMetrics controllerMetrics =
_clusterInfoAccessor.getControllerMetrics();
+ if (controllerMetrics == null) {
+ return;
+ }
+
+ // Update gauge value that indicates the delay in terms of the number of
time buckets.
+ Map<String, Long> watermarkForTable =
+ _mergeRollupWatermarks.computeIfAbsent(tableNameWithType, k -> new
ConcurrentHashMap<>());
+ watermarkForTable.compute(mergeLevel, (k, v) -> {
+ if (v == null) {
+ LOGGER.info(
+ "Creating the gauge metric for tracking the merge/roll-up task
delay for table: {} and mergeLevel: {}."
+ + "(watermarkMs={}, bufferTimeMs={}, bucketTimeMs={},
taskDelayInNumTimeBuckets={})", tableNameWithType,
+ mergeLevel, watermarkMs, bucketTimeMs, bucketTimeMs,
+ getMergeRollupTaskDelayInNumTimeBuckets(watermarkMs, bufferTimeMs,
bucketTimeMs));
+
controllerMetrics.addCallbackGaugeIfNeeded(getMetricNameForTaskDelay(tableNameWithType,
mergeLevel),
+ (() ->
getMergeRollupTaskDelayInNumTimeBuckets(watermarkForTable.getOrDefault(k, -1L),
bufferTimeMs,
+ bucketTimeMs)));
+ }
+ return watermarkMs;
+ });
+ }
+
+ /**
+ * Reset the delay metrics for the given table name.
+ *
+ * @param tableNameWithType a table name with type
+ */
+ private void resetDelayMetrics(String tableNameWithType) {
+ ControllerMetrics controllerMetrics =
_clusterInfoAccessor.getControllerMetrics();
+ if (controllerMetrics == null) {
+ return;
+ }
+
+ // Delete all the watermarks associated with the given table name
+ Map<String, Long> watermarksForTable =
_mergeRollupWatermarks.remove(tableNameWithType);
+ if (watermarksForTable != null) {
+ for (String mergeLevel : watermarksForTable.keySet()) {
+
controllerMetrics.removeGauge(getMetricNameForTaskDelay(tableNameWithType,
mergeLevel));
+ }
+ }
+ }
+
+ /**
+ * Reset the delay metrics for the given table name and merge level.
+ *
+ * @param tableNameWithType table name with type
+ * @param mergeLevel merge level
+ */
+ private void resetDelayMetrics(String tableNameWithType, String mergeLevel) {
+ ControllerMetrics controllerMetrics =
_clusterInfoAccessor.getControllerMetrics();
+ if (controllerMetrics == null) {
+ return;
+ }
+
+ // Delete all the watermarks associated with the given the table name and
the merge level.
+ Map<String, Long> watermarksForTable =
_mergeRollupWatermarks.get(tableNameWithType);
+ if (watermarksForTable != null) {
+ if (watermarksForTable.remove(mergeLevel) != null) {
+
controllerMetrics.removeGauge(getMetricNameForTaskDelay(tableNameWithType,
mergeLevel));
+ }
+ }
+ }
+
+ /**
+ * Clean up the metrics that no longer need to be emitted.
+ *
+ * We clean up the metrics for the following cases:
+ * 1. Table got deleted.
+ * 2. The current controller is no longer the leader for a table.
+ * 3. Merge task config got deleted.
+ * 4. Merge task config got modified and some merge levels got deleted.
+ *
+ * TODO: Current code will remove all metrics in case we invoke the ad-hoc
task scheduling on a single table.
+ * We will file the follow-up PR to address this issue. We need to separate
out APIs for ad-hoc scheduling and
+ * periodic scheduling. We will only enable metrics for periodic case.
+ *
+ * @param tableConfigs list of tables
+ */
+ private void cleanUpDelayMetrics(List<TableConfig> tableConfigs) {
+ Map<String, TableConfig> tableConfigMap = new HashMap<>();
+ for (TableConfig tableConfig : tableConfigs) {
+ tableConfigMap.put(tableConfig.getTableName(), tableConfig);
+ }
+
+ for (String tableNameWithType : new
ArrayList<>(_mergeRollupWatermarks.keySet())) {
+ TableConfig currentTableConfig = tableConfigMap.get(tableNameWithType);
+ // Table does not exist in the cluster or merge task config is removed
+ if (currentTableConfig == null) {
+ resetDelayMetrics(tableNameWithType);
+ continue;
+ }
+
+ // The current controller is no longer leader for this table
+ if
(!_clusterInfoAccessor.getLeaderControllerManager().isLeaderForTable(tableNameWithType))
{
+ resetDelayMetrics(tableNameWithType);
+ continue;
+ }
+
+ // Task config is modified and some merge level got removed
+ Map<String, String> taskConfigs =
currentTableConfig.getTaskConfig().getConfigsForTaskType(getTaskType());
+ Map<String, Map<String, String>> mergeLevelToConfigs =
MergeRollupTaskUtils.getLevelToConfigMap(taskConfigs);
+ Map<String, Long> tableToWatermark =
_mergeRollupWatermarks.get(tableNameWithType);
+ for (String mergeLevel : tableToWatermark.keySet()) {
+ if (!mergeLevelToConfigs.containsKey(mergeLevel)) {
+ resetDelayMetrics(tableNameWithType, mergeLevel);
+ }
+ }
+ }
+ }
+
+ private String getMetricNameForTaskDelay(String tableNameWithType, String
mergeLevel) {
+ // e.g. mergeRollupTaskDelayInNumBuckets.myTable_OFFLINE.daily
+ return MERGE_ROLLUP_TASK_DELAY_IN_NUM_BUCKETS + "." + tableNameWithType +
"." + mergeLevel;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]