This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fc358a87fc Add offset based lag metrics (#13298)
fc358a87fc is described below
commit fc358a87fc63def0b5c5a31a24bde1d9df0d9ecb
Author: Kartik Khare <[email protected]>
AuthorDate: Fri Jun 7 19:21:32 2024 +0530
Add offset based lag metrics (#13298)
* Add offset based lag metrics
* Add tests
* Refactoring
---------
Co-authored-by: Kartik Khare <[email protected]>
---
.../apache/pinot/common/metrics/ServerGauge.java | 3 +-
.../manager/realtime/IngestionDelayTracker.java | 92 +++++++++++++++++++---
.../realtime/RealtimeSegmentDataManager.java | 16 ++--
.../manager/realtime/RealtimeTableDataManager.java | 10 ++-
.../realtime/IngestionDelayTrackerTest.java | 65 +++++++++++++++
5 files changed, 167 insertions(+), 19 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
index f0a1fdd136..072c048c55 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
@@ -72,7 +72,8 @@ public enum ServerGauge implements AbstractMetrics.Gauge {
END_TO_END_REALTIME_INGESTION_DELAY_MS("milliseconds", false),
// Needed to track if valid doc id snapshots are present for faster restarts
UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT("upsertValidDocIdSnapshotCount", false),
- UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT("upsertPrimaryKeysInSnapshotCount",
false);
+ UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT("upsertPrimaryKeysInSnapshotCount",
false),
+ REALTIME_INGESTION_OFFSET_LAG("offsetLag", false);
private final String _gaugeName;
private final String _unit;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
index 84a5ea7ba0..eed1302708 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
@@ -34,6 +34,8 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.apache.pinot.common.metrics.ServerGauge;
import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.spi.stream.LongMsgOffset;
+import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,13 +83,23 @@ public class IngestionDelayTracker {
// Class to wrap supported timestamps collected for an ingested event
private static class IngestionTimestamps {
+ private final long _firstStreamIngestionTimeMs;
+ private final long _ingestionTimeMs;
IngestionTimestamps(long ingestionTimesMs, long
firstStreamIngestionTimeMs) {
_ingestionTimeMs = ingestionTimesMs;
_firstStreamIngestionTimeMs = firstStreamIngestionTimeMs;
}
- private final long _ingestionTimeMs;
- private final long _firstStreamIngestionTimeMs;
}
+
+ private static class IngestionOffsets {
+ private final StreamPartitionMsgOffset _latestOffset;
+ private final StreamPartitionMsgOffset _offset;
+ IngestionOffsets(StreamPartitionMsgOffset offset, StreamPartitionMsgOffset
latestOffset) {
+ _offset = offset;
+ _latestOffset = latestOffset;
+ }
+ }
+
// Sleep interval for scheduled executor service thread that triggers read
of ideal state
private static final int SCHEDULED_EXECUTOR_THREAD_TICK_INTERVAL_MS =
300000; // 5 minutes +/- precision in timeouts
// Once a partition is marked for verification, we wait 10 minutes to pull
its ideal state.
@@ -98,6 +110,8 @@ public class IngestionDelayTracker {
// HashMap used to store ingestion time measures for all partitions active
for the current table.
private final Map<Integer, IngestionTimestamps>
_partitionToIngestionTimestampsMap = new ConcurrentHashMap<>();
+
+ private final Map<Integer, IngestionOffsets> _partitionToOffsetMap = new
ConcurrentHashMap<>();
// We mark partitions that go from CONSUMING to ONLINE in
_partitionsMarkedForVerification: if they do not
// go back to CONSUMING in some period of time, we verify whether they are
still hosted in this server by reading
// ideal state. This is done with the goal of minimizing reading ideal state
for efficiency reasons.
@@ -174,6 +188,22 @@ public class IngestionDelayTracker {
return agedIngestionDelayMs;
}
+ private long getPartitionOffsetLag(IngestionOffsets offset) {
+ if (offset == null) {
+ return 0;
+ }
+ StreamPartitionMsgOffset currentOffset = offset._offset;
+ StreamPartitionMsgOffset latestOffset = offset._latestOffset;
+
+ // Compute aged delay for current partition
+ // TODO: Support other types of offsets
+ if (!(currentOffset instanceof LongMsgOffset && latestOffset instanceof
LongMsgOffset)) {
+ return 0;
+ }
+
+ return ((LongMsgOffset) latestOffset).getOffset() - ((LongMsgOffset)
currentOffset).getOffset();
+ }
+
/*
* Helper function to be called when we should stop tracking a given
partition. Removes the partition from
* all our maps.
@@ -182,11 +212,13 @@ public class IngestionDelayTracker {
*/
private void removePartitionId(int partitionGroupId) {
_partitionToIngestionTimestampsMap.remove(partitionGroupId);
+ _partitionToOffsetMap.remove(partitionGroupId);
// If we are removing a partition we should stop reading its ideal state.
_partitionsMarkedForVerification.remove(partitionGroupId);
_serverMetrics.removePartitionGauge(_metricName, partitionGroupId,
ServerGauge.REALTIME_INGESTION_DELAY_MS);
_serverMetrics.removePartitionGauge(_metricName, partitionGroupId,
ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS);
+ _serverMetrics.removePartitionGauge(_metricName, partitionGroupId,
ServerGauge.REALTIME_INGESTION_OFFSET_LAG);
}
/*
@@ -216,20 +248,32 @@ public class IngestionDelayTracker {
_clock = clock;
}
- /*
- * Called by RealTimeSegmentDataManagers to post ingestion time updates to
this tracker class.
+ /**
+ * Called by RealTimeSegmentDataManagers to update the ingestion delay
metrics for a given partition.
*
- * @param ingestionTimeMs ingestion time being recorded.
- * @param firstStreamIngestionTimeMs time the event was ingested in the
first stage of the ingestion pipeline.
- * @param partitionGroupId partition ID for which this ingestion time is
being recorded.
+ * @param ingestionTimeMs ingestion time being recorded.
+ * @param firstStreamIngestionTimeMs time the event was ingested in the
first stage of the ingestion pipeline.
+ * @param msgOffset message offset of the event being
ingested.
+ * @param latestOffset latest message offset in the stream.
+ * @param partitionGroupId partition ID for which the ingestion
metrics are being recorded.
*/
- public void updateIngestionDelay(long ingestionTimeMs, long
firstStreamIngestionTimeMs, int partitionGroupId) {
- // Store new measure and wipe old one for this partition
+ public void updateIngestionMetrics(long ingestionTimeMs, long
firstStreamIngestionTimeMs,
+ StreamPartitionMsgOffset msgOffset, StreamPartitionMsgOffset
latestOffset,
+ int partitionGroupId) {
if (!_isServerReadyToServeQueries.get() ||
_realTimeTableDataManager.isShutDown()) {
// Do not update the ingestion delay metrics during server startup period
// or once the table data manager has been shutdown.
return;
}
+
+ updateIngestionDelay(ingestionTimeMs, firstStreamIngestionTimeMs,
partitionGroupId);
+ updateIngestionOffsets(msgOffset, latestOffset, partitionGroupId);
+
+ // If we are consuming we do not need to track this partition for removal.
+ _partitionsMarkedForVerification.remove(partitionGroupId);
+ }
+
+ public void updateIngestionDelay(long ingestionTimeMs, long
firstStreamIngestionTimeMs, int partitionGroupId) {
if ((ingestionTimeMs < 0) && (firstStreamIngestionTimeMs < 0)) {
// If stream does not return a valid ingestion timestamps don't publish
a metric
return;
@@ -252,8 +296,25 @@ public class IngestionDelayTracker {
() -> getPartitionEndToEndIngestionDelayMs(partitionGroupId));
}
}
- // If we are consuming we do not need to track this partition for removal.
- _partitionsMarkedForVerification.remove(partitionGroupId);
+ }
+
+ public void updateIngestionOffsets(StreamPartitionMsgOffset currentOffset,
StreamPartitionMsgOffset latestOffset,
+ int partitionGroupId) {
+ if ((currentOffset == null)) {
+ // If stream does not return a valid ingestion offset don't publish a
metric
+ return;
+ }
+ IngestionOffsets previousMeasure =
+ _partitionToOffsetMap.put(partitionGroupId, new
IngestionOffsets(currentOffset, latestOffset));
+ if (previousMeasure == null) {
+ // First time we start tracking a partition we should start tracking it
via metric
+ // Only publish the metric if supported by the underlying stream. If not
supported the stream
+ // returns Long.MIN_VALUE
+ if (currentOffset != null) {
+ _serverMetrics.setOrUpdatePartitionGauge(_metricName, partitionGroupId,
+ ServerGauge.REALTIME_INGESTION_OFFSET_LAG, () ->
getPartitionIngestionOffsetLag(partitionGroupId));
+ }
+ }
}
/*
@@ -346,6 +407,15 @@ public class IngestionDelayTracker {
return getIngestionDelayMs(currentMeasure._ingestionTimeMs);
}
+ public long getPartitionIngestionOffsetLag(int partitionGroupId) {
+ // Not protected as this will only be invoked when metric is installed
which happens after server ready
+ IngestionOffsets currentMeasure =
_partitionToOffsetMap.get(partitionGroupId);
+ if (currentMeasure == null) { // Guard just in case we read the metric
without initializing it
+ return 0;
+ }
+ return getPartitionOffsetLag(currentMeasure);
+ }
+
/*
* Method to get end to end ingestion delay for a given partition.
*
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
index f50942c588..8dc7842f81 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
@@ -660,7 +660,7 @@ public class RealtimeSegmentDataManager extends
SegmentDataManager {
updateCurrentDocumentCountMetrics();
if (messageBatch.getUnfilteredMessageCount() > 0) {
- updateIngestionDelay(messageBatch.getLastMessageMetadata());
+ updateIngestionMetrics(messageBatch.getLastMessageMetadata());
_hasMessagesFetched = true;
if (streamMessageCount > 0 && _segmentLogger.isDebugEnabled()) {
_segmentLogger.debug("Indexed {} messages ({} messages read from
stream) current offset {}",
@@ -1778,10 +1778,16 @@ public class RealtimeSegmentDataManager extends
SegmentDataManager {
_partitionMetadataProvider =
_streamConsumerFactory.createPartitionMetadataProvider(_clientId,
_partitionGroupId);
}
- private void updateIngestionDelay(RowMetadata metadata) {
+ private void updateIngestionMetrics(RowMetadata metadata) {
if (metadata != null) {
-
_realtimeTableDataManager.updateIngestionDelay(metadata.getRecordIngestionTimeMs(),
- metadata.getFirstStreamRecordIngestionTimeMs(), _partitionGroupId);
+ try {
+ StreamPartitionMsgOffset latestOffset =
+
_partitionMetadataProvider.fetchStreamPartitionOffset(OffsetCriteria.LARGEST_OFFSET_CRITERIA,
5000);
+
_realtimeTableDataManager.updateIngestionMetrics(metadata.getRecordIngestionTimeMs(),
+ metadata.getFirstStreamRecordIngestionTimeMs(),
metadata.getOffset(), latestOffset, _partitionGroupId);
+ } catch (Exception e) {
+ _segmentLogger.warn("Failed to fetch latest offset for updating
ingestion delay", e);
+ }
}
}
@@ -1790,7 +1796,7 @@ public class RealtimeSegmentDataManager extends
SegmentDataManager {
*/
private void setIngestionDelayToZero() {
long currentTimeMs = System.currentTimeMillis();
- _realtimeTableDataManager.updateIngestionDelay(currentTimeMs,
currentTimeMs, _partitionGroupId);
+ _realtimeTableDataManager.updateIngestionMetrics(currentTimeMs,
currentTimeMs, null, null, _partitionGroupId);
}
// This should be done during commit? We may not always commit when we build
a segment....
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index 6c9c647b1f..3659fae1c2 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -74,6 +74,7 @@ import org.apache.pinot.spi.data.DateTimeFieldSpec;
import org.apache.pinot.spi.data.DateTimeFormatSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
import org.apache.pinot.spi.utils.TimeUtils;
@@ -265,10 +266,15 @@ public class RealtimeTableDataManager extends
BaseTableDataManager {
* Method used by RealtimeSegmentManagers to update their partition delays
*
* @param ingestionTimeMs Ingestion delay being reported.
+ * @param firstStreamIngestionTimeMs Ingestion time of the first message in
the stream.
* @param partitionGroupId Partition ID for which delay is being updated.
+ * @param offset last offset received for the partition.
+ * @param latestOffset latest upstream offset for the partition.
*/
- public void updateIngestionDelay(long ingestionTimeMs, long
firstStreamIngestionTimeMs, int partitionGroupId) {
- _ingestionDelayTracker.updateIngestionDelay(ingestionTimeMs,
firstStreamIngestionTimeMs, partitionGroupId);
+ public void updateIngestionMetrics(long ingestionTimeMs, long
firstStreamIngestionTimeMs,
+ StreamPartitionMsgOffset offset, StreamPartitionMsgOffset latestOffset,
int partitionGroupId) {
+ _ingestionDelayTracker.updateIngestionMetrics(ingestionTimeMs,
firstStreamIngestionTimeMs, offset, latestOffset,
+ partitionGroupId);
}
/*
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
index 4a1c27a506..8072731583 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
@@ -24,6 +24,8 @@ import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.spi.stream.LongMsgOffset;
+import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -253,4 +255,67 @@ public class IngestionDelayTrackerTest {
ingestionDelayTracker = createTracker();
ingestionDelayTracker.shutdown();
}
+
+ @Test
+ public void testRecordIngestionDelayOffsetWithNoAging() {
+ final int partition0 = 0;
+ final int partition1 = 1;
+
+ IngestionDelayTracker ingestionDelayTracker = createTracker();
+ // Use fixed clock so samples don't age
+ Instant now = Instant.now();
+ ZoneId zoneId = ZoneId.systemDefault();
+ Clock clock = Clock.fixed(now, zoneId);
+ ingestionDelayTracker.setClock(clock);
+
+ // Test tracking offset lag for a single partition
+ StreamPartitionMsgOffset msgOffset0 = new LongMsgOffset(100);
+ StreamPartitionMsgOffset latestOffset0 = new LongMsgOffset(200);
+ ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0,
partition0);
+
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
100);
+
+ // Test tracking offset lag for another partition
+ StreamPartitionMsgOffset msgOffset1 = new LongMsgOffset(50);
+ StreamPartitionMsgOffset latestOffset1 = new LongMsgOffset(150);
+ ingestionDelayTracker.updateIngestionOffsets(msgOffset1, latestOffset1,
partition1);
+
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition1),
100);
+
+ // Update offset lag for partition0
+ msgOffset0 = new LongMsgOffset(150);
+ latestOffset0 = new LongMsgOffset(200);
+ ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0,
partition0);
+
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
50);
+
+ ingestionDelayTracker.shutdown();
+ }
+
+ @Test
+ public void testRecordIngestionDelayOffsetWithAging() {
+ final int partition0 = 0;
+ final long partition0OffsetLag0 = 100;
+ final long partition0OffsetLag1 = 50;
+
+ IngestionDelayTracker ingestionDelayTracker = createTracker();
+
+ // With samples for a single partition, test that sample is aged as
expected
+ Instant now = Instant.now();
+ ZoneId zoneId = ZoneId.systemDefault();
+ Clock clock = Clock.fixed(now, zoneId);
+ ingestionDelayTracker.setClock(clock);
+
+ StreamPartitionMsgOffset msgOffset0 = new LongMsgOffset(100);
+ StreamPartitionMsgOffset latestOffset0 = new LongMsgOffset(200);
+ ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0,
partition0);
+
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
partition0OffsetLag0);
+
+ // Update offset lag and test aging
+ msgOffset0 = new LongMsgOffset(150);
+ latestOffset0 = new LongMsgOffset(200);
+ ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0,
partition0);
+
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
partition0OffsetLag1);
+
+
+
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
partition0OffsetLag1);
+ ingestionDelayTracker.shutdown();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]