This is an automated email from the ASF dual-hosted git repository.

kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new fc358a87fc Add offset based lag metrics (#13298)
fc358a87fc is described below

commit fc358a87fc63def0b5c5a31a24bde1d9df0d9ecb
Author: Kartik Khare <[email protected]>
AuthorDate: Fri Jun 7 19:21:32 2024 +0530

    Add offset based lag metrics (#13298)
    
    * Add offset based lag metrics
    
    * Add tests
    
    * Refactoring
    
    ---------
    
    Co-authored-by: Kartik Khare <[email protected]>
---
 .../apache/pinot/common/metrics/ServerGauge.java   |  3 +-
 .../manager/realtime/IngestionDelayTracker.java    | 92 +++++++++++++++++++---
 .../realtime/RealtimeSegmentDataManager.java       | 16 ++--
 .../manager/realtime/RealtimeTableDataManager.java | 10 ++-
 .../realtime/IngestionDelayTrackerTest.java        | 65 +++++++++++++++
 5 files changed, 167 insertions(+), 19 deletions(-)

diff --git 
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java 
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
index f0a1fdd136..072c048c55 100644
--- 
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
+++ 
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerGauge.java
@@ -72,7 +72,8 @@ public enum ServerGauge implements AbstractMetrics.Gauge {
   END_TO_END_REALTIME_INGESTION_DELAY_MS("milliseconds", false),
   // Needed to track if valid doc id snapshots are present for faster restarts
   UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT("upsertValidDocIdSnapshotCount", false),
-  UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT("upsertPrimaryKeysInSnapshotCount", 
false);
+  UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT("upsertPrimaryKeysInSnapshotCount", 
false),
+  REALTIME_INGESTION_OFFSET_LAG("offsetLag", false);
 
   private final String _gaugeName;
   private final String _unit;
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
index 84a5ea7ba0..eed1302708 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTracker.java
@@ -34,6 +34,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import org.apache.pinot.common.metrics.ServerGauge;
 import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.spi.stream.LongMsgOffset;
+import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
 import org.apache.pinot.spi.utils.builder.TableNameBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -81,13 +83,23 @@ public class IngestionDelayTracker {
 
   // Class to wrap supported timestamps collected for an ingested event
   private static class IngestionTimestamps {
+    private final long _firstStreamIngestionTimeMs;
+    private final long _ingestionTimeMs;
     IngestionTimestamps(long ingestionTimesMs, long 
firstStreamIngestionTimeMs) {
       _ingestionTimeMs = ingestionTimesMs;
       _firstStreamIngestionTimeMs = firstStreamIngestionTimeMs;
     }
-    private final long _ingestionTimeMs;
-    private final long _firstStreamIngestionTimeMs;
   }
+
+  private static class IngestionOffsets {
+    private final StreamPartitionMsgOffset _latestOffset;
+    private final StreamPartitionMsgOffset _offset;
+    IngestionOffsets(StreamPartitionMsgOffset offset, StreamPartitionMsgOffset 
latestOffset) {
+      _offset = offset;
+      _latestOffset = latestOffset;
+    }
+  }
+
   // Sleep interval for scheduled executor service thread that triggers read 
of ideal state
   private static final int SCHEDULED_EXECUTOR_THREAD_TICK_INTERVAL_MS = 
300000; // 5 minutes +/- precision in timeouts
   // Once a partition is marked for verification, we wait 10 minutes to pull 
its ideal state.
@@ -98,6 +110,8 @@ public class IngestionDelayTracker {
 
   // HashMap used to store ingestion time measures for all partitions active 
for the current table.
   private final Map<Integer, IngestionTimestamps> 
_partitionToIngestionTimestampsMap = new ConcurrentHashMap<>();
+
+  private final Map<Integer, IngestionOffsets> _partitionToOffsetMap = new 
ConcurrentHashMap<>();
   // We mark partitions that go from CONSUMING to ONLINE in 
_partitionsMarkedForVerification: if they do not
   // go back to CONSUMING in some period of time, we verify whether they are 
still hosted in this server by reading
   // ideal state. This is done with the goal of minimizing reading ideal state 
for efficiency reasons.
@@ -174,6 +188,22 @@ public class IngestionDelayTracker {
     return agedIngestionDelayMs;
   }
 
+  private long getPartitionOffsetLag(IngestionOffsets offset) {
+    if (offset == null) {
+      return 0;
+    }
+    StreamPartitionMsgOffset currentOffset = offset._offset;
+    StreamPartitionMsgOffset latestOffset = offset._latestOffset;
+
+    // Compute aged delay for current partition
+    // TODO: Support other types of offsets
+    if (!(currentOffset instanceof LongMsgOffset && latestOffset instanceof 
LongMsgOffset)) {
+      return 0;
+    }
+
+    return ((LongMsgOffset) latestOffset).getOffset() - ((LongMsgOffset) 
currentOffset).getOffset();
+  }
+
   /*
    * Helper function to be called when we should stop tracking a given 
partition. Removes the partition from
    * all our maps.
@@ -182,11 +212,13 @@ public class IngestionDelayTracker {
    */
   private void removePartitionId(int partitionGroupId) {
     _partitionToIngestionTimestampsMap.remove(partitionGroupId);
+    _partitionToOffsetMap.remove(partitionGroupId);
     // If we are removing a partition we should stop reading its ideal state.
     _partitionsMarkedForVerification.remove(partitionGroupId);
     _serverMetrics.removePartitionGauge(_metricName, partitionGroupId, 
ServerGauge.REALTIME_INGESTION_DELAY_MS);
     _serverMetrics.removePartitionGauge(_metricName, partitionGroupId,
         ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS);
+    _serverMetrics.removePartitionGauge(_metricName, partitionGroupId, 
ServerGauge.REALTIME_INGESTION_OFFSET_LAG);
   }
 
   /*
@@ -216,20 +248,32 @@ public class IngestionDelayTracker {
     _clock = clock;
   }
 
-  /*
-   * Called by RealTimeSegmentDataManagers to post ingestion time updates to 
this tracker class.
+  /**
+   * Called by RealTimeSegmentDataManagers to update the ingestion delay 
metrics for a given partition.
    *
-   * @param ingestionTimeMs ingestion time being recorded.
-   * @param firstStreamIngestionTimeMs time the event was ingested in the 
first stage of the ingestion pipeline.
-   * @param partitionGroupId partition ID for which this ingestion time is 
being recorded.
+   * @param ingestionTimeMs             ingestion time being recorded.
+   * @param firstStreamIngestionTimeMs  time the event was ingested in the 
first stage of the ingestion pipeline.
+   * @param msgOffset                   message offset of the event being 
ingested.
+   * @param latestOffset                latest message offset in the stream.
+   * @param partitionGroupId            partition ID for which the ingestion 
metrics are being recorded.
    */
-  public void updateIngestionDelay(long ingestionTimeMs, long 
firstStreamIngestionTimeMs, int partitionGroupId) {
-    // Store new measure and wipe old one for this partition
+  public void updateIngestionMetrics(long ingestionTimeMs, long 
firstStreamIngestionTimeMs,
+      StreamPartitionMsgOffset msgOffset, StreamPartitionMsgOffset 
latestOffset,
+      int partitionGroupId) {
     if (!_isServerReadyToServeQueries.get() || 
_realTimeTableDataManager.isShutDown()) {
       // Do not update the ingestion delay metrics during server startup period
       // or once the table data manager has been shutdown.
       return;
     }
+
+    updateIngestionDelay(ingestionTimeMs, firstStreamIngestionTimeMs, 
partitionGroupId);
+    updateIngestionOffsets(msgOffset, latestOffset, partitionGroupId);
+
+    // If we are consuming we do not need to track this partition for removal.
+    _partitionsMarkedForVerification.remove(partitionGroupId);
+  }
+
+  public void updateIngestionDelay(long ingestionTimeMs, long 
firstStreamIngestionTimeMs, int partitionGroupId) {
     if ((ingestionTimeMs < 0) && (firstStreamIngestionTimeMs < 0)) {
       // If stream does not return a valid ingestion timestamps don't publish 
a metric
       return;
@@ -252,8 +296,25 @@ public class IngestionDelayTracker {
             () -> getPartitionEndToEndIngestionDelayMs(partitionGroupId));
       }
     }
-    // If we are consuming we do not need to track this partition for removal.
-    _partitionsMarkedForVerification.remove(partitionGroupId);
+  }
+
+  public void updateIngestionOffsets(StreamPartitionMsgOffset currentOffset, 
StreamPartitionMsgOffset latestOffset,
+      int partitionGroupId) {
+    if ((currentOffset == null)) {
+      // If stream does not return a valid ingestion offset don't publish a 
metric
+      return;
+    }
+    IngestionOffsets previousMeasure =
+        _partitionToOffsetMap.put(partitionGroupId, new 
IngestionOffsets(currentOffset, latestOffset));
+    if (previousMeasure == null) {
+      // First time we start tracking a partition we should start tracking it 
via metric
+      // Only publish the metric if supported by the underlying stream. If not 
supported the stream
+      // returns Long.MIN_VALUE
+      if (currentOffset != null) {
+        _serverMetrics.setOrUpdatePartitionGauge(_metricName, partitionGroupId,
+            ServerGauge.REALTIME_INGESTION_OFFSET_LAG, () -> 
getPartitionIngestionOffsetLag(partitionGroupId));
+      }
+    }
   }
 
   /*
@@ -346,6 +407,15 @@ public class IngestionDelayTracker {
     return getIngestionDelayMs(currentMeasure._ingestionTimeMs);
   }
 
+  public long getPartitionIngestionOffsetLag(int partitionGroupId) {
+    // Not protected as this will only be invoked when metric is installed 
which happens after server ready
+    IngestionOffsets currentMeasure = 
_partitionToOffsetMap.get(partitionGroupId);
+    if (currentMeasure == null) { // Guard just in case we read the metric 
without initializing it
+      return 0;
+    }
+    return getPartitionOffsetLag(currentMeasure);
+  }
+
   /*
    * Method to get end to end ingestion delay for a given partition.
    *
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
index f50942c588..8dc7842f81 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
@@ -660,7 +660,7 @@ public class RealtimeSegmentDataManager extends 
SegmentDataManager {
 
     updateCurrentDocumentCountMetrics();
     if (messageBatch.getUnfilteredMessageCount() > 0) {
-      updateIngestionDelay(messageBatch.getLastMessageMetadata());
+      updateIngestionMetrics(messageBatch.getLastMessageMetadata());
       _hasMessagesFetched = true;
       if (streamMessageCount > 0 && _segmentLogger.isDebugEnabled()) {
         _segmentLogger.debug("Indexed {} messages ({} messages read from 
stream) current offset {}",
@@ -1778,10 +1778,16 @@ public class RealtimeSegmentDataManager extends 
SegmentDataManager {
     _partitionMetadataProvider = 
_streamConsumerFactory.createPartitionMetadataProvider(_clientId, 
_partitionGroupId);
   }
 
-  private void updateIngestionDelay(RowMetadata metadata) {
+  private void updateIngestionMetrics(RowMetadata metadata) {
     if (metadata != null) {
-      
_realtimeTableDataManager.updateIngestionDelay(metadata.getRecordIngestionTimeMs(),
-          metadata.getFirstStreamRecordIngestionTimeMs(), _partitionGroupId);
+      try {
+        StreamPartitionMsgOffset latestOffset =
+            
_partitionMetadataProvider.fetchStreamPartitionOffset(OffsetCriteria.LARGEST_OFFSET_CRITERIA,
 5000);
+        
_realtimeTableDataManager.updateIngestionMetrics(metadata.getRecordIngestionTimeMs(),
+            metadata.getFirstStreamRecordIngestionTimeMs(), 
metadata.getOffset(), latestOffset, _partitionGroupId);
+      } catch (Exception e) {
+        _segmentLogger.warn("Failed to fetch latest offset for updating 
ingestion delay", e);
+      }
     }
   }
 
@@ -1790,7 +1796,7 @@ public class RealtimeSegmentDataManager extends 
SegmentDataManager {
    */
   private void setIngestionDelayToZero() {
     long currentTimeMs = System.currentTimeMillis();
-    _realtimeTableDataManager.updateIngestionDelay(currentTimeMs, 
currentTimeMs, _partitionGroupId);
+    _realtimeTableDataManager.updateIngestionMetrics(currentTimeMs, 
currentTimeMs, null, null, _partitionGroupId);
   }
 
   // This should be done during commit? We may not always commit when we build 
a segment....
diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index 6c9c647b1f..3659fae1c2 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -74,6 +74,7 @@ import org.apache.pinot.spi.data.DateTimeFieldSpec;
 import org.apache.pinot.spi.data.DateTimeFormatSpec;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
 import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
 import org.apache.pinot.spi.utils.TimeUtils;
@@ -265,10 +266,15 @@ public class RealtimeTableDataManager extends 
BaseTableDataManager {
    * Method used by RealtimeSegmentManagers to update their partition delays
    *
    * @param ingestionTimeMs Ingestion delay being reported.
+   * @param firstStreamIngestionTimeMs Ingestion time of the first message in 
the stream.
    * @param partitionGroupId Partition ID for which delay is being updated.
+   * @param offset last offset received for the partition.
+   * @param latestOffset latest upstream offset for the partition.
    */
-  public void updateIngestionDelay(long ingestionTimeMs, long 
firstStreamIngestionTimeMs, int partitionGroupId) {
-    _ingestionDelayTracker.updateIngestionDelay(ingestionTimeMs, 
firstStreamIngestionTimeMs, partitionGroupId);
+  public void updateIngestionMetrics(long ingestionTimeMs, long 
firstStreamIngestionTimeMs,
+      StreamPartitionMsgOffset offset, StreamPartitionMsgOffset latestOffset, 
int partitionGroupId) {
+    _ingestionDelayTracker.updateIngestionMetrics(ingestionTimeMs, 
firstStreamIngestionTimeMs, offset, latestOffset,
+        partitionGroupId);
   }
 
   /*
diff --git 
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
 
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
index 4a1c27a506..8072731583 100644
--- 
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
+++ 
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/IngestionDelayTrackerTest.java
@@ -24,6 +24,8 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.spi.stream.LongMsgOffset;
+import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -253,4 +255,67 @@ public class IngestionDelayTrackerTest {
     ingestionDelayTracker = createTracker();
     ingestionDelayTracker.shutdown();
   }
+
+  @Test
+  public void testRecordIngestionDelayOffsetWithNoAging() {
+    final int partition0 = 0;
+    final int partition1 = 1;
+
+    IngestionDelayTracker ingestionDelayTracker = createTracker();
+    // Use fixed clock so samples don't age
+    Instant now = Instant.now();
+    ZoneId zoneId = ZoneId.systemDefault();
+    Clock clock = Clock.fixed(now, zoneId);
+    ingestionDelayTracker.setClock(clock);
+
+    // Test tracking offset lag for a single partition
+    StreamPartitionMsgOffset msgOffset0 = new LongMsgOffset(100);
+    StreamPartitionMsgOffset latestOffset0 = new LongMsgOffset(200);
+    ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0, 
partition0);
+    
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
 100);
+
+    // Test tracking offset lag for another partition
+    StreamPartitionMsgOffset msgOffset1 = new LongMsgOffset(50);
+    StreamPartitionMsgOffset latestOffset1 = new LongMsgOffset(150);
+    ingestionDelayTracker.updateIngestionOffsets(msgOffset1, latestOffset1, 
partition1);
+    
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition1),
 100);
+
+    // Update offset lag for partition0
+    msgOffset0 = new LongMsgOffset(150);
+    latestOffset0 = new LongMsgOffset(200);
+    ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0, 
partition0);
+    
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
 50);
+
+    ingestionDelayTracker.shutdown();
+  }
+
+  @Test
+  public void testRecordIngestionDelayOffsetWithAging() {
+    final int partition0 = 0;
+    final long partition0OffsetLag0 = 100;
+    final long partition0OffsetLag1 = 50;
+
+    IngestionDelayTracker ingestionDelayTracker = createTracker();
+
+    // With samples for a single partition, test that sample is aged as 
expected
+    Instant now = Instant.now();
+    ZoneId zoneId = ZoneId.systemDefault();
+    Clock clock = Clock.fixed(now, zoneId);
+    ingestionDelayTracker.setClock(clock);
+
+    StreamPartitionMsgOffset msgOffset0 = new LongMsgOffset(100);
+    StreamPartitionMsgOffset latestOffset0 = new LongMsgOffset(200);
+    ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0, 
partition0);
+    
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
 partition0OffsetLag0);
+
+    // Update offset lag and test aging
+    msgOffset0 = new LongMsgOffset(150);
+    latestOffset0 = new LongMsgOffset(200);
+    ingestionDelayTracker.updateIngestionOffsets(msgOffset0, latestOffset0, 
partition0);
+    
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
 partition0OffsetLag1);
+
+
+    
Assert.assertEquals(ingestionDelayTracker.getPartitionIngestionOffsetLag(partition0),
 partition0OffsetLag1);
+    ingestionDelayTracker.shutdown();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to