tarun11Mavani commented on code in PR #19041:
URL: https://github.com/apache/pinot/pull/19041#discussion_r3797108129
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java:
##########
@@ -246,17 +254,57 @@ public void seal()
writeSparseJsonColumn(sparseKeys);
}
- if (_coercionFailures > 0) {
- LOGGER.info("OPEN_STRUCT '{}': dropped {} values due to type coercion
failures", _columnName, _coercionFailures);
- ServerMetrics serverMetrics = ServerMetrics.get();
- if (serverMetrics != null) {
-
serverMetrics.addMeteredGlobalValue(ServerMeter.OPEN_STRUCT_TYPE_COERCION_FAILURES,
_coercionFailures);
- }
+ long totalCoercionFailures =
_coercionFailuresPerKey.values().stream().mapToLong(Long::longValue).sum();
+ if (totalCoercionFailures > 0) {
+ LOGGER.info("OPEN_STRUCT '{}': dropped {} values due to type coercion
failures (keys: {})",
+ _columnName, totalCoercionFailures, _coercionFailuresPerKey);
+ }
+ long totalInferenceFailures =
_inferenceFailuresPerKey.values().stream().mapToLong(Long::longValue).sum();
+ if (totalInferenceFailures > 0) {
+ LOGGER.info("OPEN_STRUCT '{}': {} type inference failures fell back to
STRING (keys: {})",
+ _columnName, totalInferenceFailures, _inferenceFailuresPerKey);
}
+ emitMetrics(sparseKeys.size());
emitParentColumnMetadata(sparseKeys);
}
+ private void emitMetrics(int sparseKeyCount) {
+ ServerMetrics serverMetrics = ServerMetrics.get();
+ if (serverMetrics == null || _numDocs == 0) {
+ return;
+ }
+ String col = _columnName;
+
+ long totalCoercion =
_coercionFailuresPerKey.values().stream().mapToLong(Long::longValue).sum();
+ if (totalCoercion > 0) {
+ serverMetrics.addMeteredTableValue(_tableNameWithType, col,
+ ServerMeter.OPEN_STRUCT_TYPE_COERCION_FAILURES, totalCoercion);
+ }
+ long totalInference =
_inferenceFailuresPerKey.values().stream().mapToLong(Long::longValue).sum();
+ if (totalInference > 0) {
+ serverMetrics.addMeteredTableValue(_tableNameWithType, col,
+ ServerMeter.OPEN_STRUCT_TYPE_INFERENCE_FAILURES, totalInference);
+ }
+
+ serverMetrics.setOrUpdateTableGauge(_tableNameWithType, col,
+ ServerGauge.OPEN_STRUCT_DENSE_KEY_COUNT, _resolvedDenseKeys.size());
+ serverMetrics.setOrUpdateTableGauge(_tableNameWithType, col,
+ ServerGauge.OPEN_STRUCT_SPARSE_KEY_COUNT, sparseKeyCount);
+ serverMetrics.setOrUpdateTableGauge(_tableNameWithType, col,
+ ServerGauge.OPEN_STRUCT_TOTAL_KEYS_DISCOVERED,
_presenceBitmaps.size());
Review Comment:
Renamed all four to `OPEN_STRUCT_LAST_SEGMENT_*` so the "last sealed segment
wins" semantics is explicit. `SegmentMessageHandlerFactory` now calls
`removeTableGauge`/`removeTableMeter` on table deletion for everything
recoverable from `denseKeys`.
If key level tracking is enabled(disabled by default), discovered per-key
gauges outside `denseKeys` can't be named at deletion time, which is now called
out explicitly in the javadoc rather than left silent.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
