This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/doris-cli.git
The following commit(s) were added to refs/heads/main by this push:
new dd3f417 fix(parser): keep per-operator counters when a PlanInfo block
precedes them
dd3f417 is described below
commit dd3f417ae405efcc652fc586d110433196f2ffd1
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sat May 30 21:22:57 2026 +0800
fix(parser): keep per-operator counters when a PlanInfo block precedes them
On Doris 5.0.0 profiles every operator carrying a "- PlanInfo" block lost
its
counters: the block never terminated, so the counter lines that follow it —
which share the "- key: value" shape of plan-info items — were swallowed
into
plan_info. RowsProduced went missing, so output_rows came back null and the
downstream total_scan_rows summed to 0 and serialized as null. Only
operators
without a PlanInfo block (e.g. LOCAL_EXCHANGE_OPERATOR(PASSTHROUGH))
survived,
which is why the failure looked scan-specific but was not.
Terminate the PlanInfo block by indentation — its real structural signal —
instead of line shape: the block ends as soon as a line dedents back to the
"- PlanInfo" header's indent or shallower. This covers both the Doris 3/4
layout (CommonCounters/CustomCounters group headers) and the Doris 5 layout
(counters sit directly at the PlanInfo level). A content/shape check cannot
work
because a scalar plan-info item like "- offset: 0" is indistinguishable
from a
counter like "- InputRows: 0".
Tests:
- unit regressions for the Doris 3/4 and Doris 5 operator layouts
- strengthen the real-profile test so every PlanInfo-bearing operator must
still
surface its counters
- e2e (suite_profile): assert AGGREGATION surfaces output_rows and that no
PlanInfo-bearing operator loses its counters; verified against a real
selectdb-5.0.0 cluster (24 pass / 0 fail)
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
---
src/parser/fragment_parser.rs | 142 ++++++++++++++++++++++++++++++++++--------
src/parser/profile_parser.rs | 34 ++++++++++
tests/e2e/suite_profile.sh | 15 +++++
3 files changed, 165 insertions(+), 26 deletions(-)
diff --git a/src/parser/fragment_parser.rs b/src/parser/fragment_parser.rs
index 51496b6..009852e 100644
--- a/src/parser/fragment_parser.rs
+++ b/src/parser/fragment_parser.rs
@@ -120,6 +120,7 @@ fn parse_operator(info: &mut OperatorInfo, lines: &[&str],
start: usize) -> (Ope
let mut plan_info = std::collections::HashMap::new();
let mut i = start;
let mut in_plan_info = false;
+ let mut plan_info_indent = 0usize;
while i < lines.len() {
let line = lines[i];
@@ -135,42 +136,60 @@ fn parse_operator(info: &mut OperatorInfo, lines:
&[&str], start: usize) -> (Ope
// Check for PlanInfo block
if PLAN_INFO_RE.is_match(line) {
in_plan_info = true;
+ plan_info_indent = get_indent(line);
i += 1;
continue;
}
if in_plan_info {
- // PlanInfo items are indented further
- if let Some(caps) = PLAN_INFO_ITEM_RE.captures(line) {
- let key = caps[1].trim().to_string();
- let value = caps[2].trim().to_string();
-
- // Check if this is a table reference in a scan operator
- // Doris format: "TABLE: tpch_demo.orders(orders),
PREAGGREGATION: ON"
- if key == "TABLE" || key == "table" {
- let table_val = value
- .split(',')
- .next()
- .unwrap_or(&value)
- .split('(')
- .next()
- .unwrap_or(&value)
- .trim()
- .to_string();
- if !table_val.is_empty() {
- info.table_name = Some(table_val);
+ // PlanInfo items are rendered as children of the "- PlanInfo"
node, so
+ // they are indented deeper than it. The block ends the moment a
line
+ // dedents back to the header's indent or shallower: that line is
the
+ // operator's first counter (Doris 5, where counters sit at the
PlanInfo
+ // level) or a CommonCounters:/CustomCounters: group header (Doris
3/4).
+ // We key off indentation, not line shape, on purpose — counter
lines
+ // like "- RowsProduced: sum 4, avg 4, max 4, min 4" share the
exact
+ // "- key: value" shape of plan-info items, so a shape-based check
+ // swallows every counter into plan_info and nulls out output_rows.
+ if line.trim().is_empty() {
+ // Blank lines don't terminate the block.
+ i += 1;
+ continue;
+ }
+ if get_indent(line) > plan_info_indent {
+ if let Some(caps) = PLAN_INFO_ITEM_RE.captures(line) {
+ let key = caps[1].trim().to_string();
+ let value = caps[2].trim().to_string();
+
+ // Check if this is a table reference in a scan operator
+ // Doris format: "TABLE: tpch_demo.orders(orders),
PREAGGREGATION: ON"
+ if key == "TABLE" || key == "table" {
+ let table_val = value
+ .split(',')
+ .next()
+ .unwrap_or(&value)
+ .split('(')
+ .next()
+ .unwrap_or(&value)
+ .trim()
+ .to_string();
+ if !table_val.is_empty() {
+ info.table_name = Some(table_val);
+ }
}
- }
- plan_info.insert(key, value);
- } else if line.trim().starts_with('-') &&
COUNTER_RE.is_match(line) {
- // We've exited PlanInfo and hit counters
- in_plan_info = false;
- } else {
- // Still in PlanInfo — might be a continuation
+ plan_info.insert(key, value);
+ }
+ // Deeper lines that aren't "key: value" (e.g. "- local merge
sort",
+ // "- cardinality=4") are plan-info prose we don't structure —
skip
+ // them but stay inside the block.
i += 1;
continue;
}
+
+ // Dedented to the header level or shallower: the PlanInfo block
is over.
+ // Fall through and reprocess this same line as a counter / group
header.
+ in_plan_info = false;
}
if !in_plan_info {
@@ -316,4 +335,75 @@ mod tests {
parse_agg_value("sum 57.94 KB, avg 57.94 KB, max 57.94 KB, min
57.94 KB").unwrap();
assert!(agg.sum.unwrap() > 50000.0);
}
+
+ // WHY: the diagnosis layer's output_rows / total_scan_rows are read
straight
+ // off each operator's RowsProduced counter. An operator that carries a
+ // "- PlanInfo" block must still surface its counters; if the PlanInfo
block
+ // fails to terminate it swallows every following "- key: value" counter
line
+ // and the operator silently loses output_rows. These two tests pin the
+ // termination for both profile layouts so that regression can't return.
+
+ #[test]
+ fn planinfo_block_does_not_swallow_counters_doris4() {
+ // Doris 3/4 layout: a "- PlanInfo" block, then
CommonCounters/CustomCounters
+ // group headers holding the real counters (counters indented deeper
than the
+ // group header, the group header shallower than the PlanInfo header).
+ let text = "\
+Fragment 0:
+ Pipeline 0(instance_num=1):
+ AGGREGATION_OPERATOR(nereids_id=153)(id=3):
+ - PlanInfo
+ - output: count(partial_count(*))[#9]
+ - group by: event_type
+ - cardinality=4
+ CommonCounters:
+ - BlocksProduced: sum 4, avg 0, max 1, min 0
+ - RowsProduced: sum 7, avg 0, max 1, min 0
+ CustomCounters:
+ - GetBlockFailedTime: sum 2, avg 0, max 1, min 0
+";
+ let frags = parse_merged_profile(text);
+ let op = &frags[0].pipelines[0].operators[0];
+ assert_eq!(
+ op.metrics.rows_produced.as_ref().and_then(|v| v.sum),
+ Some(7.0),
+ "RowsProduced must survive a preceding PlanInfo block"
+ );
+ // PlanInfo itself is still captured...
+ assert!(op.plan_info.contains_key("group by"));
+ // ...and a plan-info scalar must not leak into the counters map.
+ assert!(!op.all_counters.contains_key("output"));
+ }
+
+ #[test]
+ fn planinfo_block_does_not_swallow_counters_doris5() {
+ // Doris 5 layout: no CommonCounters/CustomCounters grouping — the
operator's
+ // counters sit directly under it at the same indent as the "-
PlanInfo"
+ // header. The block must still terminate so RowsProduced is captured
(this is
+ // the case in the bug report where only PlanInfo-less operators
populated).
+ let text = "\
+Fragment 0:
+ Pipeline 0(instance_num=1):
+ OLAP_SCAN_OPERATOR(nereids_id=160. table_name=sales(sales))(id=0):
+ - PlanInfo
+ - TABLE: db.sales(sales)
+ - PREDICATES: amount > 10
+ - BlocksProduced: sum 4, avg 4, max 4, min 4
+ - RowsProduced: sum 450, avg 450, max 450, min 450
+ - ScanRows: sum 500, avg 500, max 500, min 500
+";
+ let frags = parse_merged_profile(text);
+ let op = &frags[0].pipelines[0].operators[0];
+ assert_eq!(
+ op.metrics.rows_produced.as_ref().and_then(|v| v.sum),
+ Some(450.0),
+ "RowsProduced must be captured when counters sit at the PlanInfo
indent"
+ );
+ assert_eq!(
+ op.all_counters.get("ScanRows").and_then(|v| v.sum),
+ Some(500.0)
+ );
+ // PlanInfo's TABLE is still mined for the table name.
+ assert_eq!(op.info.table_name.as_deref(), Some("db.sales"));
+ }
}
diff --git a/src/parser/profile_parser.rs b/src/parser/profile_parser.rs
index b8aba9c..f52a26a 100644
--- a/src/parser/profile_parser.rs
+++ b/src/parser/profile_parser.rs
@@ -425,6 +425,40 @@ mod real_profile_tests {
"flattened operators must be sorted by exec_time_avg_ms descending"
);
+ // Regression (PlanInfo-swallows-counters): an operator that carries a
+ // "- PlanInfo" block must still surface its counters. The bug was
that the
+ // PlanInfo block never terminated, so every following counter line
(which
+ // shares the "- key: value" shape of a plan-info item) was parsed as
+ // plan-info and the operator lost RowsProduced -> output_rows -> the
+ // downstream total_scan_rows degraded to null.
+ let with_plan_info: Vec<_> = profile
+ .fragments
+ .iter()
+ .flat_map(|f| &f.pipelines)
+ .flat_map(|p| &p.operators)
+ .filter(|op| !op.plan_info.is_empty())
+ .collect();
+ assert!(
+ !with_plan_info.is_empty(),
+ "fixture must contain operators with a PlanInfo block to exercise
this path"
+ );
+ for op in &with_plan_info {
+ // Every Doris operator emits at least ExecTime, so a PlanInfo
operator
+ // with an empty counter map means the block absorbed the counters.
+ assert!(
+ !op.all_counters.is_empty(),
+ "operator {:?} carries a PlanInfo block but has no counters —
the \
+ PlanInfo parser swallowed them",
+ op.info.full_name
+ );
+ }
+ assert!(
+ with_plan_info
+ .iter()
+ .any(|op| op.metrics.rows_produced.is_some()),
+ "at least one PlanInfo-bearing operator must surface RowsProduced
(output_rows)"
+ );
+
// Regression: DetailProfile / Appendix must be section boundaries,
not bleed
// into MergedProfile and fabricate empty fragments — every parsed
fragment has
// real pipelines.
diff --git a/tests/e2e/suite_profile.sh b/tests/e2e/suite_profile.sh
index 17c8da9..5c06787 100644
--- a/tests/e2e/suite_profile.sh
+++ b/tests/e2e/suite_profile.sh
@@ -101,6 +101,14 @@ suite_profile() {
# query_stats: scanned rows ≈ what we loaded; fragment/operator counts
real.
expect_parsed "profile: get -> query_stats.total_scan_rows ≈ loaded
rows" \
'(.query_stats.total_scan_rows >= ('"${ROWS:-2000}"' * 0.9 | floor))
and (.query_stats.fragment_count>=1) and (.query_stats.operator_count>0)'
+ # Regression (PlanInfo block swallowed per-operator counters):
output_rows must
+ # populate for operators that carry a "- PlanInfo" block — this is NOT
scan
+ # specific. The bug nulled RowsProduced -> output_rows for ~every
PlanInfo
+ # bearing operator (and on Doris 5.0 the OLAP_SCAN itself carries a
PlanInfo
+ # block, so total_scan_rows above goes null too). AGGREGATION always
carries a
+ # PlanInfo block, so it must surface a non-null output_rows here.
+ expect_parsed "profile: get -> AGGREGATION operator surfaces output_rows
(PlanInfo kept its counters)" \
+ 'any(.operators[]; (.name|test("AGG")) and (.output_rows != null))'
# Fragment breakdown must be REAL: exactly the 3 fragments of this
group-by,
# count consistent with the array, and NO empty duplicates (regression
guard —
# a DetailProfile/Appendix block bleeding into MergedProfile used to
fabricate
@@ -135,6 +143,13 @@ suite_profile() {
'(.profile.fragments|length>0) and
(.profile.fragments[0].pipelines|length>0) and (.operators|length>0)'
expect_parsed "profile: --full -> an operator exposes parsed
all_counters" \
'any(.profile.fragments[].pipelines[].operators[];
(.all_counters|type=="object") and (.all_counters|length>0))'
+ # Regression (PlanInfo block swallowed counters): the block must
terminate and
+ # NOT absorb the counter lines that follow it. The fingerprint of that
bug is
+ # an operator with a non-empty plan_info but an empty/absent
all_counters
+ # (every counter, RowsProduced included, got mis-parsed as plan-info).
The
+ # --full tree exposes both maps per operator, so assert there are none.
+ expect_parsed "profile: --full -> no PlanInfo-bearing operator lost
its counters" \
+ '[.profile.fragments[].pipelines[].operators[] |
select((.plan_info|length)>0) | select((.all_counters|length)==0)] | length ==
0'
fi
# ── --raw: the raw profile text round-trips; capture it as the fixture ─
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]