This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/doris-cli.git


The following commit(s) were added to refs/heads/main by this push:
     new dd3f417  fix(parser): keep per-operator counters when a PlanInfo block 
precedes them
dd3f417 is described below

commit dd3f417ae405efcc652fc586d110433196f2ffd1
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sat May 30 21:22:57 2026 +0800

    fix(parser): keep per-operator counters when a PlanInfo block precedes them
    
    On Doris 5.0.0 profiles every operator carrying a "- PlanInfo" block lost 
its
    counters: the block never terminated, so the counter lines that follow it —
    which share the "- key: value" shape of plan-info items — were swallowed 
into
    plan_info. RowsProduced went missing, so output_rows came back null and the
    downstream total_scan_rows summed to 0 and serialized as null. Only 
operators
    without a PlanInfo block (e.g. LOCAL_EXCHANGE_OPERATOR(PASSTHROUGH)) 
survived,
    which is why the failure looked scan-specific but was not.
    
    Terminate the PlanInfo block by indentation — its real structural signal —
    instead of line shape: the block ends as soon as a line dedents back to the
    "- PlanInfo" header's indent or shallower. This covers both the Doris 3/4
    layout (CommonCounters/CustomCounters group headers) and the Doris 5 layout
    (counters sit directly at the PlanInfo level). A content/shape check cannot 
work
    because a scalar plan-info item like "- offset: 0" is indistinguishable 
from a
    counter like "- InputRows: 0".
    
    Tests:
    - unit regressions for the Doris 3/4 and Doris 5 operator layouts
    - strengthen the real-profile test so every PlanInfo-bearing operator must 
still
      surface its counters
    - e2e (suite_profile): assert AGGREGATION surfaces output_rows and that no
      PlanInfo-bearing operator loses its counters; verified against a real
      selectdb-5.0.0 cluster (24 pass / 0 fail)
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
---
 src/parser/fragment_parser.rs | 142 ++++++++++++++++++++++++++++++++++--------
 src/parser/profile_parser.rs  |  34 ++++++++++
 tests/e2e/suite_profile.sh    |  15 +++++
 3 files changed, 165 insertions(+), 26 deletions(-)

diff --git a/src/parser/fragment_parser.rs b/src/parser/fragment_parser.rs
index 51496b6..009852e 100644
--- a/src/parser/fragment_parser.rs
+++ b/src/parser/fragment_parser.rs
@@ -120,6 +120,7 @@ fn parse_operator(info: &mut OperatorInfo, lines: &[&str], 
start: usize) -> (Ope
     let mut plan_info = std::collections::HashMap::new();
     let mut i = start;
     let mut in_plan_info = false;
+    let mut plan_info_indent = 0usize;
 
     while i < lines.len() {
         let line = lines[i];
@@ -135,42 +136,60 @@ fn parse_operator(info: &mut OperatorInfo, lines: 
&[&str], start: usize) -> (Ope
         // Check for PlanInfo block
         if PLAN_INFO_RE.is_match(line) {
             in_plan_info = true;
+            plan_info_indent = get_indent(line);
             i += 1;
             continue;
         }
 
         if in_plan_info {
-            // PlanInfo items are indented further
-            if let Some(caps) = PLAN_INFO_ITEM_RE.captures(line) {
-                let key = caps[1].trim().to_string();
-                let value = caps[2].trim().to_string();
-
-                // Check if this is a table reference in a scan operator
-                // Doris format: "TABLE: tpch_demo.orders(orders), 
PREAGGREGATION: ON"
-                if key == "TABLE" || key == "table" {
-                    let table_val = value
-                        .split(',')
-                        .next()
-                        .unwrap_or(&value)
-                        .split('(')
-                        .next()
-                        .unwrap_or(&value)
-                        .trim()
-                        .to_string();
-                    if !table_val.is_empty() {
-                        info.table_name = Some(table_val);
+            // PlanInfo items are rendered as children of the "- PlanInfo" 
node, so
+            // they are indented deeper than it. The block ends the moment a 
line
+            // dedents back to the header's indent or shallower: that line is 
the
+            // operator's first counter (Doris 5, where counters sit at the 
PlanInfo
+            // level) or a CommonCounters:/CustomCounters: group header (Doris 
3/4).
+            // We key off indentation, not line shape, on purpose — counter 
lines
+            // like "- RowsProduced: sum 4, avg 4, max 4, min 4" share the 
exact
+            // "- key: value" shape of plan-info items, so a shape-based check
+            // swallows every counter into plan_info and nulls out output_rows.
+            if line.trim().is_empty() {
+                // Blank lines don't terminate the block.
+                i += 1;
+                continue;
+            }
+            if get_indent(line) > plan_info_indent {
+                if let Some(caps) = PLAN_INFO_ITEM_RE.captures(line) {
+                    let key = caps[1].trim().to_string();
+                    let value = caps[2].trim().to_string();
+
+                    // Check if this is a table reference in a scan operator
+                    // Doris format: "TABLE: tpch_demo.orders(orders), 
PREAGGREGATION: ON"
+                    if key == "TABLE" || key == "table" {
+                        let table_val = value
+                            .split(',')
+                            .next()
+                            .unwrap_or(&value)
+                            .split('(')
+                            .next()
+                            .unwrap_or(&value)
+                            .trim()
+                            .to_string();
+                        if !table_val.is_empty() {
+                            info.table_name = Some(table_val);
+                        }
                     }
-                }
 
-                plan_info.insert(key, value);
-            } else if line.trim().starts_with('-') && 
COUNTER_RE.is_match(line) {
-                // We've exited PlanInfo and hit counters
-                in_plan_info = false;
-            } else {
-                // Still in PlanInfo — might be a continuation
+                    plan_info.insert(key, value);
+                }
+                // Deeper lines that aren't "key: value" (e.g. "- local merge 
sort",
+                // "- cardinality=4") are plan-info prose we don't structure — 
skip
+                // them but stay inside the block.
                 i += 1;
                 continue;
             }
+
+            // Dedented to the header level or shallower: the PlanInfo block 
is over.
+            // Fall through and reprocess this same line as a counter / group 
header.
+            in_plan_info = false;
         }
 
         if !in_plan_info {
@@ -316,4 +335,75 @@ mod tests {
             parse_agg_value("sum 57.94 KB, avg 57.94 KB, max 57.94 KB, min 
57.94 KB").unwrap();
         assert!(agg.sum.unwrap() > 50000.0);
     }
+
+    // WHY: the diagnosis layer's output_rows / total_scan_rows are read 
straight
+    // off each operator's RowsProduced counter. An operator that carries a
+    // "- PlanInfo" block must still surface its counters; if the PlanInfo 
block
+    // fails to terminate it swallows every following "- key: value" counter 
line
+    // and the operator silently loses output_rows. These two tests pin the
+    // termination for both profile layouts so that regression can't return.
+
+    #[test]
+    fn planinfo_block_does_not_swallow_counters_doris4() {
+        // Doris 3/4 layout: a "- PlanInfo" block, then 
CommonCounters/CustomCounters
+        // group headers holding the real counters (counters indented deeper 
than the
+        // group header, the group header shallower than the PlanInfo header).
+        let text = "\
+Fragment 0:
+  Pipeline 0(instance_num=1):
+    AGGREGATION_OPERATOR(nereids_id=153)(id=3):
+       - PlanInfo
+          - output: count(partial_count(*))[#9]
+          - group by: event_type
+          - cardinality=4
+      CommonCounters:
+         - BlocksProduced: sum 4, avg 0, max 1, min 0
+         - RowsProduced: sum 7, avg 0, max 1, min 0
+      CustomCounters:
+         - GetBlockFailedTime: sum 2, avg 0, max 1, min 0
+";
+        let frags = parse_merged_profile(text);
+        let op = &frags[0].pipelines[0].operators[0];
+        assert_eq!(
+            op.metrics.rows_produced.as_ref().and_then(|v| v.sum),
+            Some(7.0),
+            "RowsProduced must survive a preceding PlanInfo block"
+        );
+        // PlanInfo itself is still captured...
+        assert!(op.plan_info.contains_key("group by"));
+        // ...and a plan-info scalar must not leak into the counters map.
+        assert!(!op.all_counters.contains_key("output"));
+    }
+
+    #[test]
+    fn planinfo_block_does_not_swallow_counters_doris5() {
+        // Doris 5 layout: no CommonCounters/CustomCounters grouping — the 
operator's
+        // counters sit directly under it at the same indent as the "- 
PlanInfo"
+        // header. The block must still terminate so RowsProduced is captured 
(this is
+        // the case in the bug report where only PlanInfo-less operators 
populated).
+        let text = "\
+Fragment 0:
+  Pipeline 0(instance_num=1):
+    OLAP_SCAN_OPERATOR(nereids_id=160. table_name=sales(sales))(id=0):
+       - PlanInfo
+          - TABLE: db.sales(sales)
+          - PREDICATES: amount > 10
+       - BlocksProduced: sum 4, avg 4, max 4, min 4
+       - RowsProduced: sum 450, avg 450, max 450, min 450
+       - ScanRows: sum 500, avg 500, max 500, min 500
+";
+        let frags = parse_merged_profile(text);
+        let op = &frags[0].pipelines[0].operators[0];
+        assert_eq!(
+            op.metrics.rows_produced.as_ref().and_then(|v| v.sum),
+            Some(450.0),
+            "RowsProduced must be captured when counters sit at the PlanInfo 
indent"
+        );
+        assert_eq!(
+            op.all_counters.get("ScanRows").and_then(|v| v.sum),
+            Some(500.0)
+        );
+        // PlanInfo's TABLE is still mined for the table name.
+        assert_eq!(op.info.table_name.as_deref(), Some("db.sales"));
+    }
 }
diff --git a/src/parser/profile_parser.rs b/src/parser/profile_parser.rs
index b8aba9c..f52a26a 100644
--- a/src/parser/profile_parser.rs
+++ b/src/parser/profile_parser.rs
@@ -425,6 +425,40 @@ mod real_profile_tests {
             "flattened operators must be sorted by exec_time_avg_ms descending"
         );
 
+        // Regression (PlanInfo-swallows-counters): an operator that carries a
+        // "- PlanInfo" block must still surface its counters. The bug was 
that the
+        // PlanInfo block never terminated, so every following counter line 
(which
+        // shares the "- key: value" shape of a plan-info item) was parsed as
+        // plan-info and the operator lost RowsProduced -> output_rows -> the
+        // downstream total_scan_rows degraded to null.
+        let with_plan_info: Vec<_> = profile
+            .fragments
+            .iter()
+            .flat_map(|f| &f.pipelines)
+            .flat_map(|p| &p.operators)
+            .filter(|op| !op.plan_info.is_empty())
+            .collect();
+        assert!(
+            !with_plan_info.is_empty(),
+            "fixture must contain operators with a PlanInfo block to exercise 
this path"
+        );
+        for op in &with_plan_info {
+            // Every Doris operator emits at least ExecTime, so a PlanInfo 
operator
+            // with an empty counter map means the block absorbed the counters.
+            assert!(
+                !op.all_counters.is_empty(),
+                "operator {:?} carries a PlanInfo block but has no counters — 
the \
+                 PlanInfo parser swallowed them",
+                op.info.full_name
+            );
+        }
+        assert!(
+            with_plan_info
+                .iter()
+                .any(|op| op.metrics.rows_produced.is_some()),
+            "at least one PlanInfo-bearing operator must surface RowsProduced 
(output_rows)"
+        );
+
         // Regression: DetailProfile / Appendix must be section boundaries, 
not bleed
         // into MergedProfile and fabricate empty fragments — every parsed 
fragment has
         // real pipelines.
diff --git a/tests/e2e/suite_profile.sh b/tests/e2e/suite_profile.sh
index 17c8da9..5c06787 100644
--- a/tests/e2e/suite_profile.sh
+++ b/tests/e2e/suite_profile.sh
@@ -101,6 +101,14 @@ suite_profile() {
       # query_stats: scanned rows ≈ what we loaded; fragment/operator counts 
real.
       expect_parsed "profile: get -> query_stats.total_scan_rows ≈ loaded 
rows" \
         '(.query_stats.total_scan_rows >= ('"${ROWS:-2000}"' * 0.9 | floor)) 
and (.query_stats.fragment_count>=1) and (.query_stats.operator_count>0)'
+      # Regression (PlanInfo block swallowed per-operator counters): 
output_rows must
+      # populate for operators that carry a "- PlanInfo" block — this is NOT 
scan
+      # specific. The bug nulled RowsProduced -> output_rows for ~every 
PlanInfo
+      # bearing operator (and on Doris 5.0 the OLAP_SCAN itself carries a 
PlanInfo
+      # block, so total_scan_rows above goes null too). AGGREGATION always 
carries a
+      # PlanInfo block, so it must surface a non-null output_rows here.
+      expect_parsed "profile: get -> AGGREGATION operator surfaces output_rows 
(PlanInfo kept its counters)" \
+        'any(.operators[]; (.name|test("AGG")) and (.output_rows != null))'
       # Fragment breakdown must be REAL: exactly the 3 fragments of this 
group-by,
       # count consistent with the array, and NO empty duplicates (regression 
guard —
       # a DetailProfile/Appendix block bleeding into MergedProfile used to 
fabricate
@@ -135,6 +143,13 @@ suite_profile() {
           '(.profile.fragments|length>0) and 
(.profile.fragments[0].pipelines|length>0) and (.operators|length>0)'
         expect_parsed "profile: --full -> an operator exposes parsed 
all_counters" \
           'any(.profile.fragments[].pipelines[].operators[]; 
(.all_counters|type=="object") and (.all_counters|length>0))'
+        # Regression (PlanInfo block swallowed counters): the block must 
terminate and
+        # NOT absorb the counter lines that follow it. The fingerprint of that 
bug is
+        # an operator with a non-empty plan_info but an empty/absent 
all_counters
+        # (every counter, RowsProduced included, got mis-parsed as plan-info). 
The
+        # --full tree exposes both maps per operator, so assert there are none.
+        expect_parsed "profile: --full -> no PlanInfo-bearing operator lost 
its counters" \
+          '[.profile.fragments[].pipelines[].operators[] | 
select((.plan_info|length)>0) | select((.all_counters|length)==0)] | length == 
0'
       fi
 
       # ── --raw: the raw profile text round-trips; capture it as the fixture ─


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to