https://github.com/SLTozer updated https://github.com/llvm/llvm-project/pull/203254
>From 603d0a10c7e910e086ec6a91cefcb1a091abdbbd Mon Sep 17 00:00:00 2001 From: Stephen Tozer <[email protected]> Date: Thu, 11 Jun 2026 11:00:50 +0100 Subject: [PATCH 1/3] [Dexter] Write expects for variables in Debugger scopes Following on from the previous patch, this patch adds support for writing expects from !value/all nodes, generating separate expects for each variable in the requested debugger scope, for each continuous range of lines it is live for. --- .../dexter/dex/evaluation/ExpectRewriter.py | 210 ++++++++++++++++-- .../Inputs/rewrite_scopes_expected.cpp | 49 ++++ .../Inputs/rewrite_scopes_list_expected.cpp | 71 ++++++ .../scripts/rewriting/rewrite_scopes.cpp | 40 ++++ .../scripts/rewriting/rewrite_scopes_list.cpp | 54 +++++ 5 files changed, 406 insertions(+), 18 deletions(-) create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py index 5f8829287efe4..134560c45e9c0 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py @@ -6,16 +6,15 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception """Utilities for using debugger output to generate expected values that match that output.""" -from collections import Counter, OrderedDict, defaultdict -from copy import deepcopy -from enum import Enum, IntEnum +from collections import defaultdict from typing import Any, Dict, List, Optional, Set, Tuple, Union from dex.dextIR import DextIR, StepIR, ValueIR from dex.evaluation.StateMatch import get_active_where_matches -from dex.test_script.Nodes import Expect, Then, Value, Where +from dex.test_script.Nodes import DexRange, Expect, Line, Then, Value, ValueAll, Where from dex.test_script.Script import DexterScript, Scope from dex.tools.Main import Context +from dex.utils.Exceptions import Error class ExpectedValueRewriter: @@ -69,6 +68,94 @@ def freeze(input): return result +class ExpectedScopeRewriter: + """Given a list of ValueIRs for all variables in a scope, generates a set of expected values for each.""" + + def __init__(self, expect: Expect, step: StepIR, values: List[ValueIR]): + self.expect = expect + self.step = step + self.values = values + self.expected_values = [ + ExpectedValueRewriter(expect, value) for value in values + ] + + +# (StartLine, StopLine) -> [(Var, ExpectedValues)] +ExpectedScopeRewrites = Dict[Optional[Tuple[int, int]], List[Tuple[str, Any]]] + + +def collect_scope_values( + step_scope_values: List[ExpectedScopeRewriter], +) -> ExpectedScopeRewrites: + if not step_scope_values: + return {} + assert all( + step_scope_values[0].step.current_location.path == sv.step.current_location.path + for sv in step_scope_values[1:] + ), "Dexter currently does not handle scope watches that span multiple files." + all_vars: Set[str] = set() + for step_scope_rewriter in step_scope_values: + all_vars.update( + ev_rewriter.root_value.expression + for ev_rewriter in step_scope_rewriter.expected_values + ) + line_sorted_steps = sorted( + step_scope_values, + key=lambda step_scope_rewriter: step_scope_rewriter.step.current_location.lineno, + ) + + # Now we have a list of expected values for each variable sorted by the lines at which they appear; we use this to + # form blocks of continuous liveness. In general, for unoptimized code we expect variables to have a single + # continuous live range. + per_range_var_unique_expected_values: ExpectedScopeRewrites = defaultdict(list) + for var in sorted(all_vars): + # Now build a list of all continuous live ranges. + continuous_live_ranges: List[Tuple[int, int, List[ExpectedValueRewriter]]] = [] + is_live = False + ever_dead = False + for step_scope_rewriter in line_sorted_steps: + line = step_scope_rewriter.step.current_location.lineno + var_ev_rewriter = next( + ( + ev_rewriter + for ev_rewriter in step_scope_rewriter.expected_values + if ev_rewriter.root_value.expression == var + ), + None, + ) + if var_ev_rewriter is None or var_ev_rewriter.expected_value is None: + is_live = False + ever_dead = True + continue + + if not is_live: + continuous_live_ranges.append((line, line, [var_ev_rewriter])) + else: + start, stop, range_evs = continuous_live_ranges[-1] + assert line >= stop + range_evs.append(var_ev_rewriter) + continuous_live_ranges[-1] = (start, line, range_evs) + is_live = True + + if not continuous_live_ranges: + continue + + if not ever_dead: + assert len(continuous_live_ranges) == 1 + per_range_var_unique_expected_values[None].append( + (var, unique_expected_values(continuous_live_ranges[0][2])) + ) + continue + + # Finally, collect the results into the per_var map. + for start, stop, expected_values in continuous_live_ranges: + per_range_var_unique_expected_values[(start, stop)].append( + (var, unique_expected_values(expected_values)) + ) + + return per_range_var_unique_expected_values + + class StepExpectRewriter: """Processes all active, unknown expects at a given debugger step and produces ExpectedValueRewriter results for each.""" @@ -82,13 +169,24 @@ def __init__(self, step: StepIR, script: DexterScript): for where_match in self.state_match.values() for expect in where_match.active_expects } - self.expect_matches: Dict[Expect, ExpectedValueRewriter] = {} + self.expect_value_matches: Dict[Expect, ExpectedValueRewriter] = {} + self.expect_scope_matches: Dict[Expect, ExpectedScopeRewriter] = {} def add_expected_values(expect: Expect, expected_value: Any, scope: Scope): - assert isinstance(expect, Value), "Non-Value expects currently unsupported" - if expect in active_expects and expected_value is None: - self.expect_matches[expect] = ExpectedValueRewriter( - expect, step.watches[expect.get_watched_expr()] + if expect not in active_expects or expected_value is not None: + return + if (expr := expect.get_watched_expr()) is not None: + self.expect_value_matches[expect] = ExpectedValueRewriter( + expect, step.watches[expr] + ) + elif (scope_name := expect.get_watched_scope()) is not None: + scope_vars = step.scope_watches.get(scope_name, []) + self.expect_scope_matches[expect] = ExpectedScopeRewriter( + expect, step, [step.watches[var] for var in scope_vars] + ) + else: + raise Error( + f"Unexpected expect without watched expression or scope: {expect}" ) script.visit_script(visit_expect=add_expected_values) @@ -104,8 +202,12 @@ def __init__(self, context: Context, dext_ir: DextIR): self.unknown_expect_rewrites: Dict[ Expect, List[Tuple[int, ExpectedValueRewriter]] ] = {} + self.scope_expect_rewrites: Dict[ + Expect, List[Tuple[int, ExpectedScopeRewriter]] + ] = {} self.new_script: Optional[DexterScript] = None self.new_expected_values: Dict[Expect, Any] = {} + self.new_expected_scopes: Dict[Expect, ExpectedScopeRewrites] = {} self.missing_expect_rewrites: List[Expect] = [] script = dext_ir.script @@ -115,15 +217,21 @@ def __init__(self, context: Context, dext_ir: DextIR): # Collect every Expect with an unknown value into the `unknown_expect_rewrites` dict. We expect all Expects in # this dict to have observed values, and don't expect to rewrite any Expects outside of this dict. - def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope): + def collect_expects_to_rewrite( + expect: Expect, expected_value: Any, scope: Scope + ): + if isinstance(expect, ValueAll): + assert expected_value is None + self.scope_expect_rewrites[expect] = [] + return assert isinstance(expect, Value), "Non-Value expects currently unsupported" if expected_value is None: self.unknown_expect_rewrites[expect] = [] - script.visit_script(visit_expect=collect_unknown_expects) + script.visit_script(visit_expect=collect_expects_to_rewrite) # If there are no expects to update, then there is no rewriting to be done - exit early. - if not self.unknown_expect_rewrites: + if not self.unknown_expect_rewrites and not self.scope_expect_rewrites: return # Populate the `unknown_expect_rewrites` dict, mapping each expect with an unknown value to its list of observed @@ -133,10 +241,20 @@ def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope): ] for step_rewriter in self.step_rewriters: step_idx = step_rewriter.step.step_index - for expect, expected_value_rewriter in step_rewriter.expect_matches.items(): + for ( + expect, + expected_value_rewriter, + ) in step_rewriter.expect_value_matches.items(): self.unknown_expect_rewrites[expect].append( (step_idx, expected_value_rewriter) ) + for ( + expect, + expected_scope_rewriter, + ) in step_rewriter.expect_scope_matches.items(): + self.scope_expect_rewrites[expect].append( + (step_idx, expected_scope_rewriter) + ) # For each unknown expect, merge the observed values into a writable "expected values" entry, which may be a # list or a single value. @@ -150,9 +268,18 @@ def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope): ) is not None } + # Do the same for unknown scope expects. + self.new_expected_scopes = { + expect: collect_scope_values( + [rewriter for idx, rewriter in expect_rewriters] + ) + for expect, expect_rewriters in self.scope_expect_rewrites.items() + } # Finally, use the new expected values to rewrite the script. - self.new_script = rewrite_script(script, self.new_expected_values) + self.new_script = rewrite_script( + script, self.new_expected_values, self.new_expected_scopes + ) self.missing_expect_rewrites = [ expect for expect in self.unknown_expect_rewrites @@ -161,7 +288,10 @@ def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope): @property def num_successful_rewrites(self): - return len(self.new_expected_values) + return len(self.new_expected_values) + sum( + sum(len(var_expects) for var_expects in new_expected_scope.values()) + for new_expected_scope in self.new_expected_scopes.values() + ) @property def num_unsuccessful_rewrites(self): @@ -169,7 +299,9 @@ def num_unsuccessful_rewrites(self): def rewrite_script( - script: DexterScript, add_expected_values: Dict[Expect, Any] + script: DexterScript, + add_expected_values: Dict[Expect, Any], + expected_scope_rewrites: Dict[Expect, ExpectedScopeRewrites], ) -> DexterScript: """Given a set of updates to apply to a provided script, returns a copy of the script_obj with the updates applied. @@ -194,12 +326,54 @@ def replace_then(then: Then, scope: Scope): new_node_child_map[scope.where] = then def replace_expect(expect: Expect, expected_value, scope: Scope): - new_expected_value = add_expected_values.get(expect) or expected_value - new_node_child_map[expect] = new_expected_value scope_where_children = new_node_child_map.setdefault(scope.where, []) assert isinstance( scope_where_children, list ), f"Unexpected child for state node {scope.where}: {scope_where_children}" + if isinstance(expect, ValueAll): + assert ( + expect in expected_scope_rewrites + ), "Script-rewriter error: Dexter missed rewriting !expect/all node." + scope_rewrites = expected_scope_rewrites[expect] + for line_range in sorted( + scope_rewrites.keys(), key=lambda lines: lines or (0, 0) + ): + var_expected_values = scope_rewrites[line_range] + # First we determine which node will be the parent for the new expect nodes; then we can start appending + # new expects to that parent's child list. + if line_range is None: + new_expect_sibling_list = scope_where_children + else: + start, stop = line_range + lines = ( + Line(start) + if start == stop + else DexRange(Line(start), Line(stop)) + ) + new_expect_parent = Where({"lines": lines}, is_and=True) + # Reuse an existing !and node if one exists... + try: + existing_parent = next( + node + for node in scope_where_children + if str(node) == str(new_expect_parent) + ) + new_expect_sibling_list = new_node_child_map.setdefault( + existing_parent, [] + ) + except StopIteration: + scope_where_children.append(new_expect_parent) + new_expect_sibling_list = new_node_child_map.setdefault( + new_expect_parent, [] + ) + for var, expected_values in var_expected_values: + new_expect = Value(var) + new_expect_sibling_list.append(new_expect) + new_node_child_map[new_expect] = expected_values + return + assert isinstance(expect, Value) + new_expected_value = add_expected_values.get(expect) or expected_value + new_node_child_map[expect] = new_expected_value scope_where_children.append(expect) script.visit_script( diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp new file mode 100644 index 0000000000000..0786712bcf65c --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp @@ -0,0 +1,49 @@ +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: %dexter_regression_test_cxx_build %s -o %t/test +// RUN: %dexter_regression_test_run --use-script --binary %t/test \ +// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s +// RUN: diff %t/results/%{s:basename} \ +// RUN: %S/Inputs/rewrite_scopes_expected.cpp + +/// Tests that we can collect the values of all available variables with +/// !value/all and produce corresponding variable expects. + +/// NB: The exact contents of this file are compared against the expect file in +/// the Inputs/ directory; any changes to this file, including comments, +/// will require updating the corresponding expected file. + +char There[] = "Here"; + +int main() { + int One = 2; + char Red[] = "Blue"; + return 0; // !dex_label ret +} + +// CHECK: total_watched_steps: 3 +// CHECK: correct_steps: 3 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 11 +// CHECK: missing_values: 0 + +/* +--- +? !where {lines: !label 'ret'} +: !value 'One': '2' + !value 'Red': + '[0]': '''B''' + '[1]': '''l''' + '[2]': '''u''' + '[3]': '''e''' + '[4]': '''\0''' + !value '::There': + '[0]': '''H''' + '[1]': '''e''' + '[2]': '''r''' + '[3]': '''e''' + '[4]': '''\0''' +... +*/ diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp new file mode 100644 index 0000000000000..8c536f08b710b --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp @@ -0,0 +1,71 @@ +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: %dexter_regression_test_cxx_build %s -o %t/test +// RUN: %dexter_regression_test_run --use-script --binary %t/test \ +// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s +// RUN: diff %t/results/%{s:basename} \ +// RUN: %S/Inputs/rewrite_scopes_list_expected.cpp + +/// Tests that !value/all creates appropriate state nodes for the different +/// variables: +/// - TopFloor is live for the entire scope of the !value/all, so appears +/// directly under the root !where +/// - Elevator is live at two disjoint positions, so appears under two different +/// !and nodes. +/// - Ground and Button become live at the same time after the !value/all +/// becomes active and for the remainder of the program, so are both placed +/// under a single shared !and. + +/// NB: The exact contents of this file are compared against the expect file in +/// the Inputs/ directory; any changes to this file, including comments, +/// will require updating the corresponding expected file. + +void ding() {} +void swapPassengers() {} + +int main() { + int TopFloor = 10; + int Ground = 0, Button = 6; // !dex_label start + for (int Elevator = Ground; Elevator < Button; ++Elevator) { + ding(); + } + swapPassengers(); + for (int Elevator = Button; Elevator < TopFloor; ++Elevator) { + ding(); + } + return 0; // !dex_label ret +} + +// CHECK: Rewrote script to add 5 expected values. + +// CHECK: total_watched_steps: 83 +// CHECK: correct_steps: 83 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 13 +// CHECK: missing_values: 0 + +/* +--- +? !where {lines: !range [!label 'start', !label 'ret']} +: !value 'TopFloor': '10' + ? !and {lines: !range [29, 36]} + : !value 'Button': '6' + !value 'Ground': '0' + ? !and {lines: 30} + : !value 'Elevator': + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + ? !and {lines: 34} + : !value 'Elevator': + - '6' + - '7' + - '8' + - '9' +... +*/ diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp new file mode 100644 index 0000000000000..fc0fec1a91bb0 --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp @@ -0,0 +1,40 @@ +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: %dexter_regression_test_cxx_build %s -o %t/test +// RUN: %dexter_regression_test_run --use-script --binary %t/test \ +// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s +// RUN: diff %t/results/%{s:basename} \ +// RUN: %S/Inputs/rewrite_scopes_expected.cpp + +/// Tests that we can collect the values of all available variables with +/// !value/all and produce corresponding variable expects. + +/// NB: The exact contents of this file are compared against the expect file in +/// the Inputs/ directory; any changes to this file, including comments, +/// will require updating the corresponding expected file. + +char There[] = "Here"; + +int main() { + int One = 2; + char Red[] = "Blue"; + return 0; // !dex_label ret +} + +// CHECK: total_watched_steps: 3 +// CHECK: correct_steps: 3 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 11 +// CHECK: missing_values: 0 + +/* +--- +!where {lines: !label ret}: + ? !value/all Locals + ? !value/all Globals + # Invalid scopes won't appear in the output. + ? !value/all NotARealScope +... +*/ diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp new file mode 100644 index 0000000000000..1291a30fb2796 --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp @@ -0,0 +1,54 @@ +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: %dexter_regression_test_cxx_build %s -o %t/test +// RUN: %dexter_regression_test_run --use-script --binary %t/test \ +// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s +// RUN: diff %t/results/%{s:basename} \ +// RUN: %S/Inputs/rewrite_scopes_list_expected.cpp + +/// Tests that !value/all creates appropriate state nodes for the different +/// variables: +/// - TopFloor is live for the entire scope of the !value/all, so appears +/// directly under the root !where +/// - Elevator is live at two disjoint positions, so appears under two different +/// !and nodes. +/// - Ground and Button become live at the same time after the !value/all +/// becomes active and for the remainder of the program, so are both placed +/// under a single shared !and. + +/// NB: The exact contents of this file are compared against the expect file in +/// the Inputs/ directory; any changes to this file, including comments, +/// will require updating the corresponding expected file. + +void ding() {} +void swapPassengers() {} + +int main() { + int TopFloor = 10; + int Ground = 0, Button = 6; // !dex_label start + for (int Elevator = Ground; Elevator < Button; ++Elevator) { + ding(); + } + swapPassengers(); + for (int Elevator = Button; Elevator < TopFloor; ++Elevator) { + ding(); + } + return 0; // !dex_label ret +} + +// CHECK: Rewrote script to add 5 expected values. + +// CHECK: total_watched_steps: 83 +// CHECK: correct_steps: 83 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 13 +// CHECK: missing_values: 0 + +/* +--- +!where {lines: !range [!label start, !label ret]}: + ? !value/all Locals +... +*/ >From 8ca700ab817be51ff8112bc31ab00d6b2b346277 Mon Sep 17 00:00:00 2001 From: Stephen Tozer <[email protected]> Date: Wed, 24 Jun 2026 13:11:27 +0100 Subject: [PATCH 2/3] Rename writer->rewriter --- .../debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py index 134560c45e9c0..7bcec6e687fe7 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py @@ -227,7 +227,6 @@ def collect_expects_to_rewrite( assert isinstance(expect, Value), "Non-Value expects currently unsupported" if expected_value is None: self.unknown_expect_rewrites[expect] = [] - script.visit_script(visit_expect=collect_expects_to_rewrite) # If there are no expects to update, then there is no rewriting to be done - exit early. >From b0e0b50537a143d40415bbcfe0f01efa853eda08 Mon Sep 17 00:00:00 2001 From: Stephen Tozer <[email protected]> Date: Wed, 24 Jun 2026 13:40:36 +0100 Subject: [PATCH 3/3] format --- .../debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py index 7bcec6e687fe7..134560c45e9c0 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectRewriter.py @@ -227,6 +227,7 @@ def collect_expects_to_rewrite( assert isinstance(expect, Value), "Non-Value expects currently unsupported" if expected_value is None: self.unknown_expect_rewrites[expect] = [] + script.visit_script(visit_expect=collect_expects_to_rewrite) # If there are no expects to update, then there is no rewriting to be done - exit early. _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
