This is an automated email from the ASF dual-hosted git repository. asorokoumov pushed a commit to branch fixup-0.7.0-rc3 in repository https://gitbox.apache.org/repos/asf/otava.git
commit 6fe39cd8f1c21ff77fc41b1e0be7292a730913b2 Author: Alex Sorokoumov <[email protected]> AuthorDate: Thu Nov 27 21:25:00 2025 -0800 OTAVA-82: Fix --help requiring a config file --- otava/main.py | 25 ++-- tests/cli_help_test.py | 355 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 370 insertions(+), 10 deletions(-) diff --git a/otava/main.py b/otava/main.py index 019aff1..f5fc9f1 100644 --- a/otava/main.py +++ b/otava/main.py @@ -515,15 +515,6 @@ def analysis_options_from_args(args: argparse.Namespace) -> AnalysisOptions: def main(): - try: - conf = config.load_config() - except ConfigError as err: - logging.error(err.message) - exit(1) - script_main(conf) - - -def script_main(conf: Config, args: List[str] = None): logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) parser = argparse.ArgumentParser(description="Change Detection for Continuous Performance Engineering") @@ -601,8 +592,22 @@ def script_main(conf: Config, args: List[str] = None): "validate", help="validates the tests and metrics defined in the configuration" ) + # Parse arguments first, before loading config + args = parser.parse_args() + + # If no command provided, just print usage and exit (no config needed) + if args.command is None: + parser.print_usage() + return + + # Now load the config only when we actually need it + try: + conf = config.load_config() + except ConfigError as err: + logging.error(err.message) + exit(1) + try: - args = parser.parse_args(args=args) otava = Otava(conf) if args.command == "list-groups": diff --git a/tests/cli_help_test.py b/tests/cli_help_test.py new file mode 100644 index 0000000..019f81b --- /dev/null +++ b/tests/cli_help_test.py @@ -0,0 +1,355 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import subprocess +import textwrap +from typing import List + +import pytest + + +def test_main_help(): + """Test that 'otava --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava [-h] + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} + ... + + Change Detection for Continuous Performance Engineering + + positional arguments: + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} + list-tests list available tests + list-metrics list available metrics for a test + list-groups list available groups of tests + analyze analyze performance test results + regressions find performance regressions + validate validates the tests and metrics defined in the + configuration + + optional arguments: + -h, --help show this help message and exit + """ + ).strip() + + actual = _run_otava(["--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_main_help_short(): + """Test that 'otava -h' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava [-h] + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} + ... + + Change Detection for Continuous Performance Engineering + + positional arguments: + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} + list-tests list available tests + list-metrics list available metrics for a test + list-groups list available groups of tests + analyze analyze performance test results + regressions find performance regressions + validate validates the tests and metrics defined in the + configuration + + optional arguments: + -h, --help show this help message and exit + """ + ).strip() + + actual = _run_otava(["-h"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_analyze_help(): + """Test that 'otava analyze --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava analyze [-h] [--update-grafana] [--update-postgres] + [--update-bigquery] + [--notify-slack NOTIFY_SLACK [NOTIFY_SLACK ...]] + [--cph-report-since DATE] + [--output {log,json,regressions_only}] + [--branch [STRING]] [--metrics LIST] [--attrs LIST] + [--since-commit STRING | --since-version STRING | --since DATE] + [--until-commit STRING | --until-version STRING | --until DATE] + [--last COUNT] [-P, --p-value PVALUE] [-M MAGNITUDE] + [--window WINDOW] [--orig-edivisive ORIG_EDIVISIVE] + tests [tests ...] + + positional arguments: + tests name of the test or group of the tests + + optional arguments: + -h, --help show this help message and exit + --update-grafana Update Grafana dashboards with appropriate annotations of change points + --update-postgres Update PostgreSQL database results with change points + --update-bigquery Update BigQuery database results with change points + --notify-slack NOTIFY_SLACK [NOTIFY_SLACK ...] + Send notification containing a summary of change points to given Slack channels + --cph-report-since DATE + Sets a limit on the date range of the Change Point History reported to Slack. Same syntax as --since. + --output {log,json,regressions_only} + Output format for the generated report. + --branch [STRING] name of the branch + --metrics LIST a comma-separated list of metrics to analyze + --attrs LIST a comma-separated list of attribute names associated with the runs (e.g. commit, branch, version); if not specified, it will be automatically filled based on available information + --since-commit STRING + the commit at the start of the time span to analyze + --since-version STRING + the version at the start of the time span to analyze + --since DATE the start of the time span to analyze; accepts ISO, and human-readable dates like '10 weeks ago' + --until-commit STRING + the commit at the end of the time span to analyze + --until-version STRING + the version at the end of the time span to analyze + --until DATE the end of the time span to analyze; same syntax as --since + --last COUNT the number of data points to take from the end of the series + -P, --p-value PVALUE maximum accepted P-value of a change-point; P denotes the probability that the change-point has been found by a random coincidence, rather than a real difference between the data distributions + -M MAGNITUDE, --magnitude MAGNITUDE + minimum accepted magnitude of a change-point computed as abs(new_mean / old_mean - 1.0); use it to filter out stupidly small changes like < 0.01 + --window WINDOW the number of data points analyzed at once; the window size affects the discriminative power of the change point detection algorithm; large windows are less susceptible to noise; however, a very large window may cause dismissing short regressions as noise so it is best to keep it short enough to include not more than a few change points (optimally at most 1) + --orig-edivisive ORIG_EDIVISIVE + use the original edivisive algorithm with no windowing and weak change points analysis improvements + """ + ).strip() + + actual = _run_otava(["analyze", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_list_tests_help(): + """Test that 'otava list-tests --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava list-tests [-h] [group ...] + + positional arguments: + group name of the group of the tests + + optional arguments: + -h, --help show this help message and exit + """ + ).strip() + + actual = _run_otava(["list-tests", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_list_metrics_help(): + """Test that 'otava list-metrics --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava list-metrics [-h] test + + positional arguments: + test name of the test + + optional arguments: + -h, --help show this help message and exit + """ + ).strip() + + actual = _run_otava(["list-metrics", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_list_groups_help(): + """Test that 'otava list-groups --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava list-groups [-h] + + optional arguments: + -h, --help show this help message and exit + """ + ).strip() + + actual = _run_otava(["list-groups", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_regressions_help(): + """Test that 'otava regressions --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava regressions [-h] [--branch [STRING]] [--metrics LIST] + [--attrs LIST] + [--since-commit STRING | --since-version STRING | --since DATE] + [--until-commit STRING | --until-version STRING | --until DATE] + [--last COUNT] [-P, --p-value PVALUE] [-M MAGNITUDE] + [--window WINDOW] [--orig-edivisive ORIG_EDIVISIVE] + tests [tests ...] + + positional arguments: + tests name of the test or group of the tests + + optional arguments: + -h, --help show this help message and exit + --branch [STRING] name of the branch + --metrics LIST a comma-separated list of metrics to analyze + --attrs LIST a comma-separated list of attribute names associated + with the runs (e.g. commit, branch, version); if not + specified, it will be automatically filled based on + available information + --since-commit STRING + the commit at the start of the time span to analyze + --since-version STRING + the version at the start of the time span to analyze + --since DATE the start of the time span to analyze; accepts ISO, + and human-readable dates like '10 weeks ago' + --until-commit STRING + the commit at the end of the time span to analyze + --until-version STRING + the version at the end of the time span to analyze + --until DATE the end of the time span to analyze; same syntax as + --since + --last COUNT the number of data points to take from the end of the + series + -P, --p-value PVALUE maximum accepted P-value of a change-point; P denotes + the probability that the change-point has been found + by a random coincidence, rather than a real difference + between the data distributions + -M MAGNITUDE, --magnitude MAGNITUDE + minimum accepted magnitude of a change-point computed + as abs(new_mean / old_mean - 1.0); use it to filter + out stupidly small changes like < 0.01 + --window WINDOW the number of data points analyzed at once; the window + size affects the discriminative power of the change + point detection algorithm; large windows are less + susceptible to noise; however, a very large window may + cause dismissing short regressions as noise so it is + best to keep it short enough to include not more than + a few change points (optimally at most 1) + --orig-edivisive ORIG_EDIVISIVE + use the original edivisive algorithm with no windowing + and weak change points analysis improvements + """ + ).strip() + + actual = _run_otava(["regressions", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_remove_annotations_help(): + """Test that 'otava remove-annotations --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava remove-annotations [-h] [--force] [tests ...] + + positional arguments: + tests name of the test or test group + + optional arguments: + -h, --help show this help message and exit + --force don't ask questions, just do it + """ + ).strip() + + actual = _run_otava(["remove-annotations", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_validate_help(): + """Test that 'otava validate --help' works without a config file.""" + expected = textwrap.dedent( + """\ + usage: otava validate [-h] + + optional arguments: + -h, --help show this help message and exit + """ + ).strip() + + actual = _run_otava(["validate", "--help"]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def test_no_command_shows_usage(): + """Test that running 'otava' without a command shows usage without requiring config.""" + expected = textwrap.dedent( + """\ + usage: otava [-h] + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} + ... + """ + ).strip() + + actual = _run_otava([]) + assert actual == expected, f"Expected:\n{expected}\n\nActual:\n{actual}" + + +def _normalize_help_output(output: str) -> str: + """ + Normalize help output by removing warnings and extra whitespace. + This makes tests more robust to environment-specific warnings. + """ + lines = output.split("\n") + filtered_lines = [] + for line in lines: + # Skip urllib3 warning lines + if "urllib3" in line or "NotOpenSSLWarning" in line or "warnings.warn" in line: + continue + # Skip importlib.metadata warning + if "importlib.metadata" in line: + continue + # Skip uv package management messages + if "Uninstalled" in line and "package" in line: + continue + if "Installed" in line and "package" in line: + continue + filtered_lines.append(line) + + # Join and normalize whitespace + result = "\n".join(filtered_lines) + # Remove leading/trailing whitespace from each line while preserving structure + return "\n".join(line.rstrip() for line in result.split("\n")).strip() + + +def _run_otava(args: List[str]) -> str: + """ + Run a help command and return normalized output. + """ + cmd = ["uv", "run", "otava"] + args + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + # Don't set OTAVA_CONFIG - help should work without config + # COLUMNS=80 controls output split over multiple lines + env=dict(os.environ, COLUMNS="80"), + ) + + if proc.returncode != 0: + pytest.fail( + f"Help command returned non-zero exit code.\n\n" + f"Command: {cmd!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + + # Combine stdout and stderr, then normalize + combined_output = proc.stdout + proc.stderr + return _normalize_help_output(combined_output) \ No newline at end of file
