This is an automated email from the ASF dual-hosted git repository. asorokoumov pushed a commit to branch config-arg-parse-fixup in repository https://gitbox.apache.org/repos/asf/otava.git
commit 113eeeb151eacab1a0d5641230c5d7af677de787 Author: Alex Sorokoumov <[email protected]> AuthorDate: Mon Dec 15 16:58:02 2025 -0800 Fixup 3 bugs in ConfigArgParse usage 1. --help no longer throws exceptions 2. ConfigArgParse is passed into the actual commands 3. Throw when passed an unrecognized CLI arg --- otava/config.py | 18 +- otava/main.py | 42 ++- tests/cli_help_test.py | 750 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/config_test.py | 17 ++ 4 files changed, 811 insertions(+), 16 deletions(-) diff --git a/otava/config.py b/otava/config.py index ca1d587..ef707a9 100644 --- a/otava/config.py +++ b/otava/config.py @@ -191,6 +191,18 @@ class NestedYAMLConfigFileParser(configargparse.ConfigFileParser): # Value must be cast to string here, so arg parser can cast from string to expected type later flattened_dict[new_key] = str(value) + def get_syntax_description(self): + return "" + + +def add_service_option_groups(parser) -> None: + """Add Graphite, Grafana, Slack, Postgres, and BigQuery option groups to a parser.""" + GraphiteConfig.add_parser_args(parser.add_argument_group('Graphite Options', 'Options for Graphite configuration')) + GrafanaConfig.add_parser_args(parser.add_argument_group('Grafana Options', 'Options for Grafana configuration')) + SlackConfig.add_parser_args(parser.add_argument_group('Slack Options', 'Options for Slack configuration')) + PostgresConfig.add_parser_args(parser.add_argument_group('PostgreSQL Options', 'Options for PostgreSQL configuration')) + BigQueryConfig.add_parser_args(parser.add_argument_group('BigQuery Options', 'Options for BigQuery configuration')) + def create_config_parser() -> configargparse.ArgumentParser: parser = configargparse.ArgumentParser( @@ -203,11 +215,7 @@ def create_config_parser() -> configargparse.ArgumentParser: allow_abbrev=False, # required for correct parsing of nested values from config file ) parser.add_argument('--config-file', is_config_file=True, help='Otava config file path', env_var="OTAVA_CONFIG") - GraphiteConfig.add_parser_args(parser.add_argument_group('Graphite Options', 'Options for Graphite configuration')) - GrafanaConfig.add_parser_args(parser.add_argument_group('Grafana Options', 'Options for Grafana configuration')) - SlackConfig.add_parser_args(parser.add_argument_group('Slack Options', 'Options for Slack configuration')) - PostgresConfig.add_parser_args(parser.add_argument_group('Postgres Options', 'Options for Postgres configuration')) - BigQueryConfig.add_parser_args(parser.add_argument_group('BigQuery Options', 'Options for BigQuery configuration')) + add_service_option_groups(parser) return parser diff --git a/otava/main.py b/otava/main.py index 1c2e1e4..09e7608 100644 --- a/otava/main.py +++ b/otava/main.py @@ -515,30 +515,41 @@ def analysis_options_from_args(args: argparse.Namespace) -> AnalysisOptions: def create_otava_cli_parser() -> argparse.ArgumentParser: + config_parser = config.create_config_parser() parser = argparse.ArgumentParser( - description="Hunts performance regressions in Fallout results", - parents=[config.create_config_parser()], + description="Change Detection for Continuous Performance Engineering", + parents=[config_parser], config_file_parser_class=config.NestedYAMLConfigFileParser, allow_abbrev=False, # required for correct parsing of nested values from config file ) subparsers = parser.add_subparsers(dest="command") - list_tests_parser = subparsers.add_parser("list-tests", help="list available tests") + list_tests_parser = subparsers.add_parser( + "list-tests", + help="list available tests", + ) + config.add_service_option_groups(list_tests_parser) list_tests_parser.add_argument("group", help="name of the group of the tests", nargs="*") list_metrics_parser = subparsers.add_parser( - "list-metrics", help="list available metrics for a test" + "list-metrics", + help="list available metrics for a test", ) + config.add_service_option_groups(list_metrics_parser) list_metrics_parser.add_argument("test", help="name of the test") - subparsers.add_parser("list-groups", help="list available groups of tests") + list_groups_parser = subparsers.add_parser( + "list-groups", + help="list available groups of tests", + ) + config.add_service_option_groups(list_groups_parser) analyze_parser = subparsers.add_parser( "analyze", help="analyze performance test results", - formatter_class=argparse.RawTextHelpFormatter, ) analyze_parser.add_argument("tests", help="name of the test or group of the tests", nargs="+") + config.add_service_option_groups(analyze_parser) analyze_parser.add_argument( "--update-grafana", help="Update Grafana dashboards with appropriate annotations of change points", @@ -576,14 +587,21 @@ def create_otava_cli_parser() -> argparse.ArgumentParser: setup_data_selector_parser(analyze_parser) setup_analysis_options_parser(analyze_parser) - regressions_parser = subparsers.add_parser("regressions", help="find performance regressions") + regressions_parser = subparsers.add_parser( + "regressions", + help="find performance regressions", + ) regressions_parser.add_argument( "tests", help="name of the test or group of the tests", nargs="+" ) + config.add_service_option_groups(regressions_parser) setup_data_selector_parser(regressions_parser) setup_analysis_options_parser(regressions_parser) - remove_annotations_parser = subparsers.add_parser("remove-annotations") + remove_annotations_parser = subparsers.add_parser( + "remove-annotations", + ) + config.add_service_option_groups(remove_annotations_parser) remove_annotations_parser.add_argument( "tests", help="name of the test or test group", nargs="*" ) @@ -591,9 +609,11 @@ def create_otava_cli_parser() -> argparse.ArgumentParser: "--force", help="don't ask questions, just do it", dest="force", action="store_true" ) - subparsers.add_parser( - "validate", help="validates the tests and metrics defined in the configuration" + validate_parser = subparsers.add_parser( + "validate", + help="validates the tests and metrics defined in the configuration", ) + config.add_service_option_groups(validate_parser) return parser @@ -603,7 +623,7 @@ def script_main(conf: Config = None, args: List[str] = None): parser = create_otava_cli_parser() try: - args, _ = parser.parse_known_args(args=args) + args = parser.parse_args(args=args) if conf is None: conf = config.load_config_from_parser_args(args) otava = Otava(conf) diff --git a/tests/cli_help_test.py b/tests/cli_help_test.py new file mode 100644 index 0000000..67ef324 --- /dev/null +++ b/tests/cli_help_test.py @@ -0,0 +1,750 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import subprocess +import os + + +def run_help_command(subcommand: str = None) -> subprocess.CompletedProcess: + """ + Invoke the installed otava CLI (expects entrypoint script) to capture --help output. + """ + if subcommand is None: + cmd = ["uv", "run", "otava", "--help"] + else: + cmd = ["uv", "run", "otava", subcommand, "--help"] + return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=dict(os.environ, COLUMNS="100")) + + +def test_otava_help_output(): + result = run_help_command() + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava [-h] [--config-file CONFIG_FILE] [--graphite-url GRAPHITE_URL] + [--grafana-url GRAFANA_URL] [--grafana-user GRAFANA_USER] + [--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN] + [--postgres-hostname POSTGRES_HOSTNAME] [--postgres-port POSTGRES_PORT] + [--postgres-username POSTGRES_USERNAME] [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] [--bigquery-credentials BIGQUERY_CREDENTIALS] + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} ... + +Change Detection for Continuous Performance Engineering + +positional arguments: + {list-tests,list-metrics,list-groups,analyze,regressions,remove-annotations,validate} + list-tests list available tests + list-metrics list available metrics for a test + list-groups list available groups of tests + analyze analyze performance test results + regressions find performance regressions + validate validates the tests and metrics defined in the configuration + +options: + -h, --help show this help message and exit + --config-file CONFIG_FILE + Otava config file path [env var: OTAVA_CONFIG] + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + +Options that start with '--' can also be set in a config file (specified via --config-file). In +general, command-line values override environment variables which override config file values +which override defaults. +""" + ) + + +# Test for analyze subcommand +def test_otava_analyze_help_output(): + result = run_help_command("analyze") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava analyze [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] [--grafana-password GRAFANA_PASSWORD] + [--slack-token SLACK_TOKEN] [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] [--update-grafana] + [--update-postgres] [--update-bigquery] + [--notify-slack NOTIFY_SLACK [NOTIFY_SLACK ...]] [--cph-report-since DATE] + [--output {log,json,regressions_only}] [--branch [STRING]] [--metrics LIST] + [--attrs LIST] [--since-commit STRING | --since-version STRING | + --since DATE] [--until-commit STRING | --until-version STRING | --until DATE] + [--last COUNT] [-P, --p-value PVALUE] [-M MAGNITUDE] [--window WINDOW] + [--orig-edivisive ORIG_EDIVISIVE] + tests [tests ...] + +positional arguments: + tests name of the test or group of the tests + +options: + -h, --help show this help message and exit + --update-grafana Update Grafana dashboards with appropriate annotations of change points + --update-postgres Update PostgreSQL database results with change points + --update-bigquery Update BigQuery database results with change points + --notify-slack NOTIFY_SLACK [NOTIFY_SLACK ...] + Send notification containing a summary of change points to given Slack + channels + --cph-report-since DATE + Sets a limit on the date range of the Change Point History reported to + Slack. Same syntax as --since. + --output {log,json,regressions_only} + Output format for the generated report. + --branch [STRING] name of the branch + --metrics LIST a comma-separated list of metrics to analyze + --attrs LIST a comma-separated list of attribute names associated with the runs (e.g. + commit, branch, version); if not specified, it will be automatically + filled based on available information + --since-commit STRING + the commit at the start of the time span to analyze + --since-version STRING + the version at the start of the time span to analyze + --since DATE the start of the time span to analyze; accepts ISO, and human-readable + dates like '10 weeks ago' + --until-commit STRING + the commit at the end of the time span to analyze + --until-version STRING + the version at the end of the time span to analyze + --until DATE the end of the time span to analyze; same syntax as --since + --last COUNT the number of data points to take from the end of the series + -P, --p-value PVALUE maximum accepted P-value of a change-point; P denotes the probability that + the change-point has been found by a random coincidence, rather than a + real difference between the data distributions + -M, --magnitude MAGNITUDE + minimum accepted magnitude of a change-point computed as abs(new_mean / + old_mean - 1.0); use it to filter out stupidly small changes like < 0.01 + --window WINDOW the number of data points analyzed at once; the window size affects the + discriminative power of the change point detection algorithm; large + windows are less susceptible to noise; however, a very large window may + cause dismissing short regressions as noise so it is best to keep it short + enough to include not more than a few change points (optimally at most 1) + --orig-edivisive ORIG_EDIVISIVE + use the original edivisive algorithm with no windowing and weak change + points analysis improvements + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) + + +# Test for regressions subcommand +def test_otava_regressions_help_output(): + result = run_help_command("regressions") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava regressions [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] [--grafana-password GRAFANA_PASSWORD] + [--slack-token SLACK_TOKEN] [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] [--branch [STRING]] + [--metrics LIST] [--attrs LIST] [--since-commit STRING | + --since-version STRING | --since DATE] [--until-commit STRING | + --until-version STRING | --until DATE] [--last COUNT] + [-P, --p-value PVALUE] [-M MAGNITUDE] [--window WINDOW] + [--orig-edivisive ORIG_EDIVISIVE] + tests [tests ...] + +positional arguments: + tests name of the test or group of the tests + +options: + -h, --help show this help message and exit + --branch [STRING] name of the branch + --metrics LIST a comma-separated list of metrics to analyze + --attrs LIST a comma-separated list of attribute names associated with the runs (e.g. + commit, branch, version); if not specified, it will be automatically + filled based on available information + --since-commit STRING + the commit at the start of the time span to analyze + --since-version STRING + the version at the start of the time span to analyze + --since DATE the start of the time span to analyze; accepts ISO, and human-readable + dates like '10 weeks ago' + --until-commit STRING + the commit at the end of the time span to analyze + --until-version STRING + the version at the end of the time span to analyze + --until DATE the end of the time span to analyze; same syntax as --since + --last COUNT the number of data points to take from the end of the series + -P, --p-value PVALUE maximum accepted P-value of a change-point; P denotes the probability that + the change-point has been found by a random coincidence, rather than a + real difference between the data distributions + -M, --magnitude MAGNITUDE + minimum accepted magnitude of a change-point computed as abs(new_mean / + old_mean - 1.0); use it to filter out stupidly small changes like < 0.01 + --window WINDOW the number of data points analyzed at once; the window size affects the + discriminative power of the change point detection algorithm; large + windows are less susceptible to noise; however, a very large window may + cause dismissing short regressions as noise so it is best to keep it short + enough to include not more than a few change points (optimally at most 1) + --orig-edivisive ORIG_EDIVISIVE + use the original edivisive algorithm with no windowing and weak change + points analysis improvements + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) + + +# Test for list-tests subcommand +def test_otava_list_tests_help_output(): + result = run_help_command("list-tests") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava list-tests [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] [--grafana-password GRAFANA_PASSWORD] + [--slack-token SLACK_TOKEN] [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] + [group ...] + +positional arguments: + group name of the group of the tests + +options: + -h, --help show this help message and exit + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) + + +# Test for list-metrics subcommand +def test_otava_list_metrics_help_output(): + result = run_help_command("list-metrics") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava list-metrics [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] [--grafana-password GRAFANA_PASSWORD] + [--slack-token SLACK_TOKEN] [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] + test + +positional arguments: + test name of the test + +options: + -h, --help show this help message and exit + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) + + +# Test for list-groups subcommand +def test_otava_list_groups_help_output(): + result = run_help_command("list-groups") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava list-groups [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] [--grafana-password GRAFANA_PASSWORD] + [--slack-token SLACK_TOKEN] [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] + +options: + -h, --help show this help message and exit + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) + + +# Test for remove-annotations subcommand +def test_otava_remove_annotations_help_output(): + result = run_help_command("remove-annotations") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava remove-annotations [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] + [--grafana-password GRAFANA_PASSWORD] [--slack-token SLACK_TOKEN] + [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] + [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] [--force] + [tests ...] + +positional arguments: + tests name of the test or test group + +options: + -h, --help show this help message and exit + --force don't ask questions, just do it + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) + + +def test_otava_validate_help_output(): + result = run_help_command("validate") + assert result.returncode == 0, ( + f"Expected exit code 0, got {result.returncode}. stderr:\n{result.stderr}" + ) + assert result.stderr == "", f"Expected empty stderr, got:\n{result.stderr}" + assert ( + result.stdout + == """\ +usage: otava validate [-h] [--graphite-url GRAPHITE_URL] [--grafana-url GRAFANA_URL] + [--grafana-user GRAFANA_USER] [--grafana-password GRAFANA_PASSWORD] + [--slack-token SLACK_TOKEN] [--postgres-hostname POSTGRES_HOSTNAME] + [--postgres-port POSTGRES_PORT] [--postgres-username POSTGRES_USERNAME] + [--postgres-password POSTGRES_PASSWORD] + [--postgres-database POSTGRES_DATABASE] + [--bigquery-project-id BIGQUERY_PROJECT_ID] + [--bigquery-dataset BIGQUERY_DATASET] + [--bigquery-credentials BIGQUERY_CREDENTIALS] + +options: + -h, --help show this help message and exit + +Graphite Options: + Options for Graphite configuration + + --graphite-url GRAPHITE_URL + Graphite server URL [env var: GRAPHITE_ADDRESS] + +Grafana Options: + Options for Grafana configuration + + --grafana-url GRAFANA_URL + Grafana server URL [env var: GRAFANA_ADDRESS] + --grafana-user GRAFANA_USER + Grafana server user [env var: GRAFANA_USER] + --grafana-password GRAFANA_PASSWORD + Grafana server password [env var: GRAFANA_PASSWORD] + +Slack Options: + Options for Slack configuration + + --slack-token SLACK_TOKEN + Slack bot token to use for sending notifications [env var: + SLACK_BOT_TOKEN] + +PostgreSQL Options: + Options for PostgreSQL configuration + + --postgres-hostname POSTGRES_HOSTNAME + PostgreSQL server hostname [env var: POSTGRES_HOSTNAME] + --postgres-port POSTGRES_PORT + PostgreSQL server port [env var: POSTGRES_PORT] + --postgres-username POSTGRES_USERNAME + PostgreSQL username [env var: POSTGRES_USERNAME] + --postgres-password POSTGRES_PASSWORD + PostgreSQL password [env var: POSTGRES_PASSWORD] + --postgres-database POSTGRES_DATABASE + PostgreSQL database name [env var: POSTGRES_DATABASE] + +BigQuery Options: + Options for BigQuery configuration + + --bigquery-project-id BIGQUERY_PROJECT_ID + BigQuery project ID [env var: BIGQUERY_PROJECT_ID] + --bigquery-dataset BIGQUERY_DATASET + BigQuery dataset [env var: BIGQUERY_DATASET] + --bigquery-credentials BIGQUERY_CREDENTIALS + BigQuery credentials file [env var: BIGQUERY_VAULT_SECRET] + + In general, command-line values override environment variables which override defaults. +""" + ) diff --git a/tests/config_test.py b/tests/config_test.py index 4f66b95..1b35c8f 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -28,6 +28,7 @@ from otava.config import ( load_config_from_file, load_config_from_parser_args, ) +from otava.main import create_otava_cli_parser from otava.test_config import CsvTestConfig, GraphiteTestConfig, HistoStatTestConfig @@ -439,3 +440,19 @@ slack: for key in env_vars: if key in os.environ: del os.environ[key] + + +def test_unknown_argument_raises_error(capsys): + """Test that unknown arguments raise an error.""" + parser = create_otava_cli_parser() + + # Unknown argument should cause SystemExit + with pytest.raises(SystemExit) as exc_info: + parser.parse_args(["--banana", "list-groups"]) + + # argparse exits with code 2 for invalid arguments + assert exc_info.value.code == 2 + + # Check error message + captured = capsys.readouterr() + assert "unrecognized arguments: --banana" in captured.err
