On Wed, Feb 14, 2018 at 2:09 PM, Brenda J. Butler <b...@mojatatu.com> wrote:
> This should be a general test architecture, and yet allow specific
> tests to be done. Introduce a plugin architecture.
>
> An individual test has 4 stages, setup/execute/verify/teardown. Each
> plugin gets a chance to run a function at each stage, plus one call
> before all the tests are called ("pre" suite) and one after all the
> tests are called ("post" suite). In addition, just before each
> command is executed, the plugin gets a chance to modify the command
> using the "adjust_command" hook. This makes the test suite quite
> flexible.
>
> Future patches will take some functionality out of the tdc.py script and
> place it in plugins.
>
> To use the plugins, place the implementation in the plugins directory
> and run tdc.py. It will notice the plugins and use them.
>
> Signed-off-by: Brenda J. Butler <b...@mojatatu.com>
Acked-by: Lucas Bates <luc...@mojatatu.com>
> ---
> tools/testing/selftests/tc-testing/TdcPlugin.py | 74 +++++++
> .../tc-testing/creating-plugins/AddingPlugins.txt | 104 ++++++++++
> .../selftests/tc-testing/plugin-lib/README-PLUGINS | 27 +++
> .../selftests/tc-testing/plugins/__init__.py | 0
> tools/testing/selftests/tc-testing/tdc.py | 221
> +++++++++++++++------
> 5 files changed, 368 insertions(+), 58 deletions(-)
> create mode 100644 tools/testing/selftests/tc-testing/TdcPlugin.py
> create mode 100644
> tools/testing/selftests/tc-testing/creating-plugins/AddingPlugins.txt
> create mode 100644
> tools/testing/selftests/tc-testing/plugin-lib/README-PLUGINS
> create mode 100644 tools/testing/selftests/tc-testing/plugins/__init__.py
>
> diff --git a/tools/testing/selftests/tc-testing/TdcPlugin.py
> b/tools/testing/selftests/tc-testing/TdcPlugin.py
> new file mode 100644
> index 000000000000..3ee9a6dacb52
> --- /dev/null
> +++ b/tools/testing/selftests/tc-testing/TdcPlugin.py
> @@ -0,0 +1,74 @@
> +#!/usr/bin/env python3
> +
> +class TdcPlugin:
> + def __init__(self):
> + super().__init__()
> + print(' -- {}.__init__'.format(self.sub_class))
> +
> + def pre_suite(self, testcount, testidlist):
> + '''run commands before test_runner goes into a test loop'''
> + self.testcount = testcount
> + self.testidlist = testidlist
> + if self.args.verbose > 1:
> + print(' -- {}.pre_suite'.format(self.sub_class))
> +
> + def post_suite(self, index):
> + '''run commands after test_runner completes the test loop
> + index is the last ordinal number of test that was attempted'''
> + if self.args.verbose > 1:
> + print(' -- {}.post_suite'.format(self.sub_class))
> +
> + def pre_case(self, test_ordinal, testid):
> + '''run commands before test_runner does one test'''
> + if self.args.verbose > 1:
> + print(' -- {}.pre_case'.format(self.sub_class))
> + self.args.testid = testid
> + self.args.test_ordinal = test_ordinal
> +
> + def post_case(self):
> + '''run commands after test_runner does one test'''
> + if self.args.verbose > 1:
> + print(' -- {}.post_case'.format(self.sub_class))
> +
> + def pre_execute(self):
> + '''run command before test-runner does the execute step'''
> + if self.args.verbose > 1:
> + print(' -- {}.pre_execute'.format(self.sub_class))
> +
> + def post_execute(self):
> + '''run command after test-runner does the execute step'''
> + if self.args.verbose > 1:
> + print(' -- {}.post_execute'.format(self.sub_class))
> +
> + def adjust_command(self, stage, command):
> + '''adjust the command'''
> + if self.args.verbose > 1:
> + print(' -- {}.adjust_command {}'.format(self.sub_class, stage))
> +
> + # if stage == 'pre':
> + # pass
> + # elif stage == 'setup':
> + # pass
> + # elif stage == 'execute':
> + # pass
> + # elif stage == 'verify':
> + # pass
> + # elif stage == 'teardown':
> + # pass
> + # elif stage == 'post':
> + # pass
> + # else:
> + # pass
> +
> + return command
> +
> + def add_args(self, parser):
> + '''Get the plugin args from the command line'''
> + self.argparser = parser
> + return self.argparser
> +
> + def check_args(self, args, remaining):
> + '''Check that the args are set correctly'''
> + self.args = args
> + if self.args.verbose > 1:
> + print(' -- {}.check_args'.format(self.sub_class))
> diff --git
> a/tools/testing/selftests/tc-testing/creating-plugins/AddingPlugins.txt
> b/tools/testing/selftests/tc-testing/creating-plugins/AddingPlugins.txt
> new file mode 100644
> index 000000000000..c18f88d09360
> --- /dev/null
> +++ b/tools/testing/selftests/tc-testing/creating-plugins/AddingPlugins.txt
> @@ -0,0 +1,104 @@
> +tdc - Adding plugins for tdc
> +
> +Author: Brenda J. Butler - b...@mojatatu.com
> +
> +ADDING PLUGINS
> +--------------
> +
> +A new plugin should be written in python as a class that inherits from
> TdcPlugin.
> +There are some examples in plugin-lib.
> +
> +The plugin can be used to add functionality to the test framework,
> +such as:
> +
> +- adding commands to be run before and/or after the test suite
> +- adding commands to be run before and/or after the test cases
> +- adding commands to be run before and/or after the execute phase of the
> test cases
> +- ability to alter the command to be run in any phase:
> + pre (the pre-suite stage)
> + prepare
> + execute
> + verify
> + teardown
> + post (the post-suite stage)
> +- ability to add to the command line args, and use them at run time
> +
> +
> +The functions in the class should follow the following interfaces:
> +
> + def __init__(self)
> + def pre_suite(self, testcount, testidlist) # see "PRE_SUITE" below
> + def post_suite(self, ordinal) # see "SKIPPING" below
> + def pre_case(self, test_ordinal, testid) # see "PRE_CASE" below
> + def post_case(self)
> + def pre_execute(self)
> + def post_execute(self)
> + def adjust_command(self, stage, command) # see "ADJUST" below
> + def add_args(self, parser) # see "ADD_ARGS" below
> + def check_args(self, args, remaining) # see "CHECK_ARGS" below
> +
> +
> +PRE_SUITE
> +
> +This method takes a testcount (number of tests to be run) and
> +testidlist (array of test ids for tests that will be run). This is
> +useful for various things, including when an exception occurs and the
> +rest of the tests must be skipped. The info is stored in the object,
> +and the post_suite method can refer to it when dumping the "skipped"
> +TAP output. The tdc.py script will do that for the test suite as
> +defined in the test case, but if the plugin is being used to run extra
> +tests on each test (eg, check for memory leaks on associated
> +co-processes) then that other tap output can be generated in the
> +post-suite method using this info passed in to the pre_suite method.
> +
> +
> +SKIPPING
> +
> +The post_suite method will receive the ordinal number of the last
> +test to be attempted. It can use this info when outputting
> +the TAP output for the extra test cases.
> +
> +
> +PRE_CASE
> +
> +The pre_case method will receive the ordinal number of the test
> +and the test id. Useful for outputing the extra test results.
> +
> +
> +ADJUST
> +
> +The adjust_command method receives a string representing
> +the execution stage and a string which is the actual command to be
> +executed. The plugin can adjust the command, based on the stage of
> +execution.
> +
> +The stages are represented by the following strings:
> +
> + 'pre'
> + 'setup'
> + 'command'
> + 'verify'
> + 'teardown'
> + 'post'
> +
> +The adjust_command method must return the adjusted command so tdc
> +can use it.
> +
> +
> +ADD_ARGS
> +
> +The add_args method receives the argparser object and can add
> +arguments to it. Care should be taken that the new arguments do not
> +conflict with any from tdc.py or from other plugins that will be used
> +concurrently.
> +
> +The add_args method should return the argparser object.
> +
> +
> +CHECK_ARGS
> +
> +The check_args method is so that the plugin can do validation on
> +the args, if needed. If there is a problem, and Exception should
> +be raised, with a string that explains the problem.
> +
> +eg: raise Exception('plugin xxx, arg -y is wrong, fix it')
> diff --git a/tools/testing/selftests/tc-testing/plugin-lib/README-PLUGINS
> b/tools/testing/selftests/tc-testing/plugin-lib/README-PLUGINS
> new file mode 100644
> index 000000000000..aa8a2669702b
> --- /dev/null
> +++ b/tools/testing/selftests/tc-testing/plugin-lib/README-PLUGINS
> @@ -0,0 +1,27 @@
> +tdc.py will look for plugins in a directory plugins off the cwd.
> +Make a set of numbered symbolic links from there to the actual plugins.
> +Eg:
> +
> +tdc.py
> +plugin-lib/
> +plugins/
> + __init__.py
> + 10-rootPlugin.py -> ../plugin-lib/rootPlugin.py
> + 20-valgrindPlugin.py -> ../plugin-lib/valgrindPlugin.py
> + 30-nsPlugin.py -> ../plugin-lib/nsPlugin.py
> +
> +
> +tdc.py will find them and use them.
> +
> +
> +rootPlugin
> + Check if the uid is root. If not, bail out.
> +
> +valgrindPlugin
> + Run the command under test with valgrind, and produce an extra set of
> TAP results for the memory tests.
> + This plugin will write files to the cwd, called vgnd-xxx.log. These
> will contain
> + the valgrind output for test xxx. Any file matching the glob
> 'vgnd-*.log' will be
> + deleted at the end of the run.
> +
> +nsPlugin
> + Run all the commands in a network namespace.
> diff --git a/tools/testing/selftests/tc-testing/plugins/__init__.py
> b/tools/testing/selftests/tc-testing/plugins/__init__.py
> new file mode 100644
> index 000000000000..e69de29bb2d1
> diff --git a/tools/testing/selftests/tc-testing/tdc.py
> b/tools/testing/selftests/tc-testing/tdc.py
> index a2624eda34db..3e6f9f2e1691 100755
> --- a/tools/testing/selftests/tc-testing/tdc.py
> +++ b/tools/testing/selftests/tc-testing/tdc.py
> @@ -11,17 +11,91 @@ import re
> import os
> import sys
> import argparse
> +import importlib
> import json
> import subprocess
> +import time
> from collections import OrderedDict
> from string import Template
>
> from tdc_config import *
> from tdc_helper import *
>
> +import TdcPlugin
>
> USE_NS = True
>
> +class PluginMgr:
> + def __init__(self, argparser):
> + super().__init__()
> + self.plugins = {}
> + self.plugin_instances = []
> + self.args = []
> + self.argparser = argparser
> +
> + # TODO, put plugins in order
> + plugindir = os.getenv('TDC_PLUGIN_DIR', './plugins')
> + for dirpath, dirnames, filenames in os.walk(plugindir):
> + for fn in filenames:
> + if (fn.endswith('.py') and
> + not fn == '__init__.py' and
> + not fn.startswith('#') and
> + not fn.startswith('.#')):
> + mn = fn[0:-3]
> + foo = importlib.import_module('plugins.' + mn)
> + self.plugins[mn] = foo
> + self.plugin_instances.append(foo.SubPlugin())
> +
> + def call_pre_suite(self, testcount, testidlist):
> + for pgn_inst in self.plugin_instances:
> + pgn_inst.pre_suite(testcount, testidlist)
> +
> + def call_post_suite(self, index):
> + for pgn_inst in reversed(self.plugin_instances):
> + pgn_inst.post_suite(index)
> +
> + def call_pre_case(self, test_ordinal, testid):
> + for pgn_inst in self.plugin_instances:
> + try:
> + pgn_inst.pre_case(test_ordinal, testid)
> + except Exception as ee:
> + print('exception {} in call to pre_case for {} plugin'.
> + format(ee, pgn_inst.__class__))
> + print('test_ordinal is {}'.format(test_ordinal))
> + print('testid is {}'.format(testid))
> + raise
> +
> + def call_post_case(self):
> + for pgn_inst in reversed(self.plugin_instances):
> + pgn_inst.post_case()
> +
> + def call_pre_execute(self):
> + for pgn_inst in self.plugin_instances:
> + pgn_inst.pre_execute()
> +
> + def call_post_execute(self):
> + for pgn_inst in reversed(self.plugin_instances):
> + pgn_inst.post_execute()
> +
> + def call_add_args(self, parser):
> + for pgn_inst in self.plugin_instances:
> + parser = pgn_inst.add_args(parser)
> + return parser
> +
> + def call_check_args(self, args, remaining):
> + for pgn_inst in self.plugin_instances:
> + pgn_inst.check_args(args, remaining)
> +
> + def call_adjust_command(self, stage, command):
> + for pgn_inst in self.plugin_instances:
> + command = pgn_inst.adjust_command(stage, command)
> + return command
> +
> + @staticmethod
> + def _make_argparser(args):
> + self.argparser = argparse.ArgumentParser(
> + description='Linux TC unit tests')
> +
>
> def replace_keywords(cmd):
> """
> @@ -33,21 +107,27 @@ def replace_keywords(cmd):
> return subcmd
>
>
> -def exec_cmd(command, nsonly=True):
> +def exec_cmd(args, pm, stage, command, nsonly=True):
> """
> Perform any required modifications on an executable command, then run
> it in a subprocess and return the results.
> """
> + if len(command.strip()) == 0:
> + return None, None
> if (USE_NS and nsonly):
> command = 'ip netns exec $NS ' + command
>
> if '$' in command:
> command = replace_keywords(command)
>
> + command = pm.call_adjust_command(stage, command)
> + if args.verbose > 0:
> + print('command "{}"'.format(command))
> proc = subprocess.Popen(command,
> shell=True,
> stdout=subprocess.PIPE,
> - stderr=subprocess.PIPE)
> + stderr=subprocess.PIPE,
> + env=ENVIR)
> (rawout, serr) = proc.communicate()
>
> if proc.returncode != 0 and len(serr) > 0:
> @@ -60,69 +140,85 @@ def exec_cmd(command, nsonly=True):
> return proc, foutput
>
>
> -def prepare_env(cmdlist):
> +def prepare_env(args, pm, stage, prefix, cmdlist):
> """
> - Execute the setup/teardown commands for a test case. Optionally
> - terminate test execution if the command fails.
> + Execute the setup/teardown commands for a test case.
> + Optionally terminate test execution if the command fails.
> """
> + if args.verbose > 0:
> + print('{}'.format(prefix))
> for cmdinfo in cmdlist:
> - if (type(cmdinfo) == list):
> + if isinstance(cmdinfo, list):
> exit_codes = cmdinfo[1:]
> cmd = cmdinfo[0]
> else:
> exit_codes = [0]
> cmd = cmdinfo
>
> - if (len(cmd) == 0):
> + if not cmd:
> continue
>
> - (proc, foutput) = exec_cmd(cmd)
> + (proc, foutput) = exec_cmd(args, pm, stage, cmd)
>
> - if proc.returncode not in exit_codes:
> - print
> - print("Could not execute:")
> - print(cmd)
> - print("\nError message:")
> - print(foutput)
> - print("\nAborting test run.")
> - # ns_destroy()
> - raise Exception('prepare_env did not complete successfully')
> + if proc and (proc.returncode not in exit_codes):
> + print('', file=sys.stderr)
> + print("{} *** Could not execute: \"{}\"".format(prefix, cmd),
> + file=sys.stderr)
> + print("\n{} *** Error message: \"{}\"".format(prefix, foutput),
> + file=sys.stderr)
> + print("\n{} *** Aborting test run.".format(prefix),
> file=sys.stderr)
> + print("\n\n{} *** stdout ***".format(proc.stdout),
> file=sys.stderr)
> + print("\n\n{} *** stderr ***".format(proc.stderr),
> file=sys.stderr)
> + raise Exception('"{}" did not complete
> successfully'.format(prefix))
>
> -def run_one_test(index, tidx):
> +def run_one_test(pm, args, index, tidx):
> result = True
> tresult = ""
> tap = ""
> + if args.verbose > 0:
> + print("\t====================\n=====> ", end="")
> print("Test " + tidx["id"] + ": " + tidx["name"])
> - prepare_env(tidx["setup"])
> - (p, procout) = exec_cmd(tidx["cmdUnderTest"])
> +
> + pm.call_pre_case(index, tidx['id'])
> + prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"])
> +
> + if (args.verbose > 0):
> + print('-----> execute stage')
> + pm.call_pre_execute()
> + (p, procout) = exec_cmd(args, pm, 'execute', tidx["cmdUnderTest"])
> exit_code = p.returncode
> + pm.call_post_execute()
>
> if (exit_code != int(tidx["expExitCode"])):
> result = False
> print("exit:", exit_code, int(tidx["expExitCode"]))
> print(procout)
> else:
> - match_pattern = re.compile(str(tidx["matchPattern"]),
> - re.DOTALL | re.MULTILINE)
> - (p, procout) = exec_cmd(tidx["verifyCmd"])
> + if args.verbose > 0:
> + print('-----> verify stage')
> + match_pattern = re.compile(
> + str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
> + (p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"])
> match_index = re.findall(match_pattern, procout)
> if len(match_index) != int(tidx["matchCount"]):
> result = False
>
> if not result:
> - tresult += "not "
> - tresult += "ok {} - {} # {}\n".format(str(index), tidx['id'],
> tidx["name"])
> + tresult += 'not '
> + tresult += 'ok {} - {} # {}\n'.format(str(index), tidx['id'],
> tidx['name'])
> tap += tresult
>
> if result == False:
> tap += procout
>
> - prepare_env(tidx["teardown"])
> + prepare_env(args, pm, 'teardown', '-----> teardown stage',
> tidx['teardown'])
> + pm.call_post_case()
> +
> index += 1
>
> return tap
>
> -def test_runner(filtered_tests, args):
> +def test_runner(pm, args, filtered_tests):
> """
> Driver function for the unit tests.
>
> @@ -135,63 +231,71 @@ def test_runner(filtered_tests, args):
> tcount = len(testlist)
> index = 1
> tap = str(index) + ".." + str(tcount) + "\n"
> + badtest = None
>
> + pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
> +
> + if args.verbose > 1:
> + print('Run tests here')
> for tidx in testlist:
> if "flower" in tidx["category"] and args.device == None:
> continue
> try:
> badtest = tidx # in case it goes bad
> - tap += run_one_test(index, tidx)
> + tap += run_one_test(pm, args, index, tidx)
> except Exception as ee:
> print('Exception {} (caught in test_runner, running test {} {}
> {})'.
> format(ee, index, tidx['id'], tidx['name']))
> break
> index += 1
>
> + # if we failed in setup or teardown,
> + # fill in the remaining tests with not ok
> count = index
> tap += 'about to flush the tap output if tests need to be skipped\n'
> if tcount + 1 != index:
> for tidx in testlist[index - 1:]:
> msg = 'skipped - previous setup or teardown failed'
> - tap += 'ok {} - {} # {} {} {} \n'.format(
> + tap += 'ok {} - {} # {} {} {}\n'.format(
> count, tidx['id'], msg, index, badtest.get('id',
> '--Unknown--'))
> count += 1
>
> tap += 'done flushing skipped test tap output\n'
> + pm.call_post_suite(index)
>
> return tap
>
>
> -def ns_create():
> +def ns_create(args, pm):
> """
> Create the network namespace in which the tests will be run and set up
> the required network devices for it.
> """
> if (USE_NS):
> cmd = 'ip netns add $NS'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
> cmd = 'ip link add $DEV0 type veth peer name $DEV1'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
> cmd = 'ip link set $DEV1 netns $NS'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
> cmd = 'ip link set $DEV0 up'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
> cmd = 'ip -n $NS link set $DEV1 up'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
> cmd = 'ip link set $DEV2 netns $NS'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
> cmd = 'ip -n $NS link set $DEV2 up'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'pre', cmd, False)
>
>
> -def ns_destroy():
> +def ns_destroy(args, pm):
> """
> Destroy the network namespace for testing (and any associated network
> devices as well)
> """
> if (USE_NS):
> cmd = 'ip netns delete $NS'
> - exec_cmd(cmd, False)
> + exec_cmd(args, pm, 'post', cmd, False)
>
>
> def has_blank_ids(idlist):
> @@ -272,10 +376,10 @@ def set_args(parser):
> return parser
>
>
> -def check_default_settings(args):
> +def check_default_settings(args, remaining, pm):
> """
> - Process any arguments overriding the default settings, and ensure the
> - settings are correct.
> + Process any arguments overriding the default settings,
> + and ensure the settings are correct.
> """
> # Allow for overriding specific settings
> global NAMES
> @@ -288,6 +392,8 @@ def check_default_settings(args):
> print("The specified tc path " + NAMES['TC'] + " does not exist.")
> exit(1)
>
> + pm.call_check_args(args, remaining)
> +
>
> def get_id_list(alltests):
> """
> @@ -301,16 +407,7 @@ def check_case_id(alltests):
> Check for duplicate test case IDs.
> """
> idl = get_id_list(alltests)
> - # print('check_case_id: idl is {}'.format(idl))
> - # answer = list()
> - # for x in idl:
> - # print('Looking at {}'.format(x))
> - # print('what the heck is idl.count(x)??? {}'.format(idl.count(x)))
> - # if idl.count(x) > 1:
> - # answer.append(x)
> - # print(' ... append it {}'.format(x))
> return [x for x in idl if idl.count(x) > 1]
> - return answer
>
>
> def does_id_exist(alltests, newid):
> @@ -403,7 +500,7 @@ def get_test_cases(args):
>
> for ff in args.file:
> if not os.path.isfile(ff):
> - print("IGNORING file " + ff + " \n\tBECAUSE does not exist.")
> + print("IGNORING file " + ff + "\n\tBECAUSE does not exist.")
> else:
> flist.append(os.path.abspath(ff))
>
> @@ -445,7 +542,7 @@ def get_test_cases(args):
> return allcatlist, allidlist, testcases_by_cats, alltestcases
>
>
> -def set_operation_mode(args):
> +def set_operation_mode(pm, args):
> """
> Load the test case data and process remaining arguments to determine
> what the script should do for this run, and call the appropriate
> @@ -486,12 +583,15 @@ def set_operation_mode(args):
> print("This script must be run with root privileges.\n")
> exit(1)
>
> - ns_create()
> + ns_create(args, pm)
>
> - catresults = test_runner(alltests, args)
> + if len(alltests):
> + catresults = test_runner(pm, args, alltests)
> + else:
> + catresults = 'No tests found\n'
> print('All test results: \n\n{}'.format(catresults))
>
> - ns_destroy()
> + ns_destroy(args, pm)
>
>
> def main():
> @@ -501,10 +601,15 @@ def main():
> """
> parser = args_parse()
> parser = set_args(parser)
> + pm = PluginMgr(parser)
> + parser = pm.call_add_args(parser)
> (args, remaining) = parser.parse_known_args()
> - check_default_settings(args)
> + args.NAMES = NAMES
> + check_default_settings(args, remaining, pm)
> + if args.verbose > 2:
> + print('args is {}'.format(args))
>
> - set_operation_mode(args)
> + set_operation_mode(pm, args)
>
> exit(0)
>
> --
> 2.15.1
>
