This patch needs to hold - you can discard it. Thanks. On Thu, Mar 11, 2010 at 11:38 AM, Martin Bligh <[email protected]> wrote:
> did that change any code, or you still want this patch applied as-is? > > On Mon, Mar 8, 2010 at 4:17 PM, Scott Zawalski <[email protected]> wrote: > > fyi this showed up way late in my inbox truty already started a code > review > > that is in progress internally for this. > > > > -Scott > > > > > > On Mon, Mar 8, 2010 at 12:48, Mike Truty <[email protected]> wrote: > >> > >> Is there a good tool that you use to generate this patch email? I'm > >> generating it by hand with git diff and there must be a better way. > >> Thanks, > >> Mike > >> We wanted to have a little more flexibility with test_importer.py for > >> chromeos. For example we wanted to import > >> all tests from client/site_tests and server/site_tests but only some > from > >> client/tests and server/tests. > >> We've added the following: > >> -The ability to pass a whitelist with -w that specifies a subset of > tests > >> within a tests/profilers/samples directory. > >> This option is incompatible with -a because if you're using -a you're > >> probably already managing your tests by > >> adding/deleting files. There is a subtle implication that if you > specify > >> -c, -t and -w (we do) tests under the > >> tests_dir not in the whitelist will be cleaned (we want this). We have > >> not created a site_test_importer.sh > >> that individually calls test_importer.py 4 different times with '-c -t > >> client/tests -w file', '-t client/site_tests', > >> '-t server/tests -w file' and '-t server/site_tests'. > >> -The ability to adjust any test attributes as a site-specific extension. > >> We will use the file site_set_attributes.py > >> if it exists and it should include the function: 'def > >> _set_attributes_custom(test, data)'. > >> We use this to adjust the following: > >> -Use the directory name instead of control-file NAME (handles multiple > >> control files (control.xx) as dirname.xx. > >> -Globally set run_verify to 0 > >> diff --git a/utils/test_importer.py b/utils/test_importer.py > >> index 4074a8e..b392366 100755 > >> --- a/utils/test_importer.py > >> +++ b/utils/test_importer.py > >> @@ -30,7 +30,7 @@ import common > >> import logging, re, os, sys, optparse, compiler > >> from autotest_lib.frontend import setup_django_environment > >> from autotest_lib.frontend.afe import models > >> -from autotest_lib.client.common_lib import control_data > >> +from autotest_lib.client.common_lib import control_data, utils > >> > >> > >> logging.basicConfig(level=logging.DEBUG) > >> @@ -78,7 +78,8 @@ def update_all(autotest_dir, add_noncompliant, > >> add_experimental, verbose): > >> db_clean_broken(autotest_dir, verbose) > >> > >> > >> -def update_samples(autotest_dir, add_noncompliant, add_experimental, > >> verbose): > >> +def update_samples(autotest_dir, add_noncompliant, add_experimental, > >> + verbose, whitelist_set=None): > >> sample_path = os.path.join(autotest_dir, 'server/samples') > >> if os.path.exists(sample_path): > >> if verbose: > >> @@ -88,7 +89,7 @@ def update_samples(autotest_dir, add_noncompliant, > >> add_experimental, verbose): > >> update_tests_in_db(tests, add_experimental=add_experimental, > >> add_noncompliant=add_noncompliant, > >> autotest_dir=autotest_dir, > >> - verbose=verbose) > >> + verbose=verbose, > whitelist_set=whitelist_set) > >> > >> > >> def db_clean_broken(autotest_dir, verbose): > >> @@ -114,10 +115,46 @@ def db_clean_broken(autotest_dir, verbose): > >> _log_or_execute(repr(profiler), profiler.delete) > >> > >> > >> +def db_clean_from_whitelist(autotest_dir, verbose, > >> + tests_dir, whitelist_set): > >> + """Remove tests from autotest_web which are under tests_dir but > >> + not called out by the whitelist. > >> + > >> + Arguments: > >> + autotest_dir: base directory > >> + verbose: for logging > >> + tests_dir: to filter tests > >> + whitelist_set: to filter tests > >> + """ > >> + tests_dir_len = len(tests_dir) > >> + for test in models.Test.objects.all(): > >> + full_path = os.path.join(autotest_dir, test.path) > >> + if full_path[:tests_dir_len] == tests_dir: > >> + if not (full_path + '\n') in whitelist_set: > >> + if verbose: > >> + print "Removing %s - not in whitelist" % > (test.path) > >> + _log_or_execute(repr(test), test.delete) > >> + > >> + # Find profilers that are no longer present > >> + if os.path.join("client", "profilers") == tests_dir: > >> + for profiler in models.Profiler.objects.all(): > >> + full_path = os.path.join(autotest_dir, "client", > "profilers", > >> + profiler.name) > >> + if not full_path in whitelist_set: > >> + if verbose: > >> + print "Removing %s - not in whitelist" % > (test.path) > >> + _log_or_execute(repr(profiler), profiler.delete) > >> + > >> + > >> def update_profilers_in_db(profilers, verbose=False, description='NA', > >> - add_noncompliant=False): > >> + add_noncompliant=False, whitelist_set=None): > >> """Update profilers in autotest_web database""" > >> for profiler in profilers: > >> + if whitelist_set and not (profiler + '\n') in whitelist_set: > >> + if verbose: > >> + print "Skipping %s - not on whitelist" % (profiler) > >> + continue > >> + > >> name = os.path.basename(profiler).rstrip(".py") > >> if not profilers[profiler]: > >> if add_noncompliant: > >> @@ -134,9 +171,17 @@ def update_profilers_in_db(profilers, > verbose=False, > >> description='NA', > >> > >> def update_tests_in_db(tests, dry_run=False, add_experimental=False, > >> add_noncompliant=False, verbose=False, > >> - autotest_dir=None): > >> + autotest_dir=None, whitelist_set=None): > >> """Update or add each test to the database""" > >> + site_set_attributes_module = utils.import_site_module( > >> + __file__, 'autotest_lib.utils.site_set_attributes') > >> + > >> for test in tests: > >> + if whitelist_set and not (test + '\n') in whitelist_set: > >> + if verbose: > >> + print "Skipping %s - not on whitelist" % (test) > >> + continue > >> + > >> new_test = models.Test.objects.get_or_create( > >> path=test.replace(autotest_dir, '').lstrip('/'))[0] > >> if verbose: > >> @@ -146,6 +191,10 @@ def update_tests_in_db(tests, dry_run=False, > >> add_experimental=False, > >> data = tests[test] > >> _set_attributes_clean(new_test, data) > >> > >> + # Custom Attribute Update > >> + if site_set_attributes_module: > >> + site_set_attributes_module._set_attributes_custom(new_test, > >> data) > >> + > >> # This only takes place if --add-noncompliant is provided on > the > >> CLI > >> if not new_test.name: > >> test_new_test = test.split('/') > >> @@ -346,6 +395,8 @@ def main(argv): > >> parser.add_option('-v', '--verbose', > >> dest='verbose', action='store_true', > default=False, > >> help='Run in verbose mode') > >> + parser.add_option('-w', '--whitelist-file', dest='whitelist_file', > >> + help='Filename for list of test names that must > >> match') > >> parser.add_option('-z', '--autotest_dir', dest='autotest_dir', > >> default=os.path.join(os.path.dirname(__file__), > >> '..'), > >> help='Autotest directory root') > >> @@ -365,14 +416,29 @@ def main(argv): > >> db_clean_broken(options.autotest_dir, options.verbose) > >> return 0 > >> > >> + whitelist_set = None > >> + if options.whitelist_file: > >> + if options.add_all: > >> + print "Cannot pass both --add-all and --whitelist-file" > >> + return 1 > >> + whitelist_path = os.path.abspath(options.whitelist_file) > >> + if not os.path.isfile(whitelist_path): > >> + print "--whitelist-file (%s) not found" % (whitelist_path) > >> + parser.print_help() > >> + return 1 > >> + elif options.verbose: > >> + print "Using whitelist file %s" % (whitelist_path) > >> + f = open(whitelist_path, 'r') > >> + whitelist_set = set(f.readlines()) > >> + f.close() > >> + > >> if options.add_all: > >> update_all(options.autotest_dir, options.add_noncompliant, > >> options.add_experimental, options.verbose) > >> if options.add_samples: > >> update_samples(options.autotest_dir, options.add_noncompliant, > >> - options.add_experimental, options.verbose) > >> - if options.clear_tests: > >> - db_clean_broken(options.autotest_dir, options.verbose) > >> + options.add_experimental, options.verbose, > >> + whitelist_set=whitelist_set) > >> if options.tests_dir: > >> options.tests_dir = os.path.abspath(options.tests_dir) > >> tests = get_tests_from_fs(options.tests_dir, > >> options.control_pattern, > >> @@ -380,12 +446,19 @@ def main(argv): > >> update_tests_in_db(tests, > >> add_experimental=options.add_experimental, > >> add_noncompliant=options.add_noncompliant, > >> autotest_dir=options.autotest_dir, > >> - verbose=options.verbose) > >> + verbose=options.verbose, > >> + whitelist_set=whitelist_set) > >> if options.profile_dir: > >> profilers = get_tests_from_fs(options.profile_dir, '.*py$') > >> update_profilers_in_db(profilers, verbose=options.verbose, > >> > add_noncompliant=options.add_noncompliant, > >> - description='NA') > >> + description='NA', > >> whitelist_set=whitelist_set) > >> + if options.clear_tests: > >> + db_clean_broken(options.autotest_dir, options.verbose) > >> + if options.whitelist_file and options.tests_dir: > >> + db_clean_from_whitelist(options.autotest_dir, > >> options.verbose, > >> + options.tests_dir, whitelist_set) > >> + > >> > >> > >> if __name__ == "__main__": > >> > >> > >> _______________________________________________ > >> Autotest mailing list > >> [email protected] > >> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest > >> > > > > > > _______________________________________________ > > Autotest mailing list > > [email protected] > > http://test.kernel.org/cgi-bin/mailman/listinfo/autotest > > > > > -- Thanks, Mike
_______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
