Is there a good tool that you use to generate this patch email? I'm
generating it by hand with git diff and there must be a better way.
Thanks,
Mike
We wanted to have a little more flexibility with test_importer.py for
chromeos. For example we wanted to import
all tests from client/site_tests and server/site_tests but only some from
client/tests and server/tests.
We've added the following:
-The ability to pass a whitelist with -w that specifies a subset of tests
within a tests/profilers/samples directory.
This option is incompatible with -a because if you're using -a you're
probably already managing your tests by
adding/deleting files. There is a subtle implication that if you specify
-c, -t and -w (we do) tests under the
tests_dir not in the whitelist will be cleaned (we want this). We have not
created a site_test_importer.sh
that individually calls test_importer.py 4 different times with '-c -t
client/tests -w file', '-t client/site_tests',
'-t server/tests -w file' and '-t server/site_tests'.
-The ability to adjust any test attributes as a site-specific extension. We
will use the file site_set_attributes.py
if it exists and it should include the function: 'def
_set_attributes_custom(test, data)'.
We use this to adjust the following:
-Use the directory name instead of control-file NAME (handles multiple
control files (control.xx) as dirname.xx.
-Globally set run_verify to 0
diff --git a/utils/test_importer.py b/utils/test_importer.py
index 4074a8e..b392366 100755
--- a/utils/test_importer.py
+++ b/utils/test_importer.py
@@ -30,7 +30,7 @@ import common
import logging, re, os, sys, optparse, compiler
from autotest_lib.frontend import setup_django_environment
from autotest_lib.frontend.afe import models
-from autotest_lib.client.common_lib import control_data
+from autotest_lib.client.common_lib import control_data, utils
logging.basicConfig(level=logging.DEBUG)
@@ -78,7 +78,8 @@ def update_all(autotest_dir, add_noncompliant,
add_experimental, verbose):
db_clean_broken(autotest_dir, verbose)
-def update_samples(autotest_dir, add_noncompliant, add_experimental,
verbose):
+def update_samples(autotest_dir, add_noncompliant, add_experimental,
+ verbose, whitelist_set=None):
sample_path = os.path.join(autotest_dir, 'server/samples')
if os.path.exists(sample_path):
if verbose:
@@ -88,7 +89,7 @@ def update_samples(autotest_dir, add_noncompliant,
add_experimental, verbose):
update_tests_in_db(tests, add_experimental=add_experimental,
add_noncompliant=add_noncompliant,
autotest_dir=autotest_dir,
- verbose=verbose)
+ verbose=verbose, whitelist_set=whitelist_set)
def db_clean_broken(autotest_dir, verbose):
@@ -114,10 +115,46 @@ def db_clean_broken(autotest_dir, verbose):
_log_or_execute(repr(profiler), profiler.delete)
+def db_clean_from_whitelist(autotest_dir, verbose,
+ tests_dir, whitelist_set):
+ """Remove tests from autotest_web which are under tests_dir but
+ not called out by the whitelist.
+
+ Arguments:
+ autotest_dir: base directory
+ verbose: for logging
+ tests_dir: to filter tests
+ whitelist_set: to filter tests
+ """
+ tests_dir_len = len(tests_dir)
+ for test in models.Test.objects.all():
+ full_path = os.path.join(autotest_dir, test.path)
+ if full_path[:tests_dir_len] == tests_dir:
+ if not (full_path + '\n') in whitelist_set:
+ if verbose:
+ print "Removing %s - not in whitelist" % (test.path)
+ _log_or_execute(repr(test), test.delete)
+
+ # Find profilers that are no longer present
+ if os.path.join("client", "profilers") == tests_dir:
+ for profiler in models.Profiler.objects.all():
+ full_path = os.path.join(autotest_dir, "client", "profilers",
+ profiler.name)
+ if not full_path in whitelist_set:
+ if verbose:
+ print "Removing %s - not in whitelist" % (test.path)
+ _log_or_execute(repr(profiler), profiler.delete)
+
+
def update_profilers_in_db(profilers, verbose=False, description='NA',
- add_noncompliant=False):
+ add_noncompliant=False, whitelist_set=None):
"""Update profilers in autotest_web database"""
for profiler in profilers:
+ if whitelist_set and not (profiler + '\n') in whitelist_set:
+ if verbose:
+ print "Skipping %s - not on whitelist" % (profiler)
+ continue
+
name = os.path.basename(profiler).rstrip(".py")
if not profilers[profiler]:
if add_noncompliant:
@@ -134,9 +171,17 @@ def update_profilers_in_db(profilers, verbose=False,
description='NA',
def update_tests_in_db(tests, dry_run=False, add_experimental=False,
add_noncompliant=False, verbose=False,
- autotest_dir=None):
+ autotest_dir=None, whitelist_set=None):
"""Update or add each test to the database"""
+ site_set_attributes_module = utils.import_site_module(
+ __file__, 'autotest_lib.utils.site_set_attributes')
+
for test in tests:
+ if whitelist_set and not (test + '\n') in whitelist_set:
+ if verbose:
+ print "Skipping %s - not on whitelist" % (test)
+ continue
+
new_test = models.Test.objects.get_or_create(
path=test.replace(autotest_dir, '').lstrip('/'))[0]
if verbose:
@@ -146,6 +191,10 @@ def update_tests_in_db(tests, dry_run=False,
add_experimental=False,
data = tests[test]
_set_attributes_clean(new_test, data)
+ # Custom Attribute Update
+ if site_set_attributes_module:
+ site_set_attributes_module._set_attributes_custom(new_test,
data)
+
# This only takes place if --add-noncompliant is provided on the
CLI
if not new_test.name:
test_new_test = test.split('/')
@@ -346,6 +395,8 @@ def main(argv):
parser.add_option('-v', '--verbose',
dest='verbose', action='store_true', default=False,
help='Run in verbose mode')
+ parser.add_option('-w', '--whitelist-file', dest='whitelist_file',
+ help='Filename for list of test names that must
match')
parser.add_option('-z', '--autotest_dir', dest='autotest_dir',
default=os.path.join(os.path.dirname(__file__),
'..'),
help='Autotest directory root')
@@ -365,14 +416,29 @@ def main(argv):
db_clean_broken(options.autotest_dir, options.verbose)
return 0
+ whitelist_set = None
+ if options.whitelist_file:
+ if options.add_all:
+ print "Cannot pass both --add-all and --whitelist-file"
+ return 1
+ whitelist_path = os.path.abspath(options.whitelist_file)
+ if not os.path.isfile(whitelist_path):
+ print "--whitelist-file (%s) not found" % (whitelist_path)
+ parser.print_help()
+ return 1
+ elif options.verbose:
+ print "Using whitelist file %s" % (whitelist_path)
+ f = open(whitelist_path, 'r')
+ whitelist_set = set(f.readlines())
+ f.close()
+
if options.add_all:
update_all(options.autotest_dir, options.add_noncompliant,
options.add_experimental, options.verbose)
if options.add_samples:
update_samples(options.autotest_dir, options.add_noncompliant,
- options.add_experimental, options.verbose)
- if options.clear_tests:
- db_clean_broken(options.autotest_dir, options.verbose)
+ options.add_experimental, options.verbose,
+ whitelist_set=whitelist_set)
if options.tests_dir:
options.tests_dir = os.path.abspath(options.tests_dir)
tests = get_tests_from_fs(options.tests_dir,
options.control_pattern,
@@ -380,12 +446,19 @@ def main(argv):
update_tests_in_db(tests,
add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir,
- verbose=options.verbose)
+ verbose=options.verbose,
+ whitelist_set=whitelist_set)
if options.profile_dir:
profilers = get_tests_from_fs(options.profile_dir, '.*py$')
update_profilers_in_db(profilers, verbose=options.verbose,
add_noncompliant=options.add_noncompliant,
- description='NA')
+ description='NA',
whitelist_set=whitelist_set)
+ if options.clear_tests:
+ db_clean_broken(options.autotest_dir, options.verbose)
+ if options.whitelist_file and options.tests_dir:
+ db_clean_from_whitelist(options.autotest_dir, options.verbose,
+ options.tests_dir, whitelist_set)
+
if __name__ == "__main__":
_______________________________________________
Autotest mailing list
[email protected]
http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
