On 08/17/2012 07:53 PM, Edward Pilatowicz wrote:
On Thu, Aug 16, 2012 at 04:09:26PM -0600, Dave Vadovszki wrote:
On 07/23/2012 03:13 PM, Edward Pilatowicz wrote:
i think that if we suite generated the two reports (one for the shipping
code and one for the test suite) that'd be ideal. anyone doing work
could easily check if they've changed the coverage of the test suite
substantially. as for hudson i think we'd just want to display code
coverage for the shipping code since in the end that's the most
important.
ed
https://cr.opensolaris.org/action/browse/pkg/dvadovsz/7185359_v2/webrev/
This outputs two separate reports for proto and tests coverage. I
also added a parameter to -c to specify xml or html reporting for
coverage.
There's an interesting snag in displaying coverage for for tests/api
where Hudosn instead shows you a page telling you how to use it's
own Remote API. I'm still trying to figure out how to disable this
"feature".
you have some commented out lines of code.
otherwise it looks good to me.
thanks for generating the two sets of reports.
ed
I haven't seen any other comments, so this is probably good to go. I
removed the lines of commented debug code. Anyone care to do a push?
Dave
$ hg export --git 2783
# HG changeset patch
# User Dave Vadovszki <[email protected]>
# Date 1345842510 21600
# Node ID 16919088d48f51d85fea2cfbb552403d2c0ae7b7
# Parent 51fb9d5c5921d5c25220e102dcf66289417e2d81
7185359 testsuite should produce xml coverage with -c
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -37,8 +37,9 @@
^src/po/pkg.pot$
^src/tests/.coverage$
^src/tests/.coverage-.*$
+^src/tests/cov_proto.*$
+^src/tests/cov_tests.*$
^src/tests/.timing_history.txt$
-^src/tests/htmlcov$
^src/tests/ro_data/signing_certs/produced/.*/.*\.csr
^src/tests/ro_data/signing_certs/produced/.*\.old
^src/tests/ro_data/signing_certs/produced/.*\.tmp
diff --git a/src/tests/README b/src/tests/README
--- a/src/tests/README
+++ b/src/tests/README
@@ -14,14 +14,25 @@
run.py can also be run standalone with a variety of options:
- Usage: run.py [-ghpv] [-b filename] [-o regexp]
+ Usage: run.py [-ghptv] [-c format] [-b filename] [-o regexp]
+ run.py [-hptvx] [-c format] [-b filename] [-s regexp] [-o
regexp]
+ -a <dir> Archive failed test cases to <dir>/$pid/$testcasename
+ -b <filename> Baseline filename
+ -c <format> Collect code coverage data in xml or html format
+ -d Show debug output, including commands run, and
outputs
+ -f Show fail/error information even when test is
expected to fail
-g Generate result baseline
-h This help message
+ -j Parallelism
+ -o <regexp> Run only tests that match regexp
-p Parseable output format
+ -q Quiet output
+ -s <regexp> Run tests starting at regexp
+ -t Generate timing info file
+ -u Enable IPS GUI tests, disabled by default
-v Verbose output
- -b <filename> Baseline filename
- -o <regexp> Run only tests that match regexp
- -c Collect code coverage data
+ -x Stop after the first baseline mismatch
+ -z <port> Lowest port the test suite should use
When adding, removing, or changing the results of test cases, make
sure to update baseline.txt and check it in. This can be done with the
diff --git a/src/tests/run.py b/src/tests/run.py
--- a/src/tests/run.py
+++ b/src/tests/run.py
@@ -214,14 +214,14 @@
return suite_list
def usage():
- print >> sys.stderr, "Usage: %s [-cghptv] [-b filename] [-o
regexp]" \
- % sys.argv[0]
- print >> sys.stderr, " %s [-chptvx] [-b filename] [-s
regexp] "\
+ print >> sys.stderr, "Usage: %s [-ghptv] [-c format] [-b
filename] "\
"[-o regexp]" % sys.argv[0]
+ print >> sys.stderr, " %s [-hptvx] [-c format] [-b
filename] "\
+ "[-s regexp] [-o regexp]" % sys.argv[0]
print >> sys.stderr, \
""" -a <dir> Archive failed test cases to <dir>/$pid/$testcasename
-b <filename> Baseline filename
- -c Collect code coverage data
+ -c <format> Collect code coverage data in xml or html format
-d Show debug output, including commands run, and outputs
-f Show fail/error information even when test is
expected to fail
-g Generate result baseline
@@ -239,6 +239,43 @@
"""
sys.exit(2)
+def generate_coverage(cov_format, includes, omits, dest):
+ if cov_format == "html":
+ cov_option = "-d"
+ cov_dest = dest + "_html"
+ elif cov_format == "xml":
+ cov_option = "-o"
+ cov_dest = dest + ".xml"
+ else:
+ raise Exception("Unsupported coverage format")
+
+ cmd = ["coverage", cov_format]
+ if includes and len(includes):
+ cmd.extend(["--include", ",".join(includes)])
+ if omits and len(omits):
+ cmd.extend(["--omit", ",".join(omits)])
+ cmd.extend([cov_option, cov_dest])
+
+ print >> sys.stderr, "Generating coverage report via: ", "
".join(cmd)
+ if subprocess.Popen(cmd).wait() != 0:
+ raise Exception("Failed to generate coverage report!")
+
+ # The coverage data file and report are most likely owned by
+ # root, if a true test run was performed. Make the files owned
+ # by the owner of the test directory, so they can be easily
+ # removed.
+ try:
+ uid, gid = os.stat(".")[4:6]
+ os.chown(cov_dest, uid, gid)
+ if cov_format == "html":
+ for f in os.listdir(cov_dest):
+ os.chown("%s/%s" % (cov_dest, f), uid, gid)
+ except EnvironmentError:
+ pass
+
+
+
+
if __name__ == "__main__":
# Make all warnings be errors.
warnings.simplefilter('error')
@@ -251,7 +288,7 @@
# If you add options here, you need to also update
setup.py's
# test_func to include said options.
#
- opts, pargs = getopt.getopt(sys.argv[1:],
"a:cdfghj:pqtuvxb:o:s:z:",
+ opts, pargs = getopt.getopt(sys.argv[1:],
"a:c:dfghj:pqtuvxb:o:s:z:",
["generate-baseline", "parseable", "port", "timing",
"verbose", "baseline-file", "only"])
except getopt.GetoptError, e:
@@ -280,6 +317,7 @@
output = OUTPUT_PARSEABLE
if opt == "-c":
do_coverage = True
+ coverage_format = arg
if opt == "-d":
pkg5unittest.g_debug_output = True
if opt == "-f":
@@ -313,6 +351,7 @@
jobs = int(arg)
if opt == "-q":
quiet = True
+
if (bailonfail or startattest) and generate:
usage()
if quiet and (output != OUTPUT_DOTS):
@@ -329,6 +368,9 @@
cov.stop()
shutil.rmtree(covdir)
cov = None
+ elif not coverage_format in ("xml", "html"):
+ print >> sys.stderr, "-c <format> must be xml or html"
+ usage()
# Allow relative archive dir, but first convert it to abs. paths.
if archive_dir is not None:
@@ -467,31 +509,32 @@
subprocess.Popen(["coverage", "combine"],
env=newenv).wait()
os.rename("%s/pkg5" % covdir, ".coverage")
shutil.rmtree(covdir)
- vp = pkg5unittest.g_proto_area +
"/usr/lib/python2.6/vendor-packages"
+
omits = [
# These mako templates fail.
"*etc_pkg_*mako",
# Complex use of module importer makes this fail.
"*sysrepo_p5p.py"
]
- print >> sys.stderr, "Generating html coverage report"
- cmd = ["coverage", "html", "--omit", ",".join(omits),
- "-d", "htmlcov"]
- print >> sys.stderr, "# %s" % " ".join(cmd)
- if subprocess.Popen(cmd).wait() != 0:
- print >> sys.stderr, \
- "Failed to generate coverage report!"
- exitval = 1
+ proto = ["{0}/*".format(pkg5unittest.g_proto_area)]
+
+ try:
+ generate_coverage(coverage_format,
+ proto,
+ omits, "cov_proto")
+ generate_coverage(coverage_format, None,
+ proto + omits, "cov_tests")
+ except Exception, e:
+ print >> sys.stderr, e
+ exitval = 1
+
# The coverage data file and report are most likely
owned by
# root, if a true test run was performed. Make the
files owned
# by the owner of the test directory, so they can be
easily
# removed.
try:
uid, gid = os.stat(".")[4:6]
- os.chown("htmlcov", uid, gid)
os.chown(".coverage", uid, gid)
- for f in os.listdir("htmlcov"):
- os.chown("htmlcov/%s" % f, uid, gid)
except EnvironmentError:
pass
sys.exit(exitval)
--
Dave Vadovszki
(303)272-2686
500 Eldorado Blvd., Bldg.5
Broomfield, CO US 80021
_______________________________________________
pkg-discuss mailing list
[email protected]
http://mail.opensolaris.org/mailman/listinfo/pkg-discuss
