Matt Jordan has submitted this change and it was merged. Change subject: Memory Debugging Improvements ......................................................................
Memory Debugging Improvements * Enable XML output from valgrind. * Display and save a summary of valgrind errors and leaks. * Enable use of contrib/valgrind/suppressions.txt if it exists to suppress expected leaks or system library errors. Added entry to .gitignore for this file. * Supply a sample suppressions file that ignores some reachable memory. * Create stdout_print for printing messages to the terminal and including in the failure message. * Switch some failure notifications to use stdout_print() so the messages are included in asterisk-test-suite-report.xml. * Create function for archiving a list of files from source folder to destination folder. Switch all archive functions to use this. Valgrind Summaries require the lxml module. If this module is not found summaries will not be produced, but valgrind XML output will still be available in the Asterisk logs directory. Change-Id: I21634673508a01eea1f489c751d3cf75ea55cf06 --- M .gitignore M README.txt A contrib/valgrind/summary-lines.xsl A contrib/valgrind/suppressions-sample.txt M lib/python/asterisk/asterisk.py M runtests.py 6 files changed, 192 insertions(+), 47 deletions(-) Approvals: Matt Jordan: Looks good to me, approved; Verified Ashley Sanders: Looks good to me, but someone else must approve diff --git a/.gitignore b/.gitignore index cf96d9e..b7f92b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc asterisk-test-suite-report.xml /astroot +/contrib/valgrind/suppressions.txt /logs diff --git a/README.txt b/README.txt index 30027f2..f6c5b80 100644 --- a/README.txt +++ b/README.txt @@ -127,6 +127,7 @@ $ make update $ make install - python-twisted + - python-lxml - pjsua - Download and build pjproject 1.x from source - http://www.pjsip.org/download.htm diff --git a/contrib/valgrind/summary-lines.xsl b/contrib/valgrind/summary-lines.xsl new file mode 100644 index 0000000..c06faaf --- /dev/null +++ b/contrib/valgrind/summary-lines.xsl @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="utf-8"?> +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> +<!-- + The XML output of this stylesheet is used internally for producing + text based summary. The output is processed by runtests.py, + TestRun._process_valgrind. +--> + +<xsl:template match="/valgrindoutput"> + <xml> + <xsl:if test="error | suppcounts/pair"> + <line>Valgrind <xsl:value-of select="tool" /> results<xsl:if test="usercomment"> - <xsl:value-of select="usercomment" /></xsl:if></line> + <line /> + </xsl:if> + <xsl:if test="error/what"> + <line>================= Errors ==================</line> + <xsl:for-each select="error[what]"> + <cols> + <xsl:attribute name="col1"><xsl:value-of select="kind"/></xsl:attribute> + <xsl:attribute name="col2"><xsl:value-of select="what"/></xsl:attribute> + </cols> + </xsl:for-each> + <line /> + </xsl:if> + <xsl:if test="error/xwhat/leakedbytes"> + <line>============== Leaked Memory ==============</line> + <xsl:if test="error[kind='Leak_DefinitelyLost']/xwhat/leakedbytes"> + <cols col1="Definitely Lost"> + <xsl:attribute name="col2"> + <xsl:value-of select="sum(error[kind='Leak_DefinitelyLost']/xwhat/leakedbytes | error[kind='Leak_IndirectlyLost']/xwhat/leakedbytes)" /> bytes + </xsl:attribute> + </cols> + </xsl:if> + <xsl:if test="error[kind='Leak_PossiblyLost']/xwhat/leakedbytes"> + <cols col1="Possible Lost"> + <xsl:attribute name="col2"> + <xsl:value-of select="sum(error[kind='Leak_PossiblyLost']/xwhat/leakedbytes)" /> bytes + </xsl:attribute> + </cols> + </xsl:if> + <xsl:if test="error[kind='Leak_StillReachable']/xwhat/leakedbytes"> + <cols col1="Reachable Memory"> + <xsl:attribute name="col2"> + <xsl:value-of select="sum(error[kind='Leak_StillReachable']/xwhat/leakedbytes)" /> bytes + </xsl:attribute> + </cols> + </xsl:if> + <line /> + </xsl:if> + <xsl:if test="suppcounts/pair"> + <line>============== Suppressions ===============</line> + <xsl:for-each select="suppcounts/pair"> + <cols> + <xsl:attribute name="col1"><xsl:value-of select="count" /></xsl:attribute> + <xsl:attribute name="col2"><xsl:value-of select="name" /></xsl:attribute> + </cols> + </xsl:for-each> + <line/> + </xsl:if> + </xml> +</xsl:template> + +</xsl:stylesheet> diff --git a/contrib/valgrind/suppressions-sample.txt b/contrib/valgrind/suppressions-sample.txt new file mode 100644 index 0000000..1ee57a7 --- /dev/null +++ b/contrib/valgrind/suppressions-sample.txt @@ -0,0 +1,38 @@ +# +# This is a sample valgrind suppressions file for Asterisk. +# +# To use this suppression file with the testsuite, copy it to: +# contrib/valgrind/suppressions.txt +# +# This is meant for use when valgrind parameters include: +# --show-leak-kinds=all +# +# Extra valgrind options are read from: +# ~/.valgrindrc, $VALGRIND_OPTS, ./.valgrindrc +# +{ + <ast_threadstorage_get> + # The threadstorage for the main thread is not cleaned by exit(). + Memcheck:Leak + match-leak-kinds: reachable + fun:calloc + ... + fun:ast_threadstorage_get +} +{ + <ast_ssl_init> + # libasteriskssl doesn't cleanup the init items. + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:ast_ssl_init +} +{ + <load_dynamic_module> + # dlclose is not run on modules at shutdown. + # This does not suppress allocations from module_load. + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:load_dynamic_module +} diff --git a/lib/python/asterisk/asterisk.py b/lib/python/asterisk/asterisk.py index 5dcb784..53342ce 100755 --- a/lib/python/asterisk/asterisk.py +++ b/lib/python/asterisk/asterisk.py @@ -349,19 +349,29 @@ self.install_configs(os.getcwd() + "/configs", deps) self._setup_configs() - cmd = [ - self.ast_binary, - "-f", "-g", "-q", "-m", "-n", - "-C", "%s" % os.path.join(self.astetcdir, "asterisk.conf") - ] + cmd_prefix = [] if os.getenv("VALGRIND_ENABLE") == "true": valgrind_path = test_suite_utils.which('valgrind') if valgrind_path: - cmd = [valgrind_path] + cmd + cmd_prefix = [ + valgrind_path, + '--xml=yes', + '--xml-file=%s' % self.get_path("astlogdir", 'valgrind.xml'), + '--xml-user-comment=%s (%s)' % ( + os.environ['TESTSUITE_ACTIVE_TEST'], self.host)] + suppression_file = 'contrib/valgrind/suppressions.txt' + if os.path.exists(suppression_file): + cmd_prefix.append('--suppressions=%s' % suppression_file) else: LOGGER.error('Valgrind not found') + cmd = cmd_prefix + [ + self.ast_binary, + "-f", "-g", "-q", "-m", "-n", + "-C", "%s" % os.path.join(self.astetcdir, "asterisk.conf") + ] + # Make the start/stop deferreds - this method will return # the start deferred, and pass the stop deferred to the AsteriskProtocol # object. The stop deferred will be raised when the Asterisk process diff --git a/runtests.py b/runtests.py index f0fac91..f745266 100755 --- a/runtests.py +++ b/runtests.py @@ -20,6 +20,12 @@ import random import select +try: + import lxml.etree as ET +except: + # Ensure ET is defined + ET = None + # Re-open stdout so it's line buffered. # This allows timely processing of piped output. newfno = os.dup(sys.stdout.fileno()) @@ -59,10 +65,15 @@ assert self.test_name.startswith('tests/') self.test_relpath = self.test_name[6:] + def stdout_print(self, msg): + self.stdout += msg + "\n" + print msg + def run(self): self.passed = False self.did_run = True start_time = time.time() + os.environ['TESTSUITE_ACTIVE_TEST'] = self.test_name cmd = [ "%s/run-test" % self.test_name, ] @@ -71,9 +82,7 @@ cmd = ["./lib/python/asterisk/test_runner.py", "%s" % self.test_name] if os.path.exists(cmd[0]) and os.access(cmd[0], os.X_OK): - msg = "Running %s ..." % cmd - print msg - self.stdout += msg + "\n" + self.stdout_print("Running %s ..." % cmd) cmd.append(str(self.ast_version).rstrip()) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -91,8 +100,7 @@ l = p.stdout.readline() if not l: break - print l - self.stdout += l + self.stdout_print(l) except IOError: pass p.wait() @@ -100,22 +108,19 @@ # Sanitize p.returncode so it's always a boolean. did_pass = (p.returncode == 0) if did_pass and not self.test_config.expect_pass: - msg = "Test passed but was expected to fail." - print msg - self.stdout += msg + "\n" + self.stdout_print("Test passed but was expected to fail.") if not did_pass and not self.test_config.expect_pass: print "Test failed as expected." - - self.__parse_run_output(self.stdout) self.passed = (did_pass == self.test_config.expect_pass) core_dumps = self._check_for_core() if (len(core_dumps)): - print "Core dumps detected; failing test" + self.stdout_print("Core dumps detected; failing test") self.passed = False self._archive_core_dumps(core_dumps) + self._process_valgrind() self._process_ref_debug() if not self.passed: @@ -130,6 +135,7 @@ except: print "Unable to clean up directory for test %s (non-fatal)" % self.test_name + self.__parse_run_output(self.stdout) print 'Test %s %s\n' % (cmd, 'timedout' if timedout else 'passed' if self.passed else 'failed') else: @@ -189,6 +195,42 @@ 'run_%d' % run_num) return (run_num, run_dir, archive_dir) + def _process_valgrind(self): + (run_num, run_dir, archive_dir) = self._find_run_dirs() + if (run_num == 0): + return + if not ET: + return + + i = 1 + while os.path.isdir(os.path.join(run_dir, 'ast%d/var/log/asterisk' % i)): + ast_dir = "%s/ast%d/var/log/asterisk" % (run_dir, i) + valgrind_xml = os.path.join(ast_dir, 'valgrind.xml') + valgrind_txt = os.path.join(ast_dir, 'valgrind-summary.txt') + + # All instances either use valgrind or not. + if not os.path.exists(valgrind_xml): + return + + dom = ET.parse(valgrind_xml) + xslt = ET.parse('contrib/valgrind/summary-lines.xsl') + transform = ET.XSLT(xslt) + newdom = transform(dom) + lines = [] + for node in newdom.getroot(): + if node.tag == 'line': + lines.append((node.text or '').strip()) + elif node.tag == 'cols': + lines.append("%s: %s" % ( + node.attrib['col1'].strip().rjust(20), + node.attrib['col2'].strip())) + + self.stdout_print("\n".join(lines)) + with open(valgrind_txt, 'a') as txtfile: + txtfile.write("\n".join(lines)) + txtfile.close() + i += 1 + def _process_ref_debug(self): (run_num, run_dir, archive_dir) = self._find_run_dirs() if (run_num == 0): @@ -216,7 +258,7 @@ stdout=dest_file, stderr=subprocess.STDOUT) except Exception, e: - print "Exception occurred while processing REF_DEBUG" + self.stdout_print("Exception occurred while processing REF_DEBUG") finally: dest_file.close() if res != 0: @@ -228,20 +270,26 @@ os.path.join(dest_dir, "refs.txt")) hardlink_or_copy(refs_in, os.path.join(dest_dir, "refs")) - print "REF_DEBUG identified leaks, mark test as failure" + self.stdout_print("REF_DEBUG identified leaks, mark test as failure") self.passed = False i += 1 + + def _archive_files(self, src_dir, dest_dir, *filenames): + for filename in filenames: + try: + srcfile = os.path.join(src_dir, filename) + if os.path.exists(srcfile): + hardlink_or_copy(srcfile, os.path.join(dest_dir, filename)) + except Exception, e: + print "Exception occurred while archiving file '%s' to %s: %s" % ( + srcfile, dest_dir, e + ) def _archive_logs(self): (run_num, run_dir, archive_dir) = self._find_run_dirs() self._archive_ast_logs(run_num, run_dir, archive_dir) self._archive_pcap_dump(run_dir, archive_dir) - if os.path.exists(os.path.join(run_dir, 'messages.txt')): - hardlink_or_copy(os.path.join(run_dir, 'messages.txt'), - os.path.join(archive_dir, 'messages.txt')) - if os.path.exists(os.path.join(run_dir, 'full.txt')): - hardlink_or_copy(os.path.join(run_dir, 'full.txt'), - os.path.join(archive_dir, 'full.txt')) + self._archive_files(run_dir, archive_dir, 'messages.txt', 'full.txt') def _archive_ast_logs(self, run_num, run_dir, archive_dir): """Archive the Asterisk logs""" @@ -250,31 +298,13 @@ ast_dir = "%s/ast%d/var/log/asterisk" % (run_dir, i) dest_dir = os.path.join(archive_dir, 'ast%d/var/log/asterisk' % i) - try: - hardlink_or_copy(ast_dir + "/messages.txt", - dest_dir + "/messages.txt") - hardlink_or_copy(ast_dir + "/full.txt", - dest_dir + "/full.txt") - if os.path.exists(ast_dir + "/mmlog"): - hardlink_or_copy(ast_dir + "/mmlog", - dest_dir + "/mmlog") - except Exception, e: - print "Exception occurred while archiving logs from %s to %s: %s" % ( - ast_dir, dest_dir, e - ) + self._archive_files(ast_dir, dest_dir, + 'messages.txt', 'full.txt', 'mmlog', + 'valgrind.xml', 'valgrind-summary.txt') i += 1 def _archive_pcap_dump(self, run_dir, archive_dir): - filename = "dumpfile.pcap" - src = os.path.join(run_dir, filename) - dst = os.path.join(archive_dir, filename) - if os.path.exists(src): - try: - hardlink_or_copy(src, dst) - except Exception, e: - print "Exeception occured while archiving pcap file from %s to %s: %s" % ( - src, dst, e - ) + self._archive_files(run_dir, archive_dir, 'dumpfile.pcap') def __check_can_run(self, ast_version): """Check tags and dependencies in the test config.""" @@ -577,6 +607,8 @@ return 0 if options.valgrind: + if not ET: + print "python lxml module not loaded, text summaries from valgrind will not be produced.\n" os.environ["VALGRIND_ENABLE"] = "true" print "Running tests for Asterisk %s ...\n" % str(ast_version) -- To view, visit https://gerrit.asterisk.org/15 To unsubscribe, visit https://gerrit.asterisk.org/settings Gerrit-MessageType: merged Gerrit-Change-Id: I21634673508a01eea1f489c751d3cf75ea55cf06 Gerrit-PatchSet: 4 Gerrit-Project: testsuite Gerrit-Branch: master Gerrit-Owner: Corey Farrell <g...@cfware.com> Gerrit-Reviewer: Ashley Sanders <asand...@digium.com> Gerrit-Reviewer: Corey Farrell <g...@cfware.com> Gerrit-Reviewer: Matt Jordan <mjor...@digium.com> -- _____________________________________________________________________ -- Bandwidth and Colocation Provided by http://www.api-digital.com -- asterisk-dev mailing list To UNSUBSCRIBE or update options visit: http://lists.digium.com/mailman/listinfo/asterisk-dev