HBASE-15938 submit-patch.py: Don't crash if there are tests with same name. Refactor: Split out flaky dashboard html template to separate file. (Apekshit)
Change-Id: Ie5875bdefbf886984a57dfc85661be2ac9592a7b Signed-off-by: stack <st...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9593a9f3 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9593a9f3 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9593a9f3 Branch: refs/heads/hbase-12439 Commit: 9593a9f39663e454a24f16c907e00e19cb65b903 Parents: 4ffea77 Author: Apekshit <apeksha...@gmail.com> Authored: Wed Jun 1 19:12:50 2016 -0700 Committer: stack <st...@apache.org> Committed: Thu Jun 2 08:55:42 2016 -0700 ---------------------------------------------------------------------- dev-support/findHangingTests.py | 14 ++- dev-support/flaky-dashboard-template.html | 122 +++++++++++++++++++++++++ dev-support/report-flakies.py | 112 +---------------------- 3 files changed, 137 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/findHangingTests.py ---------------------------------------------------------------------- diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py index 9ef8708..28f4895 100755 --- a/dev-support/findHangingTests.py +++ b/dev-support/findHangingTests.py @@ -46,14 +46,22 @@ def get_bad_tests(console_url): result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) if result1: test_case = result1.group(2) - hanging_tests.add(test_case) - all_tests.add(test_case) + if test_case in all_tests: + print ("ERROR! Multiple tests with same name '{}'. Might get wrong results " + "for this test.".format(test_case)) + else: + hanging_tests.add(test_case) + all_tests.add(test_case) result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\w*\.)*(\w*)", line) if result2: test_case = result2.group(2) - hanging_tests.remove(test_case) if "FAILURE!" in line: failed_tests.add(test_case) + if test_case not in hanging_tests: + print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results " + "for this test.".format(test_case)) + else: + hanging_tests.remove(test_case) result3 = re.match("^\s+(\w*).*\sTestTimedOut", line) if result3: test_case = result3.group(1) http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/flaky-dashboard-template.html ---------------------------------------------------------------------- diff --git a/dev-support/flaky-dashboard-template.html b/dev-support/flaky-dashboard-template.html new file mode 100644 index 0000000..77dfc86 --- /dev/null +++ b/dev-support/flaky-dashboard-template.html @@ -0,0 +1,122 @@ +<!-- + - Licensed to the Apache Software Foundation (ASF) under one + - or more contributor license agreements. See the NOTICE file + - distributed with this work for additional information + - regarding copyright ownership. The ASF licenses this file + - to you under the Apache License, Version 2.0 (the + - "License"); you may not use this file except in compliance + - with the License. You may obtain a copy of the License at + - + - http://www.apache.org/licenses/LICENSE-2.0 + - + - Unless required by applicable law or agreed to in writing, software + - distributed under the License is distributed on an "AS IS" BASIS, + - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + - See the License for the specific language governing permissions and + - limitations under the License. + --> +<!DOCTYPE html> +<html> +<head> + <title>Apache HBase Flaky Dashboard</title> + <style type="text/css"> + table { + table-layout: fixed; + } + th { + font-size: 15px; + } + td { + font-size: 18px; + vertical-align: text-top; + overflow: hidden; + white-space: nowrap; + } + .show_hide_button { + font-size: 100%; + padding: .5em 1em; + border: 0 rgba(0,0,0,0); + border-radius: 10px; + } + </style> +</head> +<body> +<p> + <img style="vertical-align:middle; display:inline-block;" height="80px" + src="https://hbase.apache.org/images/hbase_logo_with_orca_large.png";> + + <span style="font-size:50px; vertical-align:middle; display:inline-block;"> + Apache HBase Flaky Tests Dashboard + </span> +</p> +<br><br> +{% set counter = 0 %} +{% for url in results %} +{% set result = results[url] %} +{# Dedup ids since test names may duplicate across urls #} +{% set counter = counter + 1 %} + <span style="font-size:20px; font-weight:bold;">Job : {{ url |e }} + <a href="{{ url |e }}" style="text-decoration:none;">🔗</a></span> +<br/><br/> +<table> + <tr> + <th width="400px">Test Name</th> + <th width="150px">Flakyness</th> + <th width="200px">Failed/Timeout/Hanging</th> + <th>Run Ids</th> + </tr> + {% for test in result %} + {% set all = result[test]['all'] %} + {% set failed = result[test]['failed'] %} + {% set timeout = result[test]['timeout'] %} + {% set hanging = result[test]['hanging'] %} + {% set success = all.difference(failed).difference(hanging) %} + <tr> + <td>{{ test |e }}</td> + {% set flakyness = + (failed|length + hanging|length) * 100 / all|length %} + {% if flakyness == 100 %} + <td align="middle" style="background-color:#FF9999;"> + {% else %} + <td align="middle"> + {% endif %} + {{ "{:.1f}% ({} / {})".format( + flakyness, failed|length + hanging|length, all|length) }} + </td> + <td align="middle"> + {{ failed|length }} / {{ timeout|length }} / {{ hanging|length }} + </td> + <td> + {% set id = "details_" ~ test ~ "_" ~ counter %} + <button class="show_hide_button" onclick="toggle('{{ id }}')"> + show/hide</button> + <br/> + <div id="{{ id }}" + style="display: none; width:500px; white-space: normal"> + {% macro print_run_ids(url, run_ids) -%} + {% for i in run_ids %} + <a href="{{ url }}/{{ i }}">{{ i }}</a> + {% endfor %} + {%- endmacro %} + Failed : {{ print_run_ids(url, failed) }}<br/> + Timed Out : {{ print_run_ids(url, timeout) }}<br/> + Hanging : {{ print_run_ids(url, hanging) }}<br/> + Succeeded : {{ print_run_ids(url, success) }} + </div> + </td> + </tr> + {% endfor %} +</table> +<br><br><br> +{% endfor %} +<script type="text/javascript"> + function toggle(id) { + if (document.getElementById(id).style["display"] == "none") { + document.getElementById(id).style["display"] = "block"; + } else { + document.getElementById(id).style["display"] = "none"; + } + } +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/report-flakies.py ---------------------------------------------------------------------- diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py index c0d16c7..676eca3 100755 --- a/dev-support/report-flakies.py +++ b/dev-support/report-flakies.py @@ -22,6 +22,7 @@ import argparse import findHangingTests from jinja2 import Template +import os import logging import requests @@ -177,114 +178,9 @@ if args.mvn: with open("./failed", "w") as file: file.write(",".join(all_failed_tests)) - -template = Template(""" - <!DOCTYPE html> - <html> - <head> - <title>Apache HBase Flaky Dashboard</title> - <style type="text/css"> - table { - table-layout: fixed; - } - th { - font-size: 15px; - } - td { - font-size: 18px; - vertical-align: text-top; - overflow: hidden; - white-space: nowrap; - } - .show_hide_button { - font-size: 100%; - padding: .5em 1em; - border: 0 rgba(0,0,0,0); - border-radius: 10px; - } - </style> - </head> - <body> - <p> - <img style="vertical-align:middle; display:inline-block;" height="80px" - src="https://hbase.apache.org/images/hbase_logo_with_orca_large.png";> - - <span style="font-size:50px; vertical-align:middle; display:inline-block;"> - Apache HBase Flaky Tests Dashboard - </span> - </p> - <br><br> - {% set counter = 0 %} - {% for url in results %} - {% set result = results[url] %} - {# Dedup ids since test names may duplicate across urls #} - {% set counter = counter + 1 %} - <span style="font-size:20px; font-weight:bold;">Job : {{ url |e }} - <a href="{{ url |e }}" style="text-decoration:none;">🔗</a></span> - <br/><br/> - <table> - <tr> - <th width="400px">Test Name</th> - <th width="150px">Flakyness</th> - <th width="200px">Failed/Timeout/Hanging</th> - <th>Run Ids</th> - </tr> - {% for test in result %} - {% set all = result[test]['all'] %} - {% set failed = result[test]['failed'] %} - {% set timeout = result[test]['timeout'] %} - {% set hanging = result[test]['hanging'] %} - {% set success = all.difference(failed).difference(hanging) %} - <tr> - <td>{{ test |e }}</td> - {% set flakyness = - (failed|length + hanging|length) * 100 / all|length %} - {% if flakyness == 100 %} - <td align="middle" style="background-color:#FF9999;"> - {% else %} - <td align="middle"> - {% endif %} - {{ "{:.1f}% ({} / {})".format( - flakyness, failed|length + hanging|length, all|length) }} - </td> - <td align="middle"> - {{ failed|length }} / {{ timeout|length }} / {{ hanging|length }} - </td> - <td> - {% set id = "details_" ~ test ~ "_" ~ counter %} - <button class="show_hide_button" onclick="toggle('{{ id }}')"> - show/hide</button> - <br/> - <div id="{{ id }}" - style="display: none; width:500px; white-space: normal"> - {% macro print_run_ids(url, run_ids) -%} - {% for i in run_ids %} - <a href="{{ url }}/{{ i }}">{{ i }}</a> - {% endfor %} - {%- endmacro %} - Failed : {{ print_run_ids(url, failed) }}<br/> - Timed Out : {{ print_run_ids(url, timeout) }}<br/> - Hanging : {{ print_run_ids(url, hanging) }}<br/> - Succeeded : {{ print_run_ids(url, success) }} - </div> - </td> - </tr> - {% endfor %} - </table> - <br><br><br> - {% endfor %} - <script type="text/javascript"> - function toggle(id) { - if (document.getElementById(id).style["display"] == "none") { - document.getElementById(id).style["display"] = "block"; - } else { - document.getElementById(id).style["display"] = "none"; - } - } - </script> - </body> - </html> - """) +dev_support_dir = os.path.dirname(os.path.abspath(__file__)) +with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"), "r") as f: + template = Template(f.read()) with open("dashboard.html", "w") as f: f.write(template.render(results=url_to_bad_test_results))
![https://hbase.apache.org/images/hbase_logo_with_orca_large.png"](https://hbase.apache.org/images/hbase_logo_with_orca_large.png"){kind=link}