Repository: qpid-dispatch Updated Branches: refs/heads/master 2dde7030f -> 8701feb7f
DISPATCH-1210: Scraper shows unsettled messages Compute message settlement earlier and display summary totals for connections, sessions, and links in details display. Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/8701feb7 Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/8701feb7 Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/8701feb7 Branch: refs/heads/master Commit: 8701feb7f6924bdb7f126881c50d2a6368434e8f Parents: 2dde703 Author: Chuck Rolke <cro...@redhat.com> Authored: Thu Dec 6 16:42:05 2018 -0500 Committer: Chuck Rolke <cro...@redhat.com> Committed: Thu Dec 6 16:42:05 2018 -0500 ---------------------------------------------------------------------- tools/scraper/amqp_detail.py | 72 +++++++++++++++++++++++++++------------ tools/scraper/common.py | 3 ++ tools/scraper/parser.py | 18 +++++++--- tools/scraper/scraper.py | 29 ++++++++++------ 4 files changed, 84 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/amqp_detail.py ---------------------------------------------------------------------- diff --git a/tools/scraper/amqp_detail.py b/tools/scraper/amqp_detail.py index fd64234..7f02633 100755 --- a/tools/scraper/amqp_detail.py +++ b/tools/scraper/amqp_detail.py @@ -64,6 +64,9 @@ class ConnectionDetail(): # combined amqp_error frames on this connection self.amqp_errors = 0 + # unsettled transfer count + self.unsettled = 0 + # session_list holds all SessionDetail records either active or retired # Sessions for a connection are identified by the local channel number. # There may be many sessions all using the same channel number. @@ -123,6 +126,8 @@ class SessionDetail: self.amqp_errors = 0 + self.unsettled = 0 + self.channel = -1 self.peer_chan = -1 @@ -263,6 +268,10 @@ class LinkDetail(): self.amqp_errors = 0 + self.unsettled = 0 + self.unsettled_list = [] + + # paired handles self.output_handle = -1 self.input_handle = -1 @@ -295,7 +304,10 @@ class AllDetails(): # # def format_errors(self, n_errors): - return ("<span style=\"background-color:yellow\">%d</span>" % n_errors) if n_errors > 0 else "" + return ("<span style=\"background-color:yellow\">errors: %d</span>" % n_errors) if n_errors > 0 else "" + + def format_unsettled(self, n_unsettled): + return ("<span style=\"background-color:orange\">unsettled: %d</span>" % n_unsettled) if n_unsettled > 0 else "" def classify_connection(self, id): """ @@ -426,9 +438,7 @@ class AllDetails(): conn_details.unaccounted_frame_list.append(plf) continue # session required - channel = plf.data.channel # For incoming begin this is the remote channel - # For outgoing begin this is the local channel - # Assume they are the same for the time being + channel = plf.data.channel # Assume in/out channels are the same for the time being sess_details = conn_details.FindSession(channel) if sess_details == None: sess_details = SessionDetail(conn_details, conn_details.GetSeqNo(), plf.datetime) @@ -543,6 +553,32 @@ class AllDetails(): (splf.data.conn_id)) sdispmap[did] = splf + def compute_settlement(self): + for conn in self.rtr.conn_list: + id = self.rtr.conn_id(conn) + conn_detail = self.rtr.details.conn_details[id] + for sess in conn_detail.session_list: + for link in sess.link_list: + for plf in link.frame_list: + if plf.data.transfer: + tdid = plf.data.delivery_id + if plf.data.direction == "->": + rmap = sess.rx_rcvr_disposition_map + tmap = sess.rx_sndr_disposition_map + else: + rmap = sess.tx_rcvr_disposition_map + tmap = sess.tx_sndr_disposition_map + plf.data.disposition_display = self.resolve_settlement(link, plf, + rmap.get(tdid), + tmap.get(tdid)) + if common.transfer_is_possibly_unsettled(plf): + if tdid not in link.unsettled_list: + link.unsettled_list.append(tdid) + link.unsettled += 1 + sess.unsettled += 1 + conn_detail.unsettled += 1 + + def show_html(self): for conn in self.rtr.conn_list: id = self.rtr.conn_id(conn) @@ -556,8 +592,9 @@ class AllDetails(): peer = self.rtr.conn_peer_display.get(id, "") # peer container id peerconnid = self.comn.conn_peers_connid.get(id, "") # show the connection title - print("%s %s %s %s (nFrames=%d) %s<br>" % \ - (id, dir, peerconnid, peer, len(conn_frames), self.format_errors(conn_detail.amqp_errors))) + print("%s %s %s %s (nFrames=%d) %s %s<br>" % \ + (id, dir, peerconnid, peer, len(conn_frames), self.format_errors(conn_detail.amqp_errors), + self.format_unsettled(conn_detail.unsettled))) # data div print("<div id=\"%s_data\" style=\"display:none; margin-bottom: 2px; margin-left: 10px\">" % id) @@ -577,9 +614,10 @@ class AllDetails(): # show the session 'toggle goto' and title print("<a href=\"javascript:toggle_node('%s_sess_%s')\">%s%s</a>" % (id, sess.conn_epoch, text.lozenge(), text.nbsp())) - print("Session %s: channel: %s, peer channel: %s; Time: start %s, Counts: frames: %d %s<br>" % \ + print("Session %s: channel: %s, peer channel: %s; Time: start %s, Counts: frames: %d %s %s<br>" % \ (sess.conn_epoch, sess.channel, sess.peer_chan, sess.time_start, \ - sess.FrameCount(), self.format_errors(sess.amqp_errors))) + sess.FrameCount(), self.format_errors(sess.amqp_errors), + self.format_unsettled(sess.unsettled))) print("<div id=\"%s_sess_%s\" style=\"display:none; margin-bottom: 2px; margin-left: 10px\">" % (id, sess.conn_epoch)) # show the session-level frames @@ -597,7 +635,7 @@ class AllDetails(): print("<table") print("<tr><th>Link</th> <th>Dir</th> <th>Role</th> <th>Address</th> <th>Class</th> " "<th>snd-settle-mode</th> <th>rcv-settle-mode</th> <th>Start time</th> <th>Frames</th> " - "<th>AMQP errors</tr>") + "<th>AMQP errors</th> <th>Unsettled</th> </tr>") for link in sess.link_list: # show the link toggle and title showthis = ("<a href=\"javascript:toggle_node('%s_sess_%s_link_%s')\">%s</a>" % @@ -606,11 +644,12 @@ class AllDetails(): (id, sess.conn_epoch, link.session_seq, link.display_name)) role = "receiver" if link.is_receiver else "sender" print("<tr><td>%s %s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>" - "<td>%s</td><td>%d</td><td>%s</td></tr>" % \ + "<td>%s</td><td>%d</td><td>%s</td> <td>%s</td></tr>" % \ (showthis, visitthis, link.direction, role, link.first_address, (link.sender_class + '-' + link.receiver_class), link.snd_settle_mode, link.rcv_settle_mode, link.time_start, link.FrameCount(), - self.format_errors(link.amqp_errors))) + self.format_errors(link.amqp_errors), + self.format_unsettled(link.unsettled))) print("</table>") # second loop prints the link's frames for link in sess.link_list: @@ -622,17 +661,6 @@ class AllDetails(): print("<h4>Connection %s Session %s Link %s</h4>" % (id, sess.conn_epoch, link.display_name)) for plf in link.frame_list: - if plf.data.name == "transfer": - tdid = plf.data.delivery_id - if plf.data.direction == "->": - rmap = sess.rx_rcvr_disposition_map - tmap = sess.rx_sndr_disposition_map - else: - rmap = sess.tx_rcvr_disposition_map - tmap = sess.tx_sndr_disposition_map - plf.data.disposition_display = self.resolve_settlement(link, plf, - rmap.get(tdid), - tmap.get(tdid)) print(plf.adverbl_link_to(), plf.datetime, plf.data.direction, peer, plf.data.web_show_str, plf.data.disposition_display, "<br>") print("</div>") # end link <id>_sess_<conn_epoch>_link_<sess_seq> http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/common.py ---------------------------------------------------------------------- diff --git a/tools/scraper/common.py b/tools/scraper/common.py index 0a74f3c..654b8b4 100755 --- a/tools/scraper/common.py +++ b/tools/scraper/common.py @@ -140,3 +140,6 @@ class RestartRec(): self.event = _event self.datetime = _datetime +def transfer_is_possibly_unsettled(plf): + return (plf.data.transfer and + not (plf.data.transfer_settled or plf.data.final_disposition is not None)) http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/parser.py ---------------------------------------------------------------------- diff --git a/tools/scraper/parser.py b/tools/scraper/parser.py index d33a0c7..5f74266 100755 --- a/tools/scraper/parser.py +++ b/tools/scraper/parser.py @@ -123,7 +123,7 @@ class LogLineData: self.target = "" self.first = "" # undecorated number - '10' self.last = "" # undecorated number - '20' - self.settled = "" # Disposition or Transfer settled field + self.settled = "" # Disposition or Transfer settled field from log line self.disposition_state = "?absent?" self.snd_settle_mode = "" # Attach self.rcv_settle_mode = "" # Attach @@ -259,8 +259,7 @@ class DescribedType: :return: """ self.dtype = _dtype - self.oline = str(_line) - self.line = self.oline + self.line = str(_line) self.dtype_name = DescribedType.dtype_name(self.dtype) self.dtype_number = DescribedType.dtype_number(self.dtype) @@ -751,7 +750,6 @@ class ParsedLogLine(object): ParsedLogLine.server_info_key in _line or ParsedLogLine.router_ls_key in _line): raise ValueError("Line is not a candidate for parsing") - self.oline = _line # original line self.index = _log_index # router prefix 0 for A, 1 for B self.instance = _instance # router instance in log file self.lineno = _lineno # log line number @@ -950,7 +948,17 @@ def parse_log_file(fn, log_index, comn): search_for_in_progress = False rtr.container_name = line[(line.find(key2) + len(key2)):].strip().split()[0] elif key3 in line: - pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr) + pl = None + try: + pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr) + except ValueError as ve: + pass + except Exception as e: + # t, v, tb = sys.exc_info() + if hasattr(e, 'message'): + sys.stderr.write("Failed to parse file '%s', line %d : %s\n" % (fn, lineno, e.message)) + else: + sys.stderr.write("Failed to parse file '%s', line %d : %s\n" % (fn, lineno, e)) if pl is not None: if pl.data.is_router_ls: rtr.router_ls.append(pl) http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/scraper.py ---------------------------------------------------------------------- diff --git a/tools/scraper/scraper.py b/tools/scraper/scraper.py index 2e998b1..981d3ff 100755 --- a/tools/scraper/scraper.py +++ b/tools/scraper/scraper.py @@ -227,6 +227,12 @@ def main_except(argv): comn.shorteners.short_data_names.sort_customers() comn.shorteners.short_link_names.sort_customers() + # compute settlement + if not comn.args.skip_detail: + for rtrlist in comn.routers: + for rtr in rtrlist: + rtr.details.compute_settlement() + # # Start producing the output stream # @@ -334,11 +340,11 @@ def main_except(argv): # print the connection peer tables # - # +------+--------------------+-----+--------------------+-------+-------+----------+--------+ - # | View | Router | Dir | Peer | Log | N | Transfer | AMQP | - # | +-----------+--------+ +--------+-----------+ lines | links | bytes | errors | - # | | container | connid | | connid | container | | | | | - # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+ + # +------+--------------------+-----+--------------------+-------+-------+----------+--------+-------+ + # | View | Router | Dir | Peer | Log | N | Transfer | AMQP | unset | + # | +-----------+--------+ +--------+-----------+ lines | links | bytes | errors | tled | + # | | container | connid | | connid | container | | | | | | + # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+-------+ print("<a name=\"c_connections\"></a>") print("<h3>Connections</h3>") @@ -352,7 +358,7 @@ def main_except(argv): print("<h3>Connections by ConnectionId</h3>") print( "<table><tr> <th rowspan=\"2\">View</th> <th colspan=\"2\">Router</th> <th rowspan=\"2\">Dir</th> <th colspan=\"2\">Peer</th> <th rowspan=\"2\">Log lines</th> " - "<th rowspan=\"2\">N links</th><th rowspan=\"2\">Transfer bytes</th> <th rowspan=\"2\">AMQP errors</th> <th rowspan=\"2\">Open time</th> <th rowspan=\"2\">Close time</th></tr>") + "<th rowspan=\"2\">N links</th><th rowspan=\"2\">Transfer bytes</th> <th rowspan=\"2\">AMQP errors</th> <th rowspan=\"2\">Unsettled</th> <th rowspan=\"2\">Open time</th> <th rowspan=\"2\">Close time</th></tr>") print("<tr> <th>container</th> <th>connid</th> <th>connid</th> <th>container</th></tr>") tConn = 0 @@ -378,13 +384,14 @@ def main_except(argv): etime = rtr.conn_close_time.get(id, text.nbsp()) if etime != text.nbsp(): etime = etime.datetime + conn_details = rtr.details.conn_details[id] print("<tr>") print("<td> <input type=\"checkbox\" id=\"cb_sel_%s\" " % id) print("checked=\"true\" onclick=\"javascript:show_if_cb_sel_%s()\"> </td>" % (id)) print("<td>%s</td><td><a href=\"#cd_%s\">%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>" - "<td>%d</td><td>%s</td><td>%d</td><td>%s</td><td>%s</td></tr>" % + "<td>%d</td><td>%s</td><td>%d</td><td>%d</td><td>%s</td><td>%s</td></tr>" % (rid, id, id, rtr.conn_dir[id], peerconnid, peer, rtr.conn_log_lines[id], n_links, - rtr.conn_xfer_bytes[id], errs, stime, etime)) + rtr.conn_xfer_bytes[id], errs, conn_details.unsettled, stime, etime)) tLines += rtr.conn_log_lines[id] tBytes += rtr.conn_xfer_bytes[id] print( @@ -470,7 +477,7 @@ def main_except(argv): n_aborted += 1 if plf.data.flow_drain: n_drain += 1 - if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition is not None): + if common.transfer_is_possibly_unsettled(plf): if plf.data.no_parent_link: n_unsettled_no_parent += 1 # possibly unsettled else: @@ -542,7 +549,7 @@ def main_except(argv): "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" " "id=\"noteworthy_unsettled\">") for plf in tree: - if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition is not None) and not plf.data.no_parent_link: + if common.transfer_is_possibly_unsettled(plf) and not plf.data.no_parent_link: show_noteworthy_line(plf, comn) print("</div>") # possible unsettled transfers @@ -552,7 +559,7 @@ def main_except(argv): "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" " "id=\"noteworthy_unsettled_qm\">") for plf in tree: - if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition is not None) and plf.data.no_parent_link: + if common.transfer_is_possibly_unsettled(plf) and plf.data.no_parent_link: show_noteworthy_line(plf, comn) print("</div>") print("<hr>") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@qpid.apache.org For additional commands, e-mail: commits-h...@qpid.apache.org