From commits-return-47566-archive-asf-public=cust-asf.ponee.io@qpid.apache.org Thu Dec 6 22:44:21 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 9D437180674 for ; Thu, 6 Dec 2018 22:44:20 +0100 (CET) Received: (qmail 97824 invoked by uid 500); 6 Dec 2018 21:44:19 -0000 Mailing-List: contact commits-help@qpid.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@qpid.apache.org Delivered-To: mailing list commits@qpid.apache.org Received: (qmail 97815 invoked by uid 99); 6 Dec 2018 21:44:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Dec 2018 21:44:19 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A206FE128A; Thu, 6 Dec 2018 21:44:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chug@apache.org To: commits@qpid.apache.org Message-Id: <0bb51fa4427341e293071f1d63b901d8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: qpid-dispatch git commit: DISPATCH-1210: Scraper shows unsettled messages Date: Thu, 6 Dec 2018 21:44:19 +0000 (UTC) Repository: qpid-dispatch Updated Branches: refs/heads/master 2dde7030f -> 8701feb7f DISPATCH-1210: Scraper shows unsettled messages Compute message settlement earlier and display summary totals for connections, sessions, and links in details display. Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/8701feb7 Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/8701feb7 Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/8701feb7 Branch: refs/heads/master Commit: 8701feb7f6924bdb7f126881c50d2a6368434e8f Parents: 2dde703 Author: Chuck Rolke Authored: Thu Dec 6 16:42:05 2018 -0500 Committer: Chuck Rolke Committed: Thu Dec 6 16:42:05 2018 -0500 ---------------------------------------------------------------------- tools/scraper/amqp_detail.py | 72 +++++++++++++++++++++++++++------------ tools/scraper/common.py | 3 ++ tools/scraper/parser.py | 18 +++++++--- tools/scraper/scraper.py | 29 ++++++++++------ 4 files changed, 84 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/amqp_detail.py ---------------------------------------------------------------------- diff --git a/tools/scraper/amqp_detail.py b/tools/scraper/amqp_detail.py index fd64234..7f02633 100755 --- a/tools/scraper/amqp_detail.py +++ b/tools/scraper/amqp_detail.py @@ -64,6 +64,9 @@ class ConnectionDetail(): # combined amqp_error frames on this connection self.amqp_errors = 0 + # unsettled transfer count + self.unsettled = 0 + # session_list holds all SessionDetail records either active or retired # Sessions for a connection are identified by the local channel number. # There may be many sessions all using the same channel number. @@ -123,6 +126,8 @@ class SessionDetail: self.amqp_errors = 0 + self.unsettled = 0 + self.channel = -1 self.peer_chan = -1 @@ -263,6 +268,10 @@ class LinkDetail(): self.amqp_errors = 0 + self.unsettled = 0 + self.unsettled_list = [] + + # paired handles self.output_handle = -1 self.input_handle = -1 @@ -295,7 +304,10 @@ class AllDetails(): # # def format_errors(self, n_errors): - return ("%d" % n_errors) if n_errors > 0 else "" + return ("errors: %d" % n_errors) if n_errors > 0 else "" + + def format_unsettled(self, n_unsettled): + return ("unsettled: %d" % n_unsettled) if n_unsettled > 0 else "" def classify_connection(self, id): """ @@ -426,9 +438,7 @@ class AllDetails(): conn_details.unaccounted_frame_list.append(plf) continue # session required - channel = plf.data.channel # For incoming begin this is the remote channel - # For outgoing begin this is the local channel - # Assume they are the same for the time being + channel = plf.data.channel # Assume in/out channels are the same for the time being sess_details = conn_details.FindSession(channel) if sess_details == None: sess_details = SessionDetail(conn_details, conn_details.GetSeqNo(), plf.datetime) @@ -543,6 +553,32 @@ class AllDetails(): (splf.data.conn_id)) sdispmap[did] = splf + def compute_settlement(self): + for conn in self.rtr.conn_list: + id = self.rtr.conn_id(conn) + conn_detail = self.rtr.details.conn_details[id] + for sess in conn_detail.session_list: + for link in sess.link_list: + for plf in link.frame_list: + if plf.data.transfer: + tdid = plf.data.delivery_id + if plf.data.direction == "->": + rmap = sess.rx_rcvr_disposition_map + tmap = sess.rx_sndr_disposition_map + else: + rmap = sess.tx_rcvr_disposition_map + tmap = sess.tx_sndr_disposition_map + plf.data.disposition_display = self.resolve_settlement(link, plf, + rmap.get(tdid), + tmap.get(tdid)) + if common.transfer_is_possibly_unsettled(plf): + if tdid not in link.unsettled_list: + link.unsettled_list.append(tdid) + link.unsettled += 1 + sess.unsettled += 1 + conn_detail.unsettled += 1 + + def show_html(self): for conn in self.rtr.conn_list: id = self.rtr.conn_id(conn) @@ -556,8 +592,9 @@ class AllDetails(): peer = self.rtr.conn_peer_display.get(id, "") # peer container id peerconnid = self.comn.conn_peers_connid.get(id, "") # show the connection title - print("%s %s %s %s (nFrames=%d) %s
" % \ - (id, dir, peerconnid, peer, len(conn_frames), self.format_errors(conn_detail.amqp_errors))) + print("%s %s %s %s (nFrames=%d) %s %s
" % \ + (id, dir, peerconnid, peer, len(conn_frames), self.format_errors(conn_detail.amqp_errors), + self.format_unsettled(conn_detail.unsettled))) # data div print("
" % id) @@ -577,9 +614,10 @@ class AllDetails(): # show the session 'toggle goto' and title print("%s%s" % (id, sess.conn_epoch, text.lozenge(), text.nbsp())) - print("Session %s: channel: %s, peer channel: %s; Time: start %s, Counts: frames: %d %s
" % \ + print("Session %s: channel: %s, peer channel: %s; Time: start %s, Counts: frames: %d %s %s
" % \ (sess.conn_epoch, sess.channel, sess.peer_chan, sess.time_start, \ - sess.FrameCount(), self.format_errors(sess.amqp_errors))) + sess.FrameCount(), self.format_errors(sess.amqp_errors), + self.format_unsettled(sess.unsettled))) print("
" % (id, sess.conn_epoch)) # show the session-level frames @@ -597,7 +635,7 @@ class AllDetails(): print("Link Dir Role Address Class " "snd-settle-mode rcv-settle-mode Start time Frames " - "AMQP errors") + "AMQP errors Unsettled ") for link in sess.link_list: # show the link toggle and title showthis = ("%s" % @@ -606,11 +644,12 @@ class AllDetails(): (id, sess.conn_epoch, link.session_seq, link.display_name)) role = "receiver" if link.is_receiver else "sender" print("%s %s%s%s%s%s%s%s" - "%s%d%s" % \ + "%s%d%s %s" % \ (showthis, visitthis, link.direction, role, link.first_address, (link.sender_class + '-' + link.receiver_class), link.snd_settle_mode, link.rcv_settle_mode, link.time_start, link.FrameCount(), - self.format_errors(link.amqp_errors))) + self.format_errors(link.amqp_errors), + self.format_unsettled(link.unsettled))) print("") # second loop prints the link's frames for link in sess.link_list: @@ -622,17 +661,6 @@ class AllDetails(): print("

Connection %s Session %s Link %s

" % (id, sess.conn_epoch, link.display_name)) for plf in link.frame_list: - if plf.data.name == "transfer": - tdid = plf.data.delivery_id - if plf.data.direction == "->": - rmap = sess.rx_rcvr_disposition_map - tmap = sess.rx_sndr_disposition_map - else: - rmap = sess.tx_rcvr_disposition_map - tmap = sess.tx_sndr_disposition_map - plf.data.disposition_display = self.resolve_settlement(link, plf, - rmap.get(tdid), - tmap.get(tdid)) print(plf.adverbl_link_to(), plf.datetime, plf.data.direction, peer, plf.data.web_show_str, plf.data.disposition_display, "
") print("
") # end link _sess__link_ http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/common.py ---------------------------------------------------------------------- diff --git a/tools/scraper/common.py b/tools/scraper/common.py index 0a74f3c..654b8b4 100755 --- a/tools/scraper/common.py +++ b/tools/scraper/common.py @@ -140,3 +140,6 @@ class RestartRec(): self.event = _event self.datetime = _datetime +def transfer_is_possibly_unsettled(plf): + return (plf.data.transfer and + not (plf.data.transfer_settled or plf.data.final_disposition is not None)) http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/parser.py ---------------------------------------------------------------------- diff --git a/tools/scraper/parser.py b/tools/scraper/parser.py index d33a0c7..5f74266 100755 --- a/tools/scraper/parser.py +++ b/tools/scraper/parser.py @@ -123,7 +123,7 @@ class LogLineData: self.target = "" self.first = "" # undecorated number - '10' self.last = "" # undecorated number - '20' - self.settled = "" # Disposition or Transfer settled field + self.settled = "" # Disposition or Transfer settled field from log line self.disposition_state = "?absent?" self.snd_settle_mode = "" # Attach self.rcv_settle_mode = "" # Attach @@ -259,8 +259,7 @@ class DescribedType: :return: """ self.dtype = _dtype - self.oline = str(_line) - self.line = self.oline + self.line = str(_line) self.dtype_name = DescribedType.dtype_name(self.dtype) self.dtype_number = DescribedType.dtype_number(self.dtype) @@ -751,7 +750,6 @@ class ParsedLogLine(object): ParsedLogLine.server_info_key in _line or ParsedLogLine.router_ls_key in _line): raise ValueError("Line is not a candidate for parsing") - self.oline = _line # original line self.index = _log_index # router prefix 0 for A, 1 for B self.instance = _instance # router instance in log file self.lineno = _lineno # log line number @@ -950,7 +948,17 @@ def parse_log_file(fn, log_index, comn): search_for_in_progress = False rtr.container_name = line[(line.find(key2) + len(key2)):].strip().split()[0] elif key3 in line: - pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr) + pl = None + try: + pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr) + except ValueError as ve: + pass + except Exception as e: + # t, v, tb = sys.exc_info() + if hasattr(e, 'message'): + sys.stderr.write("Failed to parse file '%s', line %d : %s\n" % (fn, lineno, e.message)) + else: + sys.stderr.write("Failed to parse file '%s', line %d : %s\n" % (fn, lineno, e)) if pl is not None: if pl.data.is_router_ls: rtr.router_ls.append(pl) http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/scraper.py ---------------------------------------------------------------------- diff --git a/tools/scraper/scraper.py b/tools/scraper/scraper.py index 2e998b1..981d3ff 100755 --- a/tools/scraper/scraper.py +++ b/tools/scraper/scraper.py @@ -227,6 +227,12 @@ def main_except(argv): comn.shorteners.short_data_names.sort_customers() comn.shorteners.short_link_names.sort_customers() + # compute settlement + if not comn.args.skip_detail: + for rtrlist in comn.routers: + for rtr in rtrlist: + rtr.details.compute_settlement() + # # Start producing the output stream # @@ -334,11 +340,11 @@ def main_except(argv): # print the connection peer tables # - # +------+--------------------+-----+--------------------+-------+-------+----------+--------+ - # | View | Router | Dir | Peer | Log | N | Transfer | AMQP | - # | +-----------+--------+ +--------+-----------+ lines | links | bytes | errors | - # | | container | connid | | connid | container | | | | | - # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+ + # +------+--------------------+-----+--------------------+-------+-------+----------+--------+-------+ + # | View | Router | Dir | Peer | Log | N | Transfer | AMQP | unset | + # | +-----------+--------+ +--------+-----------+ lines | links | bytes | errors | tled | + # | | container | connid | | connid | container | | | | | | + # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+-------+ print("") print("

Connections

") @@ -352,7 +358,7 @@ def main_except(argv): print("

Connections by ConnectionId

") print( " " - "") + "") print("") tConn = 0 @@ -378,13 +384,14 @@ def main_except(argv): etime = rtr.conn_close_time.get(id, text.nbsp()) if etime != text.nbsp(): etime = etime.datetime + conn_details = rtr.details.conn_details[id] print("") print("" % (id)) print("" - "" % + "" % (rid, id, id, rtr.conn_dir[id], peerconnid, peer, rtr.conn_log_lines[id], n_links, - rtr.conn_xfer_bytes[id], errs, stime, etime)) + rtr.conn_xfer_bytes[id], errs, conn_details.unsettled, stime, etime)) tLines += rtr.conn_log_lines[id] tBytes += rtr.conn_xfer_bytes[id] print( @@ -470,7 +477,7 @@ def main_except(argv): n_aborted += 1 if plf.data.flow_drain: n_drain += 1 - if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition is not None): + if common.transfer_is_possibly_unsettled(plf): if plf.data.no_parent_link: n_unsettled_no_parent += 1 # possibly unsettled else: @@ -542,7 +549,7 @@ def main_except(argv): "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" " "id=\"noteworthy_unsettled\">") for plf in tree: - if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition is not None) and not plf.data.no_parent_link: + if common.transfer_is_possibly_unsettled(plf) and not plf.data.no_parent_link: show_noteworthy_line(plf, comn) print("") # possible unsettled transfers @@ -552,7 +559,7 @@ def main_except(argv): "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" " "id=\"noteworthy_unsettled_qm\">") for plf in tree: - if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition is not None) and plf.data.no_parent_link: + if common.transfer_is_possibly_unsettled(plf) and plf.data.no_parent_link: show_noteworthy_line(plf, comn) print("") print("
") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org For additional commands, e-mail: commits-help@qpid.apache.org
View Router Dir Peer Log linesN linksTransfer bytes AMQP errors Open time Close time
N linksTransfer bytes AMQP errors Unsettled Open time Close time
container connid connid container
%s%s%s%s%s%s%d%s%d%s%s
%d%s%d%d%s%s