qpid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c...@apache.org
Subject qpid-dispatch git commit: DISPATCH-1210: Scraper shows unsettled messages
Date Thu, 06 Dec 2018 21:44:19 GMT
Repository: qpid-dispatch
Updated Branches:
  refs/heads/master 2dde7030f -> 8701feb7f


DISPATCH-1210: Scraper shows unsettled messages

Compute message settlement earlier and display summary totals
for connections, sessions, and links in details display.


Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/8701feb7
Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/8701feb7
Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/8701feb7

Branch: refs/heads/master
Commit: 8701feb7f6924bdb7f126881c50d2a6368434e8f
Parents: 2dde703
Author: Chuck Rolke <crolke@redhat.com>
Authored: Thu Dec 6 16:42:05 2018 -0500
Committer: Chuck Rolke <crolke@redhat.com>
Committed: Thu Dec 6 16:42:05 2018 -0500

----------------------------------------------------------------------
 tools/scraper/amqp_detail.py | 72 +++++++++++++++++++++++++++------------
 tools/scraper/common.py      |  3 ++
 tools/scraper/parser.py      | 18 +++++++---
 tools/scraper/scraper.py     | 29 ++++++++++------
 4 files changed, 84 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/amqp_detail.py
----------------------------------------------------------------------
diff --git a/tools/scraper/amqp_detail.py b/tools/scraper/amqp_detail.py
index fd64234..7f02633 100755
--- a/tools/scraper/amqp_detail.py
+++ b/tools/scraper/amqp_detail.py
@@ -64,6 +64,9 @@ class ConnectionDetail():
         # combined amqp_error frames on this connection
         self.amqp_errors = 0
 
+        # unsettled transfer count
+        self.unsettled = 0
+
         # session_list holds all SessionDetail records either active or retired
         # Sessions for a connection are identified by the local channel number.
         # There may be many sessions all using the same channel number.
@@ -123,6 +126,8 @@ class SessionDetail:
 
         self.amqp_errors = 0
 
+        self.unsettled = 0
+
         self.channel = -1
         self.peer_chan = -1
 
@@ -263,6 +268,10 @@ class LinkDetail():
 
         self.amqp_errors = 0
 
+        self.unsettled = 0
+        self.unsettled_list = []
+
+
         # paired handles
         self.output_handle = -1
         self.input_handle = -1
@@ -295,7 +304,10 @@ class AllDetails():
     #
     #
     def format_errors(self, n_errors):
-        return ("<span style=\"background-color:yellow\">%d</span>" % n_errors)
if n_errors > 0 else ""
+        return ("<span style=\"background-color:yellow\">errors: %d</span>" %
n_errors) if n_errors > 0 else ""
+
+    def format_unsettled(self, n_unsettled):
+        return ("<span style=\"background-color:orange\">unsettled: %d</span>"
% n_unsettled) if n_unsettled > 0 else ""
 
     def classify_connection(self, id):
         """
@@ -426,9 +438,7 @@ class AllDetails():
                     conn_details.unaccounted_frame_list.append(plf)
                     continue
                 # session required
-                channel = plf.data.channel # For incoming begin this is the remote channel
-                                           # For outgoing begin this is the local channel
-                                           # Assume they are the same for the time being
+                channel = plf.data.channel # Assume in/out channels are the same for the
time being
                 sess_details = conn_details.FindSession(channel)
                 if sess_details == None:
                     sess_details = SessionDetail(conn_details, conn_details.GetSeqNo(), plf.datetime)
@@ -543,6 +553,32 @@ class AllDetails():
                                                  (splf.data.conn_id))
                             sdispmap[did] = splf
 
+    def compute_settlement(self):
+        for conn in self.rtr.conn_list:
+            id = self.rtr.conn_id(conn)
+            conn_detail = self.rtr.details.conn_details[id]
+            for sess in conn_detail.session_list:
+                for link in sess.link_list:
+                    for plf in link.frame_list:
+                        if plf.data.transfer:
+                            tdid = plf.data.delivery_id
+                            if plf.data.direction == "->":
+                                rmap = sess.rx_rcvr_disposition_map
+                                tmap = sess.rx_sndr_disposition_map
+                            else:
+                                rmap = sess.tx_rcvr_disposition_map
+                                tmap = sess.tx_sndr_disposition_map
+                            plf.data.disposition_display = self.resolve_settlement(link,
plf,
+                                                                                   rmap.get(tdid),
+                                                                                   tmap.get(tdid))
+                            if common.transfer_is_possibly_unsettled(plf):
+                                if tdid not in link.unsettled_list:
+                                    link.unsettled_list.append(tdid)
+                                    link.unsettled += 1
+                                    sess.unsettled += 1
+                                    conn_detail.unsettled += 1
+
+
     def show_html(self):
         for conn in self.rtr.conn_list:
             id = self.rtr.conn_id(conn)
@@ -556,8 +592,9 @@ class AllDetails():
             peer = self.rtr.conn_peer_display.get(id, "")  # peer container id
             peerconnid = self.comn.conn_peers_connid.get(id, "")
             # show the connection title
-            print("%s %s %s %s (nFrames=%d) %s<br>" % \
-                  (id, dir, peerconnid, peer, len(conn_frames), self.format_errors(conn_detail.amqp_errors)))
+            print("%s %s %s %s (nFrames=%d) %s %s<br>" % \
+                  (id, dir, peerconnid, peer, len(conn_frames), self.format_errors(conn_detail.amqp_errors),
+                   self.format_unsettled(conn_detail.unsettled)))
             # data div
             print("<div id=\"%s_data\" style=\"display:none; margin-bottom: 2px; margin-left:
10px\">" % id)
 
@@ -577,9 +614,10 @@ class AllDetails():
                 # show the session 'toggle goto' and title
                 print("<a href=\"javascript:toggle_node('%s_sess_%s')\">%s%s</a>"
%
                       (id, sess.conn_epoch, text.lozenge(), text.nbsp()))
-                print("Session %s: channel: %s, peer channel: %s; Time: start %s, Counts:
frames: %d %s<br>" % \
+                print("Session %s: channel: %s, peer channel: %s; Time: start %s, Counts:
frames: %d %s %s<br>" % \
                       (sess.conn_epoch, sess.channel, sess.peer_chan, sess.time_start, \
-                       sess.FrameCount(), self.format_errors(sess.amqp_errors)))
+                       sess.FrameCount(), self.format_errors(sess.amqp_errors),
+                       self.format_unsettled(sess.unsettled)))
                 print("<div id=\"%s_sess_%s\" style=\"display:none; margin-bottom: 2px;
margin-left: 10px\">" %
                       (id, sess.conn_epoch))
                 # show the session-level frames
@@ -597,7 +635,7 @@ class AllDetails():
                 print("<table")
                 print("<tr><th>Link</th> <th>Dir</th> <th>Role</th>
 <th>Address</th>  <th>Class</th>  "
                       "<th>snd-settle-mode</th>  <th>rcv-settle-mode</th>
 <th>Start time</th>  <th>Frames</th> "
-                      "<th>AMQP errors</tr>")
+                      "<th>AMQP errors</th> <th>Unsettled</th> </tr>")
                 for link in sess.link_list:
                     # show the link toggle and title
                     showthis = ("<a href=\"javascript:toggle_node('%s_sess_%s_link_%s')\">%s</a>"
%
@@ -606,11 +644,12 @@ class AllDetails():
                                 (id, sess.conn_epoch, link.session_seq, link.display_name))
                     role = "receiver" if link.is_receiver else "sender"
                     print("<tr><td>%s %s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>"
-                          "<td>%s</td><td>%d</td><td>%s</td></tr>"
% \
+                          "<td>%s</td><td>%d</td><td>%s</td>
<td>%s</td></tr>" % \
                           (showthis, visitthis, link.direction, role, link.first_address,
                            (link.sender_class + '-' + link.receiver_class), link.snd_settle_mode,
                            link.rcv_settle_mode, link.time_start, link.FrameCount(),
-                           self.format_errors(link.amqp_errors)))
+                           self.format_errors(link.amqp_errors),
+                           self.format_unsettled(link.unsettled)))
                 print("</table>")
                 # second loop prints the link's frames
                 for link in sess.link_list:
@@ -622,17 +661,6 @@ class AllDetails():
                     print("<h4>Connection %s Session %s Link %s</h4>" %
                           (id, sess.conn_epoch, link.display_name))
                     for plf in link.frame_list:
-                        if plf.data.name == "transfer":
-                            tdid = plf.data.delivery_id
-                            if plf.data.direction == "->":
-                                rmap = sess.rx_rcvr_disposition_map
-                                tmap = sess.rx_sndr_disposition_map
-                            else:
-                                rmap = sess.tx_rcvr_disposition_map
-                                tmap = sess.tx_sndr_disposition_map
-                            plf.data.disposition_display = self.resolve_settlement(link,
plf,
-                                                                                   rmap.get(tdid),
-                                                                                   tmap.get(tdid))
                         print(plf.adverbl_link_to(), plf.datetime, plf.data.direction, peer,
plf.data.web_show_str,
                               plf.data.disposition_display, "<br>")
                     print("</div>")  # end link <id>_sess_<conn_epoch>_link_<sess_seq>

http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/common.py
----------------------------------------------------------------------
diff --git a/tools/scraper/common.py b/tools/scraper/common.py
index 0a74f3c..654b8b4 100755
--- a/tools/scraper/common.py
+++ b/tools/scraper/common.py
@@ -140,3 +140,6 @@ class RestartRec():
         self.event = _event
         self.datetime = _datetime
 
+def transfer_is_possibly_unsettled(plf):
+    return (plf.data.transfer and
+            not (plf.data.transfer_settled or plf.data.final_disposition is not None))

http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/parser.py
----------------------------------------------------------------------
diff --git a/tools/scraper/parser.py b/tools/scraper/parser.py
index d33a0c7..5f74266 100755
--- a/tools/scraper/parser.py
+++ b/tools/scraper/parser.py
@@ -123,7 +123,7 @@ class LogLineData:
         self.target = ""
         self.first = ""  # undecorated number - '10'
         self.last = ""  # undecorated number - '20'
-        self.settled = ""  # Disposition or Transfer settled field
+        self.settled = ""  # Disposition or Transfer settled field from log line
         self.disposition_state = "?absent?"
         self.snd_settle_mode = ""  # Attach
         self.rcv_settle_mode = ""  # Attach
@@ -259,8 +259,7 @@ class DescribedType:
         :return:
         """
         self.dtype = _dtype
-        self.oline = str(_line)
-        self.line = self.oline
+        self.line = str(_line)
         self.dtype_name = DescribedType.dtype_name(self.dtype)
         self.dtype_number = DescribedType.dtype_number(self.dtype)
 
@@ -751,7 +750,6 @@ class ParsedLogLine(object):
                 ParsedLogLine.server_info_key in _line or
                 ParsedLogLine.router_ls_key in _line):
             raise ValueError("Line is not a candidate for parsing")
-        self.oline = _line  # original line
         self.index = _log_index  # router prefix 0 for A, 1 for B
         self.instance = _instance  # router instance in log file
         self.lineno = _lineno  # log line number
@@ -950,7 +948,17 @@ def parse_log_file(fn, log_index, comn):
                 search_for_in_progress = False
                 rtr.container_name = line[(line.find(key2) + len(key2)):].strip().split()[0]
             elif key3 in line:
-                pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr)
+                pl = None
+                try:
+                    pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr)
+                except ValueError as ve:
+                    pass
+                except Exception as e:
+                    # t, v, tb = sys.exc_info()
+                    if hasattr(e, 'message'):
+                        sys.stderr.write("Failed to parse file '%s', line %d : %s\n" % (fn,
lineno, e.message))
+                    else:
+                        sys.stderr.write("Failed to parse file '%s', line %d : %s\n" % (fn,
lineno, e))
                 if pl is not None:
                     if pl.data.is_router_ls:
                         rtr.router_ls.append(pl)

http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/8701feb7/tools/scraper/scraper.py
----------------------------------------------------------------------
diff --git a/tools/scraper/scraper.py b/tools/scraper/scraper.py
index 2e998b1..981d3ff 100755
--- a/tools/scraper/scraper.py
+++ b/tools/scraper/scraper.py
@@ -227,6 +227,12 @@ def main_except(argv):
     comn.shorteners.short_data_names.sort_customers()
     comn.shorteners.short_link_names.sort_customers()
 
+    # compute settlement
+    if not comn.args.skip_detail:
+        for rtrlist in comn.routers:
+            for rtr in rtrlist:
+                rtr.details.compute_settlement()
+
     #
     # Start producing the output stream
     #
@@ -334,11 +340,11 @@ def main_except(argv):
 
     # print the connection peer tables
     #
-    # +------+--------------------+-----+--------------------+-------+-------+----------+--------+
-    # | View |       Router       | Dir |       Peer         | Log   | N     | Transfer |
AMQP   |
-    # |      +-----------+--------+     +--------+-----------+ lines | links | bytes    |
errors |
-    # |      | container | connid |     | connid | container |       |       |          |
       |
-    # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+
+    # +------+--------------------+-----+--------------------+-------+-------+----------+--------+-------+
+    # | View |       Router       | Dir |       Peer         | Log   | N     | Transfer |
AMQP   | unset |
+    # |      +-----------+--------+     +--------+-----------+ lines | links | bytes    |
errors | tled  |
+    # |      | container | connid |     | connid | container |       |       |          |
       |       |
+    # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+-------+
 
     print("<a name=\"c_connections\"></a>")
     print("<h3>Connections</h3>")
@@ -352,7 +358,7 @@ def main_except(argv):
     print("<h3>Connections by ConnectionId</h3>")
     print(
         "<table><tr> <th rowspan=\"2\">View</th> <th colspan=\"2\">Router</th>
<th rowspan=\"2\">Dir</th> <th colspan=\"2\">Peer</th> <th rowspan=\"2\">Log
lines</th> "
-        "<th rowspan=\"2\">N links</th><th rowspan=\"2\">Transfer bytes</th>
<th rowspan=\"2\">AMQP errors</th> <th rowspan=\"2\">Open time</th>
<th rowspan=\"2\">Close time</th></tr>")
+        "<th rowspan=\"2\">N links</th><th rowspan=\"2\">Transfer bytes</th>
<th rowspan=\"2\">AMQP errors</th> <th rowspan=\"2\">Unsettled</th>
<th rowspan=\"2\">Open time</th> <th rowspan=\"2\">Close time</th></tr>")
     print("<tr> <th>container</th> <th>connid</th> <th>connid</th>
<th>container</th></tr>")
 
     tConn = 0
@@ -378,13 +384,14 @@ def main_except(argv):
                 etime = rtr.conn_close_time.get(id, text.nbsp())
                 if etime != text.nbsp():
                     etime = etime.datetime
+                conn_details = rtr.details.conn_details[id]
                 print("<tr>")
                 print("<td> <input type=\"checkbox\" id=\"cb_sel_%s\" " % id)
                 print("checked=\"true\" onclick=\"javascript:show_if_cb_sel_%s()\"> </td>"
% (id))
                 print("<td>%s</td><td><a href=\"#cd_%s\">%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>"
-                      "<td>%d</td><td>%s</td><td>%d</td><td>%s</td><td>%s</td></tr>"
%
+                      "<td>%d</td><td>%s</td><td>%d</td><td>%d</td><td>%s</td><td>%s</td></tr>"
%
                       (rid, id, id, rtr.conn_dir[id], peerconnid, peer, rtr.conn_log_lines[id],
n_links,
-                       rtr.conn_xfer_bytes[id], errs, stime, etime))
+                       rtr.conn_xfer_bytes[id], errs, conn_details.unsettled, stime, etime))
                 tLines += rtr.conn_log_lines[id]
                 tBytes += rtr.conn_xfer_bytes[id]
     print(
@@ -470,7 +477,7 @@ def main_except(argv):
             n_aborted += 1
         if plf.data.flow_drain:
             n_drain += 1
-        if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition
is not None):
+        if common.transfer_is_possibly_unsettled(plf):
             if plf.data.no_parent_link:
                 n_unsettled_no_parent += 1 # possibly unsettled
             else:
@@ -542,7 +549,7 @@ def main_except(argv):
           "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\"
"
           "id=\"noteworthy_unsettled\">")
     for plf in tree:
-        if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition
is not None) and not plf.data.no_parent_link:
+        if common.transfer_is_possibly_unsettled(plf) and not plf.data.no_parent_link:
             show_noteworthy_line(plf, comn)
     print("</div>")
     # possible unsettled transfers
@@ -552,7 +559,7 @@ def main_except(argv):
           "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\"
"
           "id=\"noteworthy_unsettled_qm\">")
     for plf in tree:
-        if plf.data.transfer and not (plf.data.transfer_settled or plf.data.final_disposition
is not None) and plf.data.no_parent_link:
+        if common.transfer_is_possibly_unsettled(plf) and plf.data.no_parent_link:
             show_noteworthy_line(plf, comn)
     print("</div>")
     print("<hr>")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org


Mime
View raw message