qpid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c...@apache.org
Subject [qpid-dispatch] branch master updated: DISPATCH-1538: Create link/transfer nicknames in temporal order
Date Fri, 03 Jan 2020 16:29:27 GMT
This is an automated email from the ASF dual-hosted git repository.

chug pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/qpid-dispatch.git


The following commit(s) were added to refs/heads/master by this push:
     new 2769224  DISPATCH-1538: Create link/transfer nicknames in temporal order
2769224 is described below

commit 276922442bfee310364bea894ea84ef9d793bf42
Author: Chuck Rolke <chug@apache.org>
AuthorDate: Fri Jan 3 11:26:07 2020 -0500

    DISPATCH-1538: Create link/transfer nicknames in temporal order
    
    Defer creating final web page content until all nicknames have been observed
    and sorted into timedate order.
---
 tools/scraper/common.py               |  4 +++
 tools/scraper/nicknamer.py            | 48 +++++++++++++++++++++++++++++++---
 tools/scraper/parser.py               | 49 +++++++++++++++++++++++++----------
 tools/scraper/scraper.py              | 15 ++++++++---
 tools/scraper/test_data/test_data.txt |  1 +
 5 files changed, 96 insertions(+), 21 deletions(-)

diff --git a/tools/scraper/common.py b/tools/scraper/common.py
index 3d52b1a..16e897c 100755
--- a/tools/scraper/common.py
+++ b/tools/scraper/common.py
@@ -196,3 +196,7 @@ def strings_of_proton_log(text):
                 skipping = False
                 r += elem
     return r
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+    sys.stderr.flush()
diff --git a/tools/scraper/nicknamer.py b/tools/scraper/nicknamer.py
index 8245704..540d829 100755
--- a/tools/scraper/nicknamer.py
+++ b/tools/scraper/nicknamer.py
@@ -22,20 +22,47 @@
 from collections import defaultdict
 import common
 
+class ShortNameSorter():
+    '''
+    Class to hold registered name and TOD in a list for sort purposes:
+      the lname was observed at this datetime.
+    '''
+    def __init__(self, lname, datetime):
+        self.lname = lname
+        self.datetime = datetime
+
 class ShortNames():
     '''
     Name shortener.
     The short name for display is "name_" + index(longName)
-    Embellish the display name with an html popup
+    Conditionally Embellish the display name with an html popup
     Link and endpoint names, and data are tracked separately
     Names longer than threshold are shortened
     Each class has a prefix used when the table is dumped as HTML
+    Link names and Transfer data have a 'customer' of a ParsedLogLine.
+    * The customers share the short and long name
+    * The dict index is the object name and the value is a list of log lines using that name
+    * Sorting the customers puts their usage of the name in time order
     '''
     def __init__(self, prefixText, _threshold=25):
         self.longnames = []
         self.prefix = prefixText
         self.threshold = _threshold
         self.customer_dict = defaultdict(list)
+        self.sorter = []
+
+    def register(self, lname, pll):
+        '''
+        Register a long name and the TimeOfDay it is observed
+        Memorize all names, translated or not
+        Strip leading/trailing double quotes
+        :param lname: the name
+        :param pll: ParsedLogLine where name was observed
+        :return: none
+        '''
+        if lname.startswith("\"") and lname.endswith("\""):
+            lname = lname[1:-1]
+        self.sorter.append(ShortNameSorter(lname, pll.datetime))
 
     def translate(self, lname, show_popup=False, customer=None):
         '''
@@ -43,6 +70,8 @@ class ShortNames():
         Memorize all names, translated or not
         Strip leading/trailing double quotes
         :param lname: the name
+        :param show_popup: if true then embellish returned name with long name popup
+        :param customer: optional ParsedLogLine for customer_dict
         :return: If shortened HTML string of shortened name with popup containing long name
else
         not-so-long name.
         '''
@@ -128,8 +157,21 @@ class ShortNames():
     def customers(self, sname):
         return self.customer_dict[sname]
 
-    def sorted_indexes(self):
-        return [self.longnames.index(sln) for sln in sorted(self.longnames)]
+    def sort_main(self):
+        '''
+        Create a list of longnames sorted in datetime order.
+        Then push the names into the class longnames list.
+
+        Called after registering all the names and recording their times but before
+        translating the names and using the translated results.
+        :return:
+        '''
+        temp = sorted(self.sorter, key=lambda sns: sns.datetime)
+        for sns in temp:
+            if sns.lname not in self.longnames:
+                self.longnames.append(sns.lname)
+        self.sorter = []
+
 
 class Shorteners():
     def __init__(self):
diff --git a/tools/scraper/parser.py b/tools/scraper/parser.py
index bcf9a07..d2e59ce 100755
--- a/tools/scraper/parser.py
+++ b/tools/scraper/parser.py
@@ -436,9 +436,9 @@ class ParsedLogLine(object):
             res.handle = resdict["handle"]
             res.role = "receiver" if resdict["role"] == "true" else "sender"
             res.is_receiver = res.role == "receiver"
+            # translated names handled later
             name = self.resdict_value(resdict, "name", "None")
-            res.link_short_name_popup = self.shorteners.short_link_names.translate(name,
True, customer=self)
-            res.link_short_name = self.shorteners.short_link_names.translate(name, False)
+            self.shorteners.short_link_names.register(name, self)
             tmpsrc = self.resdict_value(resdict, "source", None)
             tmptgt = self.resdict_value(resdict, "target", None)
             res.snd_settle_mode = self.sender_settle_mode_of(
@@ -470,9 +470,7 @@ class ParsedLogLine(object):
             res.snd_settle_mode = extract_name(tmpssm)
             res.rcv_settle_mode = extract_name(tmprsm)
             """
-            res.web_show_str = ("<strong>%s</strong> %s %s %s (source: %s, target:
%s, class: %s)" %
-                                (res.name, colorize_bg(res.channel_handle), res.role, res.link_short_name_popup,
-                                 res.source, res.target, res.link_class))
+            # show_str handled in post_extract
 
         elif perf == 0x13:
             # Performative: flow [channel,handle]
@@ -500,15 +498,8 @@ class ParsedLogLine(object):
             res.transfer_more = resdict.get("more", "") == "true"
             res.transfer_resume = resdict.get("resume", "") == "true"
             res.transfer_aborted = resdict.get("aborted", "") == "true"
-            self.transfer_short_name = self.shorteners.short_data_names.translate(res.transfer_bare,
customer=self)
-            showdat = "<a href=\"#%s_dump\">%s</a>" % (self.transfer_short_name,
self.transfer_short_name)
-            res.web_show_str = "<strong>%s</strong>  %s (%s) %s %s %s %s %s -
%s bytes" % (
-                res.name, colorize_bg(res.channel_handle), res.delivery_id,
-                self.highlighted("settled", res.transfer_settled, common.color_of("presettled")),
-                self.highlighted("more", res.transfer_more, common.color_of("more")),
-                self.highlighted("resume", res.transfer_resume, common.color_of("aborted")),
-                self.highlighted("aborted", res.transfer_aborted, common.color_of("aborted")),
-                showdat, res.transfer_size)
+            # translated names handled in post_extract
+            self.shorteners.short_data_names.register(res.transfer_bare, self)
 
         elif perf == 0x15:
             # Performative: disposition [channel] (role first-last)
@@ -716,6 +707,30 @@ class ParsedLogLine(object):
             res.web_show_str += (" <span style=\"background-color:%s\">error</span>
"
                                  "%s %s" % (common.color_of("errors"), condi, descr))
 
+    def post_extract_names(self):
+        perf = self.data.described_type.dtype_number
+        res = self.data
+        resdict = self.data.described_type.dict
+        if perf == 0x12:
+            # Performative:  attach [channel,handle] role name (source: src, target: tgt)
+            name = self.resdict_value(resdict, "name", "None")
+            res.link_short_name_popup = self.shorteners.short_link_names.translate(name,
True, customer=self)
+            res.link_short_name = self.shorteners.short_link_names.translate(name, False)
+            res.web_show_str = ("<strong>%s</strong> %s %s %s (source: %s, target:
%s, class: %s)" %
+                                (res.name, colorize_bg(res.channel_handle), res.role, res.link_short_name_popup,
+                                 res.source, res.target, res.link_class))
+        elif perf == 0x14:
+            # Performative: transfer [channel,handle] (id)
+            self.transfer_short_name = self.shorteners.short_data_names.translate(res.transfer_bare,
customer=self)
+            showdat = "<a href=\"#%s_dump\">%s</a>" % (self.transfer_short_name,
self.transfer_short_name)
+            res.web_show_str = "<strong>%s</strong>  %s (%s) %s %s %s %s %s -
%s bytes" % (
+                res.name, colorize_bg(res.channel_handle), res.delivery_id,
+                self.highlighted("settled", res.transfer_settled, common.color_of("presettled")),
+                self.highlighted("more", res.transfer_more, common.color_of("more")),
+                self.highlighted("resume", res.transfer_resume, common.color_of("aborted")),
+                self.highlighted("aborted", res.transfer_aborted, common.color_of("aborted")),
+                showdat, res.transfer_size)
+
     def adverbl_link_to(self):
         """
         :return: html link to the main adverbl data display for this line
@@ -843,6 +858,7 @@ class ParsedLogLine(object):
         ste = self.line.find(']')
         if ste < 0:
             print("Failed to parse line ", _lineno, " : ", _line)
+            traceback.print_exc()
             raise ValueError("'%s' not found in line %s" % ("]", self.line))
         self.data.conn_num = self.line[:ste]
         if self.data.conn_num.startswith("C"):
@@ -968,8 +984,10 @@ def parse_log_file(fn, log_index, comn):
                 except Exception as e:
                     # t, v, tb = sys.exc_info()
                     if hasattr(e, 'message'):
+                        traceback.print_exc()
                         sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis
continuing...\n" % (fn, lineno, e.message))
                     else:
+                        traceback.print_exc()
                         sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis
continuing...\n" % (fn, lineno, e))
                 if pl is not None:
                     if pl.data.is_router_ls:
@@ -997,8 +1015,10 @@ def parse_log_file(fn, log_index, comn):
                 except Exception as e:
                     # t, v, tb = sys.exc_info()
                     if hasattr(e, 'message'):
+                        traceback.print_exc()
                         sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis
continuing...\n" % (fn, lineno, e.message))
                     else:
+                        traceback.print_exc()
                         sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis
continuing...\n" % (fn, lineno, e))
                     # raise t, v, tb
             else:
@@ -1042,6 +1062,7 @@ if __name__ == "__main__":
     except:
         traceback.print_exc(file=sys.stdout)
         pass
+    comn.shorteners.short_data_names.sort_customers()
 
     print("Read two-instance file test")
     comn2 = common.Common()
diff --git a/tools/scraper/scraper.py b/tools/scraper/scraper.py
index afdd892..3dfde07 100755
--- a/tools/scraper/scraper.py
+++ b/tools/scraper/scraper.py
@@ -138,10 +138,6 @@ def main_except(argv):
         rtrs = parser.parse_log_file(arg_log_file, log_i, comn)
         comn.routers.append(rtrs)
 
-        # marshall facts about the run
-        for rtr in rtrs:
-            rtr.discover_connection_facts(comn)
-
     # Create lists of various things sorted by time
     tree = []  # log line
     ls_tree = []  # link state lines
@@ -155,6 +151,17 @@ def main_except(argv):
     ls_tree = sorted(ls_tree, key=lambda lfl: lfl.datetime)
     rr_tree = sorted(rr_tree, key=lambda lfl: lfl.datetime)
 
+    # post_extract the shortened names
+    comn.shorteners.short_link_names.sort_main()
+    comn.shorteners.short_data_names.sort_main()
+    for plf in tree:
+        plf.post_extract_names()
+
+    # marshall connection facts
+    for rtr_list in comn.routers:
+        for rtr in rtr_list:
+            rtr.discover_connection_facts(comn)
+
     # Back-propagate a router name/version/mode to each list's router0.
     # Complain if container name or version changes between instances.
     # Fill in container_id and shortened display_name tables
diff --git a/tools/scraper/test_data/test_data.txt b/tools/scraper/test_data/test_data.txt
index 32fe4ec..3cda589 100644
--- a/tools/scraper/test_data/test_data.txt
+++ b/tools/scraper/test_data/test_data.txt
@@ -25,6 +25,7 @@
 2018-08-06 13:41:42.793480 -0400 SERVER (trace) [7]:0 -> (EMPTY FRAME)
 2018-08-16 13:53:22.276203 -0400 SERVER (trace) [22]:0 <- @attach(18) [name="6216f14e-16db-4000-89eb-9716c02ef9e3-1acf0c3a-bdaa-4ce2-8786-e39c65bdb7bf",
handle=0, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0,
timeout=0, dynamic=true, dynamic-node-properties={:"x-opt-qd.address"="pulp.task.abc"}], target=@target(41)
[durable=0, timeout=0, dynamic=false], initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
 2018-08-16 13:56:59.038310 -0400 SERVER (trace) [8]:0 -> @transfer(20) [handle=0, delivery-id=1,
delivery-tag=b"\x14\x00\x00\x00\x00\x00\x00\x00", message-format=0, settled=true, aborted=true]
(/home/chug/git/qpid-dispatch/src/server.c:106)
+2018-07-16 13:56:59.038310 -0400 SERVER (trace) [8]:0 -> @transfer(20) [handle=0, delivery-id=1,
delivery-tag=b"\x14\x00\x00\x00\x00\x00\x00\x00", message-format=0, settled=true, aborted=true]
(/home/chug/git/qpid-dispatch/src/server.c:106)
 2018-08-24 14:29:26.821739 -0400 SERVER (trace) [2]:0 <- @attach(18) [name="qdlink.YoQaLsapwDzqhOL",
handle=1, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0,
expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router"], target=@target(41)
[durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router"],
initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
 2018-08-24 14:29:26.821750 -0400 SERVER (trace) [1]:0 <- @attach(18) [name="qdlink.qvRSF0ysELu13cM",
handle=2, role=false, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0,
expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router-data"], target=@target(41)
[durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router-data"],
initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
 2018-10-11 14:55:31.302512 -0400 SERVER (trace) [4]:0 -> @open(16) [container-id="A",
max-frame-size=16384, channel-max=32767, idle-time-out=60000, offered-capabilities=:"ANONYMOUS-RELAY",
properties={:product="qpid-dispatch-router", :version="1.4.0-SNAPSHOT", :"qd.conn-id"=4}]
(/home/chug/git/qpid-dispatch/src/server.c:106)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org


Mime
View raw message