allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject [35/42] git commit: [#4691] Cleaned up ModelCache debugging counters
Date Tue, 05 Feb 2013 20:23:34 GMT
[#4691] Cleaned up ModelCache debugging counters

Signed-off-by: Cory Johns <johnsca@geek.net>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/c340d113
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/c340d113
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/c340d113

Branch: refs/heads/master
Commit: c340d1134b9b1a036aed53916c2a0286329c0edd
Parents: 7fd94b9
Author: Cory Johns <johnsca@geek.net>
Authored: Tue Jan 29 22:16:39 2013 +0000
Committer: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Committed: Tue Feb 5 20:22:52 2013 +0000

----------------------------------------------------------------------
 Allura/allura/model/repo.py         |   19 ---------------
 Allura/allura/model/repo_refresh.py |    2 -
 scripts/refresh-last-commits.py     |   36 ++++-------------------------
 3 files changed, 5 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/Allura/allura/model/repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index b90b0c7..0f4352c 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -819,21 +819,6 @@ class ModelCache(object):
         self._synthetic_ids = defaultdict(set)
         self._synthetic_id_queries = defaultdict(set)
 
-        # temporary, for performance testing
-        self._query_hits = defaultdict(int)
-        self._instance_hits = defaultdict(int)
-        self._accesses = defaultdict(int)
-        self._get_calls = 0
-        self._get_walks = 0
-        self._get_walks_max = 0
-        self._get_hits = 0
-        self._get_misses = 0
-        self._build_calls = 0
-        self._build_walks = 0
-        self._build_walks_max = 0
-        self._skipped_trees = 0
-        self._processed_trees = 0
-
     def _normalize_query(self, query):
         _query = query
         if not isinstance(_query, tuple):
@@ -849,23 +834,19 @@ class ModelCache(object):
             raise AttributeError('%s has neither "query" nor "m" attribute' % cls)
 
     def get(self, cls, query):
-        self._accesses[cls] += 1
         _query = self._normalize_query(query)
         self._touch(cls, _query)
         if _query not in self._query_cache[cls]:
             val = self._model_query(cls).get(**query)
             self.set(cls, _query, val)
             return val
-        self._query_hits[cls] += 1
         _id = self._query_cache[cls][_query]
         if _id is None:
-            self._instance_hits[cls] += 1
             return None
         if _id not in self._instance_cache[cls]:
             val = self._model_query(cls).get(**query)
             self.set(cls, _query, val)
             return val
-        self._instance_hits[cls] += 1
         return self._instance_cache[cls][_id]
 
     def set(self, cls, query, val):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/Allura/allura/model/repo_refresh.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index 4bbd6e1..bdd5a1f 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -499,9 +499,7 @@ def compute_lcds(commit, cache):
         _compute_lcds(tree, cache)
 
 def _compute_lcds(tree, cache):
-    cache._processed_trees += 1
     if tree.path().strip('/') not in tree.commit.changed_paths:
-        cache._skipped_trees += 1
         return
     lcd = LastCommit.get(tree, create=True)  # auto-vivify LCD
     for x in tree.tree_ids:

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/scripts/refresh-last-commits.py
----------------------------------------------------------------------
diff --git a/scripts/refresh-last-commits.py b/scripts/refresh-last-commits.py
index b17a670..5b6e78a 100644
--- a/scripts/refresh-last-commits.py
+++ b/scripts/refresh-last-commits.py
@@ -93,11 +93,7 @@ def refresh_repo_lcds(commit_ids, options):
             with time(timings):
                 M.repo_refresh.compute_diffs(c.app.repo._id, tree_cache, commit)
             if len(timings) % 1000 == 0:
-                mt = max(timings)
-                tt = sum(timings)
-                at = tt / len(timings)
-                print '  Processed %d commits (max: %f, avg: %f, tot: %f, cl: %d)' % (
-                        len(timings), mt, at, tt, len(tree_cache))
+                _print_stats(timings, 1000)
     lcd_cache = M.repo.ModelCache(
             max_instances={M.repo.LastCommit: 4000},
             max_queries={M.repo.LastCommit: 10000},
@@ -116,39 +112,17 @@ def refresh_repo_lcds(commit_ids, options):
             # so that every commit sees the same copy
             lcd_cache.expire_new_instances(M.repo.LastCommit)
         if len(timings) % debug_step == 0:
-            _print_stats(lcd_cache, timings, debug_step, commit)
-            lcd_cache._get_walks_max = 0
-            lcd_cache._build_walks_max = 0
+            _print_stats(timings, debug_step)
     ThreadLocalORMSession.flush_all()
 
 
-def _print_stats(lcd_cache, timings, debug_step, commit):
+def _print_stats(timings, debug_step):
     mt = max(timings)
     tt = sum(timings)
     at = tt / len(timings)
     mat = sum(timings[-debug_step:]) / debug_step
-    laccs = lcd_cache._accesses[M.repo.LastCommit]
-    lqhits = lcd_cache._query_hits[M.repo.LastCommit]
-    lqavg = lqhits * 100 / laccs if laccs > 0 else 0
-    lihits = lcd_cache._instance_hits[M.repo.LastCommit]
-    liavg = lihits * 100 / laccs if laccs > 0 else 0
-    oaccs = sum([v for k,v in lcd_cache._accesses.items() if k != M.repo.LastCommit])
-    oqhits = sum([v for k,v in lcd_cache._query_hits.items() if k != M.repo.LastCommit])
-    oqavg = oqhits * 100 / oaccs if oaccs > 0 else 0
-    oihits = sum([v for k,v in lcd_cache._instance_hits.items() if k != M.repo.LastCommit])
-    oiavg = oihits * 100 / oaccs if oaccs > 0 else 0
-    gper = lcd_cache._get_hits * 100 / lcd_cache._get_calls if lcd_cache._get_calls >
0 else 0
-    print '  Processed %d commits (max: %f avg: %f mavg: %f tot: %f lq: %d/%d li: %d/%d qhit:
%d/%d ihit: %d/%d mgw: %d gh: %d mbw: %d ts: %d ps: %d)' % (
-            len(timings), mt, at, mat, tt,
-            lcd_cache.num_queries(), lcd_cache.num_queries(M.repo.LastCommit),
-            lcd_cache.num_instances(), lcd_cache.num_instances(M.repo.LastCommit),
-            oqavg, lqavg,
-            oiavg, liavg,
-            lcd_cache._get_walks_max, gper,
-            lcd_cache._build_walks_max,
-            len(lcd_cache.get(M.repo.TreesDoc, dict(_id=commit._id)).tree_ids),
-            lcd_cache._skipped_trees * 100 / lcd_cache._processed_trees)
-
+    print '  Processed %d commits (max: %f, avg: %f, mavg: %f, tot: %f)' % (
+            len(timings), mt, at, mat, tt)
 
 @contextmanager
 def time(timings):


Mime
View raw message