Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 836B5E5E6 for ; Tue, 5 Feb 2013 20:23:35 +0000 (UTC) Received: (qmail 91754 invoked by uid 500); 5 Feb 2013 20:23:35 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 91672 invoked by uid 500); 5 Feb 2013 20:23:35 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 91536 invoked by uid 99); 5 Feb 2013 20:23:35 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Feb 2013 20:23:35 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id BA68082A7AF; Tue, 5 Feb 2013 20:23:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tvansteenburgh@apache.org To: allura-commits@incubator.apache.org X-Mailer: ASF-Git Admin Mailer Subject: [35/42] git commit: [#4691] Cleaned up ModelCache debugging counters Message-Id: <20130205202334.BA68082A7AF@tyr.zones.apache.org> Date: Tue, 5 Feb 2013 20:23:34 +0000 (UTC) [#4691] Cleaned up ModelCache debugging counters Signed-off-by: Cory Johns Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/c340d113 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/c340d113 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/c340d113 Branch: refs/heads/master Commit: c340d1134b9b1a036aed53916c2a0286329c0edd Parents: 7fd94b9 Author: Cory Johns Authored: Tue Jan 29 22:16:39 2013 +0000 Committer: Tim Van Steenburgh Committed: Tue Feb 5 20:22:52 2013 +0000 ---------------------------------------------------------------------- Allura/allura/model/repo.py | 19 --------------- Allura/allura/model/repo_refresh.py | 2 - scripts/refresh-last-commits.py | 36 ++++------------------------- 3 files changed, 5 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/Allura/allura/model/repo.py ---------------------------------------------------------------------- diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py index b90b0c7..0f4352c 100644 --- a/Allura/allura/model/repo.py +++ b/Allura/allura/model/repo.py @@ -819,21 +819,6 @@ class ModelCache(object): self._synthetic_ids = defaultdict(set) self._synthetic_id_queries = defaultdict(set) - # temporary, for performance testing - self._query_hits = defaultdict(int) - self._instance_hits = defaultdict(int) - self._accesses = defaultdict(int) - self._get_calls = 0 - self._get_walks = 0 - self._get_walks_max = 0 - self._get_hits = 0 - self._get_misses = 0 - self._build_calls = 0 - self._build_walks = 0 - self._build_walks_max = 0 - self._skipped_trees = 0 - self._processed_trees = 0 - def _normalize_query(self, query): _query = query if not isinstance(_query, tuple): @@ -849,23 +834,19 @@ class ModelCache(object): raise AttributeError('%s has neither "query" nor "m" attribute' % cls) def get(self, cls, query): - self._accesses[cls] += 1 _query = self._normalize_query(query) self._touch(cls, _query) if _query not in self._query_cache[cls]: val = self._model_query(cls).get(**query) self.set(cls, _query, val) return val - self._query_hits[cls] += 1 _id = self._query_cache[cls][_query] if _id is None: - self._instance_hits[cls] += 1 return None if _id not in self._instance_cache[cls]: val = self._model_query(cls).get(**query) self.set(cls, _query, val) return val - self._instance_hits[cls] += 1 return self._instance_cache[cls][_id] def set(self, cls, query, val): http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/Allura/allura/model/repo_refresh.py ---------------------------------------------------------------------- diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py index 4bbd6e1..bdd5a1f 100644 --- a/Allura/allura/model/repo_refresh.py +++ b/Allura/allura/model/repo_refresh.py @@ -499,9 +499,7 @@ def compute_lcds(commit, cache): _compute_lcds(tree, cache) def _compute_lcds(tree, cache): - cache._processed_trees += 1 if tree.path().strip('/') not in tree.commit.changed_paths: - cache._skipped_trees += 1 return lcd = LastCommit.get(tree, create=True) # auto-vivify LCD for x in tree.tree_ids: http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c340d113/scripts/refresh-last-commits.py ---------------------------------------------------------------------- diff --git a/scripts/refresh-last-commits.py b/scripts/refresh-last-commits.py index b17a670..5b6e78a 100644 --- a/scripts/refresh-last-commits.py +++ b/scripts/refresh-last-commits.py @@ -93,11 +93,7 @@ def refresh_repo_lcds(commit_ids, options): with time(timings): M.repo_refresh.compute_diffs(c.app.repo._id, tree_cache, commit) if len(timings) % 1000 == 0: - mt = max(timings) - tt = sum(timings) - at = tt / len(timings) - print ' Processed %d commits (max: %f, avg: %f, tot: %f, cl: %d)' % ( - len(timings), mt, at, tt, len(tree_cache)) + _print_stats(timings, 1000) lcd_cache = M.repo.ModelCache( max_instances={M.repo.LastCommit: 4000}, max_queries={M.repo.LastCommit: 10000}, @@ -116,39 +112,17 @@ def refresh_repo_lcds(commit_ids, options): # so that every commit sees the same copy lcd_cache.expire_new_instances(M.repo.LastCommit) if len(timings) % debug_step == 0: - _print_stats(lcd_cache, timings, debug_step, commit) - lcd_cache._get_walks_max = 0 - lcd_cache._build_walks_max = 0 + _print_stats(timings, debug_step) ThreadLocalORMSession.flush_all() -def _print_stats(lcd_cache, timings, debug_step, commit): +def _print_stats(timings, debug_step): mt = max(timings) tt = sum(timings) at = tt / len(timings) mat = sum(timings[-debug_step:]) / debug_step - laccs = lcd_cache._accesses[M.repo.LastCommit] - lqhits = lcd_cache._query_hits[M.repo.LastCommit] - lqavg = lqhits * 100 / laccs if laccs > 0 else 0 - lihits = lcd_cache._instance_hits[M.repo.LastCommit] - liavg = lihits * 100 / laccs if laccs > 0 else 0 - oaccs = sum([v for k,v in lcd_cache._accesses.items() if k != M.repo.LastCommit]) - oqhits = sum([v for k,v in lcd_cache._query_hits.items() if k != M.repo.LastCommit]) - oqavg = oqhits * 100 / oaccs if oaccs > 0 else 0 - oihits = sum([v for k,v in lcd_cache._instance_hits.items() if k != M.repo.LastCommit]) - oiavg = oihits * 100 / oaccs if oaccs > 0 else 0 - gper = lcd_cache._get_hits * 100 / lcd_cache._get_calls if lcd_cache._get_calls > 0 else 0 - print ' Processed %d commits (max: %f avg: %f mavg: %f tot: %f lq: %d/%d li: %d/%d qhit: %d/%d ihit: %d/%d mgw: %d gh: %d mbw: %d ts: %d ps: %d)' % ( - len(timings), mt, at, mat, tt, - lcd_cache.num_queries(), lcd_cache.num_queries(M.repo.LastCommit), - lcd_cache.num_instances(), lcd_cache.num_instances(M.repo.LastCommit), - oqavg, lqavg, - oiavg, liavg, - lcd_cache._get_walks_max, gper, - lcd_cache._build_walks_max, - len(lcd_cache.get(M.repo.TreesDoc, dict(_id=commit._id)).tree_ids), - lcd_cache._skipped_trees * 100 / lcd_cache._processed_trees) - + print ' Processed %d commits (max: %f, avg: %f, mavg: %f, tot: %f)' % ( + len(timings), mt, at, mat, tt) @contextmanager def time(timings):