Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6EEADE43E for ; Wed, 6 Feb 2013 15:42:57 +0000 (UTC) Received: (qmail 86743 invoked by uid 500); 6 Feb 2013 15:42:57 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 86681 invoked by uid 500); 6 Feb 2013 15:42:57 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 85327 invoked by uid 99); 6 Feb 2013 15:42:53 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Feb 2013 15:42:53 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1E71D24140; Wed, 6 Feb 2013 15:42:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: johnsca@apache.org To: allura-commits@incubator.apache.org X-Mailer: ASF-Git Admin Mailer Subject: [1/50] git commit: [#4691] Added stepping to LCD refresh to only populate every 100th commit Message-Id: <20130206154253.1E71D24140@tyr.zones.apache.org> Date: Wed, 6 Feb 2013 15:42:53 +0000 (UTC) [#4691] Added stepping to LCD refresh to only populate every 100th commit Signed-off-by: Cory Johns Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/9a399c0d Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/9a399c0d Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/9a399c0d Branch: refs/heads/cj/4691 Commit: 9a399c0d13d00cc3a412efbe3745bd284b1756fc Parents: 320025c Author: Cory Johns Authored: Fri Nov 30 14:58:34 2012 +0000 Committer: Tim Van Steenburgh Committed: Tue Feb 5 20:22:49 2013 +0000 ---------------------------------------------------------------------- scripts/refresh-last-commits.py | 22 ++++++++++++++++------ 1 files changed, 16 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9a399c0d/scripts/refresh-last-commits.py ---------------------------------------------------------------------- diff --git a/scripts/refresh-last-commits.py b/scripts/refresh-last-commits.py index 8776010..0a59b31 100644 --- a/scripts/refresh-last-commits.py +++ b/scripts/refresh-last-commits.py @@ -1,3 +1,4 @@ +import sys import argparse import logging import re @@ -49,7 +50,6 @@ def main(options): continue ci_ids = list(reversed(list(c.app.repo.all_commit_ids()))) - #ci_ids = list(c.app.repo.all_commit_ids()) if options.clean: if options.diffs: # delete DiffInfoDocs @@ -77,6 +77,11 @@ def main(options): ThreadLocalORMSession.close_all() +def enum_step(iter, step): + for i,elem in enumerate(iter): + if i % step == 0: + yield i, elem + def refresh_repo_lcds(commit_ids, options): tree_cache = {} timings = [] @@ -95,15 +100,18 @@ def refresh_repo_lcds(commit_ids, options): lcd_cache = M.repo.ModelCache(80000) timings = [] print 'Processing last commits' - for commit_id in commit_ids: + for i, commit_id in enum_step(commit_ids, options.step): + print ' Processing commit %s...' % commit_id, + sys.stdout.flush() commit = M.repo.Commit.query.get(_id=commit_id) with time(timings): M.repo_refresh.compute_lcds(commit, lcd_cache) - if len(timings) % 100 == 0: + print 'done in %fs' % timings[-1] + if len(timings) % 10 == 0: mt = max(timings) tt = sum(timings) at = tt / len(timings) - mat = sum(timings[-100:]) / 100 + mat = sum(timings[-10:]) / 10 print ' Processed %d commits (max: %f, avg: %f, mavg: %f, tot: %f, lc: %d, lcl: %d, hits: %d, agw: %d, mgw: %d, gh: %d, abw: %d, mbw: %d, ts: %d)' % ( len(timings), mt, at, mat, tt, lcd_cache.size(), len(lcd_cache._cache[M.repo.LastCommit]), lcd_cache._hits * 100 / (lcd_cache._hits + lcd_cache._misses), @@ -112,8 +120,6 @@ def refresh_repo_lcds(commit_ids, options): len(lcd_cache.get(M.repo.TreesDoc, dict(_id=commit._id)).tree_ids)) ThreadLocalORMSession.flush_all() ThreadLocalORMSession.close_all() - #if len(timings) == 300: - # break @contextmanager @@ -164,6 +170,10 @@ def parse_options(): 'profiling output to ./refresh.profile') parser.add_argument('--diffs', action='store_true', dest='diffs', default=False, help='Refresh diffs as well as LCDs') + parser.add_argument('--all', action='store_const', dest='step', + const=1, default=100, help='Refresh diffs as well as LCDs') + parser.add_argument('--step', action='store', dest='step', + type=int, default=100, help='Refresh diffs as well as LCDs') return parser.parse_args() if __name__ == '__main__':