allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject [03/17] allura git commit: [#7837] ticket:736 Wipe out all the traces of DiffInfoDoc
Date Mon, 30 Mar 2015 19:30:28 GMT
[#7837] ticket:736 Wipe out all the traces of DiffInfoDoc


Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/d0dd4b75
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/d0dd4b75
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/d0dd4b75

Branch: refs/heads/master
Commit: d0dd4b757f7c33fc7a11a180e7cbcce8858edf42
Parents: d50d350
Author: Igor Bondarenko <jetmind2@gmail.com>
Authored: Mon Mar 2 16:18:10 2015 +0000
Committer: Dave Brondsema <dbrondsema@slashdotmedia.com>
Committed: Mon Mar 30 19:20:40 2015 +0000

----------------------------------------------------------------------
 Allura/allura/model/repo.py                     |  2 +-
 Allura/allura/model/repo_refresh.py             | 75 +-------------------
 Allura/allura/model/repository.py               |  9 ---
 Allura/allura/scripts/refresh_last_commits.py   | 24 +------
 Allura/allura/scripts/refreshrepo.py            |  7 --
 Allura/test-light.py                            |  3 +-
 ForgeSVN/forgesvn/model/svn.py                  | 47 +-----------
 .../forgesvn/tests/model/test_repository.py     |  7 --
 8 files changed, 7 insertions(+), 167 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/Allura/allura/model/repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index 645fdc5..f041116 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -21,6 +21,6 @@
 
 from .repository import SUser, SObjType
 from .repository import QSIZE, README_RE, VIEWABLE_EXTENSIONS, PYPELINE_EXTENSIONS, DIFF_SIMILARITY_THRESHOLD
-from .repository import CommitDoc, TreeDoc, LastCommitDoc, TreesDoc, DiffInfoDoc, CommitRunDoc
+from .repository import CommitDoc, TreeDoc, LastCommitDoc, TreesDoc, CommitRunDoc
 from .repository import RepoObject, Commit, Tree, Blob, LastCommit
 from .repository import ModelCache

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/Allura/allura/model/repo_refresh.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index d8ef050..1711bc3 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -31,7 +31,7 @@ from ming.orm import mapper, session, ThreadLocalORMSession
 
 from allura.lib import utils
 from allura.lib import helpers as h
-from allura.model.repository import CommitDoc, TreeDoc, TreesDoc, DiffInfoDoc
+from allura.model.repository import CommitDoc, TreeDoc, TreesDoc
 from allura.model.repository import CommitRunDoc
 from allura.model.repository import Commit, Tree, LastCommit, ModelCache
 from allura.model.index import ArtifactReferenceDoc, ShortlinkDoc
@@ -110,19 +110,10 @@ def refresh_repo(repo, all_commits=False, notify=True, new_clone=False):
 
     # Compute diffs
     cache = {}
-    # For some SCMs, we don't want to pre-compute the diffs because that
+    # For some SCMs, we don't want to pre-compute the LCDs because that
     # would be too expensive, so we skip them here and do them on-demand
     # with caching.
     if repo._refresh_precompute:
-        for i, oid in enumerate(commit_ids):
-            cid = CommitDoc.m.find(dict(_id=oid), validate=False).next()
-            ci = mapper(Commit).create(cid, dict(instrument=False))
-            ci.set_context(repo)
-            compute_diffs(repo._id, cache, ci)
-            if (i + 1) % 100 == 0:
-                log.info('Compute diffs %d: %s', (i + 1), ci._id)
-
-    if repo._refresh_precompute:
         model_cache = ModelCache()
         lcid_cache = {}
         for i, oid in enumerate(reversed(commit_ids)):
@@ -368,68 +359,6 @@ def unknown_commit_ids(all_commit_ids):
     return result
 
 
-def compute_diffs(repo_id, tree_cache, rhs_ci):
-    '''compute simple differences between a commit and its first parent'''
-    if rhs_ci.tree_id is None:
-        return tree_cache
-
-    def _update_cache(lhs_tree_ids, rhs_tree_ids):
-        # crazy cache logic that I'm not certain I understand
-        new_tree_ids = [
-            tid for tid in chain(lhs_tree_ids, rhs_tree_ids)
-            if tid not in tree_cache]
-        tree_index = dict(
-            (t._id, t) for t in TreeDoc.m.find(dict(_id={'$in': new_tree_ids}), validate=False))
-        tree_index.update(tree_cache)
-        rhs_tree_ids_set = set(rhs_tree_ids)
-        tree_cache.clear()
-        tree_cache.update(
-            (id, t) for id, t in tree_index.iteritems() if id in rhs_tree_ids_set)
-        return tree_index
-
-    empty_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[])
-    commit_info = get_commit_info(rhs_ci)
-    differences = []
-    rhs_treesdoc = TreesDoc.m.get(_id=rhs_ci._id)
-    if not rhs_treesdoc:
-        # FIXME: These sometimes don't exist for unknown reasons; they should
-        # be auto-gen'ed
-        log.error('Missing TreesDoc: %s', rhs_ci)
-        return tree_cache
-    for lhs_cid in rhs_ci.parent_ids:
-        lhs_ci = CommitDoc.m.get(_id=lhs_cid)
-        if lhs_ci is None:
-            log.error(
-                'Commit ID referenced as parent but not found: %s parent of %s', lhs_cid,
rhs_ci)
-            continue
-        lhs_treesdoc = TreesDoc.m.get(_id=lhs_cid)
-        if not lhs_treesdoc:
-            # FIXME: These sometimes don't exist for unknown reasons; they
-            # should be auto-gen'ed
-            log.error('Missing TreesDoc: %s', rhs_ci)
-            continue
-        tree_index = _update_cache(
-            lhs_treesdoc.tree_ids, rhs_treesdoc.tree_ids)
-        rhs_tree = tree_index[rhs_ci.tree_id]
-        lhs_tree = tree_index.get(lhs_ci.tree_id, empty_tree)
-        for name, lhs_id, rhs_id in _diff_trees(lhs_tree, rhs_tree, tree_index):
-            differences.append(
-                dict(name=name, lhs_id=lhs_id, rhs_id=rhs_id))
-    if not rhs_ci.parent_ids:
-        # no parents, so everything in rhs is new
-        tree_index = _update_cache([], rhs_treesdoc.tree_ids)
-        rhs_tree = tree_index[rhs_ci.tree_id]
-        for name, lhs_id, rhs_id in _diff_trees(empty_tree, rhs_tree, tree_index):
-            differences.append(
-                dict(name=name, lhs_id=lhs_id, rhs_id=rhs_id))
-    # Build the diffinfo
-    di = DiffInfoDoc(dict(
-        _id=rhs_ci._id,
-        differences=differences))
-    di.m.save()
-    return tree_cache
-
-
 def send_notifications(repo, commit_ids):
     '''Create appropriate notification and feed objects for a refresh'''
     from allura.model import Feed, Notification

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index d15394f..fc50860 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -893,15 +893,6 @@ TreesDoc = collection(
     Field('_id', str),
     Field('tree_ids', [str]))
 
-# Information about which things were added/removed in  commit
-# DiffInfoDoc._id = CommitDoc._id
-DiffInfoDoc = collection(
-    'repo_diffinfo', main_doc_session,
-    Field('_id', str),
-    Field(
-        'differences',
-        [dict(name=str, lhs_id=str, rhs_id=str)]))
-
 # List of commit runs (a run is a linear series of single-parent commits)
 # CommitRunDoc.commit_ids = [ CommitDoc._id, ... ]
 CommitRunDoc = collection(

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/Allura/allura/scripts/refresh_last_commits.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/refresh_last_commits.py b/Allura/allura/scripts/refresh_last_commits.py
index 4673a2a..095fafb 100644
--- a/Allura/allura/scripts/refresh_last_commits.py
+++ b/Allura/allura/scripts/refresh_last_commits.py
@@ -70,8 +70,6 @@ class RefreshLastCommits(ScriptTask):
                             'project(s) being refreshed before doing the refresh.')
         parser.add_argument('--dry-run', action='store_true', dest='dry_run',
                             default=False, help='Log names of projects that would have their
')
-        parser.add_argument('--diffs', action='store_true', dest='diffs',
-                            default=False, help='Refresh / clean diffs as well as LCDs')
         parser.add_argument('--limit', action='store', type=int, dest='limit',
                             default=False, help='Limit of how many commits to process')
         return parser
@@ -118,7 +116,7 @@ class RefreshLastCommits(ScriptTask):
                         ci_ids = list(
                             reversed(list(c.app.repo.all_commit_ids())))
                         if options.clean:
-                            cls._clean(ci_ids, options.diffs)
+                            cls._clean(ci_ids)
 
                         log.info('Refreshing all last commits in %r',
                                  c.app.repo)
@@ -138,16 +136,6 @@ class RefreshLastCommits(ScriptTask):
     def refresh_repo_lcds(cls, commit_ids, options):
         tree_cache = {}
         timings = []
-        if options.diffs:
-            print 'Processing diffs'
-            for i, commit_id in enumerate(commit_ids):
-                commit = M.repository.Commit.query.get(_id=commit_id)
-                with time(timings):
-                    M.repo_refresh.compute_diffs(
-                        c.app.repo._id, tree_cache, commit)
-                if i % 1000 == 0:
-                    cls._print_stats(i, timings, 1000)
-
         model_cache = M.repository.ModelCache(
             max_instances={M.repository.LastCommit: 4000},
             max_queries={M.repository.LastCommit: 4000},
@@ -171,15 +159,7 @@ class RefreshLastCommits(ScriptTask):
         ThreadLocalORMSession.flush_all()
 
     @classmethod
-    def _clean(cls, commit_ids, clean_diffs):
-        if clean_diffs:
-            # delete DiffInfoDocs
-            i = M.repository.DiffInfoDoc.m.find(
-                dict(_id={'$in': commit_ids})).count()
-            log.info("Deleting %i DiffInfoDoc docs for %i commits...",
-                     i, len(commit_ids))
-            M.repository.DiffInfoDoc.m.remove(dict(_id={'$in': commit_ids}))
-
+    def _clean(cls, commit_ids):
         # delete LastCommitDocs
         i = M.repository.LastCommitDoc.m.find(
             dict(commit_id={'$in': commit_ids})).count()

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/Allura/allura/scripts/refreshrepo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py
index 08f3fcc..a336b0d 100644
--- a/Allura/allura/scripts/refreshrepo.py
+++ b/Allura/allura/scripts/refreshrepo.py
@@ -118,13 +118,6 @@ class RefreshRepo(ScriptTask):
                                 M.repository.LastCommitDoc.m.remove(
                                     dict(commit_ids={'$in': ci_ids_chunk}))
 
-                            i = M.repository.DiffInfoDoc.m.find(
-                                {"_id": {"$in": ci_ids_chunk}}).count()
-                            if i:
-                                log.info("Deleting %i DiffInfoDoc docs...", i)
-                                M.repository.DiffInfoDoc.m.remove(
-                                    {"_id": {"$in": ci_ids_chunk}})
-
                             i = M.repository.CommitRunDoc.m.find(
                                 {"commit_ids": {"$in": ci_ids_chunk}}).count()
                             if i:

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/Allura/test-light.py
----------------------------------------------------------------------
diff --git a/Allura/test-light.py b/Allura/test-light.py
index be218b9..f095214 100644
--- a/Allura/test-light.py
+++ b/Allura/test-light.py
@@ -20,7 +20,7 @@ import sys
 from pylons import tmpl_context as c
 
 from allura.lib import helpers as h
-from allura.model.repository import CommitDoc, TreeDoc, TreesDoc, DiffInfoDoc
+from allura.model.repository import CommitDoc, TreeDoc, TreesDoc
 from allura.model.repository import LastCommitDoc, CommitRunDoc
 from allura.model.repo_refresh import refresh_repo
 
@@ -35,7 +35,6 @@ def main():
     CommitDoc.m.remove({})
     TreeDoc.m.remove({})
     TreesDoc.m.remove({})
-    DiffInfoDoc.m.remove({})
     LastCommitDoc.m.remove({})
     CommitRunDoc.m.remove({})
 

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/ForgeSVN/forgesvn/model/svn.py
----------------------------------------------------------------------
diff --git a/ForgeSVN/forgesvn/model/svn.py b/ForgeSVN/forgesvn/model/svn.py
index 44d625f..142ca6a 100644
--- a/ForgeSVN/forgesvn/model/svn.py
+++ b/ForgeSVN/forgesvn/model/svn.py
@@ -76,9 +76,6 @@ class Repository(M.Repository):
                                        'checkout_url'),
                                    dest_path=self.suggested_clone_dest_path()))
 
-    def compute_diffs(self):
-        return
-
     def latest(self, branch=None):
         if self._impl is None:
             return None
@@ -366,7 +363,7 @@ class SVNImplementation(M.RepositoryImplementation):
             oid for oid in oids if oid not in seen_oids]
 
     def refresh_commit_info(self, oid, seen_object_ids, lazy=True):
-        from allura.model.repository import CommitDoc, DiffInfoDoc
+        from allura.model.repository import CommitDoc
         ci_doc = CommitDoc.m.get(_id=oid)
         if ci_doc and lazy:
             return False
@@ -408,48 +405,6 @@ class SVNImplementation(M.RepositoryImplementation):
             except DuplicateKeyError:
                 if lazy:
                     return False
-        # Save diff info
-        di = DiffInfoDoc.make(dict(_id=ci_doc._id, differences=[]))
-        for path in log_entry.changed_paths:
-            if path.action in ('A', 'M', 'R'):
-                try:
-                    rhs_info = self._svn.info2(
-                        self._url + h.really_unicode(path.path),
-                        revision=self._revision(ci_doc._id),
-                        recurse=False)[0][1]
-                    rhs_id = self._obj_oid(ci_doc._id, rhs_info)
-                except pysvn.ClientError, e:
-                    # pysvn will sometimes misreport deleted files (D) as
-                    # something else (like A), causing info2() to raise a
-                    # ClientError since the file doesn't exist in this
-                    # revision. Set lrhs_id = None to treat like a deleted file
-                    log.info('This error was handled gracefully and logged '
-                             'for informational purposes only:\n' + str(e))
-                    rhs_id = None
-            else:
-                rhs_id = None
-            if ci_doc.parent_ids and path.action in ('D', 'M', 'R'):
-                try:
-                    lhs_info = self._svn.info2(
-                        self._url + h.really_unicode(path.path),
-                        revision=self._revision(ci_doc.parent_ids[0]),
-                        recurse=False)[0][1]
-                    lhs_id = self._obj_oid(ci_doc._id, lhs_info)
-                except pysvn.ClientError, e:
-                    # pysvn will sometimes report new files as 'M'odified,
-                    # causing info2() to raise ClientError since the file
-                    # doesn't exist in the parent revision. Set lhs_id = None
-                    # to treat like a newly added file.
-                    log.info('This error was handled gracefully and logged '
-                             'for informational purposes only:\n' + str(e))
-                    lhs_id = None
-            else:
-                lhs_id = None
-            di.differences.append(dict(
-                name=h.really_unicode(path.path),
-                lhs_id=lhs_id,
-                rhs_id=rhs_id))
-        di.m.save()
         return True
 
     def compute_tree_new(self, commit, tree_path='/'):

http://git-wip-us.apache.org/repos/asf/allura/blob/d0dd4b75/ForgeSVN/forgesvn/tests/model/test_repository.py
----------------------------------------------------------------------
diff --git a/ForgeSVN/forgesvn/tests/model/test_repository.py b/ForgeSVN/forgesvn/tests/model/test_repository.py
index 7f34f45..2c86309 100644
--- a/ForgeSVN/forgesvn/tests/model/test_repository.py
+++ b/ForgeSVN/forgesvn/tests/model/test_repository.py
@@ -1052,8 +1052,6 @@ class TestCommit(_TestWithRepo):
         self.repo._impl.commit = mock.Mock(return_value=self.ci)
         self.repo._impl.open_blob = self._unique_blobs()
         M.repo_refresh.refresh_commit_trees(self.ci, {})
-        M.repo_refresh.compute_diffs(self.repo._id, {}, self.ci)
-        # self.ci.compute_diffs()
         assert_equal(self.ci.diffs.added,
                      ['a', 'a/a', 'a/a/a', 'a/a/b', 'a/b'])
         assert (self.ci.diffs.copied
@@ -1064,7 +1062,6 @@ class TestCommit(_TestWithRepo):
         ci.parent_ids = ['foo']
         self._make_log(ci)
         M.repo_refresh.refresh_commit_trees(ci, {})
-        M.repo_refresh.compute_diffs(self.repo._id, {}, ci)
         assert_equal(ci.diffs.removed, ['a', 'a/a', 'a/a/a', 'a/a/b', 'a/b'])
         assert (ci.diffs.copied
                 == ci.diffs.changed
@@ -1080,7 +1077,6 @@ class TestCommit(_TestWithRepo):
         ci.parent_ids = ['foo']
         self._make_log(ci)
         M.repo_refresh.refresh_commit_trees(ci, {})
-        M.repo_refresh.compute_diffs(self.repo._id, {}, ci)
         assert_equal(ci.diffs.added, ['b', 'b/a', 'b/a/a', 'b/a/b', 'b/b'])
         assert_equal(ci.diffs.removed, ['a', 'a/a', 'a/a/a', 'a/a/b', 'a/b'])
         assert (ci.diffs.copied
@@ -1104,7 +1100,6 @@ class TestCommit(_TestWithRepo):
 
         self.repo._impl.commit = mock.Mock(return_value=self.ci)
         M.repo_refresh.refresh_commit_trees(self.ci, {})
-        M.repo_refresh.compute_diffs(self.repo._id, {}, self.ci)
         assert_equal(self.ci.diffs.added,
                      ['a', 'a/a', 'a/a/a', 'a/a/b', 'a/b'])
         assert (self.ci.diffs.copied
@@ -1122,7 +1117,6 @@ class TestCommit(_TestWithRepo):
         ci.parent_ids = ['foo']
         self._make_log(ci)
         M.repo_refresh.refresh_commit_trees(ci, {})
-        M.repo_refresh.compute_diffs(self.repo._id, {}, ci)
         assert_equal(ci.diffs.added, ['b', 'b/a', 'b/a/a', 'b/a/b', 'b/b'])
         assert_equal(ci.diffs.removed, ['a', 'a/a', 'a/a/a', 'a/a/b', 'a/b'])
         assert (ci.diffs.copied
@@ -1138,7 +1132,6 @@ class TestCommit(_TestWithRepo):
         ci.parent_ids = ['bar']
         self._make_log(ci)
         M.repo_refresh.refresh_commit_trees(ci, {})
-        M.repo_refresh.compute_diffs(self.repo._id, {}, ci)
         assert_equal(ci.diffs.added, [])
         assert_equal(ci.diffs.changed, [])
         assert_equal(ci.diffs.removed, ['b/a/a'])


Mime
View raw message