allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject git commit: Partial commit, WIP
Date Sun, 04 Nov 2012 23:45:20 GMT
Updated Branches:
  refs/heads/cj/4691 a2e0bea98 -> 997b38454 (forced update)


Partial commit, WIP


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/997b3845
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/997b3845
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/997b3845

Branch: refs/heads/cj/4691
Commit: 997b38454df51d9d9c399de7a8b03f76e8e6fafc
Parents: e8fad3c
Author: Cory Johns <johnsca@geek.net>
Authored: Wed Oct 10 02:28:07 2012 +0000
Committer: Cory Johns <johnsca@geek.net>
Committed: Sun Nov 4 23:44:46 2012 +0000

----------------------------------------------------------------------
 Allura/allura/model/repo.py                        |   63 +++++++--
 Allura/allura/model/repo_refresh.py                |  101 ++++++++++++++-
 Allura/allura/model/repository.py                  |   38 +++++-
 .../allura/templates/widgets/repo/tree_widget.html |   20 ++--
 ForgeGit/forgegit/model/git_repo.py                |    4 +
 ForgeHg/forgehg/model/hg.py                        |    4 +
 ForgeSVN/forgesvn/model/svn.py                     |    3 +
 7 files changed, 205 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/Allura/allura/model/repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index b82610c..65b0da9 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -7,11 +7,12 @@ from itertools import chain
 from datetime import datetime
 from collections import defaultdict
 from difflib import SequenceMatcher, unified_diff
+from operator import itemgetter
 
 from pylons import c
 import pymongo.errors
 
-from ming import Field, collection
+from ming import Field, collection, Index
 from ming import schema as S
 from ming.base import Object
 from ming.utils import LazyProperty
@@ -61,21 +62,25 @@ TreeDoc = collection(
     Field('blob_ids', [dict(name=str, id=str)]),
     Field('other_ids', [dict(name=str, id=str, type=SObjType)]))
 
-# Information about the last commit to touch a tree/blob
-# LastCommitDoc.object_id = TreeDoc._id
+# Information about the last commit to touch a tree
 LastCommitDoc = collection(
-    'repo_last_commit', project_doc_session,
-    Field('_id', str),
-    Field('object_id', str, index=True),
-    Field('name', str),
-    Field('commit_info', dict(
-        id=str,
-        date=datetime,
-        author=str,
-        author_email=str,
-        author_url=str,
-        shortlink=str,
-        summary=str)))
+    'repo_last_commit', main_doc_session,
+    Field('_id', S.ObjectId()),
+    Field('commit_ids', [str]),
+    Field('path', str),
+    Index('commit_ids', 'path'),
+    Field('entries', {
+        str:dict(
+            type=str,
+            name=str,
+            commit_info=dict(
+                id=str,
+                date=datetime,
+                author=str,
+                author_email=str,
+                author_url=str,
+                shortlink=str,
+                summary=str))}))
 
 # List of all trees contained within a commit
 # TreesDoc._id = CommitDoc._id
@@ -316,6 +321,25 @@ class Commit(RepoObject):
             cur = cur[part]
         return cur
 
+    def get_parent(self, index=0):
+        '''Get the parent of this commit.
+
+        If there is no parent commit, or if an invalid index is given,
+        returns None.
+        '''
+        try:
+            return Commit.query.get(_id=self.parent_ids[index])
+        except IndexError as e:
+            return None
+
+    def changed(self, path):
+        '''Test whether a given path was changed in this commit.'''
+        di = DiffInfoDoc.m.get(_id=self._id)
+        for change in di.differences:
+            if change.name.strip('/') == path.strip('/'):
+                return True
+        return False
+
 class Tree(RepoObject):
     # Ephemeral attrs
     repo=None
@@ -386,6 +410,15 @@ class Tree(RepoObject):
         return None, None
 
     def ls(self):
+        last_commit = LastCommitDoc.m.get(
+                commit_ids=self.commit._id,
+                path=self.path(),
+            )
+        if not last_commit:
+            from .repo_refresh import build_last_commit_doc
+            last_commit = build_last_commit_doc(self)
+        return sorted(last_commit.entries.values(), cmp=lambda a,b: cmp(b.type,a.type) or
cmp(a.name,b.name))
+
         # Load last commit info
         id_re = re.compile("^{0}:{1}:".format(
             self.repo._id,

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/Allura/allura/model/repo_refresh.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index 6e0db59..75109b9 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -2,6 +2,7 @@ import logging
 from itertools import chain
 from cPickle import dumps
 import re
+import os
 
 import bson
 
@@ -348,7 +349,7 @@ def compute_diffs(repo_id, tree_cache, rhs_ci):
             dict(name=name, lhs_id=lhs_id, rhs_id=rhs_id))
     # Set last commit data
     rhs_tree = tree_index[rhs_ci.tree_id]
-    refresh_last_commit(repo_id, '/', rhs_tree, lhs_tree, None, commit_info)
+    #refresh_last_commit(repo_id, '/', rhs_tree, lhs_tree, None, commit_info)
     # Build the diffinfo
     di = DiffInfoDoc(dict(
             _id=rhs_ci._id,
@@ -522,3 +523,101 @@ def last_known_commit_id(all_commit_ids, new_commit_ids):
     if not all_commit_ids: return None
     if not new_commit_ids: return all_commit_ids[-1]
     return all_commit_ids[all_commit_ids.index(new_commit_ids[0]) - 1]
+
+def build_last_commit_doc(tree):
+    '''
+    We need a LCD for this tree, for which there are two possibilities:
+
+      1) This tree was modified in an ancestor commit but the commit ID chain
+         was not filled in.  In this case, as we walk back up the tree, we'll
+         find a LCD record when or before we find the commit in which this tree
+         was changed.  If we find this, we save the new commit IDs in the LCD
+         record for faster access in the future.
+
+      2) The LCD record for the commit in which this tree was changed does not
+         exist.  We'll find the commit and still not have a LCD record, which
+         means we have to construct it.  The LCD record will only contain the
+         commit IDs up to the commit where the tree was most recently changed.
+
+         Constructing it differs for SVN and Git / Hg.  SVN can pull all the info
+         from a single SVN call.  Git / Hg have to walk up the tree.  (SVN could
+         walk up the tree as well, except that the TreesDoc and DiffInfoDoc
+         records are not correctly populated, making it hard to tell when a tree
+         was changed in a given commit, plus it's unnecessary.)
+
+         To walk up the tree, we have to keep track of which entries we still
+         need info about.  At each step of the walk, we check the following:
+
+           1) If the current tree has a LCD record, we can pull all the remaining
+              info we need from it, and we're done.
+
+           2) If the tree was modified in this commit, then we pull the info for
+              all changed entries, then making the parent tree the new active
+              tree and continuing the walk.  Once we have data for all entries,
+              we're done.
+    '''
+    unfilled = set([n.name for n in chain(tree.tree_ids, tree.blob_ids, tree.other_ids)])
+    tree_nodes = set([n.name for n in tree.tree_ids])
+    entires = []
+    commit_ids = []
+    commit = tree.commit
+    path = tree.path().strip('/')
+    lcd = None
+    has_changes = False
+    while unfilled:
+        last_commit = LastCommitDoc.m.get(
+                commit_ids=commit._id,
+                path=path,
+            )
+        if not has_changes:
+            # no changes found yet, so look to see if we have a matching LCD
+            # that just doesn't have all the commit_ids filled in
+            if last_commit:
+                # found a complete LCD for our tree
+                last_commit.commit_ids.extend(commit_ids)
+                last_commit.m.save()
+                return last_commit
+            # LCD is only valid for the most recent commit
+            # that changed the tree, so once we have changes,
+            # stop recording commit_ids
+            commit_ids.append(commit._id)
+
+        # look for changes to the tree in this commit,
+        # meaning our LCD is missing and must be built
+        diff_info = DiffInfoDoc.m.get(_id=commit._id)
+        diffs = set()
+        for d in diff_info.differences:
+            diffs.add(d.name)
+            node_path = os.path.dirname(d.name)
+            while node_path:
+                diffs.add(node_path)
+                node_path = os.path.dirname(node_path)
+        import ipdb; ipdb.set_trace()
+        for name in list(unfilled):
+            full_name = path + '/' + name
+            if full_name in diffs:
+                has_changes = True
+                if lcd is None:
+                    lcd = LastCommitDoc(dict(
+                                commit_ids=commit_ids,
+                                path=path,
+                                entries=dict(),
+                            ))
+                lcd.entries[name] = dict(
+                        type=name in tree_nodes and 'DIR' or 'BLOB',
+                        name=name,
+                        commit_info=get_commit_info(commit),
+                    )
+                unfilled.remove(name)
+
+        # if we have changes but this commit has an LCD for our
+        # path, it should have all the remaining info we need
+        if last_commit:
+            for name in list(unfilled):
+                lcd.entries[name] = last_commit.entries[name]
+                unfilled.remove(name)
+
+        # walk up the tree
+        commit = commit.parent()
+    lcd.m.save()
+    return lcd

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index ba8df55..6f57cf8 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -429,8 +429,42 @@ class Repository(Artifact, ActivityObject):
         with self.push_upstream_context():
             return MergeRequest.query.find(q).count()
 
-    def get_last_commit(self, obj):
-        from .repo import LastCommitDoc
+    def get_last_commit(self, tree):
+        '''Find the LastCommitDoc for the given tree.
+
+        Walks up the commit tree until either:
+
+        1) A LCD is found for the given tree.  (If the LCD was not found for the
+           tree's commit, the commits traversed while searching for it are
+           added to the LCD for faster retrieval in the future.)
+
+        2) The commit in which the tree was most recently modified is found.
+           In this case, we know that the LCD hasn't been constructed for this
+           (chain of) commit(s), and it will have to be built.
+        '''
+        from .repo import Commit, LastCommitDoc
+        last_commit = None
+        commit = tree.commit
+        path = tree.path()
+        orig_tree = tree
+        other_commit_ids = []
+        while commit is not None:
+            last_commit = LastCommitDoc.m.find(dict(
+                    commit_ids=commit._id,
+                    path=path,
+                )).first()
+            if last_commit is not None:
+                # found our LCD; add any traversed commits to it
+                last_commit.commit_ids.append(commit_ids)
+                last_commit.m.save()
+                return last_commit
+            elif commit.changed(path):
+                # tree was changed but no LCD found; have to build
+                return self.compute_last_commit(commit, path, other_commit_ids)
+            else:
+                other_commit_ids.append(commit._id)
+                commit = commit.get_parent()
+
         lc = LastCommitDoc.m.get(
             repo_id=self._id, object_id=obj._id)
         if lc is None:

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/Allura/allura/templates/widgets/repo/tree_widget.html
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/widgets/repo/tree_widget.html b/Allura/allura/templates/widgets/repo/tree_widget.html
index 09b0758..a2d8024 100644
--- a/Allura/allura/templates/widgets/repo/tree_widget.html
+++ b/Allura/allura/templates/widgets/repo/tree_widget.html
@@ -25,24 +25,24 @@
     {% for dirent in tree.ls() %}
     <tr>
       <td class="nowrap">
-        <a href="{{h.urlquote(dirent.href)}}">
-          <b data-icon="{{dirent.kind == 'DIR' and 'o' or 'n'}}" class="ico {{dirent.kind
== 'DIR' and 'folder' or 'table'}}"></b>
+        <a href="{{h.urlquote(dirent.name)}}">
+          <b data-icon="{{dirent.type == 'DIR' and 'o' or 'n'}}" class="ico {{dirent.type
== 'DIR' and 'folder' or 'table'}}"></b>
           <span>{{h.really_unicode(dirent.name)}}</span>
         </a>
       </td>
-      <td class="nowrap">{{lib.abbr_date(dirent.last_commit.date)}}</td>
+      <td class="nowrap">{{lib.abbr_date(dirent.commit_info.date)}}</td>
       <td class="nowrap">
-        {% if dirent.last_commit.author_url %}
-          <a href="{{dirent.last_commit.author_url}}">{{lib.email_gravatar(dirent.last_commit.author_email,
title=h.really_unicode(dirent.last_commit.author), size=16)}}</a>
-          <a href="{{dirent.last_commit.author_url}}">{{h.really_unicode(dirent.last_commit.author)}}</a>
+        {% if dirent.commit_info.author_url %}
+          <a href="{{dirent.commit_info.author_url}}">{{lib.email_gravatar(dirent.commit_info.author_email,
title=h.really_unicode(dirent.commit_info.author), size=16)}}</a>
+          <a href="{{dirent.commit_info.author_url}}">{{h.really_unicode(dirent.commit_info.author)}}</a>
         {% else %}
-          {{lib.email_gravatar(dirent.last_commit.author_email, title=h.really_unicode(dirent.last_commit.author),
size=16)}} {{h.really_unicode(dirent.last_commit.author)}}
+          {{lib.email_gravatar(dirent.commit_info.author_email, title=h.really_unicode(dirent.commit_info.author),
size=16)}} {{h.really_unicode(dirent.commit_info.author)}}
         {% endif %}
       </td>
       <td>
-        <a href="{{dirent.last_commit.href}}">
-          {{dirent.last_commit.shortlink}}
-          {{dirent.last_commit.summary}}
+        <a href="{{dirent.commit_info.href}}">
+          {{dirent.commit_info.shortlink}}
+          {{dirent.commit_info.summary}}
         </a>
       </td>
     </tr>

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/ForgeGit/forgegit/model/git_repo.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py
index 4624600..05a1195 100644
--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -291,6 +291,10 @@ class GitImplementation(M.RepositoryImplementation):
         tree = self.refresh_tree_info(ci.tree, set())
         return tree._id
 
+    def compute_last_commit_info(self, commit, path, other_commit_ids=None):
+        if other_commit_ids is None:
+            other_commit_ids = []
+
 class _OpenedGitBlob(object):
     CHUNK_SIZE=4096
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/ForgeHg/forgehg/model/hg.py
----------------------------------------------------------------------
diff --git a/ForgeHg/forgehg/model/hg.py b/ForgeHg/forgehg/model/hg.py
index 0212b4d..4400c18 100644
--- a/ForgeHg/forgehg/model/hg.py
+++ b/ForgeHg/forgehg/model/hg.py
@@ -294,4 +294,8 @@ class HgImplementation(M.RepositoryImplementation):
         tree = self.refresh_tree_info(fake_tree, set())
         return tree._id
 
+    def compute_last_commit_info(self, commit, path, other_commit_ids=None):
+        if other_commit_ids is None:
+            other_commit_ids = []
+
 Mapper.compile_all()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/997b3845/ForgeSVN/forgesvn/model/svn.py
----------------------------------------------------------------------
diff --git a/ForgeSVN/forgesvn/model/svn.py b/ForgeSVN/forgesvn/model/svn.py
index e8173d8..eedbb97 100644
--- a/ForgeSVN/forgesvn/model/svn.py
+++ b/ForgeSVN/forgesvn/model/svn.py
@@ -378,6 +378,9 @@ class SVNImplementation(M.RepositoryImplementation):
         trees_doc.m.save(safe=False)
         return tree_id
 
+    def compute_last_commit_info(self, commit, path, other_commit_ids=None):
+        pass
+
     def _tree_oid(self, commit_id, path):
         data = 'tree\n%s\n%s' % (commit_id, h.really_unicode(path))
         return sha1(data.encode('utf-8')).hexdigest()


Mime
View raw message