incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [43/50] git commit: [#4091] ticket:489 Fix scripts which depend on per project database_uri
Date Mon, 06 Jan 2014 15:18:33 GMT
[#4091] ticket:489 Fix scripts which depend on per project database_uri


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/0e8e6de5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/0e8e6de5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/0e8e6de5

Branch: refs/heads/cj/6992
Commit: 0e8e6de57a4fac1d27ff6e39fea66458d0c10c07
Parents: ef0d3ea
Author: Igor Bondarenko <jetmind2@gmail.com>
Authored: Thu Dec 26 12:52:27 2013 +0200
Committer: Dave Brondsema <dbrondsema@slashdotmedia.com>
Committed: Thu Jan 2 20:55:05 2014 +0000

----------------------------------------------------------------------
 scripts/migrate_project_database.py             | 95 --------------------
 scripts/migrations/003-migrate_project_roles.py | 26 ++----
 .../004-make-attachments-polymorphic.py         | 50 +++++------
 scripts/migrations/006-migrate-artifact-refs.py | 36 +++-----
 scripts/migrations/007-update-acls.py           |  4 +-
 5 files changed, 46 insertions(+), 165 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0e8e6de5/scripts/migrate_project_database.py
----------------------------------------------------------------------
diff --git a/scripts/migrate_project_database.py b/scripts/migrate_project_database.py
deleted file mode 100644
index a30af02..0000000
--- a/scripts/migrate_project_database.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#       Licensed to the Apache Software Foundation (ASF) under one
-#       or more contributor license agreements.  See the NOTICE file
-#       distributed with this work for additional information
-#       regarding copyright ownership.  The ASF licenses this file
-#       to you under the Apache License, Version 2.0 (the
-#       "License"); you may not use this file except in compliance
-#       with the License.  You may obtain a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#       Unless required by applicable law or agreed to in writing,
-#       software distributed under the License is distributed on an
-#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-#       KIND, either express or implied.  See the License for the
-#       specific language governing permissions and limitations
-#       under the License.
-
-import os
-import sys
-import shutil
-import logging
-
-from pylons import tmpl_context as c
-from ming.orm import session
-from allura import model as M
-log = logging.getLogger(__name__)
-
-MONGO_HOME=os.environ.get('MONGO_HOME', '/usr')
-MONGO_DUMP=os.path.join(MONGO_HOME, 'bin/mongodump')
-MONGO_RESTORE=os.path.join(MONGO_HOME, 'bin/mongorestore')
-SHARD_LENGTH=1
-
-def main():
-    if len(sys.argv) > 1:
-        shortnames = sys.argv[1:]
-    else:
-        shortnames = [ p.shortname for p in M.Project.query.find(dict(is_root=True)) ]
-    M.main_orm_session.clear()
-    for pname in shortnames:
-        # This needs to be a .find() instead of a .get() because of the
-        # __init__ projects, which have the same shortname but exist in
-        # multiple neighborhoods.
-        for project in M.Project.query.find(dict(shortname=pname)):
-            migrate_project_database(project)
-
-def migrate_project_database(project):
-    c.project = project
-    target_uri = M.Project.default_database_uri(project.shortname)
-    target_db = target_uri.rsplit('/')[-1]
-    if project.database_uri == target_uri:
-        log.info('Project %s is already migrated to %s', project.shortname, project.database_uri)
-        return 2
-    conn = M.session.main_doc_session.db.connection
-    host = '%s:%s' % (conn.host, conn.port)
-    dirname = os.tempnam()
-    try:
-        log.info('Backing up %s to %s', project.shortname, dirname)
-        db_uri = project.database_uri
-        db = db_uri.rsplit('/')[-1]
-        assert 0 == os.system('%s --host %s --db %s -o %s' % (
-                MONGO_DUMP, host, db, dirname))
-        assert 0 == os.system('%s --host %s --db %s %s/%s ' % (
-                MONGO_RESTORE, host, target_db, dirname, db))
-        for p in M.Project.query.find(dict(database_uri=db_uri)):
-            p.database_uri = M.Project.default_database_uri(project.shortname)
-        session(project).flush()
-        conn.drop_database(db)
-    finally:
-        if os.path.exists(dirname):
-            shutil.rmtree(dirname)
-    return 0
-
-def pm(etype, value, tb): # pragma no cover
-    import pdb, traceback
-    try:
-        from IPython.ipapi import make_session; make_session()
-        from IPython.Debugger import Pdb
-        sys.stderr.write('Entering post-mortem IPDB shell\n')
-        p = Pdb(color_scheme='Linux')
-        p.reset()
-        p.setup(None, tb)
-        p.print_stack_trace()
-        sys.stderr.write('%s: %s\n' % ( etype, value))
-        p.cmdloop()
-        p.forget()
-        # p.interaction(None, tb)
-    except ImportError:
-        sys.stderr.write('Entering post-mortem PDB shell\n')
-        traceback.print_exception(etype, value, tb)
-        pdb.post_mortem(tb)
-
-sys.excepthook = pm
-
-if __name__ == '__main__':
-    sys.exit(main())

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0e8e6de5/scripts/migrations/003-migrate_project_roles.py
----------------------------------------------------------------------
diff --git a/scripts/migrations/003-migrate_project_roles.py b/scripts/migrations/003-migrate_project_roles.py
index 51dc2d8..580f93f 100644
--- a/scripts/migrations/003-migrate_project_roles.py
+++ b/scripts/migrations/003-migrate_project_roles.py
@@ -15,30 +15,18 @@
 #       specific language governing permissions and limitations
 #       under the License.
 
-'''Merge all the OldProjectRole collections in various project databases into a
-central ProjectRole collection.
+'''Merge all the OldProjectRole collections in into a ProjectRole collection.
 '''
 import logging
 
-from pylons import tmpl_context as c
-
 from ming.orm import session, state
 from allura import model as M
 
 log = logging.getLogger(__name__)
 
-seen_databases = set()
-projects = M.Project.query.find().all()
-
-for p in projects:
-    if p.database_uri in seen_databases:
-        continue
-    seen_databases.add(p.database_uri)
-    log.info('Moving project roles in database %s to main DB',
-             p.database_uri)
-    c.project = p
-    for opr in M.OldProjectRole.query.find():
-        pr = M.ProjectRole(**state(opr).document)
-    session(opr).clear()
-    session(pr).flush()
-    session(pr).clear()
+log.info('Moving project roles in database %s to main DB', M.Project.database_uri())
+for opr in M.OldProjectRole.query.find():
+    pr = M.ProjectRole(**state(opr).document)
+session(opr).clear()
+session(pr).flush()
+session(pr).clear()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0e8e6de5/scripts/migrations/004-make-attachments-polymorphic.py
----------------------------------------------------------------------
diff --git a/scripts/migrations/004-make-attachments-polymorphic.py b/scripts/migrations/004-make-attachments-polymorphic.py
index 8839e51..0c34d58 100644
--- a/scripts/migrations/004-make-attachments-polymorphic.py
+++ b/scripts/migrations/004-make-attachments-polymorphic.py
@@ -17,8 +17,6 @@
 
 import logging
 
-from pylons import tmpl_context as c
-
 from ming.orm import ThreadLocalORMSession
 from allura import model as M
 from forgetracker import model as TM
@@ -28,31 +26,29 @@ from forgediscussion import model as DM
 log = logging.getLogger(__name__)
 
 def main():
-    dbs = dict((p.database_uri, p) for p in M.Project.query.find())
-    for db, p in sorted(dbs.items()):
-        log.info('=== Making attachments in %s polymorphic ===', db)
-        c.project = p
-        log.info('Fixing %d discussions', M.Discussion.query.find().count())
-        for d in M.Discussion.query.find():
-            for a in M.DiscussionAttachment.query.find(dict(
-                    discussion_id=d._id)):
-                log.info('%s: %s', d.url(), a.filename)
-        log.info('Fixing %d forums', DM.Forum.query.find().count())
-        for d in DM.Forum.query.find():
-            for a in DM.ForumAttachment.query.find(dict(
-                    discussion_id=d._id)):
-                log.info('%s: %s', d.url(), a.filename)
-        log.info('Fixing %d tickets', TM.Ticket.query.find().count())
-        for t in TM.Ticket.query.find():
-            for a in TM.TicketAttachment.query.find(dict(
-                    artifact_id=t._id)):
-                log.info('%s: %s', t.url(), a.filename)
-        log.info('Fixing %d wikis', WM.Page.query.find().count())
-        for p in WM.Page.query.find():
-            for a in WM.WikiAttachment.query.find(dict(
-                    artifact_id=p._id)):
-                log.info('%s: %s', p.url(), a.filename)
-        ThreadLocalORMSession.flush_all()
+    db = M.project_doc_session.db
+    log.info('=== Making attachments in %s polymorphic ===', db)
+    log.info('Fixing %d discussions', M.Discussion.query.find().count())
+    for d in M.Discussion.query.find():
+        for a in M.DiscussionAttachment.query.find(dict(
+                discussion_id=d._id)):
+            log.info('%s: %s', d.url(), a.filename)
+    log.info('Fixing %d forums', DM.Forum.query.find().count())
+    for d in DM.Forum.query.find():
+        for a in DM.ForumAttachment.query.find(dict(
+                discussion_id=d._id)):
+            log.info('%s: %s', d.url(), a.filename)
+    log.info('Fixing %d tickets', TM.Ticket.query.find().count())
+    for t in TM.Ticket.query.find():
+        for a in TM.TicketAttachment.query.find(dict(
+                artifact_id=t._id)):
+            log.info('%s: %s', t.url(), a.filename)
+    log.info('Fixing %d wikis', WM.Page.query.find().count())
+    for p in WM.Page.query.find():
+        for a in WM.WikiAttachment.query.find(dict(
+                artifact_id=p._id)):
+            log.info('%s: %s', p.url(), a.filename)
+    ThreadLocalORMSession.flush_all()
 
 if __name__ == '__main__':
     main()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0e8e6de5/scripts/migrations/006-migrate-artifact-refs.py
----------------------------------------------------------------------
diff --git a/scripts/migrations/006-migrate-artifact-refs.py b/scripts/migrations/006-migrate-artifact-refs.py
index f1afa84..dc9149c 100644
--- a/scripts/migrations/006-migrate-artifact-refs.py
+++ b/scripts/migrations/006-migrate-artifact-refs.py
@@ -19,8 +19,6 @@ import sys
 import logging
 from cPickle import loads
 
-from pylons import tmpl_context as c
-
 from allura import model as M
 
 log = logging.getLogger('allura.migrate-artifact-refs')
@@ -28,27 +26,21 @@ log = logging.getLogger('allura.migrate-artifact-refs')
 # Threads have artifact references that must be migrated to the new system
 def main():
     test = sys.argv[-1] == 'test'
-    all_projects = M.Project.query.find().all()
     log.info('Fixing artifact references in threads')
-    seen_dbs = set()
-    for project in all_projects:
-        if project.database_uri in seen_dbs: continue
-        seen_dbs.add(project.database_uri)
-        c.project = project
-        db = M.project_doc_session.db
-        for thread in db.thread.find():
-            ref = thread.pop('artifact_reference', None)
-            if ref is None: continue
-            Artifact = loads(ref['artifact_type'])
-            artifact = Artifact.query.get(_id=ref['artifact_id'])
-            M.ArtifactReference.from_artifact(artifact)
-            thread['ref_id'] = artifact.index_id()
-            if not test:
-                db.thread.save(thread)
-                log.info('saving thread %s', thread['_id'])
-            else:
-                log.info('would save thread %s', thread['_id'])
-            M.artifact_orm_session.clear()
+    db = M.project_doc_session.db
+    for thread in db.thread.find():
+        ref = thread.pop('artifact_reference', None)
+        if ref is None: continue
+        Artifact = loads(ref['artifact_type'])
+        artifact = Artifact.query.get(_id=ref['artifact_id'])
+        M.ArtifactReference.from_artifact(artifact)
+        thread['ref_id'] = artifact.index_id()
+        if not test:
+            db.thread.save(thread)
+            log.info('saving thread %s', thread['_id'])
+        else:
+            log.info('would save thread %s', thread['_id'])
+        M.artifact_orm_session.clear()
 
 if __name__ == '__main__':
     main()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0e8e6de5/scripts/migrations/007-update-acls.py
----------------------------------------------------------------------
diff --git a/scripts/migrations/007-update-acls.py b/scripts/migrations/007-update-acls.py
index 4c7aaa3..16e28a8 100644
--- a/scripts/migrations/007-update-acls.py
+++ b/scripts/migrations/007-update-acls.py
@@ -38,7 +38,7 @@ c_project =  main_db.project
 c_user = main_db.user
 c_project_role = main_db.project_role
 c.project = Object(
-    database_uri=c_project.find().next()['database_uri'])
+    database_uri=M.Project.database_uri())
 
 project_db = M.project_doc_session.db
 c_app_config = project_db.config
@@ -171,7 +171,7 @@ def _project_role(project_id, name=None, user_id=None):
         roles=[])
     c_project_role.save(doc)
     return doc
-                                 
+
 
 def simple_acl_update(doc, collection_name):
     '''Update dict-style to list-style ACL'''


Mime
View raw message