Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 45FFC10153 for ; Wed, 21 Aug 2013 20:12:28 +0000 (UTC) Received: (qmail 70612 invoked by uid 500); 21 Aug 2013 20:12:24 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 69982 invoked by uid 500); 21 Aug 2013 20:12:21 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 69413 invoked by uid 99); 21 Aug 2013 20:12:18 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Aug 2013 20:12:18 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id C26408C1EA3; Wed, 21 Aug 2013 20:12:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: brondsem@apache.org To: allura-commits@incubator.apache.org Date: Wed, 21 Aug 2013 20:12:19 -0000 Message-Id: <4893cfbcdbda431cb3168f5ae4d7f6e5@git.apache.org> In-Reply-To: <5e9ac297178240c4945d759d2ccda9db@git.apache.org> References: <5e9ac297178240c4945d759d2ccda9db@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/50] git commit: [#3154] ticket:386 Zip exported data [#3154] ticket:386 Zip exported data Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/6b635e69 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/6b635e69 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/6b635e69 Branch: refs/heads/db/3154b Commit: 6b635e69a81111e76281fb4e785bbaf19f8e932e Parents: 7b2d210 Author: Igor Bondarenko Authored: Thu Jul 4 14:55:35 2013 +0300 Committer: Dave Brondsema Committed: Wed Aug 21 18:12:22 2013 +0000 ---------------------------------------------------------------------- Allura/allura/model/project.py | 3 +++ Allura/allura/tasks/export_tasks.py | 28 ++++++++++++++++++++++------ Allura/allura/tests/test_tasks.py | 24 ++++++++++++++++++++++++ Allura/test.ini | 2 ++ 4 files changed, 51 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6b635e69/Allura/allura/model/project.py ---------------------------------------------------------------------- diff --git a/Allura/allura/model/project.py b/Allura/allura/model/project.py index 82c9a97..aaf5828 100644 --- a/Allura/allura/model/project.py +++ b/Allura/allura/model/project.py @@ -825,6 +825,9 @@ class Project(MappedClass, ActivityNode, ActivityObject): nbhd=self.neighborhood.url_prefix.strip('/'), project=self.shortname) + def bulk_export_filename(self): + return '%s.zip' % self.shortname + class AppConfig(MappedClass): """ http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6b635e69/Allura/allura/tasks/export_tasks.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py index 4c1f95a..f2f5371 100644 --- a/Allura/allura/tasks/export_tasks.py +++ b/Allura/allura/tasks/export_tasks.py @@ -17,9 +17,11 @@ import os import logging +import shutil from allura import model as M from allura.lib.decorators import task +from allura.model.repository import zipdir log = logging.getLogger(__name__) @@ -49,19 +51,33 @@ def bulk_export(project_shortname, tools): continue try: - cleanup() + zip_and_cleanup(project) except: - log.error('Error on cleanup.', exc_info=True) + log.error('Something went wrong during zipping exported data.', exc_info=True) def create_export_dir(project): """Create temporary directory for export files""" - path = os.path.join(project.bulk_export_path(), 'tmp') + zip_fn = project.bulk_export_filename() + # Name temporary directory after project shortname, + # thus zipdir() will use proper prefix inside the archive. + tmp_dir_suffix = zip_fn.split('.')[0] + path = os.path.join(project.bulk_export_path(), tmp_dir_suffix) if not os.path.exists(path): os.makedirs(path) return path -def cleanup(): - """Copy zip with export data to proper location. Remove temporary files.""" - pass +def zip_and_cleanup(project): + """Zip exported data. Copy it to proper location. Remove temporary files.""" + path = project.bulk_export_path() + zip_fn = project.bulk_export_filename() + temp = os.path.join(path, zip_fn.split('.')[0]) + zip_path_temp = os.path.join(temp, zip_fn) + zip_path = os.path.join(path, zip_fn) + + zipdir(temp, zip_path_temp) + + # cleanup + shutil.move(zip_path_temp, zip_path) + shutil.rmtree(temp) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6b635e69/Allura/allura/tests/test_tasks.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py index 74a5d18..d6f676e 100644 --- a/Allura/allura/tests/test_tasks.py +++ b/Allura/allura/tests/test_tasks.py @@ -17,12 +17,15 @@ # specific language governing permissions and limitations # under the License. +import os import operator +import shutil import sys import unittest from base64 import b64encode import logging +import tg import mock from pylons import tmpl_context as c, app_globals as g from datadiff.tools import assert_equal @@ -30,6 +33,7 @@ from nose.tools import assert_in from ming.orm import FieldProperty, Mapper from ming.orm import ThreadLocalORMSession from testfixtures import LogCapture +from IPython.testing.decorators import onlyif from alluratest.controller import setup_basic_test, setup_global_objects @@ -363,5 +367,25 @@ class TestExportTasks(unittest.TestCase): mock.call('Exporting wiki...')]) wiki_bulk_export.assert_called_once() + def test_create_export_dir(self): + project = M.Project.query.get(shortname='test') + export_path = project.bulk_export_path() + shutil.rmtree(export_path, ignore_errors=True) + path = export_tasks.create_export_dir(project) + assert_equal(path, '/tmp/bulk_export/p/test/test') + assert os.path.exists(os.path.join(export_path, project.shortname)) + shutil.rmtree(export_path, ignore_errors=True) + + @onlyif(os.path.exists(tg.config.get('scm.repos.tarball.zip_binary', '/usr/bin/zip')), 'zip binary is missing') + def test_zip_and_cleanup(self): + project = M.Project.query.get(shortname='test') + export_path = project.bulk_export_path() + shutil.rmtree(export_path, ignore_errors=True) + path = export_tasks.create_export_dir(project) + export_tasks.zip_and_cleanup(project) + assert not os.path.exists(path) + assert os.path.exists(os.path.join(export_path, 'test.zip')) + shutil.rmtree(export_path, ignore_errors=True) + Mapper.compile_all() http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6b635e69/Allura/test.ini ---------------------------------------------------------------------- diff --git a/Allura/test.ini b/Allura/test.ini index 7394359..6babfaf 100644 --- a/Allura/test.ini +++ b/Allura/test.ini @@ -102,6 +102,8 @@ scm.repos.tarball.enable = true scm.repos.tarball.root = /tmp/tarball scm.repos.tarball.url_prefix = file:// +bulk_export_path = /tmp/bulk_export/{nbhd}/{project} + support_tool_choices = wiki tickets discussion #stats.sample_rate = 0