Return-Path: X-Original-To: apmail-cloudstack-commits-archive@www.apache.org Delivered-To: apmail-cloudstack-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 85299CF14 for ; Thu, 25 Jul 2013 17:28:17 +0000 (UTC) Received: (qmail 39548 invoked by uid 500); 25 Jul 2013 17:28:17 -0000 Delivered-To: apmail-cloudstack-commits-archive@cloudstack.apache.org Received: (qmail 39531 invoked by uid 500); 25 Jul 2013 17:28:15 -0000 Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cloudstack.apache.org Delivered-To: mailing list commits@cloudstack.apache.org Received: (qmail 39515 invoked by uid 99); 25 Jul 2013 17:28:14 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 25 Jul 2013 17:28:14 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 2F5048112; Thu, 25 Jul 2013 17:28:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mlsorensen@apache.org To: commits@cloudstack.apache.org Message-Id: <0694241fe9034d6a8b010008c79ea021@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: updated refs/heads/4.2 to 9db03bf Date: Thu, 25 Jul 2013 17:28:14 +0000 (UTC) Updated Branches: refs/heads/4.2 2b0a97267 -> 9db03bf42 Summary: Make vmdata_kvm.py concurrency friendly Detail: Adds flocks, etc for calling this concurrently, since CLOUDSTACK-3484 made userdata, vmdata, start commands concurrent Signed-off-by: Marcus Sorensen 1374773177 -0600 Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/9db03bf4 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/9db03bf4 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/9db03bf4 Branch: refs/heads/4.2 Commit: 9db03bf426bfec6e0a3f2db4c72af3e48be42957 Parents: 2b0a972 Author: Marcus Sorensen Authored: Thu Jul 25 11:26:17 2013 -0600 Committer: Marcus Sorensen Committed: Thu Jul 25 11:26:17 2013 -0600 ---------------------------------------------------------------------- .../debian/config/opt/cloud/bin/vmdata_kvm.py | 100 +++++++++++++++---- 1 file changed, 82 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/9db03bf4/patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py ---------------------------------------------------------------------- diff --git a/patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py b/patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py index bf8baac..f508032 100644 --- a/patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py +++ b/patches/systemvm/debian/config/opt/cloud/bin/vmdata_kvm.py @@ -17,6 +17,8 @@ # under the License. import sys, getopt, json, os, base64 +from fcntl import flock, LOCK_EX, LOCK_UN + def main(argv): fpath = '' @@ -83,20 +85,37 @@ def createfile(ip, folder, file, data): if data is not None: data = base64.b64decode(data) + fh = open(dest, "w") + exflock(fh) if data is not None: - open(dest, "w").write(data) + fh.write(data) else: - open(dest, "w").write("") + fh.write("") + unflock(fh) + fh.close() os.chmod(dest, 0644) if folder == "metadata" or folder == "meta-data": - if not os.path.exists(metamanifestdir): + try: os.makedirs(metamanifestdir, 0755) + except OSError as e: + # error 17 is already exists, we do it this way for concurrency + if e.errno != 17: + print "failed to make directories " + metamanifestdir + " due to :" +e.strerror + sys.exit(1) if os.path.exists(metamanifest): - if not file in open(metamanifest).read(): - open(metamanifest, "a").write(file + '\n') + fh = open(metamanifest, "r+a") + exflock(fh) + if not file in fh.read(): + fh.write(file + '\n') + unflock(fh) + fh.close() else: - open(metamanifest, "w").write(file + '\n') + fh = open(metamanifest, "w") + exflock(fh) + fh.write(file + '\n') + unflock(fh) + fh.close() if os.path.exists(metamanifest): os.chmod(metamanifest, 0644) @@ -106,35 +125,80 @@ def htaccess(ip, folder, file): htaccessFolder = "/var/www/html/latest" htaccessFile = htaccessFolder + "/.htaccess" - if not os.path.exists(htaccessFolder): + try: os.mkdir(htaccessFolder,0755) + except OSError as e: + # error 17 is already exists, we do it this way for concurrency + if e.errno != 17: + print "failed to make directories " + htaccessFolder + " due to :" +e.strerror + sys.exit(1) if os.path.exists(htaccessFile): - if not entry in open(htaccessFile).read(): - open(htaccessFile, "a").write(entry + '\n') + fh = open(htaccessFile, "r+a") + exflock(fh) + if not entry in fh.read(): + fh.write(entry + '\n') + unflock(fh) + fh.close() + else: + fh = open(htaccessFile, "w") + exflock(fh) + fh.write("Options +FollowSymLinks\nRewriteEngine On\n\n") + fh.write(entry + '\n') + unflock(fh) + fh.close() entry="Options -Indexes\nOrder Deny,Allow\nDeny from all\nAllow from " + ip htaccessFolder = "/var/www/html/" + folder + "/" + ip htaccessFile = htaccessFolder+"/.htaccess" - if not os.path.exists(htaccessFolder): + try: os.makedirs(htaccessFolder,0755) - - open(htaccessFile, "w").write(entry + '\n') + except OSError as e: + # error 17 is already exists, we do it this way for sake of concurrency + if e.errno != 17: + print "failed to make directories " + htaccessFolder + " due to :" +e.strerror + sys.exit(1) + + fh = open(htaccessFile, "w") + exflock(fh) + fh.write(entry + '\n') + unflock(fh) + fh.close() if folder == "metadata" or folder == "meta-data": - entry="RewriteRule ^meta-data/(.+)$ ../" + folder + "/%{REMOTE_ADDR}/$1 [L,NC,QSA]" + entry = "RewriteRule ^meta-data/(.+)$ ../" + folder + "/%{REMOTE_ADDR}/$1 [L,NC,QSA]" htaccessFolder = "/var/www/html/latest" htaccessFile = htaccessFolder + "/.htaccess" - if not entry in open(htaccessFile).read(): - open(htaccessFile, "a").write(entry + '\n') + fh = open(htaccessFile, "r+a") + exflock(fh) + if not entry in fh.read(): + fh.write(entry + '\n') - entry="RewriteRule ^meta-data/$ ../" + folder + "/%{REMOTE_ADDR}/meta-data [L,NC,QSA]" + entry = "RewriteRule ^meta-data/$ ../" + folder + "/%{REMOTE_ADDR}/meta-data [L,NC,QSA]" - if not entry in open(htaccessFile).read(): - open(htaccessFile, "a").write(entry + '\n') + fh.seek(0) + if not entry in fh.read(): + fh.write(entry + '\n') + unflock(fh) + fh.close() +def exflock(file): + try: + flock(file, LOCK_EX) + except IOError as e: + print "failed to lock file" + file.name + " due to : " + e.strerror + sys.exit(1) + return True + +def unflock(file): + try: + flock(file, LOCK_UN) + except IOError: + print "failed to unlock file" + file.name + " due to : " + e.strerror + sys.exit(1) + return True if __name__ == "__main__": main(sys.argv[1:])