kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From danburk...@apache.org
Subject [1/2] kudu git commit: python3 support for some build tools
Date Mon, 05 Mar 2018 20:34:41 GMT
Repository: kudu
Updated Branches:
  refs/heads/master cc4e3957b -> 3bfcbd8ad


python3 support for some build tools

Adds python3 support to the following files while retaining python2 compatibility:

- gen_version_info.py
- kudu_util.py,
- push_to_asf.py

Some python build tools are still known not to be python3 compatible,
including:

- build_source_release.py
- dist_test.py
- trigger_gerrit.py.

These haven't been fixed either because they use urllib, which I can't
figure out how to make 2 and 3 compatible, or because I don't have a
dist-test compatible system with python 3 available.

Change-Id: I547d870a54c7d396d2706d63f65844df01ff7c57
Reviewed-on: http://gerrit.cloudera.org:8080/9492
Reviewed-by: Alexey Serbin <aserbin@cloudera.com>
Reviewed-by: Adar Dembo <adar@cloudera.com>
Tested-by: Dan Burkert <danburkert@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/249af9a1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/249af9a1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/249af9a1

Branch: refs/heads/master
Commit: 249af9a101462bb0ca840e20a3dc8b4a76e929fd
Parents: cc4e395
Author: Dan Burkert <danburkert@apache.org>
Authored: Mon Mar 5 09:22:46 2018 -0800
Committer: Dan Burkert <danburkert@apache.org>
Committed: Mon Mar 5 20:28:37 2018 +0000

----------------------------------------------------------------------
 build-support/build_source_release.py | 57 +++++++++++------------
 build-support/gen_version_info.py     | 14 +++---
 build-support/kudu_util.py            | 15 +++++--
 build-support/push_to_asf.py          | 72 +++++++++++++++---------------
 4 files changed, 84 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/build_source_release.py
----------------------------------------------------------------------
diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py
index bc33019..6d922d4 100755
--- a/build-support/build_source_release.py
+++ b/build-support/build_source_release.py
@@ -38,9 +38,8 @@ def check_repo_not_dirty():
                                shell=True) != 0
   if not dirty_repo:
     return
-  print "The repository does not appear to be clean."
-  print Colors.RED + "The source release will not include your local changes." + \
-      Colors.RESET
+  print("The repository does not appear to be clean.")
+  print(Colors.RED + "The source release will not include your local changes." + Colors.RESET)
   if not confirm_prompt("Continue?"):
     sys.exit(1)
 
@@ -50,23 +49,22 @@ def check_no_local_commits():
   Check that there are no local commits which haven't been pushed to the upstream
   repo via Jenkins.
   """
-  upstream_commit = check_output(GET_UPSTREAM_COMMIT_SCRIPT).strip()
-  cur_commit = check_output(["git", "rev-parse", "HEAD"]).strip()
+  upstream_commit = check_output(GET_UPSTREAM_COMMIT_SCRIPT).strip().decode('utf-8')
+  cur_commit = check_output(["git", "rev-parse", "HEAD"]).strip().decode('utf-8')
 
   if upstream_commit == cur_commit:
     return
-  print "The repository appears to have local commits:"
+  print("The repository appears to have local commits:")
   subprocess.check_call(["git", "log", "--oneline", "%s..HEAD" % upstream_commit])
 
-  print Colors.RED + "This should not be an official release!" + \
-      Colors.RESET
+  print(Colors.RED + "This should not be an official release!" + Colors.RESET)
   if not confirm_prompt("Continue?"):
     sys.exit(1)
 
 
 def get_version_number():
   """ Return the current version number of Kudu. """
-  return file(os.path.join(ROOT, "version.txt")).read().strip()
+  return open(os.path.join(ROOT, "version.txt")).read().strip()
 
 
 def create_tarball():
@@ -75,12 +73,12 @@ def create_tarball():
   if not os.path.exists(build_dir):
     os.path.makedirs(build_dir)
   tarball_path = os.path.join(build_dir, artifact_name + ".tar.gz")
-  print "Exporting source tarball..."
+  print("Exporting source tarball...")
   subprocess.check_output(["git", "archive",
                            "--prefix=%s/" % artifact_name,
                            "--output=%s" % tarball_path,
                            "HEAD"])
-  print Colors.GREEN + "Generated tarball:\t" + Colors.RESET, tarball_path
+  print(Colors.GREEN + "Generated tarball:\t" + Colors.RESET, tarball_path)
   return tarball_path
 
 
@@ -91,18 +89,18 @@ def sign_tarball(tarball_path):
 
   email = get_my_email()
   if not email.endswith("@apache.org"):
-    print Colors.YELLOW, "Your email address for the repository is not an @apache.org address."
-    print "Release signatures should typically be signed by committers with @apache.org GPG
keys."
-    print Colors.RESET,
+    print(Colors.YELLOW, "Your email address for the repository is not an @apache.org address.")
+    print("Release signatures should typically be signed by committers with @apache.org GPG
keys.")
+    print(Colors.RESET)
     if not confirm_prompt("Continue?"):
       return
 
   try:
     subprocess.check_call(["gpg", "--detach-sign", "--armor", "-u", email, tarball_path])
   except subprocess.CalledProcessError:
-    print Colors.RED + "GPG signing failed. Artifact will not be signed." + Colors.RESET
+    print(Colors.RED + "GPG signing failed. Artifact will not be signed." + Colors.RESET)
     return
-  print Colors.GREEN + "Generated signature:\t" + Colors.RESET, tarball_path + ".asc"
+  print(Colors.GREEN + "Generated signature:\t" + Colors.RESET, tarball_path + ".asc")
 
 
 def checksum_file(summer, path):
@@ -110,9 +108,12 @@ def checksum_file(summer, path):
   Calculates the checksum of the file 'path' using the provided hashlib
   digest implementation. Returns the hex form of the digest.
   """
-  with file(path, "rb") as f:
+  with open(path, "rb") as f:
     # Read the file in 4KB chunks until EOF.
-    for chunk in iter(lambda: f.read(4096), ""):
+    while True:
+      chunk = f.read(4096)
+      if not chunk:
+        break
       summer.update(chunk)
   return summer.hexdigest()
 
@@ -129,9 +130,9 @@ def gen_checksum_files(tarball_path):
   for hash_func, extension in hashes:
     digest = checksum_file(hash_func(), tarball_path)
     path = tarball_path + "." + extension
-    with file(path, "w") as f:
-      print >>f, "%s\t%s" % (digest, os.path.basename(tarball_path))
-    print Colors.GREEN + ("Generated %s:\t" % extension) + Colors.RESET, path
+    with open(path, "w") as f:
+      f.write("%s\t%s\n" % (digest, os.path.basename(tarball_path)))
+    print(Colors.GREEN + ("Generated %s:\t" % extension) + Colors.RESET, path)
 
 
 def run_rat(tarball_path):
@@ -151,25 +152,25 @@ def run_rat(tarball_path):
   try:
     rat_jar_dest = "%s/%s" % (tmpdir_path, os.path.basename(rat_url))
 
-    print "> Downloading RAT jar from " + rat_url
+    print("> Downloading RAT jar from " + rat_url)
     urllib.urlretrieve(rat_url, rat_jar_dest)
 
-    print "> Running RAT..."
+    print("> Running RAT...")
     xml = subprocess.check_output(["java", "-jar", rat_jar_dest, "-x", tarball_path])
     rat_report_dest = "%s/%s" % (tmpdir_path, "rat_report.xml")
     with open(rat_report_dest, "w") as f:
         f.write(xml)
 
-    print "> Parsing RAT report..."
+    print("> Parsing RAT report...")
     rat_report_result = subprocess.check_output(
         ["./build-support/release/check-rat-report.py",
          "./build-support/release/rat_exclude_files.txt",
          rat_report_dest],
         stderr=subprocess.STDOUT)
-    print Colors.GREEN + "RAT: LICENSES APPROVED" + Colors.RESET
+    print(Colors.GREEN + "RAT: LICENSES APPROVED" + Colors.RESET)
   except subprocess.CalledProcessError as e:
-    print Colors.RED + "RAT: LICENSES NOT APPROVED" + Colors.RESET
-    print e.output
+    print(Colors.RED + "RAT: LICENSES NOT APPROVED" + Colors.RESET)
+    print(e.output)
     raise e
   finally:
     shutil.rmtree(tmpdir_path)
@@ -185,7 +186,7 @@ def main():
   sign_tarball(tarball_path)
   run_rat(tarball_path)
 
-  print Colors.GREEN + "Release successfully generated!" + Colors.RESET
+  print(Colors.GREEN + "Release successfully generated!" + Colors.RESET)
   print
 
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/gen_version_info.py
----------------------------------------------------------------------
diff --git a/build-support/gen_version_info.py b/build-support/gen_version_info.py
index 5c2c181..c0cf6d1 100755
--- a/build-support/gen_version_info.py
+++ b/build-support/gen_version_info.py
@@ -20,11 +20,11 @@
 # This script generates a header file which contains definitions
 # for the current Kudu build (eg timestamp, git hash, etc)
 
+import hashlib
 import logging
 import optparse
 import os
 import re
-import sha
 import subprocess
 import sys
 import time
@@ -39,7 +39,7 @@ def output_up_to_date(path, id_hash):
   """
   if not os.path.exists(path):
     return False
-  f = file(path).read()
+  f = open(path).read()
   m = re.search("id_hash=(\w+)", f)
   if not m:
     return False
@@ -99,16 +99,16 @@ def main():
   # timestamp. We put this hash in a comment, and use it to check whether to
   # re-generate the file. If it hasn't changed since a previous run, we don't
   # re-write the file. This avoids having to rebuild all binaries on every build.
-  identifying_hash = sha.sha(repr((git_hash, hostname, username,
-                                   clean_repo, build_id))).hexdigest()
+  identifying_hash = hashlib.sha1(repr((git_hash, hostname, username,
+                                        clean_repo, build_id)).encode('utf-8')).hexdigest()
 
   if output_up_to_date(output_path, identifying_hash):
     return 0
   d = os.path.dirname(output_path)
   if not os.path.exists(d):
     os.makedirs(d)
-  with file(output_path, "w") as f:
-    print >>f, """
+  with open(output_path, "w") as f:
+    f.write("""
 // THIS FILE IS AUTO-GENERATED! DO NOT EDIT!
 //
 // id_hash=%(identifying_hash)s
@@ -124,7 +124,7 @@ def main():
 #define KUDU_BUILD_TYPE "%(build_type)s"
 #define KUDU_VERSION_STRING "%(version_string)s"
 #endif
-""" % locals()
+""" % locals())
   return 0
 
 if __name__ == "__main__":

http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/kudu_util.py
----------------------------------------------------------------------
diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
index 7945da5..da4dbe6 100644
--- a/build-support/kudu_util.py
+++ b/build-support/kudu_util.py
@@ -24,6 +24,12 @@ import os
 import subprocess
 import sys
 
+# Alias raw_input() to input() in Python 2.
+try:
+  input = raw_input
+except NameError:
+  pass
+
 class Colors(object):
   """ ANSI color codes. """
 
@@ -63,14 +69,14 @@ def confirm_prompt(prompt):
   if the user confirms.
   """
   while True:
-    print prompt, "[Y/n]:",
+    print(prompt + "[Y/n]:"),
 
     if not os.isatty(sys.stdout.fileno()):
-      print "Not running interactively. Assuming 'N'."
+      print("Not running interactively. Assuming 'N'.")
       return False
       pass
 
-    r = raw_input().strip().lower()
+    r = input().strip().lower()
     if r in ['y', 'yes', '']:
       return True
     elif r in ['n', 'no']:
@@ -79,4 +85,5 @@ def confirm_prompt(prompt):
 
 def get_my_email():
   """ Return the email address in the user's git config. """
-  return check_output(['git', 'config', '--get', 'user.email']).strip()
+  return check_output(['git', 'config', '--get',
+      'user.email']).strip().decode('utf-8')

http://git-wip-us.apache.org/repos/asf/kudu/blob/249af9a1/build-support/push_to_asf.py
----------------------------------------------------------------------
diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py
index f84a261..f92a3b3 100755
--- a/build-support/push_to_asf.py
+++ b/build-support/push_to_asf.py
@@ -32,6 +32,8 @@
 # changes to be pushed, warns them if they are pushing code for
 # which they weren't the committer, and performs the actual push.
 
+from __future__ import print_function
+
 import logging
 import optparse
 import re
@@ -59,15 +61,15 @@ def check_apache_remote():
   Otherwise, exits with an error message.
   """
   try:
-    url = check_output(['git', 'config', '--local', '--get', 'remote.apache.url']).strip()
+    url = check_output(['git', 'config', '--local', '--get', 'remote.apache.url']).strip().decode('utf-8')
   except subprocess.CalledProcessError:
-    print >>sys.stderr, "No remote named 'apache'. Please set one up, for example with:
"
-    print >>sys.stderr, "  git remote add apache", APACHE_REPO
+    print("No remote named 'apache'. Please set one up, for example with: ", file=sys.stderr)
+    print("  git remote add apache", APACHE_REPO, file=sys.stderr)
     sys.exit(1)
   if url != APACHE_REPO:
-    print >>sys.stderr, "Unexpected URL for remote 'apache'."
-    print >>sys.stderr, "  Got:     ", url
-    print >>sys.stderr, "  Expected:", APACHE_REPO
+    print("Unexpected URL for remote 'apache'.", file=sys.stderr)
+    print("  Got:     ", url, file=sys.stderr)
+    print("  Expected:", APACHE_REPO, file=sys.stderr)
     sys.exit(1)
 
 
@@ -77,15 +79,15 @@ def check_gerrit_remote():
   Otherwise, exits with an error message.
   """
   try:
-    url = check_output(['git', 'config', '--local', '--get', 'remote.gerrit.url']).strip()
+    url = check_output(['git', 'config', '--local', '--get', 'remote.gerrit.url']).strip().decode('utf-8')
   except subprocess.CalledProcessError:
-    print >>sys.stderr, "No remote named 'gerrit'. Please set one up following "
-    print >>sys.stderr, "the contributor guide."
+    print("No remote named 'gerrit'. Please set one up following ", file=sys.stderr)
+    print("the contributor guide.", file=sys.stderr)
     sys.exit(1)
   if not GERRIT_URL_RE.match(url):
-    print >>sys.stderr, "Unexpected URL for remote 'gerrit'."
-    print >>sys.stderr, "  Got:     ", url
-    print >>sys.stderr, "  Expected to find host '%s' in the URL" % GERRIT_HOST
+    print("Unexpected URL for remote 'gerrit'.", file=sys.stderr)
+    print("  Got:     ", url, file=sys.stderr)
+    print("  Expected to find host '%s' in the URL" % GERRIT_HOST, file=sys.stderr)
     sys.exit(1)
 
 
@@ -98,7 +100,7 @@ def fetch(remote):
 
 def get_branches(remote):
   """ Fetch a dictionary mapping branch name to SHA1 hash from the given remote. """
-  out = check_output(["git", "ls-remote", remote, "refs/heads/*"])
+  out = check_output(["git", "ls-remote", remote, "refs/heads/*"]).decode('utf-8')
   ret = {}
   for l in out.splitlines():
     sha, ref = l.split("\t")
@@ -110,7 +112,7 @@ def get_branches(remote):
 def rev_parse(rev):
   """Run git rev-parse, returning the sha1, or None if not found"""
   try:
-    return check_output(['git', 'rev-parse', rev], stderr=subprocess.STDOUT).strip()
+    return check_output(['git', 'rev-parse', rev], stderr=subprocess.STDOUT).strip().decode('utf-8')
   except subprocess.CalledProcessError:
     return None
 
@@ -123,7 +125,7 @@ def rev_list(arg):
 def describe_commit(rev):
   """ Return a one-line description of a commit. """
   return check_output(
-      ['git', 'log', '--color', '-n1', '--oneline', rev]).strip()
+      ['git', 'log', '--color', '-n1', '--oneline', rev]).strip().decode('utf-8')
 
 
 def is_fast_forward(ancestor, child):
@@ -132,7 +134,7 @@ def is_fast_forward(ancestor, child):
   could be fast-forward merged.
   """
   try:
-    merge_base = check_output(['git', 'merge-base', ancestor, child]).strip()
+    merge_base = check_output(['git', 'merge-base', ancestor, child]).strip().decode('utf-8')
   except:
     # If either of the commits is unknown, count this as a non-fast-forward.
     return False
@@ -141,7 +143,7 @@ def is_fast_forward(ancestor, child):
 
 def get_committer_email(rev):
   """ Return the email address of the committer of the given revision. """
-  return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip()
+  return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip().decode('utf-8')
 
 
 def do_update(branch, gerrit_sha, apache_sha):
@@ -153,36 +155,36 @@ def do_update(branch, gerrit_sha, apache_sha):
   # must have gotten committed to Apache outside of gerrit, and we'd need some
   # manual intervention.
   if not is_fast_forward(apache_sha, gerrit_sha):
-    print >>sys.stderr, "Cannot update branch '%s' from gerrit:" % branch
-    print >>sys.stderr, "Apache revision %s is not an ancestor of gerrit revision %s"
% (
-      apache_sha[:8], gerrit_sha[:8])
-    print >>sys.stderr, "Something must have been committed to Apache and bypassed
gerrit."
-    print >>sys.stderr, "Manual intervention is required."
+    print("Cannot update branch '%s' from gerrit:" % branch, file=sys.stderr)
+    print("Apache revision %s is not an ancestor of gerrit revision %s" % (
+      apache_sha[:8], gerrit_sha[:8]), file=sys.stderr)
+    print("Something must have been committed to Apache and bypassed gerrit.", file=sys.stderr)
+    print("Manual intervention is required.", file=sys.stderr)
     sys.exit(1)
 
   # List the commits that are going to be pushed to the ASF, so that the committer
   # can verify and "sign off".
   commits = rev_list("%s..%s" % (apache_sha, gerrit_sha))
   commits.reverse()  # Display from oldest to newest.
-  print "-" * 60
-  print Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits))
+ Colors.RESET
+  print("-" * 60)
+  print(Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits))
+ Colors.RESET)
   push_sha = None
   for sha in commits:
     oneline = describe_commit(sha)
-    print "  ", oneline
+    print("  ", oneline)
     committer = get_committer_email(sha)
     if committer != get_my_email():
-      print Colors.RED + "   !!! Committed by someone else (%s) !!!" % committer, Colors.RESET
+      print(Colors.RED + "   !!! Committed by someone else (%s) !!!" % committer, Colors.RESET)
       if not confirm_prompt(
           Colors.RED + "   !!! Are you sure you want to push on behalf of another committer?"
+ Colors.RESET):
         # Even if they don't want to push this commit, we could still push any
         # earlier commits that the user _did_ author.
         if push_sha is not None:
-          print "... will still update to prior commit %s..." % push_sha
+          print("... will still update to prior commit %s..." % push_sha)
         break
     push_sha = sha
   if push_sha is None:
-    print "Nothing to push"
+    print("Nothing to push")
     return
 
   # Everything has been confirmed. Do the actual push
@@ -190,9 +192,9 @@ def do_update(branch, gerrit_sha, apache_sha):
   if OPTIONS.dry_run:
     cmd.append('--dry-run')
   cmd.append('%s:refs/heads/%s' % (push_sha, branch))
-  print Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd)
+  print(Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd))
   subprocess.check_call(cmd)
-  print Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET
+  print(Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET)
   print
 
 
@@ -218,16 +220,16 @@ def main():
   # Check the current state of branches on Apache.
   # For each branch, we try to update it if the revisions don't match.
   apache_branches = get_branches('apache')
-  for branch, apache_sha in sorted(apache_branches.iteritems()):
+  for branch, apache_sha in sorted(apache_branches.items()):
     gerrit_sha = rev_parse("remotes/gerrit/" + branch)
-    print "Branch '%s':\t" % branch,
+    print("Branch '%s':\t" % branch)
     if gerrit_sha is None:
-      print Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET
+      print(Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET)
       continue
     if gerrit_sha == apache_sha:
-      print Colors.GREEN, "up to date", Colors.RESET
+      print(Colors.GREEN, "up to date", Colors.RESET)
       continue
-    print Colors.YELLOW, "needs update", Colors.RESET
+    print(Colors.YELLOW, "needs update", Colors.RESET)
     do_update(branch, gerrit_sha, apache_sha)
 
 


Mime
View raw message