kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject kylin git commit: KYLIN-3008 add submit-patch.py
Date Sun, 05 Nov 2017 08:21:04 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 82fc92f21 -> b3552c977


KYLIN-3008 add submit-patch.py


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3552c97
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3552c97
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3552c97

Branch: refs/heads/master
Commit: b3552c977a91af2824c7fcfe1c446b280e2940a8
Parents: 82fc92f
Author: shaofengshi <shaofengshi@apache.org>
Authored: Sun Nov 5 09:59:51 2017 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Sun Nov 5 15:07:36 2017 +0800

----------------------------------------------------------------------
 dev-support/make_patch.sh           | 156 ++++++++++++++++
 dev-support/python-requirements.txt |  21 +++
 dev-support/submit-patch.py         | 311 +++++++++++++++++++++++++++++++
 3 files changed, 488 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/b3552c97/dev-support/make_patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/make_patch.sh b/dev-support/make_patch.sh
new file mode 100755
index 0000000..8179098
--- /dev/null
+++ b/dev-support/make_patch.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Make a patch for the current branch based on its tracking branch
+
+# Process args
+while getopts "ahd:b:" opt; do
+    case "$opt" in
+        a)  addendum='-addendum'
+            ;;
+        d)
+            patch_dir=$OPTARG
+            ;;
+        b)
+            tracking_branch=$OPTARG
+            ;;
+        *)
+            echo -e "Usage: $0 [-h] [-a] [-d] <directory> \n\
+        Must be run from within the git branch to make the patch against.\n\
+        -h - display these instructions.\n\
+        -a - Add an 'addendum' prefix to the patch name.\n\
+        -b - Specify the base branch to diff from. (defaults to the tracking branch or origin
master)\n\
+        -d - specify a patch directory (defaults to ~/patches/)"
+            exit 0
+            ;;
+    esac
+done
+
+# Find what branch we are on
+branch=$(git branch |grep '*' |awk '{print $2}')
+if [ ! "$branch" ]; then
+    echo "Can't determine the git branch. Exiting." >&2
+    exit 1
+fi
+
+# Exit if git status is dirty
+git_dirty=$(git diff --shortstat 2> /dev/null | wc -l|awk {'print $1'})
+echo "git_dirty is $git_dirty"
+if [ "$git_dirty" -ne 0 ]; then
+    echo "Git status is dirty. Commit locally first.">&2
+    exit 1
+fi
+
+# Determine the tracking branch if needed.
+# If it was passed in from the command line
+# with -b then use dthat no matter what.
+if [ ! "$tracking_branch" ]; then
+  git log -n 1 origin/$branch > /dev/null 2>&1
+  status=$?
+  if [ "$status" -eq 128 ]; then
+      # Status 128 means there is no remote branch
+      tracking_branch='origin/master'
+  elif [ "$status" -eq 0 ]; then
+      # Status 0 means there is a remote branch
+      tracking_branch="origin/$branch"
+  else
+      echo "Unknown error: $?" >&2
+      exit 1
+  fi
+fi
+
+
+# Deal with invalid or missing $patch_dir
+if [ ! "$patch_dir" ]; then
+    echo -e "Patch directory not specified. Falling back to ~/patches/."
+    patch_dir=~/patches
+fi
+
+if [ ! -d "$patch_dir" ]; then
+    echo "$patch_dir does not exist. Creating it."
+    mkdir $patch_dir
+fi
+
+# Determine what to call the patch
+# Check to see if any patch exists that includes the branch name
+status=$(ls $patch_dir/*$branch* 2>/dev/null|grep -v addendum|wc -l|awk {'print $1'})
+if [ "$status" -eq 0 ]; then
+    # This is the first patch we are making for this release
+    prefix=''
+elif  [ "$status" -ge 1 ]; then
+    # At least one patch already exists -- add a version prefix
+    for i in {1..99}; do
+        # Check to see the maximum version of patch that exists
+        if [ ! -f "$patch_dir/$branch.v$i.patch" ]; then
+            version=$i
+            if [ -n "$addendum" ]; then
+                # Don't increment the patch # if it is an addendum
+                echo "Creating an addendum"
+                if [ "$version" -eq 1 ]; then
+                    # We are creating an addendum to the first version of the patch
+                    prefix=''
+                else
+                    # We are making an addendum to a different version of the patch
+                    let version=$version-1
+                    prefix=".v$version"
+                fi
+            else
+                prefix=".v$version"
+            fi
+            break
+        fi
+    done
+fi
+# If this is against a tracking branch other than master
+# include it in the patch name
+tracking_suffix=""
+if [[ $tracking_branch != "origin/master" \
+    &&  $tracking_branch != "master" ]]; then
+    tracking_suffix=".${tracking_branch#origin/}"
+fi
+
+patch_name="$branch$prefix$addendum$tracking_suffix.patch"
+
+# Do we need to make a diff?
+git diff --quiet $tracking_branch
+status=$?
+if [ "$status" -eq 0 ]; then
+    echo "There is no difference between $branch and $tracking_branch."
+    echo "No patch created."
+    exit 0
+fi
+
+# Check whether we need to squash or not
+local_commits=$(git log $tracking_branch..$branch|grep 'Author:'|wc -l|awk {'print $1'})
+if [ "$local_commits" -gt 1 ]; then
+    read -p "$local_commits commits exist only in your local branch. Interactive rebase?"
yn
+    case $yn in
+        [Yy]* )
+            git rebase -i $tracking_branch
+                ;;
+        [Nn]* )
+          echo "Creating $patch_dir/$patch_name using git diff."
+          git diff $tracking_branch > $patch_dir/$patch_name
+          exit 0
+        ;;
+    esac
+fi
+
+echo "Creating patch $patch_dir/$patch_name using git format-patch"
+git format-patch --stdout $tracking_branch > $patch_dir/$patch_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3552c97/dev-support/python-requirements.txt
----------------------------------------------------------------------
diff --git a/dev-support/python-requirements.txt b/dev-support/python-requirements.txt
new file mode 100644
index 0000000..e7fcf31
--- /dev/null
+++ b/dev-support/python-requirements.txt
@@ -0,0 +1,21 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+requests
+gitpython
+rbtools
+jinja2

http://git-wip-us.apache.org/repos/asf/kylin/blob/b3552c97/dev-support/submit-patch.py
----------------------------------------------------------------------
diff --git a/dev-support/submit-patch.py b/dev-support/submit-patch.py
new file mode 100755
index 0000000..babf869
--- /dev/null
+++ b/dev-support/submit-patch.py
@@ -0,0 +1,311 @@
+#!/usr/bin/env python
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Makes a patch for the current branch, creates/updates the review board request and uploads
new
+# patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus'
naming
+# rules. If no jira is specified, patch will be named (branch name).patch and jira and review
board
+# are not updated. Review board id is retrieved from the remote link in the jira.
+# Print help: submit-patch.py --h
+import argparse
+import getpass
+import git
+import json
+import logging
+import os
+import re
+import requests
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser(
+    epilog = "To avoid having to enter jira/review board username/password every time, setup
an "
+             "encrypted ~/.apache-cred files as follows:\n"
+             "1) Create a file with following single "
+             "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", "
+             "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n"
+             "2) Encrypt it with openssl.\n"
+             "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n"
+             "3) Delete original file.\n"
+             "Now onwards, you'll need to enter this encryption key only once per run. If
you "
+             "forget the key, simply regenerate ~/.apache-cred file again.",
+    formatter_class=argparse.RawTextHelpFormatter
+)
+parser.add_argument("-b", "--branch",
+                    help = "Branch to use for generating diff. If not specified, tracking
branch "
+                         "is used. If there is no tracking branch, error will be thrown.")
+
+# Arguments related to Jira.
+parser.add_argument("-jid", "--jira-id",
+                    help = "Jira id of the issue. If set, we deduce next patch version from
"
+                           "attachments in the jira and also upload the new patch. Script
will "
+                           "ask for jira username/password for authentication. If not set,
"
+                           "patch is named <branch>.patch.")
+
+# Arguments related to Review Board.
+parser.add_argument("-srb", "--skip-review-board",
+                    help = "Don't create/update the review board.",
+                    default = False, action = "store_true")
+parser.add_argument("--reviewers",
+                    help = "Comma separated list of users to add as reviewers.")
+
+# Misc arguments
+parser.add_argument("--patch-dir", default = "~/patches",
+                    help = "Directory to store patch files. If it doesn't exist, it will
be "
+                          "created. Default: ~/patches")
+parser.add_argument("--rb-repo", default = "kylin-git",
+                    help = "Review board repository. Default: kylin-git")
+args = parser.parse_args()
+
+# Setup logger
+logging.basicConfig()
+logger = logging.getLogger("submit-patch")
+logger.setLevel(logging.INFO)
+
+
+def log_fatal_and_exit(*arg):
+    logger.fatal(*arg)
+    sys.exit(1)
+
+
+def assert_status_code(response, expected_status_code, description):
+    if response.status_code != expected_status_code:
+        log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..",
+                           description, response.status_code, response.reason)
+
+
+# Make repo instance to interact with git repo.
+try:
+    repo = git.Repo(os.getcwd())
+    git = repo.git
+except git.exc.InvalidGitRepositoryError as e:
+    log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of
"
+                       "HBase's git repo.", e)
+
+logger.info(" Active branch: %s", repo.active_branch.name)
+# Do not proceed if there are uncommitted changes.
+if repo.is_dirty():
+    log_fatal_and_exit(" Git status is dirty. Commit locally first.")
+
+
+# Returns base branch for creating diff.
+def get_base_branch():
+    # if --branch is set, use it as base branch for computing diff. Also check that it's
a valid branch.
+    if args.branch is not None:
+        base_branch = args.branch
+        # Check that given branch exists.
+        for ref in repo.refs:
+            if ref.name == base_branch:
+                return base_branch
+        log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch)
+    else:
+        # if --branch is not set, use tracking branch as base branch for computing diff.
+        # If there is no tracking branch, log error and quit.
+        tracking_branch = repo.active_branch.tracking_branch()
+        if tracking_branch is None:
+            log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify
base "
+                               " branch for computing diff using --branch flag.")
+        logger.info(" Using tracking branch as base branch")
+        return tracking_branch.name
+
+
+# Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira
is
+# specified, patch is name (branch name).patch.
+def get_patch_name(branch):
+    if args.jira_id is None:
+        return branch + ".patch"
+
+    patch_name_prefix = args.jira_id.upper() + "." + branch
+    return get_patch_name_with_version(patch_name_prefix)
+
+
+# Fetches list of attachments from the jira, deduces next version for the patch and returns
final
+# patch name.
+def get_patch_name_with_version(patch_name_prefix):
+    # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637.
+    # Using crude way to get list of attachments.
+    url = "https://issues.apache.org/jira/browse/" + args.jira_id
+    logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url)
+    html = requests.get(url)
+    if html.status_code == 404:
+        log_fatal_and_exit(" Invalid jira id : %s", args.jira_id)
+    if html.status_code != 200:
+        log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code)
+    # Iterate over patch names starting from version 1 and return when name is not already
used.
+    content = unicode(html.content, 'utf-8')
+    for i in range(1, 1000):
+        name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch"
+        if name not in content:
+            return name
+
+
+# Validates that patch directory exists, if not, creates it.
+def validate_patch_dir(patch_dir):
+    # Create patch_dir if it doesn't exist.
+    if not os.path.exists(patch_dir):
+        logger.warn(" Patch directory doesn't exist. Creating it.")
+        os.mkdir(patch_dir)
+    else:
+        # If patch_dir exists, make sure it's a directory.
+        if not os.path.isdir(patch_dir):
+            log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.",
+                               patch_dir)
+
+
+# Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if
+# - base_branch has commits not in current branch
+# - current branch is same as base branch
+# - current branch is ahead of base_branch by more than 1 commits
+def check_diff_between_branches(base_branch):
+    only_in_base_branch = git.log("HEAD.." + base_branch, oneline = True)
+    only_in_active_branch = git.log(base_branch + "..HEAD", oneline = True)
+    if len(only_in_base_branch) != 0:
+        log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase "
+                           "and try again.", base_branch, len(only_in_base_branch.split("\n")))
+    if len(only_in_active_branch) == 0:
+        log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch)
+    if len(only_in_active_branch.split("\n")) > 1:
+        log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single
"
+                           "commit and try again.",
+                           base_branch, len(only_in_active_branch.split("\n")))
+
+
+# If ~/.apache-creds is present, load credentials from it otherwise prompt user.
+def get_credentials():
+    creds = dict()
+    creds_filepath = os.path.expanduser("~/.apache-creds")
+    if os.path.exists(creds_filepath):
+        try:
+            logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials")
+            content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath,
+                                              shell=True)
+        except subprocess.CalledProcessError as e:
+            log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..")
+        creds = json.loads(content)
+    else:
+        creds['jira_username'] = raw_input("Jira username:")
+        creds['jira_password'] = getpass.getpass("Jira password:")
+        if not args.skip_review_board:
+            creds['rb_username'] = raw_input("Review Board username:")
+            creds['rb_password'] = getpass.getpass("Review Board password:")
+    return creds
+
+
+def attach_patch_to_jira(issue_url, patch_filepath, creds):
+    # Upload patch to jira using REST API.
+    headers = {'X-Atlassian-Token': 'no-check'}
+    files = {'file': open(patch_filepath, 'rb')}
+    jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
+    attachment_url = issue_url +  "/attachments"
+    r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth)
+    assert_status_code(r, 200, "uploading patch to jira")
+
+
+def get_jira_summary(issue_url):
+    r = requests.get(issue_url + "?fields=summary")
+    assert_status_code(r, 200, "fetching jira summary")
+    return json.loads(r.content)["fields"]["summary"]
+
+
+def get_review_board_id_if_present(issue_url, rb_link_title):
+    r = requests.get(issue_url + "/remotelink")
+    assert_status_code(r, 200, "fetching remote links")
+    links = json.loads(r.content)
+    for link in links:
+        if link["object"]["title"] == rb_link_title:
+            res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"])
+            return res.group(1)
+    return None
+
+
+base_branch = get_base_branch()
+# Remove remote repo name from branch name if present. This assumes that we don't use '/'
in
+# actual branch names.
+base_branch_without_remote = base_branch.split('/')[-1]
+logger.info(" Base branch: %s", base_branch)
+
+check_diff_between_branches(base_branch)
+
+patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir))
+logger.info(" Patch directory: %s", patch_dir)
+validate_patch_dir(patch_dir)
+
+patch_filename = get_patch_name(base_branch_without_remote)
+logger.info(" Patch name: %s", patch_filename)
+patch_filepath = os.path.join(patch_dir, patch_filename)
+
+diff = git.format_patch(base_branch, stdout = True)
+with open(patch_filepath, "w") as f:
+    f.write(diff.encode('utf8'))
+
+if args.jira_id is not None:
+    creds = get_credentials()
+    issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id
+
+    attach_patch_to_jira(issue_url, patch_filepath, creds)
+
+    if not args.skip_review_board:
+        rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password'])
+
+        rb_link_title = "Review Board (" + base_branch_without_remote + ")"
+        rb_id = get_review_board_id_if_present(issue_url, rb_link_title)
+
+        # If no review board link found, create new review request and add its link to jira.
+        if rb_id is None:
+            reviews_url = "https://reviews.apache.org/api/review-requests/"
+            data = {"repository" : "kylin-git"}
+            r = requests.post(reviews_url, data = data, auth = rb_auth)
+            assert_status_code(r, 201, "creating new review request")
+            review_request = json.loads(r.content)["review_request"]
+            absolute_url = review_request["absolute_url"]
+            logger.info(" Created new review request: %s", absolute_url)
+
+            # Use jira summary as review's summary too.
+            summary = get_jira_summary(issue_url)
+            # Use commit message as description.
+            description = git.log("-1", pretty="%B")
+            update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups"
: "kylin",
+                                 "target_people" : args.reviewers, "summary" : summary,
+                                 "description" : description }
+            draft_url = review_request["links"]["draft"]["href"]
+            r = requests.put(draft_url, data = update_draft_data, auth = rb_auth)
+            assert_status_code(r, 200, "updating review draft")
+
+            draft_request = json.loads(r.content)["draft"]
+            diff_url = draft_request["links"]["draft_diffs"]["href"]
+            files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
+            r = requests.post(diff_url, files = files, auth = rb_auth)
+            assert_status_code(r, 201, "uploading diff to review draft")
+
+            r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
+            assert_status_code(r, 200, "publishing review request")
+
+            # Add link to review board in the jira.
+            remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}})
+            jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
+            r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth,
+                              headers={'Content-Type':'application/json'})
+        else:
+            logger.info(" Updating existing review board: https://reviews.apache.org/r/%s",
rb_id)
+            draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/"
+            diff_url = draft_url + "diffs/"
+            files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
+            r = requests.post(diff_url, files = files, auth = rb_auth)
+            assert_status_code(r, 201, "uploading diff to review draft")
+
+            r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
+            assert_status_code(r, 200, "publishing review request")


Mime
View raw message