bookkeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject [bookkeeper] branch master updated: ISSUE #431 ISSUE #221: merge scripts should mark milestone when merging pull requests
Date Fri, 11 Aug 2017 22:19:40 GMT
This is an automated email from the ASF dual-hosted git repository.

sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git


The following commit(s) were added to refs/heads/master by this push:
     new 1ff17d8  ISSUE #431 ISSUE #221: merge scripts should mark milestone when merging
pull requests
1ff17d8 is described below

commit 1ff17d899f9998cdefc99c47f0f38a75b99d1c0e
Author: Sijie Guo <sijie@apache.org>
AuthorDate: Fri Aug 11 15:19:32 2017 -0700

    ISSUE #431 ISSUE #221: merge scripts should mark milestone when merging pull requests
    
    Descriptions of the changes in this PR:
    
    improve merge script to do:
    
    - assign a milestone
    - add `area/` label
    - add `type/` label
    - add `release/` label
    - when a change needs to merge to a different branch, add the bug fix `release/` label
    
    This also addresses #221 to trim the template from pull request body.
    
    Author: Sijie Guo <sijie@apache.org>
    
    Reviewers: Jia Zhai <zhaijia03@gmail.com>
    
    This closes #437 from sijie/merge_script_for_bug_releases, closes #431, closes #221
---
 .github/ISSUE_TEMPLATE.md        |   6 -
 .github/PULL_REQUEST_TEMPLATE.md |  24 ++--
 dev/bk-merge-pr.py               | 254 +++++++++++++++++++++++++++++++++++++--
 3 files changed, 254 insertions(+), 30 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ede9c69..2505f24 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -29,9 +29,3 @@ Have you checked our documentation at http://bookkeeper.apache.org/ , If
you cou
 
 - What did you see instead?
 
----------------------------------------------------------------------------------
-
-Please label the issue with proper labels. It would help us triage the issues.
-
-- "release/": please mark the issue with the release. 
-- "type/": please mark the issue with corresponding issue type.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 9ecd4d7..caca761 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -2,15 +2,15 @@ Descriptions of the changes in this PR:
 
 (PR description content here)...
 
----
-Be sure to do all of the following to help us incorporate your contribution
-quickly and easily:
-
-- [ ] Make sure the PR title is formatted like:
-    `<Issue # or BOOKKEEPER-#>: Description of pull request`
-    `e.g. Issue 123: Description ...`
-    `e.g. BOOKKEEPER-1234: Description ...`
-- [ ] Make sure tests pass via `mvn clean apache-rat:check install findbugs:check`.
-- [ ] Replace `<Issue # or BOOKKEEPER-#>` in the title with the actual Issue/JIRA number.
-
----
+> ---
+> Be sure to do all of the following to help us incorporate your contribution
+> quickly and easily:
+> 
+> - [ ] Make sure the PR title is formatted like:
+>     `<Issue # or BOOKKEEPER-#>: Description of pull request`
+>     `e.g. Issue 123: Description ...`
+>     `e.g. BOOKKEEPER-1234: Description ...`
+> - [ ] Make sure tests pass via `mvn clean apache-rat:check install findbugs:check`.
+> - [ ] Replace `<Issue # or BOOKKEEPER-#>` in the title with the actual Issue/JIRA
number.
+> 
+> ---
diff --git a/dev/bk-merge-pr.py b/dev/bk-merge-pr.py
index e8f6e34..9e1ba80 100755
--- a/dev/bk-merge-pr.py
+++ b/dev/bk-merge-pr.py
@@ -92,6 +92,20 @@ def get_json(url, preview_api = False):
             print "Unable to fetch URL, exiting: %s" % url
         sys.exit(-1)
 
+def post_json(url, data):
+    try:
+        request = urllib2.Request(url, data, { 'Content-Type': 'application/json' })
+        if GITHUB_OAUTH_KEY:
+            request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)
+        return json.load(urllib2.urlopen(request))
+    except urllib2.HTTPError as e:
+        if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] ==
'0':
+            print "Exceeded the GitHub API rate limit; see the instructions in " + \
+                  "bk-merge-pr.py to configure an OAuth token for making authenticated "
+ \
+                  "GitHub requests."
+        else:
+            print "Unable to fetch URL, exiting: %s - %s" % (url, e)
+        sys.exit(-1)
 
 def fail(msg):
     print msg
@@ -126,6 +140,13 @@ def clean_up():
 def get_current_branch():
     return run_cmd("git rev-parse --abbrev-ref HEAD").replace("\n", "")
 
+def get_milestones():
+    return get_json("https://api.github.com/repos/%s/%s/milestones?state=open&sort=due_on&direction=asc"
% (GITHUB_USER, PROJECT_NAME))
+
+def get_all_labels():
+    result = get_json("https://api.github.com/repos/%s/%s/labels" % (GITHUB_USER, PROJECT_NAME))
+    return map(lambda x: x['name'], result)
+
 # merge the requested PR and return the merge hash
 def merge_pr(pr_num, target_ref, title, body, default_pr_reviewers, pr_repo_desc):
     pr_branch_name = "%s_MERGE_PR_%s" % (TEMP_BRANCH_PREFIX, pr_num)
@@ -244,12 +265,13 @@ def merge_pr(pr_num, target_ref, title, body, default_pr_reviewers,
pr_repo_desc
     print("Merge hash: %s" % merge_hash)
     return merge_hash, merge_log
 
-
-def cherry_pick(pr_num, merge_hash, default_branch):
+def ask_for_branch(default_branch):
     pick_ref = raw_input("Enter a branch name [%s]: " % default_branch)
     if pick_ref == "":
         pick_ref = default_branch
+    return pick_ref
 
+def cherry_pick(pr_num, merge_hash, pick_ref):
     pick_branch_name = "%s_PICK_PR_%s_%s" % (TEMP_BRANCH_PREFIX, pr_num, pick_ref.upper())
 
     run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
@@ -364,11 +386,12 @@ def standardize_jira_ref(text):
     """
     Standardize the jira reference commit message prefix to "PROJECT_NAME-XXX: Issue"
 
-    >>> standardize_jira_ref("%s-877: Script for generating patch for reviews" %
CAPITALIZED_PROJECT_NAME)
     'BOOKKEEPER-877: Script for generating patch for reviews'
+    'ISSUE #376: Script for generating patch for reviews'
     """
     jira_refs = []
     github_issue_refs = []
+    github_issue_ids = []
     components = []
 
     # Extract JIRA ref(s):
@@ -379,11 +402,12 @@ def standardize_jira_ref(text):
         text = text.replace(ref, '')
 
     # Extract Github Issue ref(s)
-    pattern = re.compile(r'(%s[-\s]*[0-9]{3,6})+' % GITHUB_ISSUES_NAME, re.IGNORECASE)
+    pattern = re.compile(r'(%s[-\s]*([0-9]{3,6}))+' % GITHUB_ISSUES_NAME, re.IGNORECASE)
     for ref in pattern.findall(text):
         # Add brackets, replace spaces or a dash with ' #', & convert to uppercase
-        github_issue_refs.append(re.sub(r'[-\s]+', ' #', ref.upper()))
-        text = text.replace(ref, '')
+        github_issue_refs.append(re.sub(r'[-\s]+', ' #', ref[0].upper()))
+        text = text.replace(ref[0], '')
+        github_issue_ids.append(ref[1].upper())
 
     # Extract project name component(s):
     # Look for alphanumeric chars, spaces, dashes, periods, and/or commas
@@ -410,7 +434,7 @@ def standardize_jira_ref(text):
     # Replace multiple spaces with a single space, e.g. if no jira refs and/or components
were included
     clean_text = re.sub(r'\s+', ' ', clean_text.strip())
 
-    return clean_text
+    return clean_text, github_issue_ids
 
 def get_reviewers(pr_num):
     """
@@ -450,30 +474,183 @@ def get_reviewers(pr_num):
         reviewers_emails.append('{0} <{1}>'.format(username.encode('utf8'), useremail))
     return ', '.join(reviewers_emails)
 
+def ask_release_for_github_issues(branch, labels):
+    print "=== Add release to github issues ==="
+    while True:
+        fix_releases = ask_for_labels("release/%s" % branch, labels, [])
+        if len(fix_releases) != 1:
+            print "Please choose only one release to add for branch '%s'." % branch
+            continue
+
+        print "=== Apply following releases to github issues ==" 
+        print "Fix Releases: %s" % ', '.join(fix_releases)
+        print ""
+
+        if raw_input("Would you like to add these releases to github issues? (y/n): ") ==
"y":
+            break
+    return fix_releases
+
+def ask_updates_for_github_issues(milestones, labels, issue_labels):
+    while True:
+        fix_milestone, fix_milestone_number, fix_areas, fix_types = \
+            get_updates_for_github_issues(milestones, labels, issue_labels)
+
+        print "=== Apply following milestone, area, type to github issues ==" 
+        print "Fix Types: %s" % ', '.join(fix_types)
+        print "Fix Areas: %s" % ', '.join(fix_areas)
+        print "Fix Milestone: %s" % fix_milestone
+        print ""
+
+        if raw_input("Would you like to update github issues with these labels? (y/n): ")
== "y":
+            break
+
+    return fix_milestone, fix_milestone_number, fix_areas, fix_types
+
+def get_updates_for_github_issues(milestones, labels, issue_labels):
+    # get milestone
+    default_milestone_name = milestones[0]['title']
+    milestone_list = map(lambda x: x['title'], milestones)
+    milestone_map = dict((milestone['title'], milestone['number']) for milestone in milestones)
+    fix_milestone = ""
+    while True:
+        fix_milestone = raw_input("Choose fix milestone : options are [%s] - default: [%s]:
" % (', '.join(milestone_list).strip(), default_milestone_name))
+        fix_milestone = fix_milestone.strip()
+        if fix_milestone == "":
+            fix_milestone = default_milestone_name
+            break
+        elif fix_milestone in milestone_map:
+            break
+        else:
+            print "Invalid milestone: %s." % fix_milestone
+
+    # get area
+    fix_areas = ask_for_labels("area/", labels, issue_labels)
+    
+    # get types
+    fix_types = ask_for_labels("type/", labels, issue_labels) 
+
+    return fix_milestone, milestone_map[fix_milestone], fix_areas, fix_types
+
+def ask_for_labels(prefix, labels, issue_labels):
+    issue_filtered_labels = map(lambda l: l.split('/')[1], filter(lambda x: x.startswith(prefix),
issue_labels))
+    filtered_labels = map(lambda l: l.split('/')[1], filter(lambda x: x.startswith(prefix),
labels))
+    while True:
+        fix_labels = raw_input("Choose label '%s' - options are: [%s] - default: [%s] (comma
separated): "
+            % (prefix, ', '.join(filtered_labels).strip(), ', '.join(issue_filtered_labels).strip()))
+        if fix_labels == "":
+            if not issue_filtered_labels:
+                print "Please specify a '%s' label to close the issue!" % prefix
+                continue
+            else:
+                fix_labels = issue_filtered_labels
+                break
+        fix_labels = fix_labels.replace(" ", "").split(",")
+        if not fix_labels:
+            print "Please specify a '%s' label to close the issue!" % prefix
+            continue
+        invalid_label = False
+        for label in fix_labels:
+            if label not in filtered_labels:
+                print "Invalid '%s' label: %s." % (prefix, label)
+                invalid_label = True
+                break
+        if invalid_label:
+            continue
+        else:
+            break
+    return fix_labels
+
+def get_github_issue_url(github_issue_id):
+    return "https://api.github.com/repos/%s/%s/issues/%s" % (GITHUB_USER, PROJECT_NAME, github_issue_id)

+
+def get_assignees_url(github_issue_id):
+    return "https://api.github.com/repos/%s/%s/issues/%s/assignees" % (GITHUB_USER, PROJECT_NAME,
github_issue_id) 
+
+def get_github_issue_labels(github_issue_id):
+    url = "https://api.github.com/repos/%s/%s/issues/%s/labels" % (GITHUB_USER, PROJECT_NAME,
github_issue_id) 
+    result = get_json(url)
+    return map(lambda x: x["name"], result)
+
+def add_release_to_github_issues(github_issue_ids, labels, fix_release):
+    for github_issue_id in github_issue_ids:
+        labels = add_release_to_github_issue(github_issue_id, labels, fix_release)
+    return labels
+
+def add_release_to_github_issue(github_issue_id, labels, fix_release):
+    url = get_github_issue_url(github_issue_id)
+    labels = ["release/%s" % fix_release] + labels
+    data = json.dumps({
+        'labels': labels
+    })
+    post_json(url, data)
+    return labels
+
+def update_github_issue(github_issue_id, fix_milestone_number, fix_milestone, fix_areas,
fix_types, other_labels):
+    url = get_github_issue_url(github_issue_id)
+    labels = other_labels + map(lambda x: "area/%s" % x, fix_areas)
+    labels = labels + map(lambda x: "type/%s" % x, fix_types)
+    labels.append("release/%s" % fix_milestone)
+    data = json.dumps({
+        'milestone': int(fix_milestone_number),
+        'labels': labels,
+    })
+    post_json(url, data)
+    return labels
+
+def update_github_issues(github_issue_ids, fix_milestone_number, fix_milestone, fix_areas,
fix_types, other_labels):
+    for github_issue_id in github_issue_ids:
+        labels = update_github_issue(github_issue_id, fix_milestone_number, fix_milestone,
fix_areas, fix_types, other_labels)
+    return labels
+
+def add_assignees_to_github_issues(github_issue_ids, assignees):
+    for github_issue_id in github_issue_ids:
+        add_assignees_to_github_issue(github_issue_id, assignees)
+
+def add_assignees_to_github_issue(github_issue_id, assignees):
+    url = get_assignees_url(github_issue_id)
+    data = json.dumps({
+        "assignees": assignees
+    })
+    post_json(url, data)
+
 def main():
     global original_head
 
+    if not GITHUB_OAUTH_KEY:
+        print "OAuth key is needed for merging bookkeeper pull requests."
+        print "If environment variable 'GITHUB_OAUTH_KEY' is not defined,"
+        print "then requests will be unauthenticated."
+        print "You can create an OAuth key at https://github.com/settings/tokens"
+        print "and set it to the environment variable 'GITHUB_OAUTH_KEY'."
+        print "(This token only needs the 'public_repo' scope permissions)"
+        exit(-1)
+
+    # 0. get the current state so we can go back
     original_head = get_current_branch()
 
+    # 1. retrieve milestones, labels, branches
+    milestones = get_milestones()
+    labels = get_all_labels()
     branches = get_json("%s/branches" % GITHUB_API_BASE)
     branch_names = filter(lambda x: x.startswith(RELEASE_BRANCH_PREFIX), [x['name'] for x
in branches])
     # Assumes branch names can be sorted lexicographically
     latest_branch = sorted(branch_names, reverse=True)[0]
 
+    # 2. retrieve the details for a given pull request
     pr_num = raw_input("Which pull request would you like to merge? (e.g. 34): ")
     pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
     pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num))
     pr_reviewers = get_reviewers(pr_num)
-
     url = pr["url"]
 
+    # 3. repare the title for commit message
     pr_title = pr["title"]
     commit_title = raw_input("Commit title [%s]: " % pr_title.encode("utf-8")).decode("utf-8")
     if commit_title == "":
         commit_title = pr_title
 
     # Decide whether to use the modified title or not
-    modified_title = standardize_jira_ref(commit_title)
+    modified_title, github_issue_ids = standardize_jira_ref(commit_title)
     if modified_title != commit_title:
         print "I've re-written the title as follows to match the standard format:"
         print "Original: %s" % commit_title
@@ -487,11 +664,57 @@ def main():
         print commit_title
 
     body = pr["body"]
+    modified_body = ""
+    for line in body.split('\n'):
+        if line.startswith('>'):
+            continue
+        modified_body = modified_body + line + "\n"
+    if modified_body != body:
+        print "I've re-written the body as follows to match the standard formats:"
+        print "Original: "
+        print body
+        print "Modified: "
+        print modified_body
+        result = raw_input("Would you like to use the modified body? (y/n): ")
+        if result.lower() == "y":
+            body = modified_body
+            print "Using modified body."
+        else:
+            print "Using original body."
+
     target_ref = pr["base"]["ref"]
     user_login = pr["user"]["login"]
     base_ref = pr["head"]["ref"]
     pr_repo_desc = "%s/%s" % (user_login, base_ref)
 
+    # append pr num to the github issues - we need to attach label and milestone to them
+    github_issue_ids.append(pr_num)
+
+    #
+    # 4. attach milestone, area, type and release to github issues
+    #
+
+    # get issue labels
+    issue_labels = get_github_issue_labels(pr_num)
+    # ask for fix milestone, area and type
+    fix_milestone, fix_milestone_number, fix_areas, fix_types = \
+        ask_updates_for_github_issues(milestones, labels, issue_labels)
+    # update issues with fix milestone, are and type
+    other_labels = filter(lambda x: not x.startswith("area"), issue_labels)
+    all_issue_labels = update_github_issues( \
+        github_issue_ids, \
+        fix_milestone_number, \
+        fix_milestone, \
+        fix_areas, \
+        fix_types, \
+        other_labels)
+    # add the pr author to the assignees
+    add_assignees_to_github_issues(github_issue_ids, [ user_login ])
+
+    #
+    # 5. Process the merge
+    #
+
     # Merged pull requests don't appear as merged in the GitHub API;
     # Instead, they're closed by asfgit.
     merge_commits = \
@@ -508,7 +731,8 @@ def main():
             fail("Couldn't find any merge commit for #%s, you may need to update HEAD." %
pr_num)
 
         print "Found commit %s:\n%s" % (merge_hash, message)
-        cherry_pick(pr_num, merge_hash, latest_branch)
+        
+        cherry_pick(pr_num, merge_hash, ask_for_branch(latest_branch))
         sys.exit(0)
 
     if not bool(pr["mergeable"]):
@@ -522,12 +746,18 @@ def main():
     continue_maybe("Proceed with merging pull request #%s?" % pr_num)
 
     merged_refs = [target_ref]
-
+    # proceed with the merge
     merge_hash, merge_commit_log = merge_pr(pr_num, target_ref, commit_title, body, pr_reviewers,
pr_repo_desc)
 
     pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
     while raw_input("\n%s (y/n): " % pick_prompt).lower() == "y":
-        merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]
+        pick_ref = ask_for_branch(latest_branch) 
+        branch_version = pick_ref.split('-')[1]
+        # add releases
+        fix_releases = ask_release_for_github_issues(branch_version, labels)
+        if len(fix_releases) > 0:
+            all_issue_labels = add_release_to_github_issues(github_issue_ids, all_issue_labels,
fix_releases[0])
+        merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, pick_ref)]
 
     if JIRA_IMPORTED:
         if JIRA_USERNAME and JIRA_PASSWORD:

-- 
To stop receiving notification emails like this one, please contact
['"commits@bookkeeper.apache.org" <commits@bookkeeper.apache.org>'].

Mime
View raw message