hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject hbase git commit: HBASE-16434 - Add date and count of flaky tests - Sort tests by decreasing order of flakyness - Internal links to each job's results - Correct calculation of total bad runs for a test - Fixes pylint errors
Date Fri, 19 Aug 2016 16:55:44 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 741d0a451 -> 0d6c4d92e


HBASE-16434
- Add date and count of flaky tests
- Sort tests by decreasing order of flakyness
- Internal links to each job's results
- Correct calculation of total bad runs for a test
- Fixes pylint errors

Change-Id: I12ebc32ccec14c5ff389464b4de8ae93653c876c


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d6c4d92
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d6c4d92
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d6c4d92

Branch: refs/heads/master
Commit: 0d6c4d92edd7adfbe902cbd1cce53964c8855406
Parents: 741d0a4
Author: Apekshit Sharma <appy@apache.org>
Authored: Wed Aug 17 00:47:16 2016 -0700
Committer: Apekshit Sharma <appy@apache.org>
Committed: Fri Aug 19 09:55:29 2016 -0700

----------------------------------------------------------------------
 dev-support/findHangingTests.py           | 74 ++++++++++++++------------
 dev-support/flaky-dashboard-template.html | 25 ++++++++-
 dev-support/report-flakies.py             | 54 ++++++++++++++-----
 3 files changed, 103 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6c4d92/dev-support/findHangingTests.py
----------------------------------------------------------------------
diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py
index 2daf2e3..54275df 100755
--- a/dev-support/findHangingTests.py
+++ b/dev-support/findHangingTests.py
@@ -15,14 +15,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-##
-# script to find hanging test from Jenkins build output
+
+# pylint: disable=invalid-name
+# To disable 'invalid constant name' warnings.
+
+"""
+# Script to find hanging test from Jenkins build output
 # usage: ./findHangingTests.py <url of Jenkins build console>
-#
+"""
+
 import re
-import requests
 import sys
-
+import requests
 
 # If any of these strings appear in the console output, it's a build one should probably
ignore
 # for analyzing failed/hanging tests.
@@ -31,62 +35,64 @@ BAD_RUN_STRINGS = [
     "The forked VM terminated without properly saying goodbye",  # JVM crashed.
 ]
 
-# Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully
gets
-# the build information.
-# If there is error getting console text or if there are blacklisted strings in console text,
-# then returns None.
-# Definitions:
-# All tests: All testcases which were run.
-# Hanging test: A testcase which started but never finished.
-# Failed test: Testcase which encountered any kind of failure. It can be failing atomic tests,
-#   timed out tests, etc
-# Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will be
-#   included in failed tests.
+
 def get_bad_tests(console_url):
+    """
+    Returns [[all tests], [failed tests], [timeout tests], [hanging tests]] if successfully
gets
+    the build information.
+    If there is error getting console text or if there are blacklisted strings in console
text,
+    then returns None.
+    """
     response = requests.get(console_url)
     if response.status_code != 200:
         print "Error getting consoleText. Response = {} {}".format(
             response.status_code, response.reason)
         return
 
-    all_tests = set()
-    hanging_tests = set()
-    failed_tests = set()
-    timeout_tests = set()
+    # All tests: All testcases which were run.
+    # Hanging test: A testcase which started but never finished.
+    # Failed test: Testcase which encountered any kind of failure. It can be failing atomic
tests,
+    #   timed out tests, etc
+    # Timeout test: A Testcase which encountered timeout. Naturally, all timeout tests will
be
+    #   included in failed tests.
+    all_tests_set = set()
+    hanging_tests_set = set()
+    failed_tests_set = set()
+    timeout_tests_set = set()
     for line in response.content.splitlines():
-        result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", line)
+        result1 = re.match("^Running org.apache.hadoop.hbase.(\\w*\\.)*(\\w*)", line)
         if result1:
             test_case = result1.group(2)
-            if test_case in all_tests:
-                print  ("ERROR! Multiple tests with same name '{}'. Might get wrong results
"
+            if test_case in all_tests_set:
+                print ("ERROR! Multiple tests with same name '{}'. Might get wrong results
"
                        "for this test.".format(test_case))
             else:
-                hanging_tests.add(test_case)
-                all_tests.add(test_case)
-        result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\w*\.)*(\w*)", line)
+                hanging_tests_set.add(test_case)
+                all_tests_set.add(test_case)
+        result2 = re.match("^Tests run:.*- in org.apache.hadoop.hbase.(\\w*\\.)*(\\w*)",
line)
         if result2:
             test_case = result2.group(2)
             if "FAILURE!" in line:
-                failed_tests.add(test_case)
-            if test_case not in hanging_tests:
+                failed_tests_set.add(test_case)
+            if test_case not in hanging_tests_set:
                 print  ("ERROR! No test '{}' found in hanging_tests. Might get wrong results
"
                         "for this test.".format(test_case))
             else:
-                hanging_tests.remove(test_case)
-        result3 = re.match("^\s+(\w*).*\sTestTimedOut", line)
+                hanging_tests_set.remove(test_case)
+        result3 = re.match("^\\s+(\\w*).*\\sTestTimedOut", line)
         if result3:
             test_case = result3.group(1)
-            timeout_tests.add(test_case)
+            timeout_tests_set.add(test_case)
         for bad_string in BAD_RUN_STRINGS:
             if re.match(".*" + bad_string + ".*", line):
                 print "Bad string found in build:\n > {}".format(line)
                 return
     print "Result > total tests: {:4}   failed : {:4}  timedout : {:4}  hanging : {:4}".format(
-          len(all_tests), len(failed_tests), len(timeout_tests), len(hanging_tests))
-    return [all_tests, failed_tests, timeout_tests, hanging_tests]
+        len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set))
+    return [all_tests_set, failed_tests_set, timeout_tests_set, hanging_tests_set]
 
 if __name__ == "__main__":
-    if len(sys.argv) != 2 :
+    if len(sys.argv) != 2:
         print "ERROR : Provide the jenkins job console URL as the only argument."
         sys.exit(1)
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6c4d92/dev-support/flaky-dashboard-template.html
----------------------------------------------------------------------
diff --git a/dev-support/flaky-dashboard-template.html b/dev-support/flaky-dashboard-template.html
index 77dfc86..70febb4 100644
--- a/dev-support/flaky-dashboard-template.html
+++ b/dev-support/flaky-dashboard-template.html
@@ -49,14 +49,35 @@
                   Apache HBase Flaky Tests Dashboard
               </span>
 </p>
+<span>Last updated: <b>{{datetime}}</b></span><br>
+<span>Count of flaky tests (cumulated from all jobs):
+    <b>{{bad_tests_count}}</b></span><br>
 <br><br>
+<span style="font-size:20px;"><b>List of Jobs</b></span><br>
+<br>
+{% set counter = 0 %}
+{% for url in results %}
+{% set counter = counter + 1 %}
+<a href="#job_{{ counter }}">{{ url |e }}</a>
+<br>
+{% endfor %}
+<br>
+<br>
+<span style="font-size:20px;"><b>Results</b></span><br>
+<br>
 {% set counter = 0 %}
 {% for url in results %}
 {% set result = results[url] %}
 {# Dedup ids since test names may duplicate across urls #}
 {% set counter = counter + 1 %}
-                <span style="font-size:20px; font-weight:bold;">Job : {{ url |e }}
-                <a href="{{ url |e }}" style="text-decoration:none;">&#x1f517;</a></span>
+<span id="job_{{ counter }}" style="font-weight:bold;">
+    {{ url |e }}<br>
+    <a href="{{ url |e }}">
+        Go to <img height="16px" src="https://jenkins.io/sites/default/files/jenkins_favicon.ico">
+    </a>
+    &nbsp;&nbsp;&nbsp;&nbsp;
+    <a href="#">Go to top</a>
+</span>
 <br/><br/>
 <table>
     <tr>

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6c4d92/dev-support/report-flakies.py
----------------------------------------------------------------------
diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py
index 8c93b0b..92b78cc 100755
--- a/dev-support/report-flakies.py
+++ b/dev-support/report-flakies.py
@@ -16,15 +16,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=invalid-name
+# To disable 'invalid constant name' warnings.
+# pylint: disable=import-error
+# Testing environment may not have all dependencies.
+
 """
 This script uses Jenkins REST api to collect test result(s) of given build/builds and generates
 flakyness data about unittests.
 Print help: report-flakies.py -h
 """
+
 import argparse
-from jinja2 import Template
 import logging
 import os
+import time
+from collections import OrderedDict
+from jinja2 import Template
+
 import requests
 
 import findHangingTests
@@ -115,7 +124,7 @@ all_failed_tests = set()
 all_hanging_tests = set()
 # Contains { <url> : { <bad_test> : { 'all': [<build ids>], 'failed': [<build
ids>],
 #                                     'timeout': [<build ids>], 'hanging': [<builds
ids>] } } }
-url_to_bad_test_results = {}
+url_to_bad_test_results = OrderedDict()
 
 # Iterates over each url, gets test results and prints flaky tests.
 expanded_urls = expand_multi_config_projects(args)
@@ -160,33 +169,48 @@ for url_max_build in expanded_urls:
         bad_tests.update(failed_tests.union(hanging_tests))
 
     # For each bad test, get build ids where it ran, timed out, failed or hanged.
-    test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(), 'hanging'
: set()}
+    test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(),
+                                'hanging' : set(), 'bad_count' : 0}
                          for key in bad_tests}
     for build in build_id_to_results:
         [all_tests, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build]
         for bad_test in test_to_build_ids:
+            is_bad = False
             if all_tests.issuperset([bad_test]):
                 test_to_build_ids[bad_test]["all"].add(build)
             if timeout_tests.issuperset([bad_test]):
                 test_to_build_ids[bad_test]['timeout'].add(build)
+                is_bad = True
             if failed_tests.issuperset([bad_test]):
                 test_to_build_ids[bad_test]['failed'].add(build)
+                is_bad = True
             if hanging_tests.issuperset([bad_test]):
                 test_to_build_ids[bad_test]['hanging'].add(build)
-    url_to_bad_test_results[url] = test_to_build_ids
-
-    if len(test_to_build_ids) > 0:
+                is_bad = True
+            if is_bad:
+                test_to_build_ids[bad_test]['bad_count'] += 1
+
+    # Calculate flakyness % for each test.
+    for bad_test in test_to_build_ids:
+        test_to_build_ids[bad_test]['flakyness'] = (
+            (test_to_build_ids[bad_test]['bad_count']) * 100.0 /
+            len(test_to_build_ids[bad_test]['all']))
+
+    # Sort tests in descending order by flakyness.
+    sorted_test_to_build_ids = OrderedDict(
+        sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True))
+    url_to_bad_test_results[url] = sorted_test_to_build_ids
+
+    if len(sorted_test_to_build_ids) > 0:
         print "URL: {}".format(url)
         print "{:>60}  {:10}  {:25}  {}".format(
             "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness")
-        for bad_test in test_to_build_ids:
-            failed = len(test_to_build_ids[bad_test]['failed'])
-            timeout = len(test_to_build_ids[bad_test]['timeout'])
-            hanging = len(test_to_build_ids[bad_test]['hanging'])
-            total = len(test_to_build_ids[bad_test]['all'])
+        for bad_test in sorted_test_to_build_ids:
+            test_status = sorted_test_to_build_ids[bad_test]
             print "{:>60}  {:10}  {:7} ( {:4} / {:5} / {:5} )  {:2.0f}%".format(
-                bad_test, total, failed + timeout, failed, timeout, hanging,
-                (failed + timeout) * 100.0 / total)
+                bad_test, len(test_status['all']), test_status['bad_count'],
+                len(test_status['failed']), len(test_status['timeout']),
+                len(test_status['hanging']), test_status['flakyness'])
     else:
         print "No flaky tests founds."
         if len(build_ids) == len(build_ids_without_tests_run):
@@ -218,4 +242,6 @@ with open(os.path.join(dev_support_dir, "flaky-dashboard-template.html"),
"r") a
     template = Template(f.read())
 
 with open("dashboard.html", "w") as f:
-    f.write(template.render(results=url_to_bad_test_results))
+    datetime = time.strftime("%m/%d/%Y %H:%M:%S")
+    f.write(template.render(datetime=datetime, bad_tests_count=len(all_bad_tests),
+                            results=url_to_bad_test_results))


Mime
View raw message