lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject lucene-solr:branch_7x: LUCENE-8106: reproduceJenkinsFailures.py improvements: add cmdline option parsing; sort max failures to the bottom of the report; retest at branch tip first with and then without the seed if 100% of iterations fail.
Date Wed, 14 Feb 2018 04:48:31 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x 56a7ddc3e -> 08b9de997


LUCENE-8106: reproduceJenkinsFailures.py improvements: add cmdline option parsing; sort max
failures to the bottom of the report; retest at branch tip first with and then without the
seed if 100% of iterations fail.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/08b9de99
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/08b9de99
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/08b9de99

Branch: refs/heads/branch_7x
Commit: 08b9de9970a90d31dfec07e36faf8c1bc0a6c593
Parents: 56a7ddc
Author: Steve Rowe <sarowe@apache.org>
Authored: Tue Feb 13 23:47:57 2018 -0500
Committer: Steve Rowe <sarowe@apache.org>
Committed: Tue Feb 13 23:48:20 2018 -0500

----------------------------------------------------------------------
 dev-tools/scripts/reproduceJenkinsFailures.py | 148 ++++++++++++++-------
 1 file changed, 97 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/08b9de99/dev-tools/scripts/reproduceJenkinsFailures.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index bb32123..aa11ff3 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -13,38 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import argparse
 import os
 import re
 import subprocess
 import sys
+import traceback
 import urllib.error
 import urllib.request
 from textwrap import dedent
 
-# Number of iterations per test suite
-testIters = 5
-
-usage = dedent('''\
-               Usage:\n
-                 python3 -u %s URL\n
-               Must be run from a Lucene/Solr git workspace. Downloads the Jenkins
-               log pointed to by the given URL, parses it for Git revision and failed
-               Lucene/Solr tests, checks out the Git revision in the local workspace,
-               groups the failed tests by module, then runs
-               'ant test -Dtest.dups=%d -Dtests.class="*.test1[|*.test2[...]]" ...'
-               in each module of interest, failing at the end if any of the runs fails.
-               To control the maximum number of concurrent JVMs used for each module's
-               test run, set 'tests.jvms', e.g. in ~/lucene.build.properties
-               ''' % (sys.argv[0], testIters))
-
-reHelpArg = re.compile(r'-{1,2}(?:\?|h(?:elp)?)')
-
 # Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master)
-reGitRev = re.compile(r'Checking out Revision (\S+)')
+reGitRev = re.compile(r'Checking out Revision (\S+)\s+\(refs/remotes/origin/([^)]+)')
 
 # Method example: NOTE: reproduce with: ant test  -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync
-Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST
-Dtests.asserts=true -Dtests.file.encoding=UTF-8
 # Suite example:  NOTE: reproduce with: ant test  -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27
-Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII
 reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))')
+reTestsSeed = re.compile(r'-Dtests.seed=\S+\s*')
 
 # Example: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21108/
 reJenkinsURLWithoutConsoleText = re.compile(r'https?://.*/\d+/?\Z', re.IGNORECASE)
@@ -57,12 +42,32 @@ reErrorFailure = re.compile(r'(?:errors|failures)="[^0]')
 # consoleText from Policeman Jenkins's Windows jobs fails to decode as UTF-8
 encoding = 'iso-8859-1'
 
-tests = {}
-modules = {}
-
 lastFailureCode = 0
 gitCheckoutSucceeded = False
 
+description = dedent('''\
+                     Must be run from a Lucene/Solr git workspace. Downloads the Jenkins
+                     log pointed to by the given URL, parses it for Git revision and failed
+                     Lucene/Solr tests, checks out the Git revision in the local workspace,
+                     groups the failed tests by module, then runs
+                     'ant test -Dtest.dups=%d -Dtests.class="*.test1[|*.test2[...]]" ...'
+                     in each module of interest, failing at the end if any of the runs fails.
+                     To control the maximum number of concurrent JVMs used for each module's
+                     test run, set 'tests.jvms', e.g. in ~/lucene.build.properties
+                     ''')
+defaultIters = 5
+
+def readConfig():
+  parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
+                                   description=description)
+  parser.add_argument('url', metavar='URL',
+                      help='Points to the Jenkins log to parse')
+  parser.add_argument('--no-fetch', dest='fetch', action='store_false', default=True,
+                      help='Do not run "git fetch" prior to "git checkout"')
+  parser.add_argument('--iters', dest='testIters', type=int, default=defaultIters, metavar='N',
+                      help='Number of iterations per test suite (default: %d)' % defaultIters)
+  return parser.parse_args()
+
 def runOutput(cmd):
   print('[repro] %s' % cmd)
   try:
@@ -82,8 +87,10 @@ def run(cmd, rememberFailure=True):
   return code
 
 def fetchAndParseJenkinsLog(url):
-  global revision
-  revision = None
+  global revisionFromLog
+  global branchFromLog
+  revisionFromLog = None
+  tests = {}
   print('[repro] Jenkins log URL: %s\n' % url)
   try:
     with urllib.request.urlopen(url) as consoleText:
@@ -91,8 +98,9 @@ def fetchAndParseJenkinsLog(url):
         line = rawLine.decode(encoding)
         match = reGitRev.match(line)
         if match is not None:
-          revision = match.group(1)
-          print('[repro] Revision: %s\n' % revision)
+          revisionFromLog = match.group(1)
+          branchFromLog = match.group(2)
+          print('[repro] Revision: %s\n' % revisionFromLog)
         else:
           match = reReproLine.search(line)
           if match is not None:
@@ -103,7 +111,7 @@ def fetchAndParseJenkinsLog(url):
   except urllib.error.URLError as e:
     raise RuntimeError('ERROR: fetching %s : %s' % (url, e))
   
-  if revision == None:
+  if revisionFromLog == None:
     if reJenkinsURLWithoutConsoleText.match(url):
       print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n')
       fetchAndParseJenkinsLog(url + '/consoleText')                                     
                  
@@ -112,18 +120,29 @@ def fetchAndParseJenkinsLog(url):
   if 0 == len(tests):
     print('[repro] No "reproduce with" lines found; exiting.')
     sys.exit(0)
+  return tests
 
-def prepareWorkspace():
+def prepareWorkspace(fetch, gitRef):
   global gitCheckoutSucceeded
-  code = run('git checkout %s' % revision)
+  if fetch:
+    code = run('git fetch')
+    if 0 != code:
+      raise RuntimeError('ERROR: "git fetch" failed.  See above.')
+  checkoutCmd = 'git checkout %s' % gitRef
+  code = run(checkoutCmd)
   if 0 != code:
-    raise RuntimeError('ERROR: "git checkout %s" failed.  See above.  Maybe try "git pull"?'
% revision)
+    raise RuntimeError('ERROR: "%s" failed.  See above.' % checkoutCmd)
+  if fetch:
+    code = run('git pull')
+    if 0 != code:
+      raise RuntimeError('ERROR: "git pull" failed.  See above.')
   gitCheckoutSucceeded = True
   code = run('ant clean')
   if 0 != code:
     raise RuntimeError('ERROR: "ant clean" failed.  See above.')
 
-def groupTestsByModule():
+def groupTestsByModule(tests):
+  modules = {}
   for (dir, _, files) in os.walk('.'):
     for file in files:
       match = reJavaFile.search(file)
@@ -140,9 +159,9 @@ def groupTestsByModule():
     print('[repro]    %s' % module)
     for test in modules[module]:
       print('[repro]       %s' % test)
+  return modules
 
-def runTests():
-  global lastFailureCode
+def runTests(testIters, modules, tests):
   cwd = os.getcwd()
   testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s"
-Dtests.showOutput=onerror %s'
   for module in modules:
@@ -153,13 +172,13 @@ def runTests():
     os.chdir(module)
     code = run('ant compile-test')
     try:
-      if (0 != code):
+      if 0 != code:
         raise RuntimeError("ERROR: Compile failed in %s/ with code %d.  See above." % (module,
code))
       run(testCmdline % (testIters, testIters * numTests, testList, params))
     finally:
       os.chdir(cwd)
       
-def printReport():
+def printReport(testIters, location):
   failures = {}
   for start in ('lucene/build', 'solr/build'):
     for (dir, _, files) in os.walk(start):
@@ -175,35 +194,62 @@ def printReport():
               if errorFailureMatch is not None:
                 failures[testcase] += 1
                 break
-  print("[repro] Failures:")
-  for testcase in sorted(failures):
+  print("[repro] Failures%s:" % location)
+  for testcase in sorted(failures, key=lambda t: (failures[t],t)): # sort by failure count,
then by testcase 
     print("[repro]   %d/%d failed: %s" % (failures[testcase], testIters, testcase))
+  return failures
 
-def rememberGitBranch():
-  global origGitBranch
+def getLocalGitBranch():
   origGitBranch = runOutput('git rev-parse --abbrev-ref HEAD')
-  if (origGitBranch == 'HEAD'):                     # In detached HEAD state
+  if origGitBranch == 'HEAD':                       # In detached HEAD state
     origGitBranch = runOutput('git rev-parse HEAD') # Use the SHA when not on a branch
   print('[repro] Initial local git branch/revision: %s' % origGitBranch)
+  return origGitBranch
 
 def main():
-  if 2 != len(sys.argv) or reHelpArg.match(sys.argv[1]):
-    print(usage)
-    sys.exit(0)
-  fetchAndParseJenkinsLog(sys.argv[1])
-  rememberGitBranch()
+  config = readConfig()
+  tests = fetchAndParseJenkinsLog(config.url)
+  localGitBranch = getLocalGitBranch()
 
   try:
-    prepareWorkspace()
-    groupTestsByModule()
-    runTests()
-    printReport()
+    prepareWorkspace(config.fetch, revisionFromLog)
+    modules = groupTestsByModule(tests)
+    runTests(config.testIters, modules, tests)
+    failures = printReport(config.testIters, '')
+    
+    # Retest 100% failures at the tip of the branch
+    oldTests = tests
+    tests = {}
+    for fullClass in failures:
+      testcase = fullClass[(fullClass.rindex('.') + 1):]
+      if failures[fullClass] == config.testIters:
+        tests[testcase] = oldTests[testcase]
+    if len(tests) > 0:
+      print('\n[repro] Re-testing 100%% failures at the tip of %s' % branchFromLog)
+      prepareWorkspace(False, branchFromLog)
+      modules = groupTestsByModule(tests)
+      runTests(config.testIters, modules, tests)
+      failures = printReport(config.testIters, ' at the tip of %s' % branchFromLog)
+      
+      # Retest 100% tip-of-branch failures without a seed
+      oldTests = tests
+      tests = {}
+      for fullClass in failures:
+        testcase = fullClass[(fullClass.rindex('.') + 1):]
+        if failures[fullClass] == config.testIters:
+          tests[testcase] = re.sub(reTestsSeed, '', oldTests[testcase])
+      if len(tests) > 0:
+        print('\n[repro] Re-testing 100%% failures at the tip of %s without a seed' % branchFromLog)
+        prepareWorkspace(False, branchFromLog)
+        modules = groupTestsByModule(tests)
+        runTests(config.testIters, modules, tests)
+        printReport(config.testIters, ' at the tip of %s without a seed' % branchFromLog)
   except Exception as e:
-    print('[repro] %s' % e)
+    print('[repro] %s' % traceback.format_exc())
     sys.exit(1)
   finally:
     if gitCheckoutSucceeded:
-      run('git checkout %s' % origGitBranch, rememberFailure=False) # Restore original git
branch/sha
+      run('git checkout %s' % localGitBranch, rememberFailure=False) # Restore original git
branch/sha
 
   print('[repro] Exiting with code %d' % lastFailureCode)
   sys.exit(lastFailureCode)


Mime
View raw message