maven-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tibordig...@apache.org
Subject [maven-surefire] 01/01: [SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider
Date Wed, 02 Oct 2019 15:37:35 GMT
This is an automated email from the ASF dual-hosted git repository.

tibordigana pushed a commit to branch SUREFIRE-1584
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git

commit cfe702a796be6508932513c39dd431a8af11b250
Author: Col-E <mcoley2@gmu.edu>
AuthorDate: Thu May 23 03:34:24 2019 -0400

    [SUREFIRE-1584] Add support for rerunFailingTests to JUnit5 Provider
---
 .../maven/plugin/failsafe/IntegrationTestMojo.java |   2 +-
 .../maven/plugin/surefire/SurefirePlugin.java      |   2 +-
 .../src/site/apt/featurematrix.apt.vm              |   4 +-
 .../its/JUnitPlatformRerunFailingTestsIT.java      | 281 +++++++++++++++++++++
 .../junit-platform-rerun-failing-tests/pom.xml     |  55 ++++
 .../java/junitplatform/FlakyFirstTimeTest.java     |  63 +++++
 .../src/test/java/junitplatform/PassingTest.java   |  38 +++
 .../junitplatform/JUnitPlatformProvider.java       |  40 ++-
 .../surefire/junitplatform/RunListenerAdapter.java |  57 ++++-
 9 files changed, 527 insertions(+), 15 deletions(-)

diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index d900e37..35b0d41 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -264,7 +264,7 @@ public class IntegrationTestMojo
     private String encoding;
 
     /**
-     * (JUnit 4+ providers)
+     * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
      * The number of times each failing test will be rerun. If set larger than 0, rerun failing
tests immediately after
      * they fail. If a failing test passes in any of those reruns, it will be marked as pass
and reported as a "flake".
      * However, all the failing attempts will be recorded.
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index 37bf3cf..aad9230 100644
--- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -245,7 +245,7 @@ public class SurefirePlugin
     private String encoding;
 
     /**
-     * (JUnit 4+ providers)
+     * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
      * The number of times each failing test will be rerun. If set larger than 0, rerun failing
tests immediately after
      * they fail. If a failing test passes in any of those reruns, it will be marked as pass
and reported as a "flake".
      * However, all the failing attempts will be recorded.
diff --git a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
index 521895a..05d13bc 100644
--- a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
+++ b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
@@ -42,7 +42,7 @@ Feature Matrix
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
 | custom run-listener                         |     N      |    Y     |      Y     |    Y
     |  -       |  N                   |
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
-| re-run count                                |     N      |    Y     |      Y     |    N
     |  N       |  N                   |
+| re-run count                                |     N      |    Y     |      Y     |    N
     |  N       |  Y(*3)               |
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
 | skip after failure count                    |     N      |    Y     |      Y     |    Y
     |  N       |  N                   |
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
@@ -62,3 +62,5 @@ Feature Matrix
 
    (*2) 3 extensions related to JUnit5 annotation <<<DisplayName>>>.
 
+   (*3) Since 3.0.0-M4
+   
\ No newline at end of file
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
new file mode 100644
index 0000000..6d881d3
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
@@ -0,0 +1,281 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
+ * @author Matt Coley
+ */
+public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationTestCase
+{
+    private final static String VERSION = "5.5.1";
+
+    private SurefireLauncher unpack()
+    {
+        return unpack( "/junit-platform-rerun-failing-tests" );
+    }
+
+    @Before
+    public void setup()
+    {
+        assumeJavaVersion( 1.8 );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsWithOneRetry() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5,
1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults(
5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults(
5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsTwoRetry() throws Exception
+    {
+        // Four flakes, both tests have been re-run twice
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging()
+                .addGoal( "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults(
5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest().assertTestSuiteResults(
+                5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0,
0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0,
0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsFalse() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion(
+                VERSION ).maven().withFailure().debugLogging().executeTest().assertTestSuiteResults(
5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-DforkCount=3" ).withFailure().debugLogging().executeTest().assertTestSuiteResults(
5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunOneTestClass() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults(
3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+    }
+
+    @Test
+    public void testRerunOneTestMethod() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+    }
+
+    private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+    }
+
+    private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+    }
+
+    private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+    }
+
+    private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+    {
+        verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+    }
+
+    private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+    {
+        verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+    }
+
+    private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures,
int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+        outputValidator.verifyTextInLog( "Errors:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int
failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures,
int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Flakes:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures,
int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testFailingTestOne"
);
+        outputValidator.verifyTextInLog( "ERROR" );
+        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testErrorTestOne"
);
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyStatistics( OutputValidator outputValidator, int run, int failures,
int errors, int flakes )
+    {
+        if ( flakes > 0 )
+        {
+            outputValidator.verifyTextInLog(
+                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors
+ ", Skipped: 0, Flakes: " + flakes );
+        }
+        else
+        {
+            outputValidator.verifyTextInLog(
+                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors
+ ", Skipped: 0" );
+        }
+    }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
new file mode 100644
index 0000000..af4dde0
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.maven.plugins.surefire</groupId>
+    <artifactId>junit-platform-rerun-failing-tests</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <name>Test for rerun failing tests in JUnit 5/Platform</name>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire.version}</version>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
new file mode 100644
index 0000000..5c53d6d
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/FlakyFirstTimeTest.java
@@ -0,0 +1,63 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+public class FlakyFirstTimeTest
+{
+    private static int failingCount = 0;
+
+    private static int errorCount = 0;
+
+
+    @Test
+    public void testFailingTestOne()
+    {
+        System.out.println( "Failing test" );
+        // This test will fail with only one retry, but will pass with two
+        if ( failingCount < 2 )
+        {
+            failingCount++;
+            fail( "Failing test" );
+        }
+    }
+
+    @Test
+    public void testErrorTestOne() throws Exception
+    {
+        System.out.println( "Error test" );
+        // This test will error out with only one retry, but will pass with two
+        if ( errorCount < 2 )
+        {
+            errorCount++;
+            throw new IllegalArgumentException( "..." );
+        }
+    }
+
+    @Test
+    public void testPassingTest() throws Exception
+    {
+        System.out.println( "Passing test" );
+    }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
new file mode 100644
index 0000000..4f80f9f
--- /dev/null
+++ b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/src/test/java/junitplatform/PassingTest.java
@@ -0,0 +1,38 @@
+package junitplatform;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+
+public class PassingTest
+{
+    @Test
+    public void testPassingTestOne()
+    {
+        System.out.println( "Passing test one" );
+    }
+
+    @Test
+    public void testPassingTestTwo() throws Exception
+    {
+        System.out.println( "Passing test two" );
+    }
+}
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
index 1138e4a..4cd6342 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
@@ -31,6 +31,7 @@ import static org.apache.maven.surefire.report.ConsoleOutputCapture.startCapture
 import static org.apache.maven.surefire.util.TestsToRun.fromClass;
 import static org.junit.platform.commons.util.StringUtils.isBlank;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
 import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
 
 import java.io.IOException;
@@ -60,6 +61,7 @@ import org.junit.platform.engine.Filter;
 import org.junit.platform.launcher.Launcher;
 import org.junit.platform.launcher.LauncherDiscoveryRequest;
 import org.junit.platform.launcher.TagFilter;
+import org.junit.platform.launcher.TestIdentifier;
 import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
 import org.junit.platform.launcher.core.LauncherFactory;
 
@@ -147,7 +149,26 @@ public class JUnitPlatformProvider
     private void invokeAllTests( TestsToRun testsToRun, RunListener runListener )
     {
         LauncherDiscoveryRequest discoveryRequest = buildLauncherDiscoveryRequest( testsToRun
);
-        launcher.execute( discoveryRequest, new RunListenerAdapter( runListener ) );
+        RunListenerAdapter adapter = new RunListenerAdapter( runListener );
+        launcher.execute( discoveryRequest, adapter );
+        // Rerun failing tests if requested
+        int count = parameters.getTestRequest().getRerunFailingTestsCount();
+        if ( count > 0 && adapter.hasFailingTests() )
+        {
+            for ( int i = 0; i < count; i++ )
+            {
+                // Replace the "discoveryRequest" so that it only specifies the failing tests
+                discoveryRequest = buildLauncherDiscoveryRequestForRerunFailures( adapter
);
+                // Reset adapter's recorded failures and invoke the failed tests again
+                adapter.getFailures().clear();
+                launcher.execute( discoveryRequest, adapter );
+                // If no tests fail in the rerun, we're done
+                if ( !adapter.hasFailingTests() )
+                {
+                    break;
+                }
+            }
+        }
     }
 
     private LauncherDiscoveryRequest buildLauncherDiscoveryRequest( TestsToRun testsToRun
)
@@ -161,6 +182,23 @@ public class JUnitPlatformProvider
         return builder.build();
     }
 
+    private LauncherDiscoveryRequest buildLauncherDiscoveryRequestForRerunFailures( RunListenerAdapter
adapter )
+    {
+        LauncherDiscoveryRequestBuilder builder = request().filters( filters ).configurationParameters(
+                configurationParameters );
+        // Iterate over recorded failures
+        for ( TestIdentifier identifier : adapter.getFailures().keySet() )
+        {
+            // Extract quantified test name data
+            String[] classMethodName = adapter.toClassMethodNameWithoutPlan( identifier );
+            String className = classMethodName[1];
+            String methodName = classMethodName[3];
+            // Add filter for the specific failing method
+            builder.selectors( selectMethod( className, methodName ) );
+        }
+        return builder.build();
+    }
+
     private Filter<?>[] newFilters()
     {
         List<Filter<?>> filters = new ArrayList<>();
diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
index 29f3eeb..5004f26 100644
--- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
+++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java
@@ -47,6 +47,7 @@ final class RunListenerAdapter
     implements TestExecutionListener
 {
     private final ConcurrentMap<TestIdentifier, Long> testStartTime = new ConcurrentHashMap<>();
+    private final ConcurrentMap<TestIdentifier, TestExecutionResult> failures = new
ConcurrentHashMap<>();
     private final RunListener runListener;
     private volatile TestPlan testPlan;
 
@@ -59,6 +60,7 @@ final class RunListenerAdapter
     public void testPlanExecutionStarted( TestPlan testPlan )
     {
         this.testPlan = testPlan;
+        failures.clear();
     }
 
     @Override
@@ -124,6 +126,7 @@ final class RunListenerAdapter
                     {
                         runListener.testError( createReportEntry( testIdentifier, testExecutionResult,
elapsed ) );
                     }
+                    failures.put( testIdentifier, testExecutionResult );
                     break;
                 default:
                     if ( isTest )
@@ -211,6 +214,16 @@ final class RunListenerAdapter
         return new PojoStackTraceWriter( realClassName, realMethodName, throwable );
     }
 
+    private String[] toClassMethodName( TestIdentifier testIdentifier )
+    {
+        return toClassMethodName( testIdentifier, true );
+    }
+
+    public String[] toClassMethodNameWithoutPlan( TestIdentifier testIdentifier )
+    {
+        return toClassMethodName( testIdentifier, false );
+    }
+
     /**
      * <ul>
      *     <li>[0] class name - used in stacktrace parser</li>
@@ -220,9 +233,11 @@ final class RunListenerAdapter
      * </ul>
      *
      * @param testIdentifier a class or method
+     * @param usePlan {@code true} for using the test-plan to fetch sources.
+     *                {@code false} to rely on fallbacks.
      * @return 4 elements string array
      */
-    private String[] toClassMethodName( TestIdentifier testIdentifier )
+    private String[] toClassMethodName( TestIdentifier testIdentifier, boolean usePlan )
     {
         Optional<TestSource> testSource = testIdentifier.getSource();
         String display = testIdentifier.getDisplayName();
@@ -232,16 +247,24 @@ final class RunListenerAdapter
             MethodSource methodSource = testSource.map( MethodSource.class::cast ).get();
             String realClassName = methodSource.getClassName();
 
-            String[] source = testPlan.getParent( testIdentifier )
-                    .map( this::toClassMethodName )
-                    .map( s -> new String[] { s[0], s[1] } )
-                    .orElse( new String[] { realClassName, realClassName } );
+            String[] source;
+            if ( usePlan )
+            {
+               source = testPlan.getParent( testIdentifier )
+                        .map( i -> toClassMethodName( i, usePlan ) )
+                        .map( s -> new String[] { s[0], s[1] } )
+                        .orElse( new String[] { realClassName, realClassName } );
+            }
+            else
+            {
+                source = new String[] { realClassName, realClassName };
+            }
 
             String methodName = methodSource.getMethodName();
             boolean useMethod = display.equals( methodName ) || display.equals( methodName
+ "()" );
             String resolvedMethodName = useMethod ? methodName : display;
 
-            return new String[] { source[0], source[1], methodName, resolvedMethodName };
+            return new String[] {source[0], source[1], methodName, resolvedMethodName};
         }
         else if ( testSource.filter( ClassSource.class::isInstance ).isPresent() )
         {
@@ -249,14 +272,26 @@ final class RunListenerAdapter
             String className = classSource.getClassName();
             String simpleClassName = className.substring( 1 + className.lastIndexOf( '.'
) );
             String source = display.equals( simpleClassName ) ? className : display;
-            return new String[] { className, source, null, null };
+            return new String[] {className, source, null, null};
         }
         else
         {
-            String source = testPlan.getParent( testIdentifier )
-                    .map( TestIdentifier::getDisplayName )
-                    .orElse( display );
-            return new String[] { source, source, display, display };
+            String source = !usePlan ? display : testPlan.getParent( testIdentifier )
+                    .map( TestIdentifier::getDisplayName ).orElse( display );
+            return new String[] {source, source, display, display};
         }
     }
+
+    /**
+     * @return Map of tests that failed.
+     */
+    public Map<TestIdentifier, TestExecutionResult> getFailures()
+    {
+        return failures;
+    }
+
+    public boolean hasFailingTests()
+    {
+        return !getFailures().isEmpty();
+    }
 }


Mime
View raw message