From commits-return-69246-archive-asf-public=cust-asf.ponee.io@maven.apache.org Thu Jan 4 01:00:20 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id DB73318077A for ; Thu, 4 Jan 2018 01:00:20 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id CB23C160C39; Thu, 4 Jan 2018 00:00:20 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6FECC160C1B for ; Thu, 4 Jan 2018 01:00:19 +0100 (CET) Received: (qmail 57449 invoked by uid 500); 4 Jan 2018 00:00:18 -0000 Mailing-List: contact commits-help@maven.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@maven.apache.org Delivered-To: mailing list commits@maven.apache.org Received: (qmail 57440 invoked by uid 99); 4 Jan 2018 00:00:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Jan 2018 00:00:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E24FEDFC27; Thu, 4 Jan 2018 00:00:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tibordigana@apache.org To: commits@maven.apache.org Message-Id: <598b038566f541539cf0a231f872bb1e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: maven-surefire git commit: [SUREFIRE-1372] Rerunning failing tests fails in combination with Description#createSuiteDescription Date: Thu, 4 Jan 2018 00:00:06 +0000 (UTC) Repository: maven-surefire Updated Branches: refs/heads/master 869d6f29f -> 0a81c4897 [SUREFIRE-1372] Rerunning failing tests fails in combination with Description#createSuiteDescription Filtering tests by the tuple of (Class, Method) does not work when a test suite is used as a suite may not have (Class, Method) . Filtering by the description of the failed test should however work. Related Issues: - https://github.com/cucumber/cucumber-jvm/issues/1134 - https://github.com/temyers/cucumber-jvm-parallel-plugin/issues/31 SUREFIRE-1372 Filter junit4 tests to be rerun by description Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/0a81c489 Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/0a81c489 Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/0a81c489 Branch: refs/heads/master Commit: 0a81c48971e3ee4b79b0858be7b8f4b13ece7287 Parents: 869d6f2 Author: mpkorstanje Authored: Wed May 10 23:59:31 2017 +0200 Committer: Tibor17 Committed: Thu Jan 4 00:50:29 2018 +0100 ---------------------------------------------------------------------- .../src/site/apt/examples/cucumber.apt.vm | 115 +++++++++++++++++++ .../apt/examples/rerun-failing-tests.apt.vm | 5 + maven-surefire-plugin/src/site/site.xml | 1 + .../JUnit47RerunFailingTestWithCucumberIT.java | 81 +++++++++++++ .../pom.xml | 77 +++++++++++++ .../org/sample/cucumber/FlakeCucumberTest.java | 29 +++++ .../test/java/org/sample/cucumber/StepDefs.java | 56 +++++++++ .../org/sample/cucumber/Sample.feature | 10 ++ .../common/junit4/JUnit4ProviderUtil.java | 67 ++--------- .../common/junit4/MatchDescriptions.java | 104 +++++++++++++++++ .../common/junit4/JUnit4ProviderUtilTest.java | 30 ++--- .../common/junit48/FailingMethodFilter.java | 85 -------------- .../surefire/common/junit48/FilterFactory.java | 7 +- .../maven/surefire/junit4/JUnit4Provider.java | 38 ++---- .../surefire/junitcore/JUnitCoreProvider.java | 12 +- 15 files changed, 518 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm b/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm new file mode 100644 index 0000000..e0436cc --- /dev/null +++ b/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm @@ -0,0 +1,115 @@ + ------ + Using Cucumber + ------ + M.P. Korstanje + ------ + 2010-10-10 + ------ + + ~~ Licensed to the Apache Software Foundation (ASF) under one + ~~ or more contributor license agreements. See the NOTICE file + ~~ distributed with this work for additional information + ~~ regarding copyright ownership. The ASF licenses this file + ~~ to you under the Apache License, Version 2.0 (the + ~~ "License"); you may not use this file except in compliance + ~~ with the License. You may obtain a copy of the License at + ~~ + ~~ http://www.apache.org/licenses/LICENSE-2.0 + ~~ + ~~ Unless required by applicable law or agreed to in writing, + ~~ software distributed under the License is distributed on an + ~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~~ KIND, either express or implied. See the License for the + ~~ specific language governing permissions and limitations + ~~ under the License. + + ~~ NOTE: For help with the syntax of this file, see: + ~~ http://maven.apache.org/doxia/references/apt-format.html + +Using Cucumber with JUnit + +* Configuring Cucumber JUnit + + To get started with Cucumber, you need to add the required version of Cucumber JUnit to your project: + ++---+ + + [...] + + io.cucumber + cucumber-junit + ${cucumber.version} + test + + [...] + ++---+ + + Then create an empty class that uses the Cucumber JUnit runner. + +#{if}(${project.artifactId}=="maven-surefire-plugin") + ++---+ +package org.sample.cucumber; + +import org.junit.runner.RunWith; + +import cucumber.api.CucumberOptions; +import cucumber.api.junit.Cucumber; + +@RunWith( Cucumber.class ) +public class RunCucumberTest +{ + [...] +} ++---+ + +#{else} + ++---+ +package org.sample.cucumber; + +import org.junit.runner.RunWith; + +import cucumber.api.CucumberOptions; +import cucumber.api.junit.Cucumber; + +@RunWith( Cucumber.class ) +public class RunCucumberIT +{ + [...] +} ++---+ + +#{end} + + This will execute all scenarios in the package of the runner. By default a glue code is assumed to be in the same + package. The <<<@CucumberOptions>>> annotation can be used to provide additional configuration of Cucumber. + +#{if}(${project.artifactId}=="maven-surefire-plugin") + Note that in this example the BDD scenarios are executed by the ${thisPlugin} Plugin in the <<>> phase of the + build lifecycle. + ++---+ +mvn test ++---+ + +#{else} + Note that in this example the BDD scenarios are executed by the ${thisPlugin} Plugin in the <<>> + phase of the build lifecycle. The ${thisPlugin} Plugin can be invoked by calling the <<>> phase. + ++---+ +mvn verify ++---+ +#{end} + +* Using JUnit Rules + + The Cucumber supports JUnit annotations <<<@ClassRule>>>, <<<@BeforeClass>>> and <<<@AfterClass>>>. These are + invoked around the suite of features. Using these is not recommended as it limits the portability between different + runners. Instead it is recommended to use Cucumbers `Before` and `After` hooks to setup scaffolding. + +* Using other JUnit features + + The Cucumber runner acts like a suite of a JUnit tests. As such other JUnit features such as Categories, Custom JUnit + Listeners and Reporters and re-running failed tests can all be expected to work. http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm index 8a6a8bd..c34e2a6 100644 --- a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm +++ b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm @@ -140,6 +140,11 @@ mvn -D${thisPlugin.toLowerCase()}.rerunFailingTestsCount=2 test The provider <<>> executes individual test class and consequently re-runs failed tests. The provider <<>> executes all test classes and re-runs failed tests afterwards. +* Re-run execution in Cucumber JVM + + Since of 2.21.0 the provider <<>> can rerun scenarios created by <> + 2.0.0 and higher. + * Re-run and skip execution Since of 2.19.1 you can use parameters <<>> and <<>> together. http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/maven-surefire-plugin/src/site/site.xml ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/site.xml b/maven-surefire-plugin/src/site/site.xml index 2d2e95a..f5497e2 100644 --- a/maven-surefire-plugin/src/site/site.xml +++ b/maven-surefire-plugin/src/site/site.xml @@ -41,6 +41,7 @@ + http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java new file mode 100644 index 0000000..7383783 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java @@ -0,0 +1,81 @@ +package org.apache.maven.surefire.its; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static org.apache.commons.lang3.JavaVersion.JAVA_1_7; +import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion; + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests using the JUnit 47 provider to rerun failing tests with the cucumber runner. The main + * problem that the junit4 provider has with the cucumber runner is that the junit Description + * instance created by the runner has a null test class attribute. This requires that tests are + * rerun based on their description. + * + * @author mpkorstanje + */ +public class JUnit47RerunFailingTestWithCucumberIT + extends SurefireJUnit4IntegrationTestCase { + + @Before + public void assumeJdk17() { + assumeJavaVersion(JAVA_1_7); + } + + + private SurefireLauncher unpack() { + return unpack("junit47-rerun-failing-tests-with-cucumber") + .setJUnitVersion("4.12"); + } + + @Test + public void testRerunFailingErrorTestsFalse() { + unpack() + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=" + 0) + .withFailure() + .executeTest() + .assertTestSuiteResults(1, 0, 1, 0, 0); + } + + @Test + public void testRerunFailingErrorTestsWithOneRetry() { + unpack() + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=" + 1) + .withFailure() + .executeTest() + .assertTestSuiteResults(1, 0, 1, 0, 0); + } + + @Test + public void testRerunFailingErrorTestsTwoRetry() { + unpack() + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=" + 2) + .executeTest() + .assertTestSuiteResults(1, 0, 0, 0, 2); + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml new file mode 100644 index 0000000..24ee294 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml @@ -0,0 +1,77 @@ + + + + + 4.0.0 + + + org.apache.maven.surefire + it-parent + 1.0 + + + junit47-rerun-failing-tests-with-cucumber + Test for rerun failing cucumber tests in JUnit 47 + + + 2.0.0 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + org.apache.maven.surefire + surefire-junit47 + ${surefire.version} + + + + + + + + + junit + junit + ${junit.version} + test + + + io.cucumber + cucumber-java + ${cucumber.version} + test + + + io.cucumber + cucumber-junit + ${cucumber.version} + test + + + + http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java new file mode 100644 index 0000000..ecda87b --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.sample.cucumber; + +import cucumber.api.junit.Cucumber; +import org.junit.runner.RunWith; + +@RunWith( Cucumber.class ) +public class FlakeCucumberTest +{ + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java new file mode 100644 index 0000000..d253222 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.sample.cucumber; + +import cucumber.api.java.en.Given; +import cucumber.api.java.en.Then; +import cucumber.api.java.en.When; + +import static org.junit.Assert.fail; + +public class StepDefs +{ + private static int testFailures = 0; + + @Given( "^I have some code$" ) + public void I_have_some_code() + throws Throwable + { + // do nothing + } + + @When( "^I run test$" ) + public void I_run_test() + throws Throwable + { + // do nothing + } + + @Then( "^I get a flake$" ) + public void I_get_a_flake() + throws Throwable + { + // This test will error out with only one retry, but will pass with two + if( testFailures < 2 ) { + testFailures++; + fail( "failing the test on purpose." ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature new file mode 100644 index 0000000..d5b5dac --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature @@ -0,0 +1,10 @@ +Feature: Sample + + In order to use Maven + As a user + I want to do tests. + + Scenario: Do a flake test + Given I have some code + When I run test + Then I get a flake http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java index 075d9d9..6b94a64 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -19,19 +19,14 @@ package org.apache.maven.surefire.common.junit4; * under the License. */ -import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import org.apache.maven.surefire.testset.TestSetFailedException; - import org.junit.runner.Description; +import org.junit.runner.manipulation.Filter; import org.junit.runner.notification.Failure; -import static org.apache.maven.surefire.common.junit4.JUnit4Reflector.createRequest; import static org.apache.maven.surefire.util.internal.StringUtils.isBlank; import static org.junit.runner.Description.TEST_MECHANISM; @@ -44,74 +39,30 @@ import static org.junit.runner.Description.TEST_MECHANISM; */ public final class JUnit4ProviderUtil { - private JUnit4ProviderUtil() { throw new IllegalStateException( "Cannot instantiate." ); } /** - * Organize all the failures in previous run into a map between test classes and corresponding failing test methods - * - * @param allFailures all the failures in previous run - * @param testClassLoader ClassLoader used for test classes - * @return a map between failing test classes and their corresponding failing test methods - */ - public static Map, Set> generateFailingTests( List allFailures, - ClassLoader testClassLoader ) - throws TestSetFailedException - { - Map, Set> testClassMethods = new HashMap, Set>(); - Set failingTests = generateFailingTests( allFailures ); - for ( ClassMethod classMethod: failingTests ) - { - try - { - Class testClassObj = Class.forName( classMethod.getClazz(), false, testClassLoader ); - Set failingMethods = testClassMethods.get( testClassObj ); - if ( failingMethods == null ) - { - failingMethods = new HashSet(); - testClassMethods.put( testClassObj, failingMethods ); - } - failingMethods.add( classMethod.getMethod() ); - } - catch ( ClassNotFoundException e ) - { - throw new TestSetFailedException( "Unable to create test class '" + classMethod.getClazz() + "'", e ); - } - } - return testClassMethods; - } - - /** - * Get all test methods from a list of Failures + * Get all descriptions from a list of Failures * * @param allFailures the list of failures for a given test class - * @return the list of test methods + * @return the list of descriptions */ - public static Set generateFailingTests( List allFailures ) + public static Set generateFailingTestDescriptions( List allFailures ) { - Set failingMethods = new HashSet(); + Set failingTestDescriptions = new HashSet(); for ( Failure failure : allFailures ) { Description description = failure.getDescription(); if ( description.isTest() && !isFailureInsideJUnitItself( description ) ) { - ClassMethod classMethod = cutTestClassAndMethod( description ); - if ( classMethod.isValid() ) - { - failingMethods.add( classMethod ); - } + failingTestDescriptions.add( description ); } } - return failingMethods; - } - - public static Description createSuiteDescription( Collection> classes ) - { - return createRequest( classes.toArray( new Class[classes.size()] ) ).getRunner().getDescription(); + return failingTestDescriptions; } public static boolean isFailureInsideJUnitItself( Description failure ) @@ -153,4 +104,8 @@ public final class JUnit4ProviderUtil return isBlank( s ) ? null : s; } + public static Filter createMatchAnyDescriptionFilter( Iterable descriptions ) + { + return new MatchDescriptions( descriptions ); + } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java new file mode 100644 index 0000000..3275f7f --- /dev/null +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java @@ -0,0 +1,104 @@ +package org.apache.maven.surefire.common.junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.runner.Description; +import org.junit.runner.manipulation.Filter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Only run test methods in the given failure set. + * + * @author mpkorstanje + */ +public final class MatchDescriptions + extends Filter +{ + private final List filters = new ArrayList(); + + public MatchDescriptions( Iterable descriptions ) + { + for ( Description description : descriptions ) + { + filters.add( matchDescription( description ) ); + } + } + + @Override + public boolean shouldRun( Description description ) + { + for ( Filter filter : filters ) + { + if ( filter.shouldRun( description ) ) + { + return true; + } + } + return false; + } + + @Override + public String describe() + { + StringBuilder description = new StringBuilder( "Matching description " ); + for ( int i = 0; i < filters.size(); i++ ) + { + description.append( filters.get( i ).describe() ); + if ( i != filters.size() - 1 ) + { + description.append( " OR " ); + } + } + return description.toString(); + } + + private static Filter matchDescription( final Description desiredDescription ) + { + return new Filter() + { + @Override + public boolean shouldRun( Description description ) + { + if ( description.isTest() ) + { + return desiredDescription.equals( description ); + } + + for ( Description each : description.getChildren() ) + { + if ( shouldRun( each ) ) + { + return true; + } + } + + return false; + } + + @Override + public String describe() + { + return String.format( "Method %s", desiredDescription.getDisplayName() ); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java index b05a562..7a2e6da 100644 --- a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java +++ b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java @@ -20,14 +20,11 @@ package org.apache.maven.surefire.common.junit4; */ import junit.framework.TestCase; -import org.apache.maven.surefire.testset.TestSetFailedException; import org.junit.runner.Description; import org.junit.runner.notification.Failure; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.*; @@ -38,9 +35,9 @@ import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.*; public class JUnit4ProviderUtilTest extends TestCase { - public void testGenerateFailingTests() throws TestSetFailedException + public void testGenerateFailingTestDescriptions() { - List failures = new ArrayList( ); + List failures = new ArrayList(); Description test1Description = Description.createTestDescription( T1.class, "testOne" ); Description test2Description = Description.createTestDescription( T1.class, "testTwo" ); @@ -54,21 +51,14 @@ public class JUnit4ProviderUtilTest failures.add( new Failure( test4Description, new AssertionError() ) ); failures.add( new Failure( test5Description, new RuntimeException() ) ); - Map, Set> result = generateFailingTests( failures, getClass().getClassLoader() ); + Set result = generateFailingTestDescriptions( failures ); - assertEquals( 2, result.size() ); - Set resultForT1 = result.get( T1.class ); - Set resultForT2 = result.get( T2.class ); + assertEquals( 4, result.size() ); - Set expectedResultForT1 = new HashSet(); - expectedResultForT1.add( "testOne" ); - expectedResultForT1.add( "testTwo" ); - Set expectedResultForT2 = new HashSet(); - expectedResultForT2.add( "testThree" ); - expectedResultForT2.add( "testFour" ); - - assertEquals( expectedResultForT1, resultForT1 ); - assertEquals( expectedResultForT2, resultForT2 ); + assertTrue( result.contains( test1Description) ); + assertTrue( result.contains( test2Description) ); + assertTrue( result.contains( test3Description) ); + assertTrue( result.contains( test4Description) ); } public void testIllegalTestDescription$NegativeTest() @@ -97,12 +87,12 @@ public class JUnit4ProviderUtilTest assertEquals( T1.class.getName(), classMethod.getClazz() ); } - class T1 + private static class T1 { } - class T2 + private static class T2 { } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java deleted file mode 100644 index e69e7d0..0000000 --- a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.apache.maven.surefire.common.junit48; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.maven.shared.utils.io.SelectorUtils; -import org.junit.runner.Description; -import org.junit.runner.manipulation.Filter; - -import java.util.Map; -import java.util.Set; - -/** - * Only run test methods in the given input map, indexed by test class - */ -final class FailingMethodFilter - extends Filter -{ - // Map from Class -> List of method names. Are the method names hashed to include the signature? - private final Map, Set> failingClassMethodPatterns; - - public FailingMethodFilter( Map, Set> failingClassMethodPatterns ) - { - this.failingClassMethodPatterns = failingClassMethodPatterns; - } - - @Override - public boolean shouldRun( Description description ) - { - return isDescriptionMatch( description ); - } - - private boolean isDescriptionMatch( Description description ) - { - if ( description.getTestClass() == null || description.getMethodName() == null ) - { - for ( Description childrenDescription : description.getChildren() ) - { - if ( isDescriptionMatch( childrenDescription ) ) - { - return true; - } - } - return false; - } - - Set testMethodPatterns = failingClassMethodPatterns.get( description.getTestClass() ); - String testMethod = description.getMethodName(); - return testMethodPatterns != null && matchMethod( testMethodPatterns, testMethod ); - } - - private static boolean matchMethod( Set patterns, String methodName ) - { - for ( String pattern : patterns ) - { - if ( SelectorUtils.match( pattern, methodName ) ) - { - return true; - } - } - return false; - } - - @Override - public String describe() - { - return "By failing class method"; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java index 228ca6d..6ebff47 100644 --- a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java +++ b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java @@ -19,14 +19,15 @@ package org.apache.maven.surefire.common.junit48; * under the License. */ +import org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil; import org.apache.maven.surefire.group.match.GroupMatcher; import org.apache.maven.surefire.group.parse.GroupMatcherParser; import org.apache.maven.surefire.group.parse.ParseException; import org.apache.maven.surefire.testset.TestListResolver; +import org.junit.runner.Description; import org.junit.runner.manipulation.Filter; import java.util.Map; -import java.util.Set; import static org.apache.maven.surefire.booter.ProviderParameterNames.TESTNG_EXCLUDEDGROUPS_PROP; import static org.apache.maven.surefire.booter.ProviderParameterNames.TESTNG_GROUPS_PROP; @@ -118,9 +119,9 @@ public class FilterFactory return new MethodFilter( resolver ); } - public Filter createFailingMethodFilter( Map, Set> failingClassMethodMap ) + public Filter createMatchAnyDescriptionFilter( Iterable descriptions ) { - return new FailingMethodFilter( failingClassMethodMap ); + return JUnit4ProviderUtil.createMatchAnyDescriptionFilter( descriptions ); } public Filter and( Filter filter1, Filter filter2 ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index ad29224..c31a019 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -22,7 +22,6 @@ package org.apache.maven.surefire.junit4; import org.apache.maven.surefire.booter.Command; import org.apache.maven.surefire.booter.CommandListener; import org.apache.maven.surefire.booter.CommandReader; -import org.apache.maven.surefire.common.junit4.ClassMethod; import org.apache.maven.surefire.common.junit4.JUnit4RunListener; import org.apache.maven.surefire.common.junit4.JUnit4TestChecker; import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener; @@ -46,7 +45,6 @@ import org.junit.runner.Request; import org.junit.runner.Result; import org.junit.runner.Runner; import org.junit.runner.manipulation.Filter; -import org.junit.runner.notification.RunNotifier; import org.junit.runner.notification.StoppedByUserException; import java.util.Collection; @@ -55,7 +53,9 @@ import java.util.Set; import static java.lang.reflect.Modifier.isAbstract; import static java.lang.reflect.Modifier.isInterface; import static org.apache.maven.surefire.booter.CommandReader.getReader; -import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.createMatchAnyDescriptionFilter; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTestDescriptions; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.isFailureInsideJUnitItself; import static org.apache.maven.surefire.common.junit4.JUnit4Reflector.createDescription; import static org.apache.maven.surefire.common.junit4.JUnit4Reflector.createIgnored; import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.rethrowAnyTestMechanismFailures; @@ -67,7 +67,6 @@ import static org.apache.maven.surefire.testset.TestListResolver.optionallyWildc import static org.apache.maven.surefire.util.TestsToRun.fromClass; import static org.apache.maven.surefire.util.internal.ObjectUtils.systemProps; import static org.junit.runner.Request.aClass; -import static org.junit.runner.Request.method; /** * @author Kristian Rosenvold @@ -261,7 +260,6 @@ public class JUnit4Provider } private void executeWithRerun( Class clazz, Notifier notifier ) - throws TestSetFailedException { JUnitTestFailureListener failureListener = new JUnitTestFailureListener(); notifier.addListener( failureListener ); @@ -286,12 +284,10 @@ public class JUnit4Provider notifier.copyListenersTo( rerunNotifier ); for ( int i = 0; i < rerunFailingTestsCount && !failureListener.getAllFailures().isEmpty(); i++ ) { - Set failedTests = generateFailingTests( failureListener.getAllFailures() ); + Set failures = generateFailingTestDescriptions( failureListener.getAllFailures() ); failureListener.reset(); - if ( !failedTests.isEmpty() ) - { - executeFailedMethod( rerunNotifier, failedTests ); - } + Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures ); + execute( clazz, rerunNotifier, failureDescriptionFilter ); } } } @@ -371,24 +367,6 @@ public class JUnit4Provider } } - private void executeFailedMethod( RunNotifier notifier, Set failedMethods ) - throws TestSetFailedException - { - for ( ClassMethod failedMethod : failedMethods ) - { - try - { - Class methodClass = Class.forName( failedMethod.getClazz(), true, testClassLoader ); - String methodName = failedMethod.getMethod(); - method( methodClass, methodName ).getRunner().run( notifier ); - } - catch ( ClassNotFoundException e ) - { - throw new TestSetFailedException( "Unable to create test class '" + failedMethod.getClazz() + "'", e ); - } - } - } - /** * JUnit error: test count includes one test-class as a suite which has filtered out all children. * Then the child test has a description "initializationError0(org.junit.runner.manipulation.Filter)" @@ -422,6 +400,10 @@ public class JUnit4Provider private static boolean hasFilteredOutAllChildren( Description description ) { + if ( isFailureInsideJUnitItself( description ) ) + { + return true; + } String name = description.getDisplayName(); // JUnit 4.0: initializationError0; JUnit 4.12: initializationError. if ( name == null ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index a4c061e..a652515 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -39,16 +39,15 @@ import org.apache.maven.surefire.util.RunOrderCalculator; import org.apache.maven.surefire.util.ScanResult; import org.apache.maven.surefire.util.ScannerFilter; import org.apache.maven.surefire.util.TestsToRun; +import org.junit.runner.Description; import org.junit.runner.manipulation.Filter; -import org.junit.runner.notification.Failure; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import static org.apache.maven.surefire.booter.CommandReader.getReader; -import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTestDescriptions; import static org.apache.maven.surefire.common.junit4.JUnit4RunListenerFactory.createCustomListeners; import static org.apache.maven.surefire.common.junit4.Notifier.pureNotifier; import static org.apache.maven.surefire.junitcore.ConcurrentRunListener.createInstance; @@ -167,12 +166,11 @@ public class JUnitCoreProvider JUnitCoreWrapper rerunCore = new JUnitCoreWrapper( rerunNotifier, jUnitCoreParameters, consoleStream ); for ( int i = 0; i < rerunFailingTestsCount && !testFailureListener.getAllFailures().isEmpty(); i++ ) { - List failures = testFailureListener.getAllFailures(); - Map, Set> failingTests = generateFailingTests( failures, testClassLoader ); + Set failures = generateFailingTestDescriptions( testFailureListener.getAllFailures() ); testFailureListener.reset(); FilterFactory filterFactory = new FilterFactory( testClassLoader ); - Filter failingMethodsFilter = filterFactory.createFailingMethodFilter( failingTests ); - rerunCore.execute( testsToRun, failingMethodsFilter ); + Filter failureDescriptionFilter = filterFactory.createMatchAnyDescriptionFilter( failures ); + rerunCore.execute( testsToRun, failureDescriptionFilter ); } } }