maven-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tibordig...@apache.org
Subject [18/51] [partial] maven-surefire git commit: [SUREFIRE-1471] Too long Windows path cause CI issues. Renamed surefire-intergation-tests to surefire-its.
Date Mon, 05 Feb 2018 22:07:31 GMT
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/TimeoutForkedTestIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/TimeoutForkedTestIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/TimeoutForkedTestIT.java
new file mode 100644
index 0000000..fdcea20
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/TimeoutForkedTestIT.java
@@ -0,0 +1,43 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Test
+ *
+ * @author <a href="mailto:dfabulich@apache.org">Dan Fabulich</a>
+ */
+public class TimeoutForkedTestIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testTimeoutForked()
+        throws Exception
+    {
+        unpack( "/timeout-forked" ).addGoal( "-DsleepLength=10000" ).addGoal(
+            "-DforkTimeout=1" ).maven().withFailure().executeTest();
+        // SUREFIRE-468 test that had to be reverted due to SUREFIRE-705
+        //assertFalse( getSurefireReportsFile( "TEST-timeoutForked.BasicTest.xml" ).exists() );
+        // assertFalse( getSurefireReportsFile( "timeoutForked.BasicTest.txt" ).exists() );
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/TwoTestCasesIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/TwoTestCasesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/TwoTestCasesIT.java
new file mode 100644
index 0000000..c218769
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/TwoTestCasesIT.java
@@ -0,0 +1,130 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.maven.plugins.surefire.report.ReportTestSuite;
+import org.apache.maven.surefire.its.fixture.*;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test running two test cases; confirms reporting works correctly
+ *
+ * @author <a href="mailto:dfabulich@apache.org">Dan Fabulich</a>
+ */
+public class TwoTestCasesIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testTwoTestCases()
+        throws Exception
+    {
+        unpack( "junit-twoTestCases" )
+                .sysProp( "testNgVersion", "5.7" )
+                .sysProp( "testNgClassifier", "jdk15" )
+                .executeTest()
+                .verifyErrorFreeLog()
+                .assertTestSuiteResults( 2, 0, 0, 0 );
+    }
+
+    /**
+     * Runs two tests encapsulated in a suite
+     */
+    @Test
+    public void testTwoTestCaseSuite()
+        throws Exception
+    {
+        final OutputValidator outputValidator = unpack( "junit-twoTestCaseSuite" )
+                                                        .sysProp( "testNgVersion", "5.7" )
+                                                        .sysProp( "testNgClassifier", "jdk15" )
+                                                        .executeTest();
+        outputValidator.verifyErrorFreeLog().assertTestSuiteResults( 2, 0, 0, 0 );
+        List<ReportTestSuite> reports = HelperAssertions.extractReports( outputValidator.getBaseDir() );
+        Set<String> classNames = extractClassNames( reports );
+        assertContains( classNames, "junit.twoTestCaseSuite.BasicTest" );
+        assertContains( classNames, "junit.twoTestCaseSuite.TestTwo" );
+        assertEquals( "wrong number of classes", 2, classNames.size() );
+        IntegrationTestSuiteResults results = HelperAssertions.parseReportList( reports );
+        HelperAssertions.assertTestSuiteResults( 2, 0, 0, 0, results );
+    }
+
+    private void assertContains( Set<String> set, String expected )
+    {
+        if ( set.contains( expected ) )
+        {
+            return;
+        }
+        fail( "Set didn't contain " + expected );
+    }
+
+    private Set<String> extractClassNames( List<ReportTestSuite> reports )
+    {
+        HashSet<String> classNames = new HashSet<String>();
+        for ( ReportTestSuite suite : reports )
+        {
+            classNames.add( suite.getFullClassName() );
+        }
+        return classNames;
+    }
+
+    @Test
+    public void testJunit4Suite()
+        throws Exception
+    {
+        final OutputValidator outputValidator = unpack( "junit4-twoTestCaseSuite" )
+                                                        .sysProp( "testNgVersion", "5.7" )
+                                                        .sysProp( "testNgClassifier", "jdk15" )
+                                                        .executeTest();
+        outputValidator.verifyErrorFreeLog().assertTestSuiteResults( 2, 0, 0, 0 );
+
+        List<ReportTestSuite> reports =
+            HelperAssertions.extractReports( outputValidator.getBaseDir() );
+        Set<String> classNames = extractClassNames( reports );
+        assertContains( classNames, "twoTestCaseSuite.BasicTest" );
+        assertContains( classNames, "twoTestCaseSuite.Junit4TestTwo" );
+        assertEquals( "wrong number of classes", 2, classNames.size() );
+        IntegrationTestSuiteResults results = HelperAssertions.parseReportList( reports );
+        HelperAssertions.assertTestSuiteResults( 2, 0, 0, 0, results );
+    }
+
+    @Test
+    public void testTestNGSuite()
+        throws Exception
+    {
+        final OutputValidator outputValidator = unpack( "testng-twoTestCaseSuite" )
+                                                        .sysProp( "testNgVersion", "5.7" )
+                                                        .sysProp( "testNgClassifier", "jdk15" )
+                                                        .executeTest();
+        outputValidator.verifyErrorFreeLog().assertTestSuiteResults( 2, 0, 0, 0 );
+        List<ReportTestSuite> reports = HelperAssertions.extractReports( outputValidator.getBaseDir() );
+        Set<String> classNames = extractClassNames( reports );
+        assertContains( classNames, "testng.two.TestNGTestTwo" );
+        assertContains( classNames, "testng.two.TestNGSuiteTest" );
+        assertEquals( "wrong number of classes", 2, classNames.size() );
+        IntegrationTestSuiteResults results = HelperAssertions.parseReportList( reports );
+        HelperAssertions.assertTestSuiteResults( 2, 0, 0, 0, results );
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/UmlautDirIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/UmlautDirIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/UmlautDirIT.java
new file mode 100644
index 0000000..8848dc2
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/UmlautDirIT.java
@@ -0,0 +1,64 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.surefire.its.fixture.MavenLauncher;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Test a directory with an umlaut
+ *
+ * @author <a href="mailto:dfabulich@apache.org">Dan Fabulich</a>
+ */
+public class UmlautDirIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testUmlaut()
+        throws Exception
+    {
+        specialUnpack().executeTest().verifyErrorFreeLog().assertTestSuiteResults( 1, 0, 0, 0 );
+    }
+
+    @Test
+    public void testUmlautIsolatedClassLoader()
+        throws Exception
+    {
+        specialUnpack().useSystemClassLoader( false ).executeTest().assertTestSuiteResults( 1, 0, 0, 0 );
+    }
+
+    SurefireLauncher specialUnpack()
+        throws VerificationException, IOException
+    {
+        SurefireLauncher unpack = unpack( "junit-pathWithUmlaut" );
+        MavenLauncher maven = unpack.maven();
+
+        File dest = new File( maven.getUnpackedAt().getParentFile().getPath(), "/junit-pathWith\u00DCmlaut" );
+        maven.moveUnpackTo( dest );
+        return unpack;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/UnicodeTestNamesIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/UnicodeTestNamesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/UnicodeTestNamesIT.java
new file mode 100644
index 0000000..64db960
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/UnicodeTestNamesIT.java
@@ -0,0 +1,75 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the LicenseUni.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.apache.maven.surefire.its.fixture.TestFile;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeFalse;
+
+/**
+ * Verifies unicode filenames pass through correctly.
+ */
+public class UnicodeTestNamesIT
+        extends SurefireJUnit4IntegrationTestCase
+{
+    private static final String TXT_REPORT = "junit.twoTestCases.\u800C\u7D22\u5176\u60C5Test.txt";
+    private static final String XML_REPORT = "TEST-junit.twoTestCases.\u800C\u7D22\u5176\u60C5Test.xml";
+
+    @Test
+    public void checkFileNamesWithUnicode()
+    {
+        SurefireLauncher unpacked = unpack( "unicode-testnames" );
+        File basedir = unpacked.getUnpackedAt();
+
+        unpacked.execute( "clean" );
+
+        File xxyz = new File( basedir, "src/test/java/junit/twoTestCases/XXYZTest.java" );
+        File dest = new File( basedir, "src/test/java/junit/twoTestCases/\u800C\u7D22\u5176\u60C5Test.java" );
+
+        //noinspection ResultOfMethodCallIgnored
+        dest.delete();
+        assertTrue( xxyz.renameTo( dest ) );
+
+        assertTrue( dest.exists() );
+        assumeFalse( new File( basedir, "src/test/java/junit/twoTestCases/????Test.java" ).exists() );
+
+        OutputValidator outputValidator =
+                unpacked.executeTest()
+                        .assertTestSuiteResults( 2, 0, 0, 0 );
+
+        TestFile surefireReportFile = outputValidator.getSurefireReportsFile( TXT_REPORT );
+        assertTrue( surefireReportFile.exists() );
+        surefireReportFile.assertContainsText( "junit.twoTestCases.????Test" );
+
+        TestFile surefireXmlReportFile = outputValidator.getSurefireReportsXmlFile( XML_REPORT );
+        assertTrue( surefireXmlReportFile.exists() );
+        assertFalse( surefireXmlReportFile.readFileToString().isEmpty() );
+        surefireXmlReportFile.assertContainsText( "junit.twoTestCases.\u800C\u7D22\u5176\u60C5Test" );
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/UseIsolatedClassLoaderIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/UseIsolatedClassLoaderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/UseIsolatedClassLoaderIT.java
new file mode 100644
index 0000000..b87f603
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/UseIsolatedClassLoaderIT.java
@@ -0,0 +1,38 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Test useSystemClassLoader option
+ *
+ * @author <a href="mailto:dfabulich@apache.org">Dan Fabulich</a>
+ */
+public class UseIsolatedClassLoaderIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testUseSystemClassLoader()
+    {
+        executeErrorFreeTest( "/isolated-classloader", 1 );
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIT.java
new file mode 100644
index 0000000..66fa3df
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIT.java
@@ -0,0 +1,134 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.surefire.its.fixture.*;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test working directory configuration, SUREFIRE-416
+ *
+ * @author <a href="mailto:dfabulich@apache.org">Dan Fabulich</a>
+ * @author <a href="mailto:krosenvold@apache.org">Kristian Rosenvold</a>
+ */
+public class WorkingDirectoryIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+
+    @Test
+    public void testWorkingDirectory()
+        throws Exception
+    {
+        final SurefireLauncher unpack = getUnpacked();
+        final OutputValidator child = getPreparedChild( unpack );
+        unpack.executeTest().verifyErrorFreeLog();
+        child.assertTestSuiteResults( 1, 0, 0, 0 );
+        verifyOutputDirectory( child );
+    }
+
+    @Test
+    public void testWorkingDirectoryNoFork()
+        throws Exception
+    {
+        final SurefireLauncher unpack = getUnpacked();
+        final OutputValidator child = getPreparedChild( unpack );
+        unpack.forkNever().executeTest().verifyErrorFreeLog();
+        child.assertTestSuiteResults( 1, 0, 0, 0 );
+        verifyOutputDirectory( child );
+    }
+
+    @Test
+    public void testWorkingDirectoryChildOnly()
+        throws Exception
+    {
+        final SurefireLauncher unpack = getUnpacked();
+        final SurefireLauncher child = unpack.getSubProjectLauncher( "child" );
+        //child.getTargetFile( "out.txt" ).delete();
+        final OutputValidator outputValidator = child.executeTest().assertTestSuiteResults( 1, 0, 0, 0 );
+        verifyOutputDirectory( outputValidator );
+    }
+
+    @Test
+    public void testWorkingDirectoryChildOnlyNoFork()
+        throws Exception
+    {
+
+        final SurefireLauncher unpack = getUnpacked();
+        final SurefireLauncher child = unpack.getSubProjectLauncher( "child" );
+        //child.getTargetFile( "out.txt" ).delete();
+        final OutputValidator outputValidator = child.forkNever().executeTest().assertTestSuiteResults( 1, 0, 0, 0 );
+        verifyOutputDirectory( outputValidator );
+    }
+
+    private SurefireLauncher getUnpacked()
+        throws VerificationException, IOException
+    {
+        return unpack( "working-directory" );
+    }
+
+    private OutputValidator getPreparedChild( SurefireLauncher unpack )
+        throws VerificationException
+    {
+        final OutputValidator child = unpack.getSubProjectValidator( "child" );
+        getOutFile( child ).delete();
+        return child;
+    }
+
+
+    private TestFile getOutFile( OutputValidator child )
+    {
+        return child.getTargetFile( "out.txt" );
+    }
+
+    public void verifyOutputDirectory( OutputValidator childTestDir )
+        throws IOException
+    {
+        final TestFile outFile = getOutFile( childTestDir );
+        assertTrue( "out.txt doesn't exist: " + outFile.getAbsolutePath(), outFile.exists() );
+        Properties p = new Properties();
+        FileInputStream is = outFile.getFileInputStream();
+        p.load( is );
+        is.close();
+        String userDirPath = p.getProperty( "user.dir" );
+        assertNotNull( "user.dir was null in property file", userDirPath );
+        File userDir = new File( userDirPath );
+        // test if not a symlink
+        if ( childTestDir.getBaseDir().getCanonicalFile().equals( childTestDir.getBaseDir().getAbsoluteFile() ) )
+        {
+            assertTrue( "wrong user.dir ! symlink ",
+                        childTestDir.getBaseDir().getAbsolutePath().equalsIgnoreCase( userDir.getAbsolutePath() ) );
+        }
+        else
+        {
+            assertEquals( "wrong user.dir symlink ", childTestDir.getBaseDir().getCanonicalPath(),
+                          userDir.getCanonicalPath() );
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIsInvalidPropertyIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIsInvalidPropertyIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIsInvalidPropertyIT.java
new file mode 100644
index 0000000..1415b6e
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryIsInvalidPropertyIT.java
@@ -0,0 +1,39 @@
+package org.apache.maven.surefire.its;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Test when the configured working directory is an invalid property, SUREFIRE-715
+ *
+ * @author <a href="mailto:krosenvold@apache.org">Kristian Rosenvold</a>
+ */
+public class WorkingDirectoryIsInvalidPropertyIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testWorkingDirectory()
+        throws Exception
+    {
+        unpack( "working-directory-is-invalid-property" ).maven().withFailure().executeTest().verifyTextInLog(
+            "workingDirectory cannot be null" );
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryMissingIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryMissingIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryMissingIT.java
new file mode 100644
index 0000000..5247160
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/WorkingDirectoryMissingIT.java
@@ -0,0 +1,39 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Test when the configured working directory does not exist, SUREFIRE-607
+ *
+ * @author <a href="mailto:stephenc@apache.org">Stephen Connolly</a>
+ */
+public class WorkingDirectoryMissingIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testWorkingDirectory()
+    {
+        unpack( "working-directory-missing" ).executeTest().verifyErrorFreeLog();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java
new file mode 100644
index 0000000..e8ee58b
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/XmlReporterRunTimeIT.java
@@ -0,0 +1,75 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.plugins.surefire.report.ReportTestSuite;
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+import static org.apache.maven.surefire.its.fixture.HelperAssertions.extractReports;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+
+/**
+ * Test reported runtime
+ *
+ * @author Kristian Rosenvold
+ */
+public class XmlReporterRunTimeIT
+    extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testForkModeAlways()
+        throws Exception
+    {
+        // just generate .surefire-<hash> in order to apply runOrder
+        unpack( "/runorder-parallel" )
+            .executeTest()
+            .verifyErrorFree( 9 );
+
+        // now assert test results match expected values
+        OutputValidator outputValidator = unpack( "/runorder-parallel" )
+            .executeTest()
+            .verifyErrorFree( 9 );
+
+        for ( ReportTestSuite report : extractReports( outputValidator.getBaseDir() ) )
+        {
+            if ( "runorder.parallel.Test1".equals( report.getFullClassName() ) )
+            {
+                // should be 6f but because of having Windows sleep discrepancy it is 5.95f
+                assertThat( "runorder.parallel.Test1 report.getTimeElapsed found:" + report.getTimeElapsed(),
+                            report.getTimeElapsed(), allOf( greaterThanOrEqualTo( 5.95f ), lessThan( 7f ) ) );
+            }
+            else if ( "runorder.parallel.Test2".equals( report.getFullClassName() ) )
+            {
+                // should be 5f but because of having Windows sleep discrepancy it is 4.95f
+                assertThat( "runorder.parallel.Test2 report.getTimeElapsed found:" + report.getTimeElapsed(),
+                            report.getTimeElapsed(), allOf( greaterThanOrEqualTo( 4.95f ), lessThan( 6f ) ) );
+            }
+            else
+            {
+                System.out.println( "report = " + report );
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Configuration.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Configuration.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Configuration.java
new file mode 100644
index 0000000..dcd8184
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Configuration.java
@@ -0,0 +1,29 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @author <a href="mailto:tibordigana@apache.org">Tibor Digana (tibor17)</a>
+ * @since 2.19
+ */
+public enum Configuration
+{
+    TEST, INCLUDES, INCLUDES_FILE, INCLUDES_EXCLUDES, INCLUDES_EXCLUDES_FILE
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/FailsafeOutputValidator.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/FailsafeOutputValidator.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/FailsafeOutputValidator.java
new file mode 100644
index 0000000..b96cf9b
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/FailsafeOutputValidator.java
@@ -0,0 +1,47 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.it.VerificationException;
+
+public class FailsafeOutputValidator
+    extends OutputValidator
+{
+    public FailsafeOutputValidator( OutputValidator source )
+    {
+        super( source.verifier );
+    }
+
+    @Override
+    public OutputValidator verifyErrorFree( int total )
+    {
+        try
+        {
+            verifier.verifyErrorFreeLog();
+            this.assertIntegrationTestSuiteResults( total, 0, 0, 0 );
+            return this;
+        }
+        catch ( VerificationException e )
+        {
+            throw new SurefireVerifierException( e );
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
new file mode 100644
index 0000000..ed0d40c
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/HelperAssertions.java
@@ -0,0 +1,174 @@
+package org.apache.maven.surefire.its.fixture;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
+import org.apache.maven.plugin.surefire.log.api.PrintStreamLogger;
+import org.apache.maven.plugins.surefire.report.ReportTestSuite;
+import org.apache.maven.plugins.surefire.report.SurefireReportParser;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+@SuppressWarnings( { "JavaDoc" } )
+public class HelperAssertions
+{
+    /**
+     * assert that the reports in the specified testDir have the right summary statistics
+     */
+    public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, File testDir )
+    {
+        IntegrationTestSuiteResults suite = parseTestResults( testDir );
+        assertTestSuiteResults( total, errors, failures, skipped, suite );
+    }
+
+    public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes, File testDir )
+    {
+        IntegrationTestSuiteResults suite = parseTestResults( testDir );
+        assertTestSuiteResults( total, errors, failures, skipped, flakes, suite );
+    }
+
+    public static void assertTestSuiteResults( int total, File testDir )
+    {
+        IntegrationTestSuiteResults suite = parseTestResults( testDir );
+        assertTestSuiteResults( total, suite );
+    }
+
+    /**
+     * assert that the reports in the specified testDir have the right summary statistics
+     */
+    public static void assertIntegrationTestSuiteResults( int total, int errors, int failures, int skipped,
+                                                          File testDir )
+    {
+        IntegrationTestSuiteResults suite = parseIntegrationTestResults( testDir );
+        assertTestSuiteResults( total, errors, failures, skipped, suite );
+    }
+
+    public static void assertIntegrationTestSuiteResults( int total, File testDir )
+    {
+        IntegrationTestSuiteResults suite = parseIntegrationTestResults( testDir );
+        assertTestSuiteResults( total, suite );
+    }
+
+    public static void assertTestSuiteResults( int total, int errors, int failures, int skipped,
+                                               IntegrationTestSuiteResults actualSuite )
+    {
+        assertEquals( "wrong number of tests", total, actualSuite.getTotal() );
+        assertEquals( "wrong number of errors", errors, actualSuite.getErrors() );
+        assertEquals( "wrong number of failures", failures, actualSuite.getFailures() );
+        assertEquals( "wrong number of skipped", skipped, actualSuite.getSkipped() );
+    }
+
+    public static void assertTestSuiteResults( int total, IntegrationTestSuiteResults actualSuite )
+    {
+        assertEquals( "wrong number of tests", total, actualSuite.getTotal() );
+    }
+
+    public static void assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes,
+                                               IntegrationTestSuiteResults actualSuite )
+    {
+        assertTestSuiteResults(total, errors, failures, skipped, actualSuite);
+        assertEquals( "wrong number of flaky tests", flakes, actualSuite.getFlakes() );
+    }
+
+    public static IntegrationTestSuiteResults parseTestResults( File... testDirs )
+    {
+        List<ReportTestSuite> reports = extractReports( testDirs );
+        return parseReportList( reports );
+    }
+
+    public static IntegrationTestSuiteResults parseIntegrationTestResults( File... testDirs )
+    {
+        List<ReportTestSuite> reports = extractITReports( testDirs );
+        return parseReportList( reports );
+    }
+
+    /**
+     * Converts a list of ReportTestSuites into an IntegrationTestSuiteResults object, suitable for summary assertions
+     */
+    public static IntegrationTestSuiteResults parseReportList( List<ReportTestSuite> reports )
+    {
+        assertTrue( "No reports!", !reports.isEmpty() );
+        int total = 0, errors = 0, failures = 0, skipped = 0, flakes = 0;
+        for ( ReportTestSuite report : reports )
+        {
+            total += report.getNumberOfTests();
+            errors += report.getNumberOfErrors();
+            failures += report.getNumberOfFailures();
+            skipped += report.getNumberOfSkipped();
+            flakes += report.getNumberOfFlakes();
+        }
+        return new IntegrationTestSuiteResults( total, errors, failures, skipped, flakes );
+    }
+
+    public static List<ReportTestSuite> extractReports( File... testDirs )
+    {
+        List<File> reportsDirs = new ArrayList<File>();
+        for ( File testDir : testDirs )
+        {
+            File reportsDir = new File( testDir, "target/surefire-reports" );
+            assertTrue( "Reports directory is missing: " + reportsDir.getAbsolutePath(), reportsDir.exists() );
+            reportsDirs.add( reportsDir );
+        }
+        ConsoleLogger logger = new PrintStreamLogger( System.out );
+        SurefireReportParser parser = new SurefireReportParser( reportsDirs, Locale.getDefault(), logger );
+        try
+        {
+            return parser.parseXMLReportFiles();
+        }
+        catch ( Exception e )
+        {
+            throw new RuntimeException( "Couldn't parse XML reports", e );
+        }
+    }
+
+    public static List<ReportTestSuite> extractITReports( File... testDirs )
+    {
+        List<File> reportsDirs = new ArrayList<File>();
+        for ( File testDir : testDirs )
+        {
+            File reportsDir = new File( testDir, "target/failsafe-reports" );
+            assertTrue( "Reports directory is missing: " + reportsDir.getAbsolutePath(), reportsDir.exists() );
+            reportsDirs.add( reportsDir );
+        }
+        ConsoleLogger logger = new PrintStreamLogger( System.out );
+        SurefireReportParser parser = new SurefireReportParser( reportsDirs, Locale.getDefault(), logger );
+        try
+        {
+            return parser.parseXMLReportFiles();
+        }
+        catch ( Exception e )
+        {
+            throw new RuntimeException( "Couldn't parse XML reports", e );
+        }
+    }
+
+    public static void assumeJavaVersion( double expectedVersion )
+    {
+        String thisVersion = System.getProperty( "java.specification.version" );
+        assumeTrue( "java.specification.version: " + thisVersion,
+                Double.valueOf( thisVersion ) >= expectedVersion );
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
new file mode 100644
index 0000000..f147281
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/IntegrationTestSuiteResults.java
@@ -0,0 +1,91 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+public class IntegrationTestSuiteResults
+{
+    private int total, errors, failures, skipped, flakes;
+
+    public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped )
+    {
+        this.total = total;
+        this.errors = errors;
+        this.failures = failures;
+        this.skipped = skipped;
+    }
+
+    public IntegrationTestSuiteResults( int total, int errors, int failures, int skipped, int flakes )
+    {
+        this(total, errors, failures, skipped);
+        this.flakes = flakes;
+    }
+
+    public int getTotal()
+    {
+        return total;
+    }
+
+    public void setTotal( int total )
+    {
+        this.total = total;
+    }
+
+    public int getErrors()
+    {
+        return errors;
+    }
+
+    public void setErrors( int errors )
+    {
+        this.errors = errors;
+    }
+
+    public int getFailures()
+    {
+        return failures;
+    }
+
+    public void setFailures( int failures )
+    {
+        this.failures = failures;
+    }
+
+    public int getSkipped()
+    {
+        return skipped;
+    }
+
+    public void setSkipped( int skipped )
+    {
+        this.skipped = skipped;
+    }
+
+    public int getFlakes()
+    {
+        return flakes;
+    }
+
+    public void setFlakes( int flakes )
+    {
+        this.flakes = flakes;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncher.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncher.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncher.java
new file mode 100755
index 0000000..bd12dda
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncher.java
@@ -0,0 +1,504 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.commons.lang.text.StrSubstitutor;
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.it.Verifier;
+import org.apache.maven.it.util.ResourceExtractor;
+import org.apache.maven.shared.utils.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+
+import static java.util.Collections.unmodifiableList;
+
+/**
+ * Encapsulate all needed features to start a maven run
+ * <br>
+ *
+ * @author Kristian Rosenvold
+ */
+public class MavenLauncher
+{
+    private final List<String> cliOptions = new ArrayList<String>();
+
+    private final List<String> goals = new ArrayList<String>();
+
+    private final Map<String, String> envvars = new HashMap<String, String>();
+
+    private File unpackedAt;
+
+    private Verifier verifier;
+
+    private OutputValidator validator;
+
+    private final Class testCaseBeingRun;
+
+    private final String resourceName;
+
+    private final String suffix;
+
+    private final String[] cli;
+
+    private boolean expectFailure;
+
+    public MavenLauncher( Class testClass, String resourceName, String suffix, String[] cli )
+    {
+        this.testCaseBeingRun = testClass;
+        this.resourceName = resourceName;
+        this.suffix = suffix != null ? suffix : "";
+        this.cli = cli == null ? null : cli.clone();
+        resetGoals();
+        resetCliOptions();
+    }
+
+    public MavenLauncher( Class testClass, String resourceName, String suffix )
+    {
+        this( testClass, resourceName, suffix, null );
+    }
+
+    public File getUnpackedAt()
+    {
+        return ensureUnpacked();
+    }
+
+    private File ensureUnpacked()
+    {
+        if ( unpackedAt == null )
+        {
+            unpackedAt = simpleExtractResources( testCaseBeingRun, resourceName );
+        }
+        return unpackedAt;
+    }
+
+    public void moveUnpackTo( File dest )
+        throws IOException
+    {
+        FileUtils.deleteDirectory( dest );
+        //noinspection ResultOfMethodCallIgnored
+        getUnpackedAt().renameTo( dest );
+        unpackedAt = dest;
+    }
+
+    public void resetGoals()
+    {
+        goals.clear();
+    }
+
+    void addCliOption( String cliOption )
+    {
+        cliOptions.add( cliOption );
+    }
+
+
+
+    private StackTraceElement findTopElemenent( StackTraceElement[] stackTrace, Class testClassToLookFor )
+    {
+        StackTraceElement bestmatch = null;
+        for ( StackTraceElement stackTraceElement : stackTrace )
+        {
+            if ( stackTraceElement.getClassName().equals( testClassToLookFor.getName() ) )
+            {
+                bestmatch = stackTraceElement;
+            }
+        }
+        return bestmatch;
+    }
+
+    StackTraceElement[] getStackTraceElements()
+    {
+        try
+        {
+            throw new RuntimeException();
+        }
+        catch ( RuntimeException e )
+        {
+            return e.getStackTrace();
+        }
+    }
+
+    public void reset()
+    {
+        resetGoals();
+        resetCliOptions();
+    }
+
+    private void resetCliOptions()
+    {
+        cliOptions.clear();
+    }
+
+    public MavenLauncher getSubProjectLauncher( String subProject )
+        throws VerificationException
+    {
+        MavenLauncher mavenLauncher =
+            new MavenLauncher( testCaseBeingRun, resourceName + File.separator + subProject, suffix, cli );
+        mavenLauncher.unpackedAt = new File( ensureUnpacked(), subProject );
+        return mavenLauncher;
+    }
+
+    public OutputValidator getSubProjectValidator( String subProject )
+        throws VerificationException
+    {
+        final File subFile = getValidator().getSubFile( subProject );
+        return new OutputValidator( new Verifier( subFile.getAbsolutePath() ) );
+    }
+
+
+    public MavenLauncher addEnvVar( String key, String value )
+    {
+        envvars.put( key, value );
+        return this;
+    }
+
+    public MavenLauncher assertNotPresent( String subFile )
+    {
+        getVerifier().assertFileNotPresent( getValidator().getSubFile( subFile ).getAbsolutePath() );
+        return this;
+    }
+
+    public MavenLauncher showErrorStackTraces()
+    {
+        addCliOption( "-e" );
+        return this;
+    }
+
+    public MavenLauncher debugLogging()
+    {
+        addCliOption( "-X" );
+        return this;
+    }
+
+    public MavenLauncher failNever()
+    {
+        addCliOption( "-fn" );
+        return this;
+    }
+
+    public MavenLauncher offline()
+    {
+        addCliOption( "-o" );
+        return this;
+    }
+
+    public MavenLauncher skipClean()
+    {
+        writeGoal( "-Dclean.skip=true" );
+        return this;
+    }
+
+    public MavenLauncher addGoal( String goal )
+    {
+        writeGoal( goal );
+        return this;
+    }
+
+    public FailsafeOutputValidator executeVerify()
+    {
+        return new FailsafeOutputValidator( conditionalExec( "verify" ) );
+    }
+
+    public OutputValidator executeTest()
+    {
+        return conditionalExec( "test" );
+    }
+
+    List<String> getGoals()
+    {
+        return unmodifiableList( goals );
+    }
+
+    private void writeGoal( String newGoal )
+    {
+        if ( newGoal != null && newGoal.startsWith( "-D" ) )
+        {
+            final String sysPropKey =
+                    newGoal.contains( "=" ) ? newGoal.substring( 0, newGoal.indexOf( '=' ) ) : newGoal;
+
+            final String sysPropStarter = sysPropKey + "=";
+
+            for ( ListIterator<String> it = goals.listIterator(); it.hasNext(); )
+            {
+                String goal = it.next();
+                if ( goal.equals( sysPropKey ) || goal.startsWith( sysPropStarter ) )
+                {
+                    System.out.printf( "[WARNING] System property already exists '%s'. Overriding to '%s'.\n",
+                                             goal, newGoal );
+                    it.set( newGoal );
+                    return;
+                }
+            }
+        }
+        goals.add( newGoal );
+    }
+
+    private OutputValidator conditionalExec(String goal)
+    {
+        OutputValidator verify;
+        try
+        {
+            verify = execute( goal );
+        }
+        catch ( SurefireVerifierException exc )
+        {
+            if ( expectFailure )
+            {
+                return getValidator();
+            }
+            else
+            {
+                throw exc;
+            }
+        }
+        if ( expectFailure )
+        {
+            throw new RuntimeException( "Expecting build failure, got none!" );
+        }
+        return verify;
+
+    }
+
+    public MavenLauncher withFailure()
+    {
+        this.expectFailure = true;
+        return this;
+    }
+
+
+    public OutputValidator execute( String goal )
+    {
+        addGoal( goal );
+        return executeCurrentGoals();
+    }
+
+    public OutputValidator executeCurrentGoals()
+    {
+
+        String userLocalRepo = System.getProperty( "user.localRepository" );
+        String testBuildDirectory = System.getProperty( "testBuildDirectory" );
+        boolean useInterpolatedSettings = Boolean.getBoolean( "useInterpolatedSettings" );
+
+        try
+        {
+            if ( useInterpolatedSettings )
+            {
+                File interpolatedSettings = new File( testBuildDirectory, "interpolated-settings" );
+
+                if ( !interpolatedSettings.exists() )
+                {
+                    // hack "a la" invoker plugin to download dependencies from local repo
+                    // and not download from central
+
+                    Map<String, String> values = new HashMap<String, String>( 1 );
+                    values.put( "localRepositoryUrl", toUrl( userLocalRepo ) );
+                    StrSubstitutor strSubstitutor = new StrSubstitutor( values );
+
+                    String fileContent = FileUtils.fileRead( new File( testBuildDirectory, "settings.xml" ) );
+
+                    String filtered = strSubstitutor.replace( fileContent );
+
+                    FileUtils.fileWrite( interpolatedSettings.getAbsolutePath(), filtered );
+
+                }
+
+                addCliOption( "-s " + interpolatedSettings.getCanonicalPath() );
+            }
+            getVerifier().setCliOptions( cliOptions );
+
+            getVerifier().executeGoals( goals, envvars );
+            return getValidator();
+        }
+        catch ( IOException e )
+        {
+            throw new SurefireVerifierException( e.getMessage(), e );
+        }
+        catch ( VerificationException e )
+        {
+            throw new SurefireVerifierException( e.getMessage(), e );
+        }
+        finally
+        {
+            getVerifier().resetStreams();
+        }
+    }
+
+    private static String toUrl( String filename )
+    {
+        /*
+         * NOTE: Maven fails to properly handle percent-encoded "file:" URLs (WAGON-111) so don't use File.toURI() here
+         * as-is but use the decoded path component in the URL.
+         */
+        String url = "file://" + new File( filename ).toURI().getPath();
+        if ( url.endsWith( "/" ) )
+        {
+            url = url.substring( 0, url.length() - 1 );
+        }
+        return url;
+    }
+
+    public MavenLauncher activateProfile( String profile )
+    {
+        return addGoal( "-P" + profile );
+    }
+
+    public MavenLauncher sysProp( String variable, String value )
+    {
+        return addGoal( "-D" + variable + "=" + value );
+    }
+
+    public MavenLauncher sysProp( Map<String, String> properties )
+    {
+        for ( Map.Entry<String, String> property : properties.entrySet() )
+        {
+            sysProp( property.getKey(), property.getValue() );
+        }
+        return this;
+    }
+
+    public MavenLauncher sysProp( String variable, boolean value )
+    {
+        return addGoal( "-D" + variable + "=" + value );
+    }
+
+    public MavenLauncher sysProp( String variable, int value )
+    {
+        return addGoal( "-D" + variable + "=" + value );
+    }
+
+    public MavenLauncher sysProp( String variable, double value )
+    {
+        return addGoal( "-D" + variable + "=" + value );
+    }
+
+    public MavenLauncher showExceptionMessages()
+    {
+        addCliOption( "-e" );
+        return this;
+    }
+
+    public MavenLauncher deleteSiteDir()
+    {
+        try
+        {
+            FileUtils.deleteDirectory( getValidator().getSubFile( "site" ) );
+        }
+        catch ( IOException e )
+        {
+            throw new SurefireVerifierException( e );
+        }
+        return this;
+    }
+
+    public OutputValidator getValidator()
+    {
+        if ( validator == null )
+        {
+            this.validator = new OutputValidator( getVerifier() );
+        }
+        return validator;
+    }
+
+    public void setForkJvm( boolean forkJvm ) {
+        getVerifier().setForkJvm( forkJvm );
+    }
+
+    private Verifier getVerifier()
+    {
+        if ( verifier == null )
+        {
+            try
+            {
+                verifier =
+                    cli == null
+                    ? new Verifier( ensureUnpacked().getAbsolutePath(), null, false )
+                    : new Verifier( ensureUnpacked().getAbsolutePath(), null, false, cli );
+            }
+            catch ( VerificationException e )
+            {
+                throw new RuntimeException( e );
+            }
+        }
+        return verifier;
+    }
+    
+    private File simpleExtractResources( Class<?> cl, String resourcePath )
+    {
+        if ( !resourcePath.startsWith( "/" ) )
+        {
+            resourcePath = "/" + resourcePath;
+        }
+        File tempDir = getUnpackDir();
+        File testDir = new File( tempDir, resourcePath );
+        try
+        {
+            File parentPom = new File( tempDir.getParentFile(), "pom.xml" );
+            if (!parentPom.exists()){
+                URL resource = cl.getResource( "/pom.xml" );
+                FileUtils.copyURLToFile( resource, parentPom );
+            }
+
+            FileUtils.deleteDirectory( testDir );
+            File file = ResourceExtractor.extractResourceToDestination( cl, resourcePath, tempDir, true );
+            return file.getCanonicalFile();
+        }
+        catch ( IOException e )
+        {
+            throw new RuntimeException( e );
+        }
+
+    }
+
+    File getUnpackDir()
+    {
+        String tempDirPath = System.getProperty( "maven.test.tmpdir", System.getProperty( "java.io.tmpdir" ) );
+        return new File( tempDirPath,
+                         testCaseBeingRun.getSimpleName() + "_" + getTestMethodName() + suffix );
+    }
+
+    public File getArtifactPath( String gid, String aid, String version, String ext )
+    {
+        return new File( verifier.getArtifactPath( gid, aid, version, ext ) );
+    }
+
+    String getTestMethodName()
+    {
+        // dirty. Im sure we can use junit4 rules to attach testname to thread instead
+        StackTraceElement[] stackTrace = getStackTraceElements();
+        StackTraceElement topInTestClass;
+        topInTestClass = findTopElemenent( stackTrace, testCaseBeingRun );
+        if ( topInTestClass == null )
+        {
+            // Look in superclass...
+            topInTestClass = findTopElemenent( stackTrace, testCaseBeingRun.getSuperclass() );
+        }
+        if ( topInTestClass != null )
+        {
+            return topInTestClass.getMethodName();
+        }
+        throw new IllegalStateException( "Cannot find " + testCaseBeingRun.getName() + "in stacktrace" );
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncherTest.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncherTest.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncherTest.java
new file mode 100644
index 0000000..42657c0
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/MavenLauncherTest.java
@@ -0,0 +1,47 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.CoreMatchers.hasItems;
+
+/**
+ * @author <a href="mailto:tibordigana@apache.org">Tibor Digana (tibor17)</a>
+ * @since 2.20
+ */
+public class MavenLauncherTest
+{
+    @Test
+    public void shouldNotDuplicateSystemProperties()
+    {
+        MavenLauncher launcher = new MavenLauncher( getClass(), "", "" )
+                                         .addGoal( "-DskipTests" )
+                                         .addGoal( "-Dx=a" )
+                                         .addGoal( "-DskipTests" )
+                                         .addGoal( "-Dx=b" );
+
+        assertThat( launcher.getGoals(), hasItems( "-Dx=b", "-DskipTests" ) );
+
+        assertThat( launcher.getGoals().size(), is( 2 ) );
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
new file mode 100644
index 0000000..a76f86f
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
@@ -0,0 +1,230 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.it.Verifier;
+import org.hamcrest.Matcher;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * A specialized verifier that enforces a standard use case for surefire IT's
+ *
+ * @author Kristian Rosenvold
+ */
+public class OutputValidator
+{
+    protected final Verifier verifier;
+
+    protected final File baseDir;
+
+    public OutputValidator( Verifier verifier )
+    {
+        this.verifier = verifier;
+        this.baseDir = new File( verifier.getBasedir() );
+    }
+
+    public OutputValidator verifyTextInLog( String text )
+    {
+        try
+        {
+            verifier.verifyTextInLog( text );
+        }
+        catch ( VerificationException e )
+        {
+            throw new SurefireVerifierException( e );
+        }
+        return this;
+    }
+
+
+    public OutputValidator verifyErrorFreeLog()
+    {
+        try
+        {
+            verifier.verifyErrorFreeLog();
+        }
+        catch ( VerificationException e )
+        {
+            throw new SurefireVerifierException( e );
+        }
+        return this;
+    }
+
+    public OutputValidator verifyErrorFree( int total )
+    {
+        try
+        {
+            verifier.verifyErrorFreeLog();
+            this.assertTestSuiteResults( total, 0, 0, 0 );
+            return this;
+        }
+        catch ( VerificationException e )
+        {
+            throw new SurefireVerifierException( e );
+        }
+    }
+
+    public OutputValidator assertThatLogLine( Matcher<String> line, Matcher<Integer> nTimes )
+        throws VerificationException
+    {
+        int counter = 0;
+        for ( String log : loadLogLines() )
+        {
+            if ( line.matches( log ) )
+            {
+                counter++;
+            }
+        }
+        assertThat( "log pattern does not match nTimes", counter, nTimes );
+        return this;
+    }
+
+    public Collection<String> loadLogLines()
+        throws VerificationException
+    {
+        return verifier.loadFile( verifier.getBasedir(), verifier.getLogFileName(), false );
+    }
+
+    public List<String> loadFile( File file, Charset charset )
+    {
+        //noinspection unchecked
+        try
+        {
+            return FileUtils.readLines( file, charset.name() );
+        }
+        catch ( IOException e )
+        {
+            throw new SurefireVerifierException( e );
+        }
+    }
+
+    public String getBasedir()
+    {
+        return verifier.getBasedir();
+    }
+
+    /**
+     * Returns a file, referenced from the extracted root (where pom.xml is located)
+     *
+     * @param path The subdirectory under basedir
+     * @return A file
+     */
+    public File getSubFile( String path )
+    {
+        return new File( getBasedir(), path );
+    }
+
+    public OutputValidator assertTestSuiteResults( int total, int errors, int failures, int skipped )
+    {
+        HelperAssertions.assertTestSuiteResults( total, errors, failures, skipped, baseDir );
+        return this;
+    }
+
+    public OutputValidator assertTestSuiteResults( int total, int errors, int failures, int skipped, int flakes )
+    {
+        HelperAssertions.assertTestSuiteResults( total, errors, failures, skipped, flakes, baseDir );
+        return this;
+    }
+
+    public OutputValidator assertTestSuiteResults( int total )
+    {
+        HelperAssertions.assertTestSuiteResults( total, baseDir );
+        return this;
+    }
+
+    public OutputValidator assertIntegrationTestSuiteResults( int total, int errors, int failures, int skipped )
+    {
+        HelperAssertions.assertIntegrationTestSuiteResults( total, errors, failures, skipped, baseDir );
+        return this;
+    }
+
+    public OutputValidator assertIntegrationTestSuiteResults( int total )
+    {
+        HelperAssertions.assertIntegrationTestSuiteResults( total, baseDir );
+        return this;
+    }
+
+    public TestFile getTargetFile( String modulePath, String fileName )
+    {
+        File targetDir = getSubFile( modulePath + "/target" );
+        return new TestFile( new File( targetDir, fileName ), this );
+    }
+
+    public TestFile getTargetFile( String fileName )
+    {
+        File targetDir = getSubFile( "target" );
+        return new TestFile( new File( targetDir, fileName ), this );
+    }
+
+    public TestFile getSurefireReportsFile( String fileName )
+    {
+        File targetDir = getSurefireReportsDirectory();
+        return new TestFile( new File( targetDir, fileName ), this );
+    }
+
+    public TestFile getSurefireReportsXmlFile( String fileName )
+    {
+        File targetDir = getSurefireReportsDirectory();
+        return new TestFile( new File( targetDir, fileName ), Charset.forName( "UTF-8" ), this );
+    }
+
+    public File getSurefireReportsDirectory()
+    {
+        return getSubFile( "target/surefire-reports" );
+    }
+
+    public TestFile getSiteFile( String fileName )
+    {
+        File targetDir = getSubFile( "target/site" );
+        return new TestFile( new File( targetDir, fileName ), this );
+    }
+
+    public File getBaseDir()
+    {
+        return baseDir;
+    }
+
+    public boolean stringsAppearInSpecificOrderInLog( String[] strings )
+        throws VerificationException
+    {
+        int i = 0;
+        for ( String line : loadLogLines() )
+        {
+            if ( line.startsWith( strings[i] ) )
+            {
+                if ( i == strings.length - 1 )
+                {
+                    return true;
+                }
+                ++i;
+            }
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Settings.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Settings.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Settings.java
new file mode 100644
index 0000000..28013fb
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/Settings.java
@@ -0,0 +1,72 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @author <a href="mailto:tibordigana@apache.org">Tibor Digana (tibor17)</a>
+ * @since 2.19
+ */
+public enum Settings
+{
+    JUNIT4_TEST( TestFramework.JUNIT4, Configuration.TEST ),
+    JUNIT47_TEST( TestFramework.JUNIT47, Configuration.TEST ),
+    JUNIT4_INCLUDES( TestFramework.JUNIT4, Configuration.INCLUDES ),
+    JUNIT47_INCLUDES( TestFramework.JUNIT47, Configuration.INCLUDES ),
+    JUNIT4_INCLUDES_EXCLUDES( TestFramework.JUNIT4, Configuration.INCLUDES_EXCLUDES ),
+    JUNIT47_INCLUDES_EXCLUDES( TestFramework.JUNIT47, Configuration.INCLUDES_EXCLUDES ),
+    JUNIT4_INCLUDES_FILE( TestFramework.JUNIT4, Configuration.INCLUDES_FILE ),
+    JUNIT47_INCLUDES_FILE( TestFramework.JUNIT47, Configuration.INCLUDES_FILE ),
+    JUNIT4_INCLUDES_EXCLUDES_FILE( TestFramework.JUNIT4, Configuration.INCLUDES_EXCLUDES_FILE ),
+    JUNIT47_INCLUDES_EXCLUDES_FILE( TestFramework.JUNIT47, Configuration.INCLUDES_EXCLUDES_FILE ),
+    TestNG_TEST( TestFramework.TestNG, Configuration.TEST ),
+    TestNG_INCLUDES( TestFramework.TestNG, Configuration.INCLUDES ),
+    TestNG_INCLUDES_EXCLUDES( TestFramework.TestNG, Configuration.INCLUDES_EXCLUDES ),
+    TestNG_INCLUDES_FILE( TestFramework.TestNG, Configuration.INCLUDES_FILE ),
+    TestNG_INCLUDES_EXCLUDES_FILE( TestFramework.TestNG, Configuration.INCLUDES_EXCLUDES_FILE );
+
+    private final TestFramework framework;
+    private final Configuration configuration;
+
+    Settings( TestFramework framework, Configuration configuration )
+    {
+        this.framework = framework;
+        this.configuration = configuration;
+    }
+
+    public String path()
+    {
+        return name().replace( '_', '-' ).toLowerCase();
+    }
+
+    public String profile()
+    {
+        return path();
+    }
+
+    public TestFramework getFramework()
+    {
+        return framework;
+    }
+
+    public Configuration getConfiguration()
+    {
+        return configuration;
+    }
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireJUnit4IntegrationTestCase.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireJUnit4IntegrationTestCase.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireJUnit4IntegrationTestCase.java
new file mode 100644
index 0000000..ee04dbc
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireJUnit4IntegrationTestCase.java
@@ -0,0 +1,60 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Contains commonly used features for most tests, encapsulating
+ * common use cases.
+ * <br>
+ * Also includes thread-safe access to the extracted resource
+ * files, which AbstractSurefireIntegrationTestClass does not.
+ * Thread safe only for running in "classes" mode.
+ *
+ * @author Kristian Rosenvold
+ */
+public abstract class SurefireJUnit4IntegrationTestCase
+{
+    public OutputValidator executeErrorFreeTest( String sourceName, int total )
+    {
+        return unpack( sourceName ).executeTest().verifyErrorFree( total );
+    }
+
+    public SurefireLauncher unpack( String sourceName )
+    {
+        return unpack( getClass(), sourceName, "" );
+    }
+
+    public SurefireLauncher unpack( String sourceName, String suffix )
+    {
+        return unpack( getClass(), sourceName, suffix );
+    }
+
+    public static SurefireLauncher unpack( Class testClass, String sourceName, String suffix, String[] cli )
+    {
+        MavenLauncher mavenLauncher = new MavenLauncher( testClass, sourceName, suffix, cli );
+        return new SurefireLauncher( mavenLauncher );
+    }
+
+    public static SurefireLauncher unpack( Class testClass, String sourceName, String suffix )
+    {
+        return unpack( testClass, sourceName, suffix, null );
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
----------------------------------------------------------------------
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
new file mode 100755
index 0000000..76d96e0
--- /dev/null
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
@@ -0,0 +1,492 @@
+package org.apache.maven.surefire.its.fixture;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.it.VerificationException;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+
+/**
+ * Encapsulate all needed features to start a surefire run
+ * <br>
+ * Also includes thread-safe access to the extracted resource
+ * files
+ *
+ * @author Kristian Rosenvold                                 -
+ */
+public final class SurefireLauncher
+{
+    public static final String EXT_JDK_HOME_KEY = "jdk.home";
+
+    public static final String EXT_JDK_HOME = System.getProperty( EXT_JDK_HOME_KEY );
+
+    private static final File JAVA_HOME = javaHome();
+
+    private final MavenLauncher mavenLauncher;
+
+    private final String surefireVersion = System.getProperty( "surefire.version" );
+
+    public SurefireLauncher( MavenLauncher mavenLauncher )
+    {
+        this.mavenLauncher = mavenLauncher;
+        reset();
+    }
+
+    public MavenLauncher maven()
+    {
+        return mavenLauncher;
+    }
+
+    String getTestMethodName()
+    {
+        return mavenLauncher.getTestMethodName();
+    }
+
+    public void reset()
+    {
+        mavenLauncher.reset();
+        for ( String s : getInitialGoals() )
+        {
+            mavenLauncher.addGoal( s );
+        }
+        setInProcessJavaHome();
+    }
+
+    private static File javaHome()
+    {
+        String javaHome = isBlank( EXT_JDK_HOME ) ? System.getenv( "JAVA_HOME" ) : EXT_JDK_HOME;
+        if ( isBlank( javaHome ) )
+        {
+            javaHome = System.getProperty( "java.home" );
+            File jre = new File( javaHome );
+            if ( "jre".equals( jre.getName() ) )
+            {
+                javaHome = jre.getParent();
+            }
+        }
+
+        try
+        {
+            File javaHomeAsDir = new File( javaHome ).getCanonicalFile();
+            if ( !javaHomeAsDir.isDirectory() )
+            {
+                throw new RuntimeException( javaHomeAsDir.getAbsolutePath() + " is not a JAVA_HOME directory." );
+            }
+            System.out.println( "Using JAVA_HOME=" + javaHomeAsDir.getAbsolutePath() + " in forked launcher." );
+            return javaHomeAsDir;
+        }
+        catch ( IOException e )
+        {
+            throw new RuntimeException( e );
+        }
+    }
+
+    private void setInProcessJavaHome()
+    {
+        setLauncherJavaHome( JAVA_HOME.getPath() );
+    }
+
+    public SurefireLauncher setLauncherJavaHome( String javaHome )
+    {
+        mavenLauncher.addEnvVar( "JAVA_HOME", javaHome );
+        return this;
+    }
+
+    public SurefireLauncher getSubProjectLauncher( String subProject )
+        throws VerificationException
+    {
+        return new SurefireLauncher( mavenLauncher.getSubProjectLauncher( subProject ) );
+    }
+
+    public OutputValidator getSubProjectValidator( String subProject )
+        throws VerificationException
+    {
+        return mavenLauncher.getSubProjectValidator( subProject );
+    }
+
+    public SurefireLauncher addEnvVar( String key, String value )
+    {
+        mavenLauncher.addEnvVar( key, value );
+        return this;
+    }
+
+    public SurefireLauncher setMavenOpts(String opts){
+        addEnvVar( "MAVEN_OPTS", opts );
+        return this;
+    }
+
+    private List<String> getInitialGoals()
+    {
+        List<String> goals = new ArrayList<>();
+
+        goals.add( "-Dsurefire.version=" + surefireVersion );
+
+        String jacocoAgent = System.getProperty( "jacoco.agent", "" );
+        goals.add( "-Djacoco.agent=" + jacocoAgent );
+
+        return goals;
+    }
+
+    public SurefireLauncher showErrorStackTraces()
+    {
+        mavenLauncher.showErrorStackTraces();
+        return this;
+    }
+
+    public SurefireLauncher debugLogging()
+    {
+        mavenLauncher.debugLogging();
+        return this;
+    }
+
+    @SuppressWarnings( "UnusedDeclaration" )
+    public SurefireLauncher debugSurefireFork()
+    {
+        mavenLauncher.sysProp( "maven.surefire.debug", "true" );
+        return this;
+    }
+
+    public SurefireLauncher failNever()
+    {
+        mavenLauncher.failNever();
+        return this;
+    }
+
+    public SurefireLauncher groups( String groups )
+    {
+        mavenLauncher.sysProp( "groups", groups );
+        return this;
+    }
+
+    public SurefireLauncher addGoal( String goal )
+    {
+        mavenLauncher.addGoal( goal );
+        return this;
+    }
+
+    public OutputValidator executeTest()
+    {
+        return mavenLauncher.execute( "test" );
+    }
+
+    public OutputValidator executeInstall()
+        throws VerificationException
+    {
+        return mavenLauncher.execute( "install" );
+    }
+
+
+    public FailsafeOutputValidator executeVerify()
+    {
+        OutputValidator verify = execute( "verify" );
+        return new FailsafeOutputValidator( verify );
+    }
+
+    public OutputValidator execute( String goal )
+    {
+        return mavenLauncher.execute( goal );
+    }
+
+    public OutputValidator executeSurefireReport()
+    {
+        return mavenLauncher.execute( "surefire-report:report" );
+    }
+
+
+    public OutputValidator executeCurrentGoals()
+    {
+        return mavenLauncher.executeCurrentGoals();
+    }
+
+
+    public SurefireLauncher printSummary( boolean printsummary )
+    {
+        mavenLauncher.sysProp( "printSummary", printsummary );
+        return this;
+    }
+
+    public SurefireLauncher redirectToFile( boolean redirect )
+    {
+        mavenLauncher.sysProp( "maven.test.redirectTestOutputToFile", redirect );
+        return this;
+    }
+
+    public SurefireLauncher forkOnce()
+    {
+        return forkMode( "once" );
+    }
+
+    public SurefireLauncher forkNever()
+    {
+        return forkMode( "never" );
+    }
+
+    public SurefireLauncher forkAlways()
+    {
+        return forkMode( "always" );
+    }
+
+    public SurefireLauncher forkPerTest()
+    {
+        return forkMode( "pertest" );
+    }
+
+    public SurefireLauncher forkPerThread()
+    {
+        return forkMode( "perthread" );
+    }
+
+    public SurefireLauncher threadCount( int threadCount )
+    {
+        mavenLauncher.sysProp( "threadCount", threadCount );
+        return this;
+    }
+
+    public SurefireLauncher forkCount( int forkCount )
+    {
+        mavenLauncher.sysProp( "forkCount", forkCount );
+        return this;
+    }
+
+    public SurefireLauncher reuseForks( boolean reuseForks )
+    {
+        mavenLauncher.sysProp( "reuseForks", reuseForks );
+        return this;
+    }
+
+    public SurefireLauncher forkMode( String forkMode )
+    {
+        mavenLauncher.sysProp( "forkMode", forkMode );
+        return this;
+    }
+
+    public SurefireLauncher runOrder( String runOrder )
+    {
+        mavenLauncher.sysProp( "surefire.runOrder", runOrder );
+        return this;
+    }
+
+    public SurefireLauncher failIfNoTests( boolean fail )
+    {
+        mavenLauncher.sysProp( "failIfNoTests", fail );
+        return this;
+    }
+
+
+    public SurefireLauncher mavenTestFailureIgnore( boolean fail )
+    {
+        mavenLauncher.sysProp( "maven.test.failure.ignore", fail );
+        return this;
+    }
+
+    public SurefireLauncher failIfNoSpecifiedTests( boolean fail )
+    {
+        mavenLauncher.sysProp( "surefire.failIfNoSpecifiedTests", fail );
+        return this;
+    }
+
+    public SurefireLauncher useSystemClassLoader( boolean useSystemClassLoader )
+    {
+        mavenLauncher.sysProp( "useSystemClassLoader", useSystemClassLoader );
+        return this;
+    }
+
+    public SurefireLauncher activateProfile( String profile )
+    {
+        mavenLauncher.activateProfile( profile );
+        return this;
+    }
+
+
+    protected String getSurefireVersion()
+    {
+        return surefireVersion;
+    }
+
+    public SurefireLauncher disablePerCoreThreadCount()
+    {
+        mavenLauncher.sysProp( "perCoreThreadCount", false );
+        return this;
+    }
+
+    public SurefireLauncher disableParallelOptimization()
+    {
+        mavenLauncher.sysProp( "parallelOptimized", "false" );
+        return this;
+    }
+
+    public SurefireLauncher parallel( String parallel )
+    {
+        mavenLauncher.sysProp( "parallel", parallel );
+        return this;
+    }
+
+    public SurefireLauncher parallelSuites()
+    {
+        return parallel( "suites" );
+    }
+
+    public SurefireLauncher parallelClasses()
+    {
+        return parallel( "classes" );
+    }
+
+    public SurefireLauncher parallelMethods()
+    {
+        return parallel( "methods" );
+    }
+
+    public SurefireLauncher parallelBoth()
+    {
+        return parallel( "both" );
+    }
+
+    public SurefireLauncher parallelSuitesAndClasses()
+    {
+        return parallel( "suitesAndClasses" );
+    }
+
+    public SurefireLauncher parallelSuitesAndMethods()
+    {
+        return parallel( "suitesAndMethods" );
+    }
+
+    public SurefireLauncher parallelClassesAndMethods()
+    {
+        return parallel( "classesAndMethods" );
+    }
+
+    public SurefireLauncher parallelAll()
+    {
+        return parallel( "all" );
+    }
+
+    public SurefireLauncher useUnlimitedThreads()
+    {
+        mavenLauncher.sysProp( "useUnlimitedThreads", true );
+        return this;
+    }
+
+    public SurefireLauncher threadCountSuites( int count )
+    {
+        mavenLauncher.sysProp( "threadCountSuites", count );
+        return this;
+    }
+
+    public SurefireLauncher threadCountClasses( int count )
+    {
+        mavenLauncher.sysProp( "threadCountClasses", count );
+        return this;
+    }
+
+    public SurefireLauncher threadCountMethods( int count )
+    {
+        mavenLauncher.sysProp( "threadCountMethods", count );
+        return this;
+    }
+
+    public SurefireLauncher parallelTestsTimeoutInSeconds( double timeout )
+    {
+        mavenLauncher.sysProp( "surefire.parallel.timeout", timeout );
+        return this;
+    }
+
+    public SurefireLauncher parallelTestsTimeoutForcedInSeconds( double timeout )
+    {
+        mavenLauncher.sysProp( "surefire.parallel.forcedTimeout", timeout );
+        return this;
+    }
+
+    public SurefireLauncher argLine( String value )
+    {
+        mavenLauncher.sysProp( "argLine", value );
+        return this;
+    }
+
+    public SurefireLauncher sysProp( String variable, String value )
+    {
+        mavenLauncher.sysProp( variable, value );
+        return this;
+    }
+
+    public SurefireLauncher setJUnitVersion( String version )
+    {
+        mavenLauncher.sysProp( "junit.version", version );
+        return this;
+    }
+
+    public SurefireLauncher setGroups( String groups )
+    {
+        mavenLauncher.sysProp( "groups", groups );
+        return this;
+    }
+
+    public SurefireLauncher setExcludedGroups( String excludedGroups )
+    {
+        mavenLauncher.sysProp( "excludedGroups", excludedGroups );
+        return this;
+    }
+
+
+    public File getUnpackedAt()
+    {
+        return mavenLauncher.getUnpackedAt();
+    }
+
+    public SurefireLauncher addFailsafeReportOnlyGoal()
+    {
+        mavenLauncher.addGoal( getReportPluginGoal( ":failsafe-report-only" ) );
+        return this;
+    }
+
+    public SurefireLauncher addSurefireReportGoal()
+    {
+        mavenLauncher.addGoal( getReportPluginGoal( "report" ) );
+        return this;
+    }
+
+    public SurefireLauncher addSurefireReportOnlyGoal()
+    {
+        mavenLauncher.addGoal( getReportPluginGoal( "report-only" ) );
+        return this;
+    }
+
+    private String getReportPluginGoal( String goal )
+    {
+        return "org.apache.maven.plugins:maven-surefire-report-plugin:" + getSurefireVersion() + ":" + goal;
+    }
+
+    public SurefireLauncher setTestToRun( String basicTest )
+    {
+        mavenLauncher.sysProp( "test", basicTest );
+        return this;
+    }
+
+    public SurefireLauncher setForkJvm()
+    {
+        mavenLauncher.setForkJvm( true );
+        return this;
+    }
+}


Mime
View raw message