[SUREFIRE-756] Allow ability to capture executed random runOrder for re-play purposes
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
index 332c311..d77b3cf 100644
--- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
+++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -311,6 +311,22 @@
private String runOrder;
/**
+ * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}.
+ * <br>
+ * <br>
+ * If no seeds are set and {@code failsafe.runOrder} is set to {@code random}, then the seed used will be
+ * outputted (search for "To reproduce ordering use flag -Dfailsafe.runOrder.random.seed").
+ * <br>
+ * <br>
+ * To deterministically reproduce any random test order that was run before, simply set the seed to
+ * be the same value.
+ *
+ * @since 3.0.0-M6
+ */
+ @Parameter( property = "failsafe.runOrder.random.seed" )
+ private Long runOrderRandomSeed;
+
+ /**
* A file containing include patterns, each in a next line. Blank lines, or lines starting with # are ignored.
* If {@code includes} are also specified, these patterns are appended. Example with path, simple and regex
* includes:
@@ -892,6 +908,18 @@
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return runOrderRandomSeed;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
+ @Override
public File getIncludesFile()
{
return includesFile;
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
index 64975b4..5c2eea3 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -865,6 +865,10 @@
public abstract void setRunOrder( String runOrder );
+ public abstract Long getRunOrderRandomSeed();
+
+ public abstract void setRunOrderRandomSeed( Long runOrderRandomSeed );
+
protected abstract void handleSummary( RunResult summary, Exception firstForkException )
throws MojoExecutionException, MojoFailureException;
@@ -1129,6 +1133,7 @@
warnIfNotApplicableSkipAfterFailureCount();
warnIfIllegalTempDir();
warnIfForkCountIsZero();
+ printDefaultSeedIfNecessary();
}
return true;
}
@@ -1285,7 +1290,7 @@
ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration();
provider.addProviderProperties();
RunOrderParameters runOrderParameters =
- new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ) );
+ new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRunOrderRandomSeed() );
if ( isNotForking() )
{
@@ -3051,6 +3056,17 @@
}
}
+ private void printDefaultSeedIfNecessary()
+ {
+ if ( getRunOrderRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) )
+ {
+ setRunOrderRandomSeed( System.nanoTime() );
+ getConsoleLogger().info(
+ "Tests will run in random order. To reproduce ordering use flag -D"
+ + getPluginName() + ".runOrder.random.seed=" + getRunOrderRandomSeed() );
+ }
+ }
+
final class TestNgProviderInfo
implements ProviderInfo
{
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
index c8aee3c..1e00f16 100644
--- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
+++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -56,6 +56,7 @@
import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID;
import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER;
import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION;
+import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER_RANDOM_SEED;
import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY;
import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST;
import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT;
@@ -161,6 +162,7 @@
{
properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) );
properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() );
+ properties.setProperty( RUN_ORDER_RANDOM_SEED, runOrderParameters.getRunOrderRandomSeed() );
}
ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration();
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
index 7d5343c..c9dbf5c 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
@@ -840,6 +840,18 @@
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return null;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+
+ }
+
+ @Override
protected void handleSummary( RunResult summary, Exception firstForkException )
{
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
index a08224c..06e52be 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
@@ -2382,6 +2382,18 @@
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return null;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+
+ }
+
+ @Override
protected void handleSummary( RunResult summary, Exception firstForkException )
{
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
index afcb642..1c9b89f 100644
--- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
+++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
@@ -710,6 +710,18 @@
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return null;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+
+ }
+
+ @Override
public String[] getDependenciesToScan()
{
return dependenciesToScan;
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
index f2b4fdc..34ed33c 100644
--- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
+++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -293,6 +293,22 @@
private String runOrder;
/**
+ * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}.
+ * <br>
+ * <br>
+ * If no seeds are set and {@code surefire.runOrder} is set to {@code random}, then the seed used will be
+ * outputted (search for "To reproduce ordering use flag -Dsurefire.runOrder.random.seed").
+ * <br>
+ * <br>
+ * To deterministically reproduce any random test order that was run before, simply set the seed to
+ * be the same value.
+ *
+ * @since 3.0.0-M6
+ */
+ @Parameter( property = "surefire.runOrder.random.seed" )
+ private Long runOrderRandomSeed;
+
+ /**
* A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
* also specified, these patterns are appended. Example with path, simple and regex includes:
* <pre><code>
@@ -795,6 +811,18 @@
}
@Override
+ public Long getRunOrderRandomSeed()
+ {
+ return runOrderRandomSeed;
+ }
+
+ @Override
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
+ @Override
public File getIncludesFile()
{
return includesFile;
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
index 1fd2c3f..07e3b3e 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
@@ -31,16 +31,34 @@
private File runStatisticsFile;
+ private Long runOrderRandomSeed;
+
public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile )
{
this.runOrder = runOrder;
this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = null;
}
public RunOrderParameters( String runOrder, File runStatisticsFile )
{
this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = null;
+ }
+
+ public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long runOrderRandomSeed )
+ {
+ this.runOrder = runOrder;
+ this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
+ public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrderRandomSeed )
+ {
+ this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
+ this.runStatisticsFile = runStatisticsFile;
+ this.runOrderRandomSeed = runOrderRandomSeed;
}
public static RunOrderParameters alphabetical()
@@ -53,6 +71,16 @@
return runOrder;
}
+ public Long getRunOrderRandomSeed()
+ {
+ return runOrderRandomSeed;
+ }
+
+ public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+ {
+ this.runOrderRandomSeed = runOrderRandomSeed;
+ }
+
public File getRunStatisticsFile()
{
return runStatisticsFile;
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
index c897bb7..8ced06a 100644
--- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
+++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
@@ -28,6 +28,7 @@
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Random;
/**
* Applies the final runorder of the tests
@@ -45,12 +46,21 @@
private final int threadCount;
+ private final Random random;
+
public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
{
this.runOrderParameters = runOrderParameters;
this.threadCount = threadCount;
this.runOrder = runOrderParameters.getRunOrder();
this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null;
+ Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed();
+ if ( runOrderRandomSeed == null )
+ {
+ runOrderRandomSeed = System.nanoTime();
+ runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
+ }
+ this.random = new Random( runOrderRandomSeed );
}
@Override
@@ -72,7 +82,7 @@
{
if ( RunOrder.RANDOM.equals( runOrder ) )
{
- Collections.shuffle( testClasses );
+ Collections.shuffle( testClasses, random );
}
else if ( RunOrder.FAILEDFIRST.equals( runOrder ) )
{
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
index fa664be..ae950df 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -46,6 +46,7 @@
public static final String SOURCE_DIRECTORY = "testSuiteDefinitionTestSourceDirectory";
public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory";
public static final String RUN_ORDER = "runOrder";
+ public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed";
public static final String RUN_STATISTICS_FILE = "runStatisticsFile";
public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles";
public static final String PROVIDER_CONFIGURATION = "providerConfiguration";
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
index 4ec8cec..6a6cfae 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -102,6 +102,7 @@
final List<String> testSuiteXmlFiles = properties.getStringList( TEST_SUITE_XML_FILES );
final File testClassesDirectory = properties.getFileProperty( TEST_CLASSES_DIRECTORY );
final String runOrder = properties.getProperty( RUN_ORDER );
+ final Long runOrderRandomSeed = properties.getLongProperty( RUN_ORDER_RANDOM_SEED );
final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE );
final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT );
@@ -111,7 +112,8 @@
properties.getBooleanProperty( FAILIFNOTESTS ), runOrder );
RunOrderParameters runOrderParameters
- = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ) );
+ = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ),
+ runOrderRandomSeed );
TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier );
TestRequest testSuiteDefinition =
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
index 7f4c355..b10b322 100644
--- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
+++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -178,10 +178,11 @@
return null;
}
//Can't use the constructor with the RunOrder parameter. Using it causes some integration tests to fail.
- Class<?>[] arguments = { String.class, File.class };
+ Class<?>[] arguments = { String.class, File.class, Long.class };
Constructor<?> constructor = getConstructor( this.runOrderParameters, arguments );
File runStatisticsFile = runOrderParameters.getRunStatisticsFile();
- return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile );
+ return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile,
+ runOrderParameters.getRunOrderRandomSeed() );
}
private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo )
diff --git a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
index d236d6a..44e4627 100644
--- a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
+++ b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
@@ -119,6 +119,20 @@
assertTrue( isCalled( foo ) );
}
+ public void testRunOrderParametersWithRunOrderRandomSeed()
+ {
+ SurefireReflector surefireReflector = getReflector();
+ Object foo = getFoo();
+
+ // Arbitrary random seed that should be ignored because RunOrder is not RANDOM
+ Long runOrderRandomSeed = 5L;
+
+ RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ),
+ runOrderRandomSeed );
+ surefireReflector.setRunOrderParameters( foo, runOrderParameters );
+ assertTrue( isCalled( foo ) );
+ }
+
public void testNullRunOrderParameters()
{
SurefireReflector surefireReflector = getReflector();
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
index 430c9bf..0032e8d 100644
--- a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
@@ -19,6 +19,7 @@
* under the License.
*/
+import java.util.Arrays;
import java.util.Calendar;
import org.apache.maven.it.VerificationException;
import org.apache.maven.surefire.its.fixture.OutputValidator;
@@ -61,6 +62,43 @@
}
@Test
+ public void testRandomJUnit4DifferentSeed()
+ throws Exception
+ {
+ long seed = 0L;
+ OutputValidator validator = executeWithRandomOrder( "junit4", seed );
+ String[] expected = validator.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER );
+ for ( long i = seed; i < 5 + seed; i++ )
+ {
+ OutputValidator validator2 = executeWithRandomOrder( "junit4", i );
+ String[] observed = validator2.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER );
+ if ( ! Arrays.equals( expected, observed ) )
+ {
+ return;
+ }
+ }
+ throw new VerificationException( "All random orders with the different seeds produced the same orders" );
+ }
+
+ @Test
+ public void testRandomJUnit4SameSeed()
+ throws Exception
+ {
+ long seed = 0L;
+ OutputValidator validator = executeWithRandomOrder( "junit4", seed );
+ String[] expected = validator.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER );
+ for ( long i = 0; i < 5; i++ )
+ {
+ OutputValidator validator2 = executeWithRandomOrder( "junit4", seed );
+ String[] observed = validator2.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER );
+ if ( ! Arrays.equals( expected, observed ) )
+ {
+ throw new VerificationException( "Random orders with the same seed produced different orders" );
+ }
+ }
+ }
+
+ @Test
public void testReverseAlphabeticalJUnit4()
throws Exception
{
@@ -149,6 +187,17 @@
.verifyErrorFree( 3 );
}
+ private OutputValidator executeWithRandomOrder( String profile, long seed )
+ {
+ return unpack()
+ .activateProfile( profile )
+ .forkMode( getForkMode() )
+ .runOrder( "random" )
+ .runOrderRandomSeed( String.valueOf( seed ) )
+ .executeTest()
+ .verifyErrorFree( 3 );
+ }
+
protected String getForkMode()
{
return "once";
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
index d36c6a6..14b360c 100644
--- a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -223,6 +224,32 @@
return baseDir;
}
+ public String[] getStringsOrderInLog( String[] strings )
+ throws VerificationException
+ {
+ String[] retArr = new String[strings.length];
+ List<String> strList = new ArrayList<String>( Arrays.asList( strings ) );
+ int i = 0;
+ for ( String line : loadLogLines() )
+ {
+ for ( int j = 0; j < strList.size(); j++ )
+ {
+ if ( line.startsWith( strList.get( j ) ) )
+ {
+ retArr[i] = strList.get( j );
+ ++i;
+ if ( i == strings.length )
+ {
+ return retArr;
+ }
+ strList.remove( j );
+ break;
+ }
+ }
+ }
+ return retArr;
+ }
+
public boolean stringsAppearInSpecificOrderInLog( String[] strings )
throws VerificationException
{
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
index a5970d4..0342172 100755
--- a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
+++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java
@@ -289,6 +289,12 @@
return this;
}
+ public SurefireLauncher runOrderRandomSeed( String runOrderRandomSeed )
+ {
+ mavenLauncher.sysProp( "surefire.runOrder.random.seed", runOrderRandomSeed );
+ return this;
+ }
+
public SurefireLauncher failIfNoTests( boolean fail )
{
mavenLauncher.sysProp( "failIfNoTests", fail );
