[SUREFIRE-1766] Surefire does not display TestNG data provider values on command line
diff --git a/pom.xml b/pom.xml
index 7cde7cd..06361d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -668,6 +668,7 @@
<maxJdkVersion>${maven.compiler.target}</maxJdkVersion>
<excludes>
<exclude>org.junit.platform:junit-platform-commons</exclude>
+ <exclude>org.assertj:assertj-core</exclude>
</excludes>
</enforceBytecodeVersion>
</rules>
diff --git a/surefire-providers/surefire-testng/pom.xml b/surefire-providers/surefire-testng/pom.xml
index 2dddddc..539cdcd 100644
--- a/surefire-providers/surefire-testng/pom.xml
+++ b/surefire-providers/surefire-testng/pom.xml
@@ -17,76 +17,93 @@
~ under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-providers</artifactId>
- <version>3.0.0-SNAPSHOT</version>
- </parent>
+ <parent>
+ <groupId>org.apache.maven.surefire</groupId>
+ <artifactId>surefire-providers</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ </parent>
- <artifactId>surefire-testng</artifactId>
+ <artifactId>surefire-testng</artifactId>
- <name>SureFire TestNG Runner</name>
- <description>SureFire TestNG Runner</description>
+ <name>SureFire TestNG Runner</name>
+ <description>SureFire TestNG Runner</description>
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.surefire</groupId>
- <artifactId>common-java5</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-testng-utils</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <version>5.10</version>
- <classifier>jdk15</classifier>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>jacoco-agent</id>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <propertyName>jacoco.agent</propertyName>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <argLine>${jvm.args.tests} ${jacoco.agent}</argLine>
- </configuration>
- <dependencies>
- <dependency>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-shadefire</artifactId>
- <version>3.0.0-M4</version> <!-- ${shadedVersion}, but resolved due to https://issues.apache.org/jira/browse/MRELEASE-799 -->
- </dependency>
- </dependencies>
- </plugin>
- </plugins>
- </build>
+ <artifactId>common-java5</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.surefire</groupId>
+ <artifactId>surefire-testng-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>5.10</version>
+ <classifier>jdk15</classifier>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-reflect</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>jacoco-agent</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <propertyName>jacoco.agent</propertyName>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <argLine>${jvm.args.tests} ${jacoco.agent}</argLine>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.surefire</groupId>
+ <artifactId>surefire-shadefire</artifactId>
+ <!-- ${shadedVersion}, but resolved due to https://issues.apache.org/jira/browse/MRELEASE-799 -->
+ <version>3.0.0-M4</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java
index 83c9d2b..1c483a5 100644
--- a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java
+++ b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java
@@ -32,6 +32,8 @@
import org.testng.ITestListener;
import org.testng.ITestResult;
+import java.util.Arrays;
+
import static org.apache.maven.surefire.report.SimpleReportEntry.ignored;
import static org.apache.maven.surefire.report.SimpleReportEntry.withException;
@@ -67,13 +69,13 @@
{
String clazz = result.getTestClass().getName();
String group = groupString( result.getMethod().getGroups(), clazz );
- reporter.testStarting( new CategorizedReportEntry( clazz, result.getName(), group ) );
+ reporter.testStarting( new CategorizedReportEntry( clazz, testName( result ), group ) );
}
@Override
public void onTestSuccess( ITestResult result )
{
- ReportEntry report = new SimpleReportEntry( result.getTestClass().getName(), null, result.getName(), null );
+ ReportEntry report = new SimpleReportEntry( result.getTestClass().getName(), null, testName( result ), null );
reporter.testSucceeded( report );
}
@@ -81,10 +83,9 @@
public void onTestFailure( ITestResult result )
{
IClass clazz = result.getTestClass();
- ReportEntry report = withException( clazz.getName(), null, result.getName(), null,
- new PojoStackTraceWriter( clazz.getRealClass().getName(),
- result.getMethod().getMethodName(),
- result.getThrowable() ) );
+ ReportEntry report = withException( clazz.getName(), null, testName( result ), null,
+ new PojoStackTraceWriter( clazz.getRealClass().getName(), result.getMethod().getMethodName(),
+ result.getThrowable() ) );
reporter.testFailed( report );
}
@@ -95,7 +96,7 @@
//noinspection ThrowableResultOfMethodCallIgnored
Throwable t = result.getThrowable();
String reason = t == null ? null : t.getMessage();
- ReportEntry report = ignored( result.getTestClass().getName(), null, result.getName(), null, reason );
+ ReportEntry report = ignored( result.getTestClass().getName(), null, testName( result ), null, reason );
reporter.testSkipped( report );
}
@@ -103,10 +104,9 @@
public void onTestFailedButWithinSuccessPercentage( ITestResult result )
{
IClass clazz = result.getTestClass();
- ReportEntry report = withException( clazz.getName(), null, result.getName(), null,
- new PojoStackTraceWriter( clazz.getRealClass().getName(),
- result.getMethod().getMethodName(),
- result.getThrowable() ) );
+ ReportEntry report = withException( clazz.getName(), null, testName( result ), null,
+ new PojoStackTraceWriter( clazz.getRealClass().getName(), result.getMethod().getMethodName(),
+ result.getThrowable() ) );
reporter.testSucceeded( report );
}
@@ -183,4 +183,18 @@
//onTestSuccess( result );
}
+ /**
+ * Acquire a better representation of the test name that includes parameters and the invocation count, if there are
+ * any parameters
+ *
+ * @param result the test result to extract from
+ * @return a descriptive name for the test
+ */
+ private static String testName( ITestResult result )
+ {
+ Object[] parameters = result.getParameters();
+ String name = result.getName();
+ return parameters == null || parameters.length == 0
+ ? name : name + Arrays.toString( parameters ) + "(" + result.getMethod().getCurrentInvocationCount() + ")";
+ }
}
diff --git a/surefire-providers/surefire-testng/src/test/java/org/apache/maven/surefire/testng/TestNGReporterTest.java b/surefire-providers/surefire-testng/src/test/java/org/apache/maven/surefire/testng/TestNGReporterTest.java
new file mode 100644
index 0000000..36a7600
--- /dev/null
+++ b/surefire-providers/surefire-testng/src/test/java/org/apache/maven/surefire/testng/TestNGReporterTest.java
@@ -0,0 +1,240 @@
+package org.apache.maven.surefire.testng;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.maven.surefire.report.CategorizedReportEntry;
+import org.apache.maven.surefire.report.RunListener;
+import org.apache.maven.surefire.report.SimpleReportEntry;
+import org.mockito.ArgumentCaptor;
+import org.testng.ITestClass;
+import org.testng.ITestNGMethod;
+import org.testng.ITestResult;
+
+import static org.fest.assertions.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+import static org.powermock.reflect.Whitebox.invokeMethod;
+
+/**
+ * Tests for {@link TestNGReporter}.
+ */
+public class TestNGReporterTest extends TestCase
+{
+ public void testParameterizedTestName() throws Exception
+ {
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ when( method.getCurrentInvocationCount() ).thenReturn( 3 );
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getParameters() ).thenReturn( new String[] { "val1", "val2" } );
+ when( testResult.getMethod() ).thenReturn( method );
+ String testName = invokeMethod( TestNGReporter.class, "testName", testResult );
+ assertThat( testName )
+ .isEqualTo( "myTest[val1, val2](3)" );
+ }
+
+ public void testWithoutParameterizedTestName() throws Exception
+ {
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getMethod() ).thenReturn( method );
+ String testName = invokeMethod( TestNGReporter.class, "testName", testResult );
+ assertThat( testName )
+ .isEqualTo( "myTest" );
+ }
+
+ public void testOnTestStart()
+ {
+ ITestClass cls = mock( ITestClass.class );
+ when( cls.getName() ).thenReturn( "pkg.MyClass" );
+
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ when( method.getCurrentInvocationCount() ).thenReturn( 3 );
+ when( method.getGroups() ).thenReturn( new String[0] );
+
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getTestClass() ).thenReturn( cls );
+ when( testResult.getMethod() ).thenReturn( method );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getParameters() ).thenReturn( new String[] { "val1", "val2" } );
+
+ RunListener listener = mock( RunListener.class );
+ TestNGReporter reporter = new TestNGReporter( listener );
+ reporter.onTestStart( testResult );
+
+ ArgumentCaptor<CategorizedReportEntry> reportEntry = ArgumentCaptor.forClass( CategorizedReportEntry.class );
+ verify( listener ).testStarting( reportEntry.capture() );
+ verifyNoMoreInteractions( listener );
+
+ assertThat( reportEntry.getValue().getSourceName() )
+ .isEqualTo( "pkg.MyClass" );
+
+ assertThat( reportEntry.getValue().getName() )
+ .isEqualTo( "myTest[val1, val2](3)" );
+ }
+
+ public void testOnTestSuccess()
+ {
+ ITestClass cls = mock( ITestClass.class );
+ when( cls.getName() ).thenReturn( "pkg.MyClass" );
+
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ when( method.getCurrentInvocationCount() ).thenReturn( 3 );
+
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getTestClass() ).thenReturn( cls );
+ when( testResult.getMethod() ).thenReturn( method );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getParameters() ).thenReturn( new String[] { "val1", "val2" } );
+
+ RunListener listener = mock( RunListener.class );
+ TestNGReporter reporter = new TestNGReporter( listener );
+ reporter.onTestSuccess( testResult );
+
+ ArgumentCaptor<SimpleReportEntry> reportEntry = ArgumentCaptor.forClass( SimpleReportEntry.class );
+ verify( listener ).testSucceeded( reportEntry.capture() );
+ verifyNoMoreInteractions( listener );
+
+ assertThat( reportEntry.getValue().getSourceName() )
+ .isEqualTo( "pkg.MyClass" );
+
+ assertThat( reportEntry.getValue().getName() )
+ .isEqualTo( "myTest[val1, val2](3)" );
+ }
+
+ public void testOnTestFailure()
+ {
+ Exception stackTrace = new Exception();
+
+ ITestClass cls = mock( ITestClass.class );
+ when( cls.getName() ).thenReturn( getClass().getName() );
+
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ when( method.getCurrentInvocationCount() ).thenReturn( 1 );
+ when( method.getMethodName() ).thenReturn( "myTest" );
+
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getThrowable() ).thenReturn( stackTrace );
+ when( cls.getRealClass() ).thenReturn( getClass() );
+ when( testResult.getTestClass() ).thenReturn( cls );
+ when( testResult.getMethod() ).thenReturn( method );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getParameters() ).thenReturn( new String[] { "val1", "val2" } );
+
+ RunListener listener = mock( RunListener.class );
+ TestNGReporter reporter = new TestNGReporter( listener );
+ reporter.onTestFailure( testResult );
+
+ ArgumentCaptor<SimpleReportEntry> reportEntry = ArgumentCaptor.forClass( SimpleReportEntry.class );
+ verify( listener ).testFailed( reportEntry.capture() );
+ verifyNoMoreInteractions( listener );
+
+ assertThat( reportEntry.getValue().getSourceName() )
+ .isEqualTo( getClass().getName() );
+
+ assertThat( reportEntry.getValue().getName() )
+ .isEqualTo( "myTest[val1, val2](1)" );
+
+ assertThat( reportEntry.getValue().getStackTraceWriter() )
+ .isNotNull();
+
+ assertThat( reportEntry.getValue().getStackTraceWriter().getThrowable().getTarget() )
+ .isSameAs( stackTrace );
+ }
+
+ public void testOnSkippedTest()
+ {
+ Exception stackTrace = new Exception( "test skip reason" );
+
+ ITestClass cls = mock( ITestClass.class );
+ when( cls.getName() ).thenReturn( getClass().getName() );
+
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ when( method.getCurrentInvocationCount() ).thenReturn( 1 );
+
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getThrowable() ).thenReturn( stackTrace );
+ when( testResult.getTestClass() ).thenReturn( cls );
+ when( testResult.getMethod() ).thenReturn( method );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getParameters() ).thenReturn( new String[] { "val1", "val2" } );
+
+ RunListener listener = mock( RunListener.class );
+ TestNGReporter reporter = new TestNGReporter( listener );
+ reporter.onTestSkipped( testResult );
+
+ ArgumentCaptor<SimpleReportEntry> reportEntry = ArgumentCaptor.forClass( SimpleReportEntry.class );
+ verify( listener ).testSkipped( reportEntry.capture() );
+ verifyNoMoreInteractions( listener );
+
+ assertThat( reportEntry.getValue().getSourceName() )
+ .isEqualTo( getClass().getName() );
+
+ assertThat( reportEntry.getValue().getName() )
+ .isEqualTo( "myTest[val1, val2](1)" );
+
+ assertThat( reportEntry.getValue().getMessage() )
+ .isEqualTo( stackTrace.getMessage() );
+ }
+
+ public void testOnTestFailedButWithinSuccessPercentage()
+ {
+ Exception stackTrace = new Exception();
+
+ ITestClass cls = mock( ITestClass.class );
+ when( cls.getName() ).thenReturn( getClass().getName() );
+
+ ITestNGMethod method = mock( ITestNGMethod.class );
+ when( method.getCurrentInvocationCount() ).thenReturn( 1 );
+ when( method.getMethodName() ).thenReturn( "myTest" );
+
+ ITestResult testResult = mock( ITestResult.class );
+ when( testResult.getThrowable() ).thenReturn( stackTrace );
+ when( cls.getRealClass() ).thenReturn( getClass() );
+ when( testResult.getTestClass() ).thenReturn( cls );
+ when( testResult.getMethod() ).thenReturn( method );
+ when( testResult.getName() ).thenReturn( "myTest" );
+ when( testResult.getParameters() ).thenReturn( new String[] { "val1", "val2" } );
+
+ RunListener listener = mock( RunListener.class );
+ TestNGReporter reporter = new TestNGReporter( listener );
+ reporter.onTestFailedButWithinSuccessPercentage( testResult );
+
+ ArgumentCaptor<SimpleReportEntry> reportEntry = ArgumentCaptor.forClass( SimpleReportEntry.class );
+ verify( listener ).testSucceeded( reportEntry.capture() );
+ verifyNoMoreInteractions( listener );
+
+ assertThat( reportEntry.getValue().getSourceName() )
+ .isEqualTo( getClass().getName() );
+
+ assertThat( reportEntry.getValue().getName() )
+ .isEqualTo( "myTest[val1, val2](1)" );
+
+ assertThat( reportEntry.getValue().getStackTraceWriter() )
+ .isNotNull();
+
+ assertThat( reportEntry.getValue().getStackTraceWriter().getThrowable().getTarget() )
+ .isSameAs( stackTrace );
+ }
+}
