differenciate between errors and failures for junitlauncher
diff --git a/WHATSNEW b/WHATSNEW
index 3cba7a7..d8279a3 100644
--- a/WHATSNEW
+++ b/WHATSNEW
@@ -32,6 +32,10 @@
path even if it was not a file separator.
Bugzilla Report 69680
+ * <junitlauncher>'s legacy formatters now separate errors from
+ failures like their <junit> counterparts did.
+ Bugzilla Report 69687
+
Other changes:
--------------
diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
index be70e94..9376882 100644
--- a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
+++ b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
@@ -19,6 +19,7 @@
import org.apache.tools.ant.Project;
import org.apache.tools.ant.util.FileUtils;
+import org.junit.platform.engine.TestExecutionResult;
import org.junit.platform.engine.TestSource;
import org.junit.platform.engine.support.descriptor.ClassSource;
import org.junit.platform.launcher.TestIdentifier;
@@ -187,6 +188,10 @@
}
+ protected static boolean isFailure(final TestExecutionResult executionResult) {
+ return executionResult.getThrowable().orElse(null) instanceof AssertionError;
+ }
+
/*
A "store" for sysout/syserr content that gets sent to the AbstractJUnitResultFormatter.
This store first uses a relatively decent sized in-memory buffer for storing the sysout/syserr
diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
index 7583d78..5f1f591 100644
--- a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
+++ b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
@@ -65,6 +65,7 @@
final Stats stats = entry.getValue();
final StringBuilder sb = new StringBuilder("Tests run: ").append(stats.numTestsRun.get());
sb.append(", Failures: ").append(stats.numTestsFailed.get());
+ sb.append(", Errors: ").append(stats.numTestsWithError.get());
sb.append(", Skipped: ").append(stats.numTestsSkipped.get());
sb.append(", Aborted: ").append(stats.numTestsAborted.get());
sb.append(", Time elapsed: ");
@@ -188,7 +189,11 @@
break;
}
case FAILED: {
- sb.append(" FAILED");
+ if (isFailure(testExecutionResult)) {
+ sb.append(" FAILED");
+ } else {
+ sb.append(" Caused an ERROR");
+ }
appendThrowable(sb, testExecutionResult);
break;
}
@@ -214,7 +219,11 @@
break;
}
case FAILED: {
- parentClassStats.numTestsFailed.incrementAndGet();
+ if (isFailure(testExecutionResult)) {
+ parentClassStats.numTestsFailed.incrementAndGet();
+ } else {
+ parentClassStats.numTestsWithError.incrementAndGet();
+ }
break;
}
}
@@ -264,6 +273,7 @@
private final TestIdentifier testIdentifier;
private final AtomicLong numTestsRun = new AtomicLong(0);
private final AtomicLong numTestsFailed = new AtomicLong(0);
+ private final AtomicLong numTestsWithError = new AtomicLong(0);
private final AtomicLong numTestsSkipped = new AtomicLong(0);
private final AtomicLong numTestsAborted = new AtomicLong(0);
private final long startedAt;
diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
index f72822d..ca3f7f4 100644
--- a/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
+++ b/src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
@@ -55,6 +55,7 @@
private final Map<TestIdentifier, Stats> testIds = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<String>> skipped = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> failed = new ConcurrentHashMap<>();
+ private final Map<TestIdentifier, Optional<Throwable>> errored = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> aborted = new ConcurrentHashMap<>();
private TestPlan testPlan;
@@ -62,6 +63,7 @@
private long testPlanEndedAt = -1;
private final AtomicLong numTestsRun = new AtomicLong(0);
private final AtomicLong numTestsFailed = new AtomicLong(0);
+ private final AtomicLong numTestsErrored = new AtomicLong(0);
private final AtomicLong numTestsSkipped = new AtomicLong(0);
private final AtomicLong numTestsAborted = new AtomicLong(0);
private boolean useLegacyReportingName = true;
@@ -126,8 +128,13 @@
break;
}
case FAILED: {
- this.numTestsFailed.incrementAndGet();
- this.failed.put(testIdentifier, testExecutionResult.getThrowable());
+ if (isFailure(testExecutionResult)) {
+ this.numTestsFailed.incrementAndGet();
+ this.failed.put(testIdentifier, testExecutionResult.getThrowable());
+ } else {
+ this.numTestsErrored.incrementAndGet();
+ this.errored.put(testIdentifier, testExecutionResult.getThrowable());
+ }
break;
}
}
@@ -168,6 +175,7 @@
private static final String ELEM_TESTCASE = "testcase";
private static final String ELEM_SKIPPED = "skipped";
private static final String ELEM_FAILURE = "failure";
+ private static final String ELEM_ERROR = "error";
private static final String ELEM_ABORTED = "aborted";
private static final String ELEM_SYSTEM_OUT = "system-out";
private static final String ELEM_SYSTEM_ERR = "system-err";
@@ -180,6 +188,7 @@
private static final String ATTR_TIMESTAMP = "timestamp";
private static final String ATTR_NUM_ABORTED = "aborted";
private static final String ATTR_NUM_FAILURES = "failures";
+ private static final String ATTR_NUM_ERRORS = "errors";
private static final String ATTR_NUM_TESTS = "tests";
private static final String ATTR_NUM_SKIPPED = "skipped";
private static final String ATTR_MESSAGE = "message";
@@ -208,6 +217,7 @@
writeAttribute(writer, ATTR_TIMESTAMP, timestamp);
writeAttribute(writer, ATTR_NUM_TESTS, String.valueOf(numTestsRun.longValue()));
writeAttribute(writer, ATTR_NUM_FAILURES, String.valueOf(numTestsFailed.longValue()));
+ writeAttribute(writer, ATTR_NUM_ERRORS, String.valueOf(numTestsErrored.longValue()));
writeAttribute(writer, ATTR_NUM_SKIPPED, String.valueOf(numTestsSkipped.longValue()));
writeAttribute(writer, ATTR_NUM_ABORTED, String.valueOf(numTestsAborted.longValue()));
@@ -239,7 +249,7 @@
void writeTestCase(final XMLStreamWriter writer) throws XMLStreamException {
for (final Map.Entry<TestIdentifier, Stats> entry : testIds.entrySet()) {
final TestIdentifier testId = entry.getKey();
- if (!testId.isTest() && !failed.containsKey(testId)) {
+ if (!testId.isTest() && !failed.containsKey(testId) && !errored.containsKey(testId)) {
// only interested in test methods unless there was a failure,
// in which case we want the exception reported
// (https://bz.apache.org/bugzilla/show_bug.cgi?id=63850)
@@ -283,6 +293,8 @@
writeSkipped(writer, testId);
// failed element if the test failed
writeFailed(writer, testId);
+ // error element if the test caused an error
+ writeErrored(writer, testId);
// aborted element if the test was aborted
writeAborted(writer, testId);
@@ -306,8 +318,21 @@
if (!failed.containsKey(testIdentifier)) {
return;
}
- writer.writeStartElement(ELEM_FAILURE);
- final Optional<Throwable> cause = failed.get(testIdentifier);
+ writeFailedOrErrored(writer, ELEM_FAILURE, failed.get(testIdentifier));
+ }
+
+ private void writeErrored(final XMLStreamWriter writer, final TestIdentifier testIdentifier) throws XMLStreamException {
+ if (!errored.containsKey(testIdentifier)) {
+ return;
+ }
+ writeFailedOrErrored(writer, ELEM_ERROR, errored.get(testIdentifier));
+ }
+
+ private void writeFailedOrErrored(final XMLStreamWriter writer,
+ final String elementName,
+ final Optional<Throwable> cause)
+ throws XMLStreamException {
+ writer.writeStartElement(elementName);
if (cause.isPresent()) {
final Throwable t = cause.get();
final String message = t.getMessage();
diff --git a/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java b/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
index 3735de8..cd42209 100644
--- a/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
+++ b/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
@@ -20,18 +20,36 @@
import org.apache.tools.ant.Project;
import org.junit.Test;
import org.junit.platform.engine.ConfigurationParameters;
+import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestExecutionResult;
+import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
+import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.TestPlan;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.function.Function;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.sax.SAXSource;
+import javax.xml.transform.stream.StreamSource;
import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertThat;
public class LegacyXmlResultFormatterTest {
@@ -57,6 +75,56 @@
assertThat(result, containsString(ENCODED));
}
+ @Test
+ public void properlyReportsFailures() throws Exception {
+ properlyReportsFailuresAndErrors(new AssertionError("failed", null), true);
+ }
+
+ @Test
+ public void properlyReportsErrors() throws Exception {
+ properlyReportsFailuresAndErrors(new NullPointerException("failed"), false);
+ }
+
+ private void properlyReportsFailuresAndErrors(Throwable errorOrFailure,
+ boolean shouldBeFailure)
+ throws Exception {
+
+ final TestPlan plan = startTest(false);
+ final TestDescriptor testDescriptor = new DummyTestDescriptor("failure");
+ final TestIdentifier test = TestIdentifier.from(testDescriptor);
+ f.executionStarted(test);
+ final TestExecutionResult testResult = TestExecutionResult.failed(errorOrFailure);
+ f.executionFinished(test, testResult);
+ final String result = finishTest(plan);
+
+ final int expectedFailureCount = shouldBeFailure ? 1 : 0;
+ final int expectedErrorCount = shouldBeFailure ? 0 : 1;
+
+ final StreamSource source = new StreamSource() {
+ @Override
+ public Reader getReader() {
+ return new StringReader(result);
+ }
+ };
+ final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ final InputSource is = SAXSource.sourceToInputSource(source);
+ final DocumentBuilder b = dbf.newDocumentBuilder();
+ final Document doc = b.parse(is);
+ final Element suite = doc.getDocumentElement();
+ assertThat(suite.getTagName(), equalTo("testsuite"));
+ assertThat(suite.getAttribute("failures"), equalTo(expectedFailureCount + ""));
+ assertThat(suite.getAttribute("errors"), equalTo(expectedErrorCount + ""));
+ final NodeList testCases = suite.getElementsByTagName("testcase");
+ assertThat(testCases.getLength(), equalTo(1));
+ final Node testCase = testCases.item(0);
+ assertThat(testCase, instanceOf(Element.class));
+ final Element testCaseElement = (Element) testCase;
+ NodeList failureElements = testCaseElement.getElementsByTagName("failure");
+ assertThat(failureElements.getLength(), equalTo(expectedFailureCount));
+ NodeList errorElements = testCaseElement.getElementsByTagName("error");
+ assertThat(errorElements.getLength(), equalTo(expectedErrorCount));
+ }
+
private TestPlan startTest(final boolean withProperties) {
f.setContext(new TestExecutionContext() {
@Override
@@ -111,4 +179,15 @@
return new String(bos.toByteArray(), StandardCharsets.UTF_8);
}
}
+
+ private static class DummyTestDescriptor extends AbstractTestDescriptor {
+ private DummyTestDescriptor(String displayName) {
+ super(UniqueId.forEngine("testengine"), displayName);
+ }
+
+ @Override
+ public TestDescriptor.Type getType() {
+ return TestDescriptor.Type.TEST;
+ }
+ }
}
