Show tests on TC board that were stable and have become flaky (#159)
Signed-off-by: Ivan Rakov <ivan.glukos@gmail.com>
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java
index b80d67b..6be47bc 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java
@@ -114,6 +114,20 @@
return Strings.isNullOrEmpty(srvCode) ? ITcBotConfig.DEFAULT_SERVER_CODE : srvCode;
}
+ /** {@inheritDoc} */
+ @Override public Integer flakyRate() {
+ Integer flakyRate = getConfig().flakyRate();
+
+ return flakyRate == null || flakyRate < 0 || flakyRate > 100 ? ITcBotConfig.DEFAULT_FLAKY_RATE : flakyRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double confidence() {
+ Double confidence = getConfig().confidence();
+
+ return confidence == null || confidence < 0 || confidence > 1 ? ITcBotConfig.DEFAULT_CONFIDENCE : confidence;
+ }
+
@Override
public ITrackedBranchesConfig getTrackedBranches() {
return getConfig();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index e96a0be..fe1ef2b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -30,11 +30,13 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssueType;
import org.apache.ignite.ci.jobs.CheckQueueJob;
@@ -64,11 +66,14 @@
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.history.InvocationData;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch;
+import static org.apache.ignite.tcignited.history.RunStatus.RES_FAILURE;
+import static org.apache.ignite.tcignited.history.RunStatus.RES_OK;
/**
*
@@ -471,15 +476,49 @@
type = IssueType.newFailure;
final String flakyComments = runStat.getFlakyComments();
- if (!Strings.isNullOrEmpty(flakyComments)) {
- if (runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) == null) {
- logger.info("Skipping registering new issue for test fail:" +
- " Test seems to be flaky " + name + ": " + flakyComments);
-
- firstFailedBuildId = null;
- }
- else
+ if (!Strings.isNullOrEmpty(flakyComments) &&
+ runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) != null)
type = IssueType.newFailureForFlakyTest;
+ }
+ }
+
+ double flakyRate = 0;
+
+ if (firstFailedBuildId == null || type == null) {
+ List<Invocation> invocations = runStat.getInvocations().
+ filter(invocation -> invocation != null && invocation.status() != InvocationData.MISSING)
+ .collect(Collectors.toList());
+
+ int confidenceOkTestsRow = Math.max(1,
+ (int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - cfg.flakyRate() / 100.0)));
+
+ if (invocations.size() >= confidenceOkTestsRow * 2) {
+ List<Invocation> lastInvocations =
+ invocations.subList(invocations.size() - confidenceOkTestsRow * 2, invocations.size());
+
+ int stableTestRuns = 0;
+
+ for (int i = 0; i < confidenceOkTestsRow; i++) {
+ if (lastInvocations.get(i).status() == RES_OK.getCode())
+ stableTestRuns++;
+ else
+ break;
+ }
+
+ if (stableTestRuns == confidenceOkTestsRow) {
+ long failedTestRuns = 0;
+
+ for (int i = confidenceOkTestsRow; i < confidenceOkTestsRow * 2; i++) {
+ if (lastInvocations.get(i).status() == RES_FAILURE.getCode())
+ failedTestRuns++;
+ }
+
+ flakyRate = (double) failedTestRuns / confidenceOkTestsRow * 100;
+
+ if (flakyRate > cfg.flakyRate()) {
+ type = IssueType.newTestWithHighFlakyRate;
+ firstFailedBuildId = lastInvocations.get(confidenceOkTestsRow).buildId();
+ }
}
}
}
@@ -501,6 +540,7 @@
issue.trackedBranchName = trackedBranch;
issue.displayName = testFailure.testName;
issue.webUrl = testFailure.webUrl;
+ issue.flakyRate = flakyRate;
issue.buildTags.addAll(suiteTags);
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
index a8ae186..f6c1f7b 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
@@ -98,6 +98,14 @@
return ITcBotConfig.DEFAULT_SERVER_CODE;
}
+ @Override public Integer flakyRate() {
+ return DEFAULT_FLAKY_RATE;
+ }
+
+ @Override public Double confidence() {
+ return DEFAULT_CONFIDENCE;
+ }
+
@Override public ITrackedBranchesConfig getTrackedBranches() {
return tracked;
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java
index bc5e4c9..aeb07fc 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java
@@ -106,8 +106,8 @@
Map<String, String> buildWoChanges = new TreeMap<String, String>() {
{
- put("testFailedShoudlBeConsideredAsFlaky", "0000011111");
- put("testFlakyStableFailure", "0000011111111111");
+ put("testFailedShouldBeConsideredAsFlaky", "0000011111");
+ put("testFlakyStableFailure", "0000010101100101");
}
};
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java b/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java
index 6923122..d765130 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java
@@ -47,6 +47,8 @@
/** Display type. for issue. Kept for backward compatibilty with older records without type code. */
private String displayType;
+ public double flakyRate;
+
@Nullable
public String trackedBranchName;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java
index 4bfbba6..9042e1e 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java
@@ -19,6 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -43,6 +44,7 @@
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.SingleBuildRunCtx;
+import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.BlameCandidate;
import org.apache.ignite.tcbot.engine.defect.DefectCompacted;
import org.apache.ignite.tcbot.engine.defect.DefectFirstBuild;
@@ -63,6 +65,10 @@
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ITest;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
+import org.apache.ignite.tcignited.history.IRunHistory;
+
+import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING;
+import static org.apache.ignite.tcignited.history.RunStatus.RES_OK;
public class BoardService {
@Inject IIssuesStorage issuesStorage;
@@ -74,6 +80,7 @@
@Inject IStringCompactor compactor;
@Inject BuildChainProcessor buildChainProcessor;
@Inject IUserStorage userStorage;
+ @Inject ITcBotConfig cfg;
/**
* @param creds Credentials.
@@ -118,7 +125,7 @@
rebuild = !freshRebuild.isEmpty() ? freshRebuild.stream().findFirst() : Optional.empty();
for (DefectIssue issue : cause.issues()) {
- BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue);
+ BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue, firstBuild.buildTypeId());
defectUi.addIssue(issueUi);
}
@@ -134,7 +141,7 @@
public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn,
Optional<FatBuildCompacted> rebuild,
- DefectIssue issue) {
+ DefectIssue issue, int projectId) {
Optional<ITest> testResult;
String issueType = compactor.getStringFromId(issue.issueTypeCode());
@@ -176,15 +183,47 @@
if (test.isIgnoredTest() || test.isMutedTest())
status = IssueResolveStatus.IGNORED;
+ else if (IssueType.newTestWithHighFlakyRate.code().equals(issueType)) {
+ int fullSuiteNameAndFullTestName = issue.testNameCid();
+
+ int branchName = rebuild.get().branchName();
+
+ IRunHistory runStat = tcIgn.getTestRunHist(fullSuiteNameAndFullTestName, projectId, branchName);
+
+ if (runStat == null)
+ status = IssueResolveStatus.UNKNOWN;
+ else {
+ List<Integer> runResults = runStat.getLatestRunResults();
+ if (runResults == null)
+ status = IssueResolveStatus.UNKNOWN;
+ else {
+ int confidenceOkTestsRow = Math.max(1,
+ (int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - issue.getFlakyRate() / 100.0)));
+ Collections.reverse(runResults);
+ int okTestRow = 0;
+
+ for (Integer run : runResults) {
+ if (run == null || run == RES_MISSING.getCode())
+ continue;
+ if (run == RES_OK.getCode() && (okTestRow < confidenceOkTestsRow))
+ okTestRow++;
+ else
+ break;
+ }
+
+ status = okTestRow >= confidenceOkTestsRow ? IssueResolveStatus.FIXED : IssueResolveStatus.FAILING;
+ }
+ }
+ }
else
status = test.isFailedTest(compactor) ? IssueResolveStatus.FAILING : IssueResolveStatus.FIXED;
FatBuildCompacted fatBuildCompacted = rebuild.get();
Long testNameId = test.getTestId();
- String projectId = fatBuildCompacted.projectId(compactor);
+ String RebuildProjectId = fatBuildCompacted.projectId(compactor);
String branchName = fatBuildCompacted.branchName(compactor);
- webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, projectId, branchName);
+ webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, RebuildProjectId, branchName);
}
else {
//exception for new test. removal of test means test is fixed
@@ -244,6 +283,7 @@
int issueTypeCid = compactor.getStringId(issue.type);
Integer testNameCid = compactor.getStringIdIfPresent(testName);
int trackedBranchCid = compactor.getStringId(issue.trackedBranchName);
+ double flakyRate = issue.flakyRate;
int tcSrvCodeCid = compactor.getStringId(srvCode);
defectStorage.merge(tcSrvCodeCid, srvId, fatBuild,
@@ -252,7 +292,7 @@
defect.trackedBranchCidSetIfEmpty(trackedBranchCid);
- defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid);
+ defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid, flakyRate);
defect.removeOldVerBlameCandidates();
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java
index 3c9af9a..1d38a21 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java
@@ -30,9 +30,21 @@
/** Default server code. */
public String DEFAULT_SERVER_CODE = "apache";
+ /** Default flaky rate. */
+ public Integer DEFAULT_FLAKY_RATE = 20;
+
+ /** Default confidence. */
+ public Double DEFAULT_CONFIDENCE = 0.95;
+
/** */
public String primaryServerCode();
+ /** */
+ public Integer flakyRate();
+
+ /** */
+ public Double confidence();
+
/**
* @return Tracked branches configuration for TC Bot.
*/
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java
index 818ba5a..0fd0f24 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java
@@ -38,6 +38,12 @@
/** Primary server ID. */
@Nullable private String primaryServerCode;
+ /** Flaky rate to consider test as a flaky test. */
+ @Nullable private Integer flakyRate;
+
+ /** Сonfidence (used with flaky tests). */
+ @Nullable private Double confidence;
+
/** Additional list Servers to be used for validation of PRs, but not for tracking any branches. */
private List<TcServerConfig> tcServers = new ArrayList<>();
@@ -83,6 +89,20 @@
return primaryServerCode;
}
+ /**
+ * @return Flaky rate to consider test as a flaky test.
+ */
+ @Nullable public Integer flakyRate() {
+ return flakyRate;
+ }
+
+ /**
+ * @return Сonfidence.
+ */
+ @Nullable public Double confidence() {
+ return confidence;
+ }
+
public Optional<TcServerConfig> getTcConfig(String code) {
return tcServers.stream().filter(s -> Objects.equals(code, s.getCode())).findAny();
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java
index fd271ce..568296f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java
@@ -33,8 +33,8 @@
this.build = build;
}
- public DefectFirstBuild addIssue(int typeCid, Integer testNameCid) {
- issues.add(new DefectIssue(typeCid, testNameCid));
+ public DefectFirstBuild addIssue(int typeCid, Integer testNameCid, double flakyRate) {
+ issues.add(new DefectIssue(typeCid, testNameCid, flakyRate));
return this;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java
index 5ee60c7..726332f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java
@@ -24,9 +24,12 @@
private int issueTypeCode;
private int testOrSuiteName;
- public DefectIssue(int issueTypeCode, Integer testNameCid) {
+ private double flakyRate;
+
+ public DefectIssue(int issueTypeCode, Integer testNameCid, double flakyRate) {
this.issueTypeCode = issueTypeCode;
testOrSuiteName = testNameCid;
+ this.flakyRate = flakyRate;
}
/** {@inheritDoc} */
@@ -52,4 +55,8 @@
public int issueTypeCode() {
return issueTypeCode;
}
+
+ public double getFlakyRate() {
+ return flakyRate;
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java
index b979475..415f481 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java
@@ -27,9 +27,11 @@
import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING;
public class EventTemplates {
- private static final int OK = RES_OK.getCode();
- private static final int FAIL = RES_FAILURE.getCode();
- private static final int MISSING = RES_MISSING.getCode();
+ public static final int OK = RES_OK.getCode();
+ public static final int FAIL = RES_FAILURE.getCode();
+ public static final int MISSING = RES_MISSING.getCode();
+ public static final int OK_OR_FAILURE = RES_OK_OR_FAILURE.getCode();
+ public static final int CRITICAL_FAILURE = RES_CRITICAL_FAILURE.getCode();
public static final EventTemplate newFailure = new EventTemplate(
new int[]{OK, OK, OK, OK, OK},
@@ -37,8 +39,8 @@
);
public static final EventTemplate newCriticalFailure = new EventTemplate(
- new int[]{RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode()},
- new int[]{RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode()}
+ new int[]{OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE},
+ new int[]{CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE}
);
public static final EventTemplate newContributedTestFailure = new EventTemplate(
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java
index b7b6cce..25171c4 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java
@@ -34,7 +34,10 @@
newCriticalFailure("newCriticalFailure", "New Critical Failure"),
/** New trusted suite failure. */
- newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure");
+ newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure"),
+
+ /** New failure for flaky test. */
+ newTestWithHighFlakyRate("newTestWithHighFlakyRate", "Test with high flaky rate");
/** Code. */
private final String code;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index ac11fb0..92088e6 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -544,6 +544,10 @@
return compactor.getStringFromId(projectId);
}
+ public int projectId() {
+ return projectId;
+ }
+
public List<ProblemOccurrence> problems(IStringCompactor compactor) {
if (this.problems == null)
return Collections.emptyList();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java
index b54bc53..ff60b0a 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java
@@ -16,18 +16,22 @@
*/
package org.apache.ignite.tcignited.history;
+import java.util.stream.Stream;
import javax.annotation.Nullable;
import java.util.List;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
/**
* Test or Build run statistics.
*/
public interface IRunHistory {
- /**
- *
- */
+
public boolean isFlaky();
+ public Iterable<Invocation> invocations();
+
+ public Stream<Invocation> getInvocations();
+
@Nullable
List<Integer> getLatestRunResults();
