package org.apache.maven.surefire.its;

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.maven.surefire.its.fixture.OutputValidator;
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
import org.apache.maven.surefire.its.fixture.SurefireLauncher;
import org.junit.Before;
import org.junit.Test;

import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion;

/**
 * JUnit4 RunListener Integration Test.
 *
 * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
 * @author Matt Coley
 */
@SuppressWarnings( "checkstyle:magicnumber" )
public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationTestCase
{
    private static final String VERSION = "5.5.1";

    private SurefireLauncher unpack()
    {
        return unpack( "/junit-platform-rerun-failing-tests" );
    }

    @Before
    public void setup()
    {
        assumeJavaVersion( 1.8 );
    }

    @Test
    public void testRerunFailingErrorTestsWithOneRetry()
    {
        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" )
                .withFailure()
                .debugLogging()
                .executeTest()
                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
        verifyFailuresOneRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
                "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
        verifyFailuresOneRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
        verifyFailuresOneRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
        verifyFailuresOneRetryAllClasses( outputValidator );
    }

    @Test
    public void testRerunFailingErrorTestsTwoRetry()
    {
        // Four flakes, both tests have been re-run twice
        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );

        verifyFailuresTwoRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest().assertTestSuiteResults(
                5, 0, 0, 0, 4 );

        verifyFailuresTwoRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );

        verifyFailuresTwoRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );

        verifyFailuresTwoRetryAllClasses( outputValidator );
    }

    @Test
    public void testRerunFailingErrorTestsFalse()
    {
        OutputValidator outputValidator = unpack().setJUnitVersion(
                VERSION ).maven().withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );

        verifyFailuresNoRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-DforkCount=3" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );

        verifyFailuresNoRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal( "-Dparallel=methods" ).addGoal(
                "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1,
                1, 0, 0 );

        verifyFailuresNoRetryAllClasses( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal( "-Dparallel=classes" ).addGoal(
                "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1,
                1, 0, 0 );

        verifyFailuresNoRetryAllClasses( outputValidator );
    }

    @Test
    public void testRerunOneTestClass()
    {
        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );

        verifyFailuresOneRetryOneClass( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 3, 1,
                1, 0, 0 );

        verifyFailuresOneRetryOneClass( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
                "-DuseUnlimitedThreads=true" ).addGoal(
                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 3, 1,
                1, 0, 0 );

        verifyFailuresOneRetryOneClass( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
                "-DuseUnlimitedThreads=true" ).addGoal(
                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 3, 1,
                1, 0, 0 );

        verifyFailuresOneRetryOneClass( outputValidator );
    }

    @Test
    public void testRerunOneTestMethod()
    {
        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
                .assertTestSuiteResults( 1, 0, 1, 0, 0 );

        verifyFailuresOneRetryOneMethod( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
                .assertTestSuiteResults( 1, 0, 1, 0, 0 );

        verifyFailuresOneRetryOneMethod( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
                "-DuseUnlimitedThreads=true" ).addGoal(
                "-Dtest=FlakyFirstTimeTest#testFailing*" )
                .withFailure()
                .debugLogging()
                .executeTest()
                .assertTestSuiteResults( 1, 0, 1, 0, 0 );

        verifyFailuresOneRetryOneMethod( outputValidator );

        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
                "-DuseUnlimitedThreads=true" ).addGoal(
                "-Dtest=FlakyFirstTimeTest#testFailing*" )
                .withFailure()
                .debugLogging()
                .executeTest()
                .assertTestSuiteResults( 1, 0, 1, 0, 0 );

        verifyFailuresOneRetryOneMethod( outputValidator );
    }

    private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
    {
        verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
    }

    private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
    {
        verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
    }

    private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
    {
        verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
    }

    private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
    {
        verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
    }

    private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
    {
        verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
    }

    private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
                                         int flakes )
    {
        outputValidator.verifyTextInLog( "Failures:" );
        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );

        outputValidator.verifyTextInLog( "Errors:" );
        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );

        verifyStatistics( outputValidator, run, failures, errors, flakes );
    }

    private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
                                             int flakes )
    {
        outputValidator.verifyTextInLog( "Failures:" );
        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );

        verifyStatistics( outputValidator, run, failures, errors, flakes );
    }

    private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors,
                                         int flakes )
    {
        outputValidator.verifyTextInLog( "Flakes:" );
        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
        outputValidator.verifyTextInLog( "Run 3: PASS" );

        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );

        verifyStatistics( outputValidator, run, failures, errors, flakes );
    }

    private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
    {
        outputValidator.verifyTextInLog( "Failures:" );
        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testFailingTestOne" );
        outputValidator.verifyTextInLog( "ERROR" );
        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testErrorTestOne" );

        verifyStatistics( outputValidator, run, failures, errors, flakes );
    }

    private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
    {
        if ( flakes > 0 )
        {
            outputValidator.verifyTextInLog(
                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: "
                            + flakes );
        }
        else
        {
            outputValidator.verifyTextInLog(
                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
        }
    }
}