commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/ResultsCommand.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.commons.rng.examples.stress;

import org.apache.commons.rng.simple.RandomSource;

import picocli.CommandLine.Command;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Formatter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * Specification for the "results" command.
 *
 * <p>This command creates a summary of the results from known test applications.</p>
 *
 * <ul>
 *   <li>Dieharder
 *   <li>Test U01 (BigCrush, Crush, SmallCrush)
 * </ul>
 */
@Command(name = "results",
         description = {"Collate results from stress test applications."})
class ResultsCommand implements Callable<Void> {
    /** The pattern to identify the RandomSource in the stress test result header. */
    private static final Pattern RANDOM_SOURCE_PATTERN = Pattern.compile("^# RandomSource: (.*)");
    /** The pattern to identify the RNG in the stress test result header. */
    private static final Pattern RNG_PATTERN = Pattern.compile("^# RNG: (.*)");
    /** The pattern to identify the test duration in the stress test result footer. */
    private static final Pattern TEST_DURATION_PATTERN = Pattern.compile("^# Test duration:");
    /** The pattern to identify the Dieharder test format. */
    private static final Pattern DIEHARDER_PATTERN = Pattern.compile("^# *dieharder version");
    /** The pattern to identify a Dieharder failed test result. */
    private static final Pattern DIEHARDER_FAILED_PATTERN = Pattern.compile("FAILED *$");
    /** The pattern to identify the Test U01 test format. */
    private static final Pattern TESTU01_PATTERN = Pattern.compile("^ *Version: TestU01");
    /** The pattern to identify the Test U01 summary header. */
    private static final Pattern TESTU01_SUMMARY_PATTERN = Pattern.compile("^========= Summary results of (\\S*) ");
    /** The pattern to identify the Test U01 failed test result. */
    private static final Pattern TESTU01_TEST_RESULT_PATTERN = Pattern.compile("^  ?(\\d+  .*)    ");
    /** The pattern to identify the Test U01 test starting entry. */
    private static final Pattern TESTU01_STARTING_PATTERN = Pattern.compile("^ *Starting (\\S*)");
    /** The name of the Dieharder sums test. */
    private static final String DIEHARDER_SUMS = "diehard_sums";
    /** The string identifying a bit-reversed generator. */
    private static final String BIT_REVERSED = "Bit-reversed";
    /** Character '\'. */
    private static final char FORWARD_SLASH = '\\';
    /** Character '/'. */
    private static final char BACK_SLASH = '\\';
    /** Character '|'. */
    private static final char PIPE = '|';
    /** The column name for the RNG. */
    private static final String COLUMN_RNG = "RNG";

    /** The standard options. */
    @Mixin
    private StandardOptions reusableOptions;

    /** The results files. */
    @Parameters(arity = "1..*",
                description = "The results files.",
                paramLabel = "<file>")
    private List<File> resultsFiles = new ArrayList<>();

    /** The file output prefix. */
    @Option(names = {"-o", "--out"},
            description = "The output file (default: stdout).")
    private File fileOutput;

    /** The output format. */
    @Option(names = {"-f", "--format"},
            description = {"Output format (default: ${DEFAULT-VALUE}).",
                           "Valid values: ${COMPLETION-CANDIDATES}."})
    private ResultsCommand.OutputFormat outputFormat = OutputFormat.TXT;

    /** The flag to show failed tests. */
    @Option(names = {"--failed"},
            description = {"Output failed tests (support varies by format).",
                           "CSV: List individual test failures.",
                           "APT: Count of systematic test failures."})
    private boolean showFailedTests;

    /** The flag to include the Dieharder sums test. */
    @Option(names = {"--include-sums"},
            description = "Include Dieharder sums test.")
    private boolean includeDiehardSums;

    /** The common file path prefix used when outputting file paths. */
    @Option(names = {"--path-prefix"},
            description = {"Common path prefix.",
                           "If specified this will replace the common prefix from all " +
                           "files when the path is output, e.g. for the APT report."})
    private String pathPrefix = "";

    /** The flag to include the Dieharder sums test. */
    @Option(names = {"-i", "--ignore"},
            description = "Ignore partial results.")
    private boolean ignorePartialResults;

    /**
     * The output mode for the results.
     */
    enum OutputFormat {
        /** Comma-Separated Values (CSV) output. */
        CSV,
        /** Almost Plain Text (APT) output. */
        APT,
        /** Text table output. */
        TXT,
        /** Systematic failures output. */
        FAILURES,
    }

    /**
     * The test application file format.
     */
    enum TestFormat {
        /** Dieharder. */
        DIEHARDER,
        /** Test U01. */
        TESTU01,
    }

    /**
     * Encapsulate the results of a test application.
     */
    private static class TestResult {
        /** The result file. */
        private final File resultFile;
        /** The random source. */
        private final RandomSource randomSource;
        /** Flag indicating the generator was bit reversed. */
        private final boolean bitReversed;
        /** The test application format. */
        private final TestFormat testFormat;
        /** The names of the failed tests. */
        private final List<String> failedTests = new ArrayList<>();
        /** The test application name. */
        private String testApplicationName;
        /** Flag to indicate results are complete (i.e. not still in progress). */
        private boolean complete;

        /**
         * @param resultFile the result file
         * @param randomSource the random source
         * @param bitReversed the bit reversed flag
         * @param testFormat the test format
         */
        TestResult(File resultFile,
                   RandomSource randomSource,
                   boolean bitReversed,
                   TestFormat testFormat) {
            this.resultFile = resultFile;
            this.randomSource = randomSource;
            this.bitReversed = bitReversed;
            this.testFormat = testFormat;
        }

        /**
         * Adds the failed test.
         *
         * @param testId the test id
         */
        void addFailedTest(String testId) {
            failedTests.add(testId);
        }

        /**
         * Gets the result file.
         *
         * @return the result file
         */
        File getResultFile() {
            return resultFile;
        }

        /**
         * Gets the random source.
         *
         * @return the random source
         */
        RandomSource getRandomSource() {
            return randomSource;
        }

        /**
         * Checks if the generator was bit reversed.
         *
         * @return true if bit reversed
         */
        boolean isBitReversed() {
            return bitReversed;
        }

        /**
         * Gets the test format.
         *
         * @return the test format
         */
        TestFormat getTestFormat() {
            return testFormat;
        }

        /**
         * Gets the failed tests.
         *
         * @return the failed tests
         */
        List<String> getFailedTests() {
            return failedTests;
        }

        /**
         * Gets the failure count.
         *
         * @return the failure count
         */
        int getFailureCount() {
            return failedTests.size();
        }

        /**
         * Gets the failure count as a string. This will be negative if the test is not complete.
         *
         * @return the failure count
         */
        String getFailureCountString() {
            return (isComplete()) ? Integer.toString(failedTests.size()) : "-" + failedTests.size();
        }

        /**
         * Sets the test application name.
         *
         * @param testApplicationName the new test application name
         */
        void setTestApplicationName(String testApplicationName) {
            this.testApplicationName = testApplicationName;
        }

        /**
         * Gets the test application name.
         *
         * @return the test application name
         */
        String getTestApplicationName() {
            return testApplicationName == null ? getTestFormat().toString() : testApplicationName;
        }

        /**
         * Checks if the test result is complete.
         *
         * @return true if complete
         */
        boolean isComplete() {
            return complete;
        }

        /**
         * Sets the complete flag.
         *
         * @param complete the new complete
         */
        void setComplete(boolean complete) {
            this.complete = complete;
        }
    }

    /**
     * Reads the results files and outputs in the chosen format.
     */
    @Override
    public Void call() {
        LogUtils.setLogLevel(reusableOptions.logLevel);

        // Read the results
        final List<TestResult> results = readResults();

        try (OutputStream out = createOutputStream()) {
            switch (outputFormat) {
            case CSV:
                writeCSVData(out, results);
                break;
            case APT:
                writeAPT(out, results);
                break;
            case TXT:
                writeTXT(out, results);
                break;
            case FAILURES:
                writeFailures(out, results);
                break;
            default:
                throw new ApplicationException("Unknown output format: " + outputFormat);
            }
        } catch (final IOException ex) {
            throw new ApplicationException("IO error: " + ex.getMessage(), ex);
        }
        return null;
    }

    /**
     * Read the results.
     *
     * @return the results
     */
    private List<TestResult> readResults() {
        final ArrayList<TestResult> results = new ArrayList<>();
        for (final File resultFile : resultsFiles) {
            readResults(results, resultFile);
        }
        return results;
    }

    /**
     * Read the file and extract any test results.
     *
     * @param results Results.
     * @param resultFile Result file.
     */
    private void readResults(List<TestResult> results,
                             File resultFile) {
        final List<String> contents = readFileContents(resultFile);
        // Files may have multiple test results per file (i.e. appended output)
        final List<List<String>> outputs = splitContents(contents);
        if (outputs.isEmpty()) {
            LogUtils.error("No test output in file: %s", resultFile);
        } else {
            for (final List<String> testOutput : outputs) {
                results.add(readResult(resultFile, testOutput));
            }
        }
    }

    /**
     * Read the file contents.
     *
     * @param resultFile Result file.
     * @return the file contents
     * @throws ApplicationException If the file cannot be read.
     */
    private static List<String> readFileContents(File resultFile) {
        final ArrayList<String> contents = new ArrayList<>();
        try (BufferedReader reader = Files.newBufferedReader(resultFile.toPath())) {
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                contents.add(line);
            }
        } catch (final IOException ex) {
            throw new ApplicationException("Failed to read file contents: " + resultFile, ex);
        }
        return contents;
    }

    /**
     * Split the file contents into separate test output. This is used in the event
     * that results have been appended to the same file.
     *
     * @param contents File contents.
     * @return the test output
     */
    private static List<List<String>> splitContents(List<String> contents) {
        final ArrayList<List<String>> testOutputs = new ArrayList<>();
        // Assume each output begins with e.g. # RandomSource: SPLIT_MIX_64
        // Note each beginning.
        int begin = -1;
        for (int i = 0; i < contents.size(); i++) {
            if (RANDOM_SOURCE_PATTERN.matcher(contents.get(i)).matches()) {
                if (begin >= 0) {
                    testOutputs.add(contents.subList(begin, i));
                }
                begin = i;
            }
        }
        if (begin >= 0) {
            testOutputs.add(contents.subList(begin, contents.size()));
        }
        return testOutputs;
    }

    /**
     * Read the file into a test result.
     *
     * @param resultFile Result file.
     * @param testOutput Test output.
     * @return the test result
     */
    private TestResult readResult(File resultFile,
                                  List<String> testOutput) {
        // Use an iterator for a single pass over the test output
        final ListIterator<String> iter = testOutput.listIterator();

        // Identify the RandomSource and bit reversed flag from the header:
        final RandomSource randomSource = getRandomSource(iter);
        final boolean bitReversed = getBitReversed(iter);
        // Identify the test application format
        final TestFormat testFormat = getTestFormat(iter);

        // Read the application results
        final TestResult testResult = new TestResult(resultFile, randomSource, bitReversed, testFormat);
        if (testFormat == TestFormat.DIEHARDER) {
            readDieharder(iter, testResult);
        } else {
            readTestU01(iter, testResult);
        }
        return testResult;
    }

    /**
     * Gets the random source from the output header.
     *
     * @param iter Iterator of the test output.
     * @return the random source
     */
    private static RandomSource getRandomSource(Iterator<String> iter) {
        while (iter.hasNext()) {
            final Matcher matcher = RANDOM_SOURCE_PATTERN.matcher(iter.next());
            if (matcher.matches()) {
                return RandomSource.valueOf(matcher.group(1));
            }
        }
        throw new ApplicationException("Failed to find RandomSource header line");
    }

    /**
     * Gets the bit-reversed flag from the output header.
     *
     * @param iter Iterator of the test output.
     * @return the bit-reversed flag
     */
    private static boolean getBitReversed(Iterator<String> iter) {
        while (iter.hasNext()) {
            final Matcher matcher = RNG_PATTERN.matcher(iter.next());
            if (matcher.matches()) {
                return matcher.group(1).contains(BIT_REVERSED);
            }
        }
        throw new ApplicationException("Failed to find RNG header line");
    }

    /**
     * Gets the test format from the output.
     *
     * @param iter Iterator of the test output.
     * @return the test format
     */
    private static TestFormat getTestFormat(Iterator<String> iter) {
        while (iter.hasNext()) {
            final String line = iter.next();
            if (DIEHARDER_PATTERN.matcher(line).find()) {
                return TestFormat.DIEHARDER;
            }
            if (TESTU01_PATTERN.matcher(line).find()) {
                return TestFormat.TESTU01;
            }
        }
        throw new ApplicationException("Failed to identify the test application format");
    }

    /**
     * Read the result output from the Dieharder test application.
     *
     * @param iter Iterator of the test output.
     * @param testResult Test result.
     */
    private void readDieharder(Iterator<String> iter,
                               TestResult testResult) {
        // Dieharder results are printed in-line using the following format:
        //#=============================================================================#
        //        test_name   |ntup| tsamples |psamples|  p-value |Assessment
        //#=============================================================================#
        //   diehard_birthdays|   0|       100|     100|0.97484422|  PASSED
        //   ...
        //        diehard_oqso|   0|   2097152|     100|0.00000000|  FAILED

        testResult.setTestApplicationName("Dieharder");

        // Identify any line containing FAILED and then get the test ID using
        // the first two columns (test_name + ntup).
        while (iter.hasNext()) {
            final String line = iter.next();
            if (DIEHARDER_FAILED_PATTERN.matcher(line).find()) {
                // Optionally include the flawed Dieharder sums test
                if (!includeDiehardSums && line.contains(DIEHARDER_SUMS)) {
                    continue;
                }
                final int index1 = line.indexOf('|');
                final int index2 = line.indexOf('|', index1 + 1);
                testResult.addFailedTest(line.substring(0, index1).trim() + ":" +
                                         line.substring(index1 + 1, index2).trim());
            } else if (TEST_DURATION_PATTERN.matcher(line).find()) {
                testResult.setComplete(true);
                break;
            }
        }
    }

    /**
     * Read the result output from the Test U01 test application.
     *
     * <p>Test U01 outputs a summary of results at the end of the test output. If this cannot
     * be found the method will throw an exception unless partial results are allowed.</p>
     *
     * @param iter Iterator of the test output.
     * @param testResult Test result.
     * @throws ApplicationException If the TestU01 summary cannot be found.
     */
    private void readTestU01(ListIterator<String> iter,
                             TestResult testResult) {
        // Results are summarised at the end of the file:
        //========= Summary results of BigCrush =========
        //
        //Version:          TestU01 1.2.3
        //Generator:        stdin
        //Number of statistics:  155
        //Total CPU time:   06:14:50.43
        //The following tests gave p-values outside [0.001, 0.9990]:
        //(eps  means a value < 1.0e-300):
        //(eps1 means a value < 1.0e-15):
        //
        //      Test                          p-value
        //----------------------------------------------
        // 1  SerialOver, r = 0                eps
        // 3  CollisionOver, t = 2           1 - eps1
        // 5  CollisionOver, t = 3           1 - eps1
        // 7  CollisionOver, t = 7             eps

        // Identify the summary line
        final String testSuiteName = skipToTestU01Summary(iter);

        // This may not be present if the results are not complete
        if (testSuiteName == null) {
            // Rewind
            while (iter.hasPrevious()) {
                iter.previous();
            }
            updateTestU01ApplicationName(iter, testResult);
            return;
        }

        setTestU01ApplicationName(testResult, testSuiteName);

        // Read test results using the entire line except the p-value for the test Id
        // Note:
        // This will count sub-parts of the same test as distinct failures.
        while (iter.hasNext()) {
            final String line = iter.next();
            final Matcher matcher = TESTU01_TEST_RESULT_PATTERN.matcher(line);
            if (matcher.find()) {
                testResult.addFailedTest(matcher.group(1).trim());
            } else if (TEST_DURATION_PATTERN.matcher(line).find()) {
                testResult.setComplete(true);
                break;
            }
        }
    }

    /**
     * Sets the Test U01 application name using the provide test suite name.
     *
     * @param testResult Test result.
     * @param testSuiteName Test suite name.
     */
    private static void setTestU01ApplicationName(TestResult testResult, String testSuiteName) {
        testResult.setTestApplicationName("TestU01 (" + testSuiteName + ")");
    }

    /**
     * Skip to the Test U01 result summary.
     *
     * <p>If this cannot be found the method will throw an exception unless partial results
     * are allowed.</p>
     *
     * @param iter Iterator of the test output.
     * @return the name of the test suite
     * @throws ApplicationException If the TestU01 summary cannot be found.
     */
    private String skipToTestU01Summary(Iterator<String> iter) {
        final String testSuiteName = findMatcherGroup1(iter, TESTU01_SUMMARY_PATTERN);
        // Allow the partial result to be ignored
        if (testSuiteName != null || ignorePartialResults) {
            return testSuiteName;
        }
        throw new ApplicationException("Failed to identify the Test U01 result summary");
    }

    /**
     * Update the test application name from the Test U01 results. This can be used to identify
     * the test suite from the start of the results in the event that the results summary has
     * not been output.
     *
     * @param iter Iterator of the test output.
     * @param testResult Test result.
     */
    private static void updateTestU01ApplicationName(Iterator<String> iter,
                                                     TestResult testResult) {
        final String testSuiteName = findMatcherGroup1(iter, TESTU01_STARTING_PATTERN);
        if (testSuiteName != null) {
            setTestU01ApplicationName(testResult, testSuiteName);
        }
    }

    /**
     * Create a matcher for each item in the iterator and if a match is identified return
     * group 1.
     *
     * @param iter Iterator of text to match.
     * @param pattern Pattern to match.
     * @return the string (or null)
     */
    private static String findMatcherGroup1(Iterator<String> iter,
                                            Pattern pattern) {
        while (iter.hasNext()) {
            final String line = iter.next();
            final Matcher matcher = pattern.matcher(line);
            if (matcher.find()) {
                return matcher.group(1);
            }
        }
        return null;
    }

    /**
     * Creates the output stream. This will not be buffered.
     *
     * @return the output stream
     */
    private OutputStream createOutputStream() {
        if (fileOutput != null) {
            try {
                Files.newOutputStream(fileOutput.toPath());
            } catch (final IOException ex) {
                throw new ApplicationException("Failed to create output: " + fileOutput, ex);
            }
        }
        return new FilterOutputStream(System.out) {
            @Override
            public void close() {
                // Do not close stdout
            }
        };
    }

    /**
     * Write the results to a table in Comma Separated Value (CSV) format.
     *
     * @param out Output stream.
     * @param results Results.
     * @throws IOException Signals that an I/O exception has occurred.
     */
    private void writeCSVData(OutputStream out,
                              List<TestResult> results) throws IOException {
        // Sort by columns
        Collections.sort(results, (o1, o2) -> {
            int result = Integer.compare(o1.getRandomSource().ordinal(), o2.getRandomSource().ordinal());
            if (result != 0) {
                return result;
            }
            result = Boolean.compare(o1.isBitReversed(), o2.isBitReversed());
            if (result != 0) {
                return result;
            }
            result = o1.getTestApplicationName().compareTo(o2.getTestApplicationName());
            if (result != 0) {
                return result;
            }
            return Integer.compare(o1.getFailureCount(), o2.getFailureCount());
        });

        try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
            output.write("RandomSource,Bit-reversed,Test,Failures");
            if (showFailedTests) {
                output.write(",Failed");
            }
            output.newLine();
            for (final TestResult result : results) {
                output.write(result.getRandomSource().toString());
                output.write(',');
                output.write(Boolean.toString(result.isBitReversed()));
                output.write(',');
                output.write(result.getTestApplicationName());
                output.write(',');
                output.write(result.getFailureCountString());
                // Optionally write out failed test names.
                if (showFailedTests) {
                    output.write(',');
                    output.write(result.getFailedTests().stream().collect(Collectors.joining("|")));
                }
                output.newLine();
            }
        }
    }

    /**
     * Write the results using the Almost Plain Text (APT) format for a table. This
     * table can be included in the documentation for the Commons RNG site.
     *
     * @param out Output stream.
     * @param results Results.
     * @throws IOException Signals that an I/O exception has occurred.
     */
    private void writeAPT(OutputStream out,
                          List<TestResult> results) throws IOException {
        // Identify all:
        // RandomSources, bit-reversed, test names,
        final List<RandomSource> randomSources = getRandomSources(results);
        final List<Boolean> bitReversed = getBitReversed(results);
        final List<String> testNames = getTestNames(results);

        // Identify the common path prefix to be replaced
        final int prefixLength = (pathPrefix.isEmpty()) ? 0 : findCommonPathPrefixLength(results);

        // Create a function to update the file path and then output the failure count
        // as a link to the file using the APT format.
        final Function<TestResult, String> toAPTString = result -> {
            String path = result.getResultFile().getPath();
            // Remove common path prefix
            path = path.substring(prefixLength);
            // Build the APT relative link
            final StringBuilder sb = new StringBuilder()
                .append("{{{").append(pathPrefix).append(path).append('}')
                .append(result.getFailureCountString()).append("}}");
            // Convert to web-link name separators
            for (int i = 0; i < sb.length(); i++) {
                if (sb.charAt(i) == BACK_SLASH) {
                    sb.setCharAt(i, FORWARD_SLASH);
                }
            }
            return sb.toString();
        };

        // Create columns for RandomSource, bit-reversed, each test name.
        // Make bit-reversed column optional if no generators are bit reversed.
        final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);

        final String header = createAPTHeader(showBitReversedColumn, testNames);
        final String separator = createAPTSeparator(header);

        // Output
        try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
            // For the first line using '*' (centre) character instead of '+' (left align)
            output.write(separator.replace('+', '*'));
            output.write(header);
            output.newLine();
            output.write(separator);

            // This will collate results for each combination of 'RandomSource + bitReversed'
            for (final RandomSource randomSource : randomSources) {
                for (final boolean reversed : bitReversed) {
                    output.write('|');
                    writeAPTColumn(output, randomSource.toString());
                    if (showBitReversedColumn) {
                        writeAPTColumn(output, Boolean.toString(reversed));
                    }
                    for (final String testName : testNames) {
                        final List<TestResult> testResults = getTestResults(results, randomSource, reversed, testName);
                        String text = testResults.stream()
                                                 .map(toAPTString)
                                                 .collect(Collectors.joining(", "));
                        // Add systematic failures in brackets
                        if (showFailedTests) {
                            text += " (" + getSystematicFailures(testResults).size() + ")";
                            writeAPTColumn(output, text);
                        }
                    }
                    output.newLine();
                    output.write(separator);
                }
            }
        }
    }

    /**
     * Gets the random sources present in the results.
     *
     * @param results Results.
     * @return the random sources
     */
    private static List<RandomSource> getRandomSources(List<TestResult> results) {
        final EnumSet<RandomSource> set = EnumSet.noneOf(RandomSource.class);
        for (final TestResult result : results) {
            set.add(result.getRandomSource());
        }
        final ArrayList<RandomSource> list = new ArrayList<>(set);
        Collections.sort(list);
        return list;
    }

    /**
     * Gets the bit-reversed options present in the results.
     *
     * @param results Results.
     * @return the bit-reversed options
     */
    private static List<Boolean> getBitReversed(List<TestResult> results) {
        final ArrayList<Boolean> list = new ArrayList<>(2);
        if (results.isEmpty()) {
            // Default to no bit-reversed results
            list.add(Boolean.FALSE);
        } else {
            final boolean first = results.get(0).isBitReversed();
            list.add(first);
            for (final TestResult result : results) {
                if (first != result.isBitReversed()) {
                    list.add(!first);
                    break;
                }
            }
        }
        Collections.sort(list);
        return list;
    }

    /**
     * Gets the test names present in the results.
     *
     * @param results Results.
     * @return the test names
     */
    private static List<String> getTestNames(List<TestResult> results) {
        final HashSet<String> set = new HashSet<>();
        for (final TestResult result : results) {
            set.add(result.getTestApplicationName());
        }
        final ArrayList<String> list = new ArrayList<>(set);
        Collections.sort(list);
        return list;
    }

    /**
     * Find the common path prefix for all result files. This is returned as the length of the
     * common prefix.
     *
     * @param results Results.
     * @return the length
     */
    private static int findCommonPathPrefixLength(List<TestResult> results) {
        if (results.isEmpty()) {
            return 0;
        }
        // Find the first prefix
        final String prefix1 = getPathPrefix(results.get(0));
        int length = prefix1.length();
        for (int i = 1; i < results.size() && length != 0; i++) {
            final String prefix2 = getPathPrefix(results.get(i));
            // Update
            final int size = Math.min(prefix2.length(), length);
            length = 0;
            while (length < size && prefix1.charAt(length) == prefix2.charAt(length)) {
                length++;
            }
        }
        return length;
    }

    /**
     * Gets the path prefix.
     *
     * @param testResult Test result.
     * @return the path prefix (or the empty string)
     */
    private static String getPathPrefix(TestResult testResult) {
        final String parent = testResult.getResultFile().getParent();
        return parent == null ? "" : parent;
    }

    /**
     * Creates the APT header.
     *
     * @param showBitReversedColumn Set to true to the show bit reversed column.
     * @param testNames Test names.
     * @return the header
     */
    private static String createAPTHeader(boolean showBitReversedColumn,
                                          List<String> testNames) {
        final StringBuilder sb = new StringBuilder(100).append("|| RNG identifier ||");
        if (showBitReversedColumn) {
            sb.append(" Bit-reversed ||");
        }
        for (final String name : testNames) {
            sb.append(' ').append(name).append(" ||");
        }
        return sb.toString();
    }

    /**
     * Creates the APT separator for each table row.
     *
     * <p>The separator is created using the '+' character to left align the columns.
     *
     * @param header Header.
     * @return the separator
     */
    private static String createAPTSeparator(String header) {
        // Replace everything with '-' except '|' which is replaced with "*-" for the first
        // character, "+-" for all other occurrences except "-+" at the end
        final StringBuilder sb = new StringBuilder(header);
        for (int i = 0; i < header.length(); i++) {
            if (sb.charAt(i) == PIPE) {
                sb.setCharAt(i, i == 0 ? '*' : '+');
                sb.setCharAt(i + 1,  '-');
            } else {
                sb.setCharAt(i,  '-');
            }
        }
        // Fix the end
        sb.setCharAt(header.length() - 2, '-');
        sb.setCharAt(header.length() - 1, '+');
        sb.append(System.lineSeparator());
        return sb.toString();
    }

    /**
     * Write the column text to the output.
     *
     * @param output Output.
     * @param text Text.
     * @throws IOException Signals that an I/O exception has occurred.
     */
    private static void writeAPTColumn(BufferedWriter output,
                                       String text) throws IOException {
        output.write(' ');
        output.write(text);
        output.write(" |");
    }

    /**
     * Gets the test results that match the arguments.
     *
     * @param results Results.
     * @param randomSource Random source.
     * @param bitReversed Bit reversed flag.
     * @param testName Test name.
     * @return the matching results
     */
    private static List<TestResult> getTestResults(List<TestResult> results,
                                                   RandomSource randomSource,
                                                   boolean bitReversed,
                                                   String testName) {
        final ArrayList<TestResult> list = new ArrayList<>();
        for (final TestResult result : results) {
            if (result.getRandomSource() == randomSource &&
                result.bitReversed == bitReversed &&
                result.getTestApplicationName().equals(testName)) {
                list.add(result);
            }
        }
        return list;
    }

    /**
     * Gets the systematic failures (tests that fail in every test result).
     *
     * @param results Results.
     * @return the systematic failures
     */
    private static List<String> getSystematicFailures(List<TestResult> results) {
        final HashMap<String, Integer> map = new HashMap<>();
        for (final TestResult result : results) {
            // Ignore partial results
            if (!result.isComplete()) {
                continue;
            }
            // Some named tests can fail more than once on different statistics.
            // For example TestU01 BigCrush LongestHeadRun can output in the summary:
            // 86  LongestHeadRun, r = 0            eps
            // 86  LongestHeadRun, r = 0          1 - eps1
            // This will be counted as 2 failed tests. For the purpose of systematic
            // failures the name of the test is the same and should be counted once.
            final HashSet<String> unique = new HashSet<>(result.getFailedTests());
            for (String test : unique) {
                map.merge(test, 1, (i, j) -> i + j);
            }
        }
        final int completeCount = (int) results.stream().filter(TestResult::isComplete).count();
        return map.entrySet().stream().filter(e -> e.getValue() == completeCount)
                                      .map(Entry::getKey)
                                      .collect(Collectors.toList());
    }

    /**
     * Write the results as a text table.
     *
     * @param out Output stream.
     * @param results Results.
     * @throws IOException Signals that an I/O exception has occurred.
     */
    private static void writeTXT(OutputStream out,
                                 List<TestResult> results) throws IOException {
        // Identify all:
        // RandomSources, bit-reversed, test names,
        final List<RandomSource> randomSources = getRandomSources(results);
        final List<Boolean> bitReversed = getBitReversed(results);
        final List<String> testNames = getTestNames(results);

        // Create columns for RandomSource, bit-reversed, each test name.
        // Make bit-reversed column optional if no generators are bit reversed.
        final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);

        final List<List<String>> columns = createTXTColumns(testNames, showBitReversedColumn);

        // Add all data
        // This will collate results for each combination of 'RandomSource + bitReversed'
        for (final RandomSource randomSource : randomSources) {
            for (final boolean reversed : bitReversed) {
                int i = 0;
                columns.get(i++).add(randomSource.toString());
                if (showBitReversedColumn) {
                    columns.get(i++).add(Boolean.toString(reversed));
                }
                for (final String testName : testNames) {
                    final List<TestResult> testResults = getTestResults(results, randomSource,
                            reversed, testName);
                    columns.get(i++).add(testResults.stream()
                                                    .map(TestResult::getFailureCountString)
                                                    .collect(Collectors.joining(",")));
                    columns.get(i++).add(String.valueOf(getSystematicFailures(testResults).size()));
                }
            }
        }

        writeColumns(out, columns);
    }

    /**
     * Creates the columns for the text output.
     *
     * @param testNames the test names
     * @param showBitReversedColumn Set to true to show the bit reversed column
     * @return the list of columns
     */
    private static List<List<String>> createTXTColumns(final List<String> testNames,
        final boolean showBitReversedColumn) {
        final ArrayList<List<String>> columns = new ArrayList<>();
        columns.add(createColumn(COLUMN_RNG));
        if (showBitReversedColumn) {
            columns.add(createColumn(BIT_REVERSED));
        }
        for (final String testName : testNames) {
            columns.add(createColumn(testName));
            columns.add(createColumn("∩"));
        }
        return columns;
    }

    /**
     * Creates the column.
     *
     * @param columnName Column name.
     * @return the list
     */
    private static List<String> createColumn(String columnName) {
        final ArrayList<String> list = new ArrayList<>();
        list.add(columnName);
        return list;
    }


    /**
     * Creates the text format from column widths.
     *
     * @param columns Columns.
     * @return the text format string
     */
    private static String createTextFormatFromColumnWidths(final List<List<String>> columns) {
        final StringBuilder sb = new StringBuilder();
        try (Formatter formatter = new Formatter(sb)) {
            for (int i = 0; i < columns.size(); i++) {
                if (i != 0) {
                    sb.append('\t');
                }
                formatter.format("%%-%ds", getColumnWidth(columns.get(i)));
            }
        }
        sb.append(System.lineSeparator());
        return sb.toString();
    }

    /**
     * Gets the column width using the maximum length of the column items.
     *
     * @param column Column.
     * @return the column width
     */
    private static int getColumnWidth(List<String> column) {
        int width = 0;
        for (final String text : column) {
            width = Math.max(width, text.length());
        }
        return width;
    }

    /**
     * Write the columns as fixed width text to the output.
     *
     * @param out Output stream.
     * @param columns Columns
     * @throws IOException Signals that an I/O exception has occurred.
     */
    private static void writeColumns(OutputStream out,
                                     List<List<String>> columns) throws IOException {
        // Create format using the column widths
        final String format = createTextFormatFromColumnWidths(columns);

        // Output
        try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
             Formatter formatter = new Formatter(output)) {
            final int rows = columns.get(0).size();
            final Object[] args = new Object[columns.size()];
            for (int row = 0; row < rows; row++) {
                for (int i = 0; i < args.length; i++) {
                    args[i] = columns.get(i).get(row);
                }
                formatter.format(format, args);
            }
        }
    }

    /**
     * Write the systematic failures as a text table.
     *
     * @param out Output stream.
     * @param results Results.
     * @throws IOException Signals that an I/O exception has occurred.
     */
    private static void writeFailures(OutputStream out,
                                      List<TestResult> results) throws IOException {
        // Identify all:
        // RandomSources, bit-reversed, test names,
        final List<RandomSource> randomSources = getRandomSources(results);
        final List<Boolean> bitReversed = getBitReversed(results);
        final List<String> testNames = getTestNames(results);

        // Create columns for RandomSource, bit-reversed, each test name.
        // Make bit-reversed column optional if no generators are bit reversed.
        final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE);

        final List<List<String>> columns = createFailuresColumns(testNames, showBitReversedColumn);

        final AlphaNumericComparator cmp = new AlphaNumericComparator();

        // Add all data for each combination of 'RandomSource + bitReversed'
        for (final RandomSource randomSource : randomSources) {
            for (final boolean reversed : bitReversed) {
                for (final String testName : testNames) {
                    final List<TestResult> testResults = getTestResults(results, randomSource,
                            reversed, testName);
                    final List<String> failures = getSystematicFailures(testResults);
                    if (failures.isEmpty()) {
                        continue;
                    }
                    Collections.sort(failures, cmp);
                    for (String failed : failures) {
                        int i = 0;
                        columns.get(i++).add(randomSource.toString());
                        if (showBitReversedColumn) {
                            columns.get(i++).add(Boolean.toString(reversed));
                        }
                        columns.get(i++).add(testName);
                        columns.get(i).add(failed);
                    }
                }
            }
        }

        writeColumns(out, columns);
    }

    /**
     * Creates the columns for the failures output.
     *
     * @param testNames the test names
     * @param showBitReversedColumn Set to true to show the bit reversed column
     * @return the list of columns
     */
    private static List<List<String>> createFailuresColumns(final List<String> testNames,
        final boolean showBitReversedColumn) {
        final ArrayList<List<String>> columns = new ArrayList<>();
        columns.add(createColumn(COLUMN_RNG));
        if (showBitReversedColumn) {
            columns.add(createColumn(BIT_REVERSED));
        }
        columns.add(createColumn("Test Suite"));
        columns.add(createColumn("Test"));
        return columns;
    }
}