| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.commons.rng.examples.stress; |
| |
| import org.apache.commons.rng.simple.RandomSource; |
| |
| import picocli.CommandLine.Command; |
| import picocli.CommandLine.Mixin; |
| import picocli.CommandLine.Option; |
| import picocli.CommandLine.Parameters; |
| |
| import java.io.BufferedReader; |
| import java.io.BufferedWriter; |
| import java.io.File; |
| import java.io.FilterOutputStream; |
| import java.io.IOException; |
| import java.io.OutputStream; |
| import java.io.OutputStreamWriter; |
| import java.nio.charset.StandardCharsets; |
| import java.nio.file.Files; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.EnumSet; |
| import java.util.Formatter; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.LinkedHashSet; |
| import java.util.List; |
| import java.util.ListIterator; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.concurrent.Callable; |
| import java.util.function.Function; |
| import java.util.function.Supplier; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| import java.util.stream.Collectors; |
| |
| /** |
| * Specification for the "results" command. |
| * |
| * <p>This command creates a summary of the results from known test applications.</p> |
| * |
| * <ul> |
| * <li>Dieharder |
| * <li>Test U01 (BigCrush, Crush, SmallCrush) |
| * </ul> |
| */ |
| @Command(name = "results", |
| description = {"Collate results from stress test applications."}) |
| class ResultsCommand implements Callable<Void> { |
| /** The pattern to identify the RandomSource in the stress test result header. */ |
| private static final Pattern RANDOM_SOURCE_PATTERN = Pattern.compile("^# RandomSource: (.*)"); |
| /** The pattern to identify the RNG in the stress test result header. */ |
| private static final Pattern RNG_PATTERN = Pattern.compile("^# RNG: (.*)"); |
| /** The pattern to identify the test exit code in the stress test result footer. */ |
| private static final Pattern TEST_EXIT_PATTERN = Pattern.compile("^# Exit value: (\\d+)"); |
| /** The pattern to identify the Dieharder test format. */ |
| private static final Pattern DIEHARDER_PATTERN = Pattern.compile("^# *dieharder version"); |
| /** The pattern to identify a Dieharder failed test result. */ |
| private static final Pattern DIEHARDER_FAILED_PATTERN = Pattern.compile("FAILED *$"); |
| /** The pattern to identify the Test U01 test format. */ |
| private static final Pattern TESTU01_PATTERN = Pattern.compile("^ *Version: TestU01"); |
| /** The pattern to identify the Test U01 summary header. */ |
| private static final Pattern TESTU01_SUMMARY_PATTERN = Pattern.compile("^========= Summary results of (\\S*) "); |
| /** The pattern to identify the Test U01 failed test result. */ |
| private static final Pattern TESTU01_TEST_RESULT_PATTERN = Pattern.compile("^ ?(\\d+ .*) "); |
| /** The pattern to identify the Test U01 test starting entry. */ |
| private static final Pattern TESTU01_STARTING_PATTERN = Pattern.compile("^ *Starting (\\S*)"); |
| /** The pattern to identify the PractRand test format. */ |
| private static final Pattern PRACTRAND_PATTERN = Pattern.compile("PractRand version"); |
| /** The pattern to identify the PractRand output byte size. */ |
| private static final Pattern PRACTRAND_OUTPUT_SIZE_PATTERN = Pattern.compile("\\(2\\^(\\d+) bytes\\)"); |
| /** The pattern to identify a PractRand failed test result. */ |
| private static final Pattern PRACTRAND_FAILED_PATTERN = Pattern.compile("FAIL *!* *$"); |
| /** The name of the Dieharder sums test. */ |
| private static final String DIEHARDER_SUMS = "diehard_sums"; |
| /** The string identifying a bit-reversed generator. */ |
| private static final String BIT_REVERSED = "Bit-reversed"; |
| /** Character '\'. */ |
| private static final char FORWARD_SLASH = '\\'; |
| /** Character '/'. */ |
| private static final char BACK_SLASH = '\\'; |
| /** Character '|'. */ |
| private static final char PIPE = '|'; |
| /** The column name for the RNG. */ |
| private static final String COLUMN_RNG = "RNG"; |
| /** Constant for conversion of bytes to binary units, prefixed with a space. */ |
| private static final String[] BINARY_UNITS = {" B", " KiB", " MiB", " GiB", " TiB", " PiB", " EiB"}; |
| |
| /** The standard options. */ |
| @Mixin |
| private StandardOptions reusableOptions; |
| |
| /** The results files. */ |
| @Parameters(arity = "1..*", |
| description = "The results files.", |
| paramLabel = "<file>") |
| private List<File> resultsFiles = new ArrayList<>(); |
| |
| /** The file output prefix. */ |
| @Option(names = {"-o", "--out"}, |
| description = "The output file (default: stdout).") |
| private File fileOutput; |
| |
| /** The output format. */ |
| @Option(names = {"-f", "--format"}, |
| description = {"Output format (default: ${DEFAULT-VALUE}).", |
| "Valid values: ${COMPLETION-CANDIDATES}."}) |
| private ResultsCommand.OutputFormat outputFormat = OutputFormat.TXT; |
| |
| /** The flag to show failed tests. */ |
| @Option(names = {"--failed"}, |
| description = {"Output failed tests (support varies by format).", |
| "CSV: List individual test failures.", |
| "APT: Count of systematic test failures."}) |
| private boolean showFailedTests; |
| |
| /** The flag to include the Dieharder sums test. */ |
| @Option(names = {"--include-sums"}, |
| description = "Include Dieharder sums test.") |
| private boolean includeDiehardSums; |
| |
| /** The common file path prefix used when outputting file paths. */ |
| @Option(names = {"--path-prefix"}, |
| description = {"Common path prefix.", |
| "If specified this will replace the common prefix from all " + |
| "files when the path is output, e.g. for the APT report."}) |
| private String pathPrefix = ""; |
| |
| /** The flag to ignore partial results. */ |
| @Option(names = {"-i", "--ignore"}, |
| description = "Ignore partial results.") |
| private boolean ignorePartialResults; |
| |
| /** The flag to delete partial results files. */ |
| @Option(names = {"--delete"}, |
| description = {"Delete partial results files.", |
| "This is not reversible!"}) |
| private boolean deletePartialResults; |
| |
| /** |
| * The output mode for the results. |
| */ |
| enum OutputFormat { |
| /** Comma-Separated Values (CSV) output. */ |
| CSV, |
| /** Almost Plain Text (APT) output. */ |
| APT, |
| /** Text table output. */ |
| TXT, |
| /** Systematic failures output. */ |
| FAILURES, |
| } |
| |
| /** |
| * The test application file format. |
| */ |
| enum TestFormat { |
| /** Dieharder. */ |
| DIEHARDER, |
| /** Test U01. */ |
| TESTU01, |
| /** PractRand. */ |
| PRACTRAND, |
| } |
| |
| /** |
| * Encapsulate the results of a test application. |
| */ |
| private static class TestResult { |
| /** The result file. */ |
| private final File resultFile; |
| /** The random source. */ |
| private final RandomSource randomSource; |
| /** Flag indicating the generator was bit reversed. */ |
| private final boolean bitReversed; |
| /** The test application format. */ |
| private final TestFormat testFormat; |
| /** The names of the failed tests. */ |
| private final List<String> failedTests = new ArrayList<>(); |
| /** The test application name. */ |
| private String testApplicationName; |
| /** |
| * Store the exit code. |
| * Initialised to {@link Integer#MIN_VALUE}. Exit values are expected to be 8-bit numbers |
| * with zero for success. |
| */ |
| private int exitCode = Integer.MIN_VALUE; |
| |
| /** |
| * @param resultFile the result file |
| * @param randomSource the random source |
| * @param bitReversed the bit reversed flag |
| * @param testFormat the test format |
| */ |
| TestResult(File resultFile, |
| RandomSource randomSource, |
| boolean bitReversed, |
| TestFormat testFormat) { |
| this.resultFile = resultFile; |
| this.randomSource = randomSource; |
| this.bitReversed = bitReversed; |
| this.testFormat = testFormat; |
| } |
| |
| /** |
| * Adds the failed test. |
| * |
| * @param testId the test id |
| */ |
| void addFailedTest(String testId) { |
| failedTests.add(testId); |
| } |
| |
| /** |
| * Gets the result file. |
| * |
| * @return the result file |
| */ |
| File getResultFile() { |
| return resultFile; |
| } |
| |
| /** |
| * Gets the random source. |
| * |
| * @return the random source |
| */ |
| RandomSource getRandomSource() { |
| return randomSource; |
| } |
| |
| /** |
| * Checks if the generator was bit reversed. |
| * |
| * @return true if bit reversed |
| */ |
| boolean isBitReversed() { |
| return bitReversed; |
| } |
| |
| /** |
| * Gets the test format. |
| * |
| * @return the test format |
| */ |
| TestFormat getTestFormat() { |
| return testFormat; |
| } |
| |
| /** |
| * Gets the failed tests. |
| * |
| * @return the failed tests |
| */ |
| List<String> getFailedTests() { |
| return failedTests; |
| } |
| |
| /** |
| * Gets the failure count. |
| * |
| * @return the failure count |
| */ |
| int getFailureCount() { |
| return failedTests.size(); |
| } |
| |
| /** |
| * Gets the a failure summary string. |
| * |
| * <p>The default implementation is the count of the number of failures. |
| * This will be negative if the test is not complete.</p> |
| * |
| * @return the failure summary |
| */ |
| String getFailureSummaryString() { |
| return (isComplete()) ? Integer.toString(failedTests.size()) : "-" + failedTests.size(); |
| } |
| |
| /** |
| * Sets the test application name. |
| * |
| * @param testApplicationName the new test application name |
| */ |
| void setTestApplicationName(String testApplicationName) { |
| this.testApplicationName = testApplicationName; |
| } |
| |
| /** |
| * Gets the test application name. |
| * |
| * @return the test application name |
| */ |
| String getTestApplicationName() { |
| return testApplicationName == null ? String.valueOf(getTestFormat()) : testApplicationName; |
| } |
| |
| /** |
| * Checks if the test result is complete. |
| * This is {@code true} only if the exit code was found and is zero. |
| * |
| * @return true if complete |
| */ |
| boolean isComplete() { |
| return exitCode == 0; |
| } |
| |
| /** |
| * Sets the exit code flag. |
| * |
| * @param exitCode the new exit code |
| */ |
| void setExitCode(int exitCode) { |
| this.exitCode = exitCode; |
| } |
| } |
| |
| /** |
| * Encapsulate the results of a PractRand test application. This is a specialisation that |
| * allows handling PractRand results which are linked to the output length used by the RNG. |
| */ |
| private static class PractRandTestResult extends TestResult { |
| /** The length of the RNG output used to generate failed tests. */ |
| private int lengthExponent; |
| |
| /** |
| * @param resultFile the result file |
| * @param randomSource the random source |
| * @param bitReversed the bit reversed flag |
| * @param testFormat the test format |
| */ |
| PractRandTestResult(File resultFile, RandomSource randomSource, boolean bitReversed, TestFormat testFormat) { |
| super(resultFile, randomSource, bitReversed, testFormat); |
| } |
| |
| /** |
| * Gets the length of the RNG output used to generate failed tests. |
| * If this is zero then no failures occurred. |
| * |
| * @return the length exponent |
| */ |
| int getLengthExponent() { |
| return lengthExponent; |
| } |
| |
| /** |
| * Sets the length of the RNG output used to generate failed tests. |
| * |
| * @param lengthExponent the length exponent |
| */ |
| void setLengthExponent(int lengthExponent) { |
| this.lengthExponent = lengthExponent; |
| } |
| |
| /** |
| * {@inheritDoc} |
| * |
| * <p>The PractRand summary is the exponent of the length of byte output from the RNG where |
| * a failure occurred. |
| */ |
| @Override |
| String getFailureSummaryString() { |
| return lengthExponent == 0 ? "-" : Integer.toString(lengthExponent); |
| } |
| } |
| |
| /** |
| * Reads the results files and outputs in the chosen format. |
| */ |
| @Override |
| public Void call() { |
| LogUtils.setLogLevel(reusableOptions.logLevel); |
| |
| // Read the results |
| final List<TestResult> results = readResults(); |
| |
| if (deletePartialResults) { |
| deleteIfIncomplete(results); |
| return null; |
| } |
| |
| try (OutputStream out = createOutputStream()) { |
| switch (outputFormat) { |
| case CSV: |
| writeCSVData(out, results); |
| break; |
| case APT: |
| writeAPT(out, results); |
| break; |
| case TXT: |
| writeTXT(out, results); |
| break; |
| case FAILURES: |
| writeFailures(out, results); |
| break; |
| default: |
| throw new ApplicationException("Unknown output format: " + outputFormat); |
| } |
| } catch (final IOException ex) { |
| throw new ApplicationException("IO error: " + ex.getMessage(), ex); |
| } |
| return null; |
| } |
| |
| /** |
| * Read the results. |
| * |
| * @return the results |
| */ |
| private List<TestResult> readResults() { |
| final ArrayList<TestResult> results = new ArrayList<>(); |
| for (final File resultFile : resultsFiles) { |
| readResults(results, resultFile); |
| } |
| return results; |
| } |
| |
| /** |
| * Read the file and extract any test results. |
| * |
| * @param results Results. |
| * @param resultFile Result file. |
| * @throws ApplicationException If the results cannot be parsed. |
| */ |
| private void readResults(List<TestResult> results, |
| File resultFile) { |
| final List<String> contents = readFileContents(resultFile); |
| // Files may have multiple test results per file (i.e. appended output) |
| final List<List<String>> outputs = splitContents(contents); |
| if (outputs.isEmpty()) { |
| LogUtils.error("No test output in file: %s", resultFile); |
| } else { |
| for (final List<String> testOutput : outputs) { |
| final TestResult result = readResult(resultFile, testOutput); |
| if (!result.isComplete()) { |
| LogUtils.info("Partial results in file: %s", resultFile); |
| if (ignorePartialResults) { |
| continue; |
| } |
| } |
| results.add(result); |
| } |
| } |
| } |
| |
| /** |
| * Read the file contents. |
| * |
| * @param resultFile Result file. |
| * @return the file contents |
| * @throws ApplicationException If the file cannot be read. |
| */ |
| private static List<String> readFileContents(File resultFile) { |
| final ArrayList<String> contents = new ArrayList<>(); |
| try (BufferedReader reader = Files.newBufferedReader(resultFile.toPath())) { |
| for (String line = reader.readLine(); line != null; line = reader.readLine()) { |
| contents.add(line); |
| } |
| } catch (final IOException ex) { |
| throw new ApplicationException("Failed to read file contents: " + resultFile, ex); |
| } |
| return contents; |
| } |
| |
| /** |
| * Split the file contents into separate test output. This is used in the event |
| * that results have been appended to the same file. |
| * |
| * @param contents File contents. |
| * @return the test output |
| */ |
| private static List<List<String>> splitContents(List<String> contents) { |
| final ArrayList<List<String>> testOutputs = new ArrayList<>(); |
| // Assume each output begins with e.g. # RandomSource: SPLIT_MIX_64 |
| // Note each beginning. |
| int begin = -1; |
| for (int i = 0; i < contents.size(); i++) { |
| if (RANDOM_SOURCE_PATTERN.matcher(contents.get(i)).matches()) { |
| if (begin >= 0) { |
| testOutputs.add(contents.subList(begin, i)); |
| } |
| begin = i; |
| } |
| } |
| if (begin >= 0) { |
| testOutputs.add(contents.subList(begin, contents.size())); |
| } |
| return testOutputs; |
| } |
| |
| /** |
| * Read the file into a test result. |
| * |
| * @param resultFile Result file. |
| * @param testOutput Test output. |
| * @return the test result |
| * @throws ApplicationException If the result cannot be parsed. |
| */ |
| private TestResult readResult(File resultFile, |
| List<String> testOutput) { |
| // Use an iterator for a single pass over the test output |
| final ListIterator<String> iter = testOutput.listIterator(); |
| |
| // Identify the RandomSource and bit reversed flag from the header: |
| final RandomSource randomSource = getRandomSource(resultFile, iter); |
| final boolean bitReversed = getBitReversed(resultFile, iter); |
| |
| // Identify the test application format. This may return null. |
| final TestFormat testFormat = getTestFormat(resultFile, iter); |
| |
| // Read the application results |
| final TestResult testResult = createTestResult(resultFile, randomSource, bitReversed, testFormat); |
| if (testFormat == TestFormat.DIEHARDER) { |
| readDieharder(iter, testResult); |
| } else if (testFormat == TestFormat.TESTU01) { |
| readTestU01(resultFile, iter, testResult); |
| } else { |
| readPractRand(iter, (PractRandTestResult) testResult); |
| } |
| return testResult; |
| } |
| |
| /** |
| * Creates the test result. |
| * |
| * @param resultFile Result file. |
| * @param randomSource Random source. |
| * @param bitReversed True if the random source was bit reversed. |
| * @param testFormat Test format. |
| * @return the test result |
| */ |
| private static TestResult createTestResult(File resultFile, RandomSource randomSource, |
| boolean bitReversed, TestFormat testFormat) { |
| return testFormat == TestFormat.PRACTRAND ? |
| new PractRandTestResult(resultFile, randomSource, bitReversed, testFormat) : |
| new TestResult(resultFile, randomSource, bitReversed, testFormat); |
| } |
| |
| /** |
| * Gets the random source from the output header. |
| * |
| * @param resultFile Result file (for the exception message). |
| * @param iter Iterator of the test output. |
| * @return the random source |
| * @throws ApplicationException If the RandomSource header line cannot be found. |
| */ |
| private static RandomSource getRandomSource(File resultFile, Iterator<String> iter) { |
| while (iter.hasNext()) { |
| final Matcher matcher = RANDOM_SOURCE_PATTERN.matcher(iter.next()); |
| if (matcher.matches()) { |
| return RandomSource.valueOf(matcher.group(1)); |
| } |
| } |
| throw new ApplicationException("Failed to find RandomSource header line: " + resultFile); |
| } |
| |
| /** |
| * Gets the bit-reversed flag from the output header. |
| * |
| * @param resultFile Result file (for the exception message). |
| * @param iter Iterator of the test output. |
| * @return the bit-reversed flag |
| * @throws ApplicationException If the RNG header line cannot be found. |
| */ |
| private static boolean getBitReversed(File resultFile, Iterator<String> iter) { |
| while (iter.hasNext()) { |
| final Matcher matcher = RNG_PATTERN.matcher(iter.next()); |
| if (matcher.matches()) { |
| return matcher.group(1).contains(BIT_REVERSED); |
| } |
| } |
| throw new ApplicationException("Failed to find RNG header line: " + resultFile); |
| } |
| |
| /** |
| * Gets the test format from the output. This scans the stdout produced by a test application. |
| * If it is not recognised this may be a valid partial result or an unknown result. Throw |
| * an exception if not allowing partial results, otherwise log an error. |
| * |
| * @param resultFile Result file (for the exception message). |
| * @param iter Iterator of the test output. |
| * @return the test format (or null) |
| * @throws ApplicationException If the test format cannot be found. |
| */ |
| private TestFormat getTestFormat(File resultFile, Iterator<String> iter) { |
| while (iter.hasNext()) { |
| final String line = iter.next(); |
| if (DIEHARDER_PATTERN.matcher(line).find()) { |
| return TestFormat.DIEHARDER; |
| } |
| if (TESTU01_PATTERN.matcher(line).find()) { |
| return TestFormat.TESTU01; |
| } |
| if (PRACTRAND_PATTERN.matcher(line).find()) { |
| return TestFormat.PRACTRAND; |
| } |
| } |
| if (!ignorePartialResults) { |
| throw new ApplicationException("Failed to identify the test application format: " + resultFile); |
| } |
| LogUtils.error("Failed to identify the test application format: %s", resultFile); |
| return null; |
| } |
| |
| /** |
| * Read the result output from the Dieharder test application. |
| * |
| * @param iter Iterator of the test output. |
| * @param testResult Test result. |
| */ |
| private void readDieharder(Iterator<String> iter, |
| TestResult testResult) { |
| // Dieharder results are printed in-line using the following format: |
| //#=============================================================================# |
| // test_name |ntup| tsamples |psamples| p-value |Assessment |
| //#=============================================================================# |
| // diehard_birthdays| 0| 100| 100|0.97484422| PASSED |
| // ... |
| // diehard_oqso| 0| 2097152| 100|0.00000000| FAILED |
| |
| testResult.setTestApplicationName("Dieharder"); |
| |
| // Identify any line containing FAILED and then get the test ID using |
| // the first two columns (test_name + ntup). |
| while (iter.hasNext()) { |
| final String line = iter.next(); |
| if (DIEHARDER_FAILED_PATTERN.matcher(line).find()) { |
| // Optionally include the flawed Dieharder sums test |
| if (!includeDiehardSums && line.contains(DIEHARDER_SUMS)) { |
| continue; |
| } |
| final int index1 = line.indexOf('|'); |
| final int index2 = line.indexOf('|', index1 + 1); |
| testResult.addFailedTest(line.substring(0, index1).trim() + ":" + |
| line.substring(index1 + 1, index2).trim()); |
| } else if (findExitCode(testResult, line)) { |
| return; |
| } |
| } |
| } |
| |
| /** |
| * Find the exit code in the line. Update the test result with the code if found. |
| * |
| * @param testResult Test result. |
| * @param line Line from the test result output. |
| * @return true, if the exit code was found |
| */ |
| private static boolean findExitCode(TestResult testResult, String line) { |
| final Matcher matcher = TEST_EXIT_PATTERN.matcher(line); |
| if (matcher.find()) { |
| testResult.setExitCode(Integer.parseInt(matcher.group(1))); |
| return true; |
| } |
| return false; |
| } |
| |
| /** |
| * Read the result output from the Test U01 test application. |
| * |
| * <p>Test U01 outputs a summary of results at the end of the test output. If this cannot |
| * be found the method will throw an exception unless partial results are allowed.</p> |
| * |
| * @param resultFile Result file (for the exception message). |
| * @param iter Iterator of the test output. |
| * @param testResult Test result. |
| * @throws ApplicationException If the TestU01 summary cannot be found. |
| */ |
| private void readTestU01(File resultFile, |
| ListIterator<String> iter, |
| TestResult testResult) { |
| // Results are summarised at the end of the file: |
| //========= Summary results of BigCrush ========= |
| // |
| //Version: TestU01 1.2.3 |
| //Generator: stdin |
| //Number of statistics: 155 |
| //Total CPU time: 06:14:50.43 |
| //The following tests gave p-values outside [0.001, 0.9990]: |
| //(eps means a value < 1.0e-300): |
| //(eps1 means a value < 1.0e-15): |
| // |
| // Test p-value |
| //---------------------------------------------- |
| // 1 SerialOver, r = 0 eps |
| // 3 CollisionOver, t = 2 1 - eps1 |
| // 5 CollisionOver, t = 3 1 - eps1 |
| // 7 CollisionOver, t = 7 eps |
| |
| // Identify the summary line |
| final String testSuiteName = skipToTestU01Summary(resultFile, iter); |
| |
| // This may not be present if the results are not complete |
| if (testSuiteName == null) { |
| // Rewind |
| while (iter.hasPrevious()) { |
| iter.previous(); |
| } |
| updateTestU01ApplicationName(iter, testResult); |
| return; |
| } |
| |
| setTestU01ApplicationName(testResult, testSuiteName); |
| |
| // Read test results using the entire line except the p-value for the test Id |
| // Note: |
| // This will count sub-parts of the same test as distinct failures. |
| while (iter.hasNext()) { |
| final String line = iter.next(); |
| final Matcher matcher = TESTU01_TEST_RESULT_PATTERN.matcher(line); |
| if (matcher.find()) { |
| testResult.addFailedTest(matcher.group(1).trim()); |
| } else if (findExitCode(testResult, line)) { |
| return; |
| } |
| } |
| } |
| |
| /** |
| * Sets the Test U01 application name using the provide test suite name. |
| * |
| * @param testResult Test result. |
| * @param testSuiteName Test suite name. |
| */ |
| private static void setTestU01ApplicationName(TestResult testResult, String testSuiteName) { |
| testResult.setTestApplicationName("TestU01 (" + testSuiteName + ")"); |
| } |
| |
| /** |
| * Skip to the Test U01 result summary. |
| * |
| * <p>If this cannot be found the method will throw an exception unless partial results |
| * are allowed.</p> |
| * |
| * @param resultFile Result file (for the exception message). |
| * @param iter Iterator of the test output. |
| * @return the name of the test suite |
| * @throws ApplicationException If the TestU01 summary cannot be found. |
| */ |
| private String skipToTestU01Summary(File resultFile, Iterator<String> iter) { |
| final String testSuiteName = findMatcherGroup1(iter, TESTU01_SUMMARY_PATTERN); |
| // Allow the partial result to be ignored |
| if (testSuiteName != null || ignorePartialResults) { |
| return testSuiteName; |
| } |
| throw new ApplicationException("Failed to identify the Test U01 result summary: " + resultFile); |
| } |
| |
| /** |
| * Update the test application name from the Test U01 results. This can be used to identify |
| * the test suite from the start of the results in the event that the results summary has |
| * not been output. |
| * |
| * @param iter Iterator of the test output. |
| * @param testResult Test result. |
| */ |
| private static void updateTestU01ApplicationName(Iterator<String> iter, |
| TestResult testResult) { |
| final String testSuiteName = findMatcherGroup1(iter, TESTU01_STARTING_PATTERN); |
| if (testSuiteName != null) { |
| setTestU01ApplicationName(testResult, testSuiteName); |
| } |
| } |
| |
| /** |
| * Create a matcher for each item in the iterator and if a match is identified return |
| * group 1. |
| * |
| * @param iter Iterator of text to match. |
| * @param pattern Pattern to match. |
| * @return the string (or null) |
| */ |
| private static String findMatcherGroup1(Iterator<String> iter, |
| Pattern pattern) { |
| while (iter.hasNext()) { |
| final String line = iter.next(); |
| final Matcher matcher = pattern.matcher(line); |
| if (matcher.find()) { |
| return matcher.group(1); |
| } |
| } |
| return null; |
| } |
| |
| /** |
| * Read the result output from the PractRand test application. |
| * |
| * @param iter Iterator of the test output. |
| * @param testResult Test result. |
| */ |
| private static void readPractRand(Iterator<String> iter, |
| PractRandTestResult testResult) { |
| // PractRand results are printed for blocks of byte output that double in size. |
| // The results report 'unusual' and 'suspicious' output and the test stops |
| // at the first failure. |
| // Results are typically reported as the size of output that failed, e.g. |
| // the JDK random fails at 256KiB. |
| // |
| // rng=RNG_stdin32, seed=0xfc6c7332 |
| // length= 128 kilobytes (2^17 bytes), time= 5.9 seconds |
| // no anomalies in 118 test result(s) |
| // |
| // rng=RNG_stdin32, seed=0xfc6c7332 |
| // length= 256 kilobytes (2^18 bytes), time= 7.6 seconds |
| // Test Name Raw Processed Evaluation |
| // [Low1/64]Gap-16:A R= +12.4 p = 8.7e-11 VERY SUSPICIOUS |
| // [Low1/64]Gap-16:B R= +13.4 p = 3.0e-11 FAIL |
| // [Low8/32]DC6-9x1Bytes-1 R= +7.6 p = 5.7e-4 unusual |
| // ...and 136 test result(s) without anomalies |
| |
| testResult.setTestApplicationName("PractRand"); |
| |
| // Store the exponent of the current output length. |
| int exp = 0; |
| |
| // Identify any line containing FAIL and then get the test name using |
| // the first token in the line. |
| while (iter.hasNext()) { |
| String line = iter.next(); |
| final Matcher matcher = PRACTRAND_OUTPUT_SIZE_PATTERN.matcher(line); |
| if (matcher.find()) { |
| // Store the current output length |
| exp = Integer.parseInt(matcher.group(1)); |
| } else if (PRACTRAND_FAILED_PATTERN.matcher(line).find()) { |
| // Record the output length where failures occurred. |
| testResult.setLengthExponent(exp); |
| // Add the failed test name. This does not include the length exponent |
| // allowing meta-processing of systematic failures. |
| // Remove initial whitespace |
| line = line.trim(); |
| final int index = line.indexOf(' '); |
| testResult.addFailedTest(line.substring(0, index)); |
| } else if (findExitCode(testResult, line)) { |
| return; |
| } |
| } |
| } |
| |
| /** |
| * Delete any result file if incomplete. |
| * |
| * @param results Results. |
| * @throws ApplicationException If a file could not be deleted. |
| */ |
| private static void deleteIfIncomplete(List<TestResult> results) { |
| results.forEach(ResultsCommand::deleteIfIncomplete); |
| } |
| |
| /** |
| * Delete the result file if incomplete. |
| * |
| * @param result Test result. |
| * @throws ApplicationException If the file could not be deleted. |
| */ |
| private static void deleteIfIncomplete(TestResult result) { |
| if (!result.isComplete()) { |
| try { |
| Files.delete(result.getResultFile().toPath()); |
| LogUtils.info("Deleted file: %s", result.getResultFile()); |
| } catch (IOException ex) { |
| throw new ApplicationException("Failed to delete file: " + result.getResultFile(), ex); |
| } |
| } |
| } |
| |
| /** |
| * Creates the output stream. This will not be buffered. |
| * |
| * @return the output stream |
| */ |
| private OutputStream createOutputStream() { |
| if (fileOutput != null) { |
| try { |
| Files.newOutputStream(fileOutput.toPath()); |
| } catch (final IOException ex) { |
| throw new ApplicationException("Failed to create output: " + fileOutput, ex); |
| } |
| } |
| return new FilterOutputStream(System.out) { |
| @Override |
| public void close() { |
| // Do not close stdout |
| } |
| }; |
| } |
| |
| /** |
| * Write the results to a table in Comma Separated Value (CSV) format. |
| * |
| * @param out Output stream. |
| * @param results Results. |
| * @throws IOException Signals that an I/O exception has occurred. |
| */ |
| private void writeCSVData(OutputStream out, |
| List<TestResult> results) throws IOException { |
| // Sort by columns |
| Collections.sort(results, (o1, o2) -> { |
| int result = Integer.compare(o1.getRandomSource().ordinal(), o2.getRandomSource().ordinal()); |
| if (result != 0) { |
| return result; |
| } |
| result = Boolean.compare(o1.isBitReversed(), o2.isBitReversed()); |
| if (result != 0) { |
| return result; |
| } |
| result = o1.getTestApplicationName().compareTo(o2.getTestApplicationName()); |
| if (result != 0) { |
| return result; |
| } |
| return Integer.compare(o1.getFailureCount(), o2.getFailureCount()); |
| }); |
| |
| try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) { |
| output.write("RandomSource,Bit-reversed,Test,Failures"); |
| if (showFailedTests) { |
| output.write(",Failed"); |
| } |
| output.newLine(); |
| for (final TestResult result : results) { |
| output.write(result.getRandomSource().toString()); |
| output.write(','); |
| output.write(Boolean.toString(result.isBitReversed())); |
| output.write(','); |
| output.write(result.getTestApplicationName()); |
| output.write(','); |
| output.write(result.getFailureSummaryString()); |
| // Optionally write out failed test names. |
| if (showFailedTests) { |
| output.write(','); |
| output.write(result.getFailedTests().stream().collect(Collectors.joining("|"))); |
| } |
| output.newLine(); |
| } |
| } |
| } |
| |
| /** |
| * Write the results using the Almost Plain Text (APT) format for a table. This |
| * table can be included in the documentation for the Commons RNG site. |
| * |
| * @param out Output stream. |
| * @param results Results. |
| * @throws IOException Signals that an I/O exception has occurred. |
| */ |
| private void writeAPT(OutputStream out, |
| List<TestResult> results) throws IOException { |
| // Identify all: |
| // RandomSources, bit-reversed, test names, |
| final List<RandomSource> randomSources = getRandomSources(results); |
| final List<Boolean> bitReversed = getBitReversed(results); |
| final List<String> testNames = getTestNames(results); |
| |
| // Identify the common path prefix to be replaced |
| final int prefixLength = (pathPrefix.isEmpty()) ? 0 : findCommonPathPrefixLength(results); |
| |
| // Create a function to update the file path and then output the failure count |
| // as a link to the file using the APT format. |
| final Function<TestResult, String> toAPTString = result -> { |
| String path = result.getResultFile().getPath(); |
| // Remove common path prefix |
| path = path.substring(prefixLength); |
| // Build the APT relative link |
| final StringBuilder sb = new StringBuilder() |
| .append("{{{").append(pathPrefix).append(path).append('}') |
| .append(result.getFailureSummaryString()).append("}}"); |
| // Convert to web-link name separators |
| for (int i = 0; i < sb.length(); i++) { |
| if (sb.charAt(i) == BACK_SLASH) { |
| sb.setCharAt(i, FORWARD_SLASH); |
| } |
| } |
| return sb.toString(); |
| }; |
| |
| // Create columns for RandomSource, bit-reversed, each test name. |
| // Make bit-reversed column optional if no generators are bit reversed. |
| final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE); |
| |
| final String header = createAPTHeader(showBitReversedColumn, testNames); |
| final String separator = createAPTSeparator(header); |
| |
| // Output |
| try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) { |
| // For the first line using '*' (centre) character instead of '+' (left align) |
| output.write(separator.replace('+', '*')); |
| output.write(header); |
| output.newLine(); |
| output.write(separator); |
| |
| final StringBuilder sb = new StringBuilder(); |
| |
| // This will collate results for each combination of 'RandomSource + bitReversed' |
| for (final RandomSource randomSource : randomSources) { |
| for (final boolean reversed : bitReversed) { |
| // Highlight in bold a RNG with no systematic failures |
| boolean highlight = true; |
| |
| // Buffer the column output |
| sb.setLength(0); |
| |
| if (showBitReversedColumn) { |
| writeAPTColumn(sb, Boolean.toString(reversed), false); |
| } |
| for (final String testName : testNames) { |
| final List<TestResult> testResults = getTestResults(results, randomSource, reversed, testName); |
| String text = testResults.stream() |
| .map(toAPTString) |
| .collect(Collectors.joining(", ")); |
| // Summarise the failures across all tests |
| final String summary = getFailuresSummary(testResults); |
| if (summary.length() != 0) { |
| // Identify RNGs with no systematic failures |
| highlight = false; |
| if (showFailedTests) { |
| // Add summary in brackets |
| text += " (" + summary + ")"; |
| } |
| } |
| writeAPTColumn(sb, text, false); |
| } |
| |
| output.write('|'); |
| writeAPTColumn(output, randomSource.toString(), highlight); |
| output.write(sb.toString()); |
| output.newLine(); |
| output.write(separator); |
| } |
| } |
| } |
| } |
| |
| /** |
| * Gets the random sources present in the results. |
| * |
| * @param results Results. |
| * @return the random sources |
| */ |
| private static List<RandomSource> getRandomSources(List<TestResult> results) { |
| final EnumSet<RandomSource> set = EnumSet.noneOf(RandomSource.class); |
| for (final TestResult result : results) { |
| set.add(result.getRandomSource()); |
| } |
| final ArrayList<RandomSource> list = new ArrayList<>(set); |
| Collections.sort(list); |
| return list; |
| } |
| |
| /** |
| * Gets the bit-reversed options present in the results. |
| * |
| * @param results Results. |
| * @return the bit-reversed options |
| */ |
| private static List<Boolean> getBitReversed(List<TestResult> results) { |
| final ArrayList<Boolean> list = new ArrayList<>(2); |
| if (results.isEmpty()) { |
| // Default to no bit-reversed results |
| list.add(Boolean.FALSE); |
| } else { |
| final boolean first = results.get(0).isBitReversed(); |
| list.add(first); |
| for (final TestResult result : results) { |
| if (first != result.isBitReversed()) { |
| list.add(!first); |
| break; |
| } |
| } |
| } |
| Collections.sort(list); |
| return list; |
| } |
| |
| /** |
| * Gets the test names present in the results. These are returned in encountered order. |
| * |
| * @param results Results. |
| * @return the test names |
| */ |
| private static List<String> getTestNames(List<TestResult> results) { |
| // Enforce encountered order with a linked hash set. |
| final Set<String> set = new LinkedHashSet<>(); |
| for (final TestResult result : results) { |
| set.add(result.getTestApplicationName()); |
| } |
| return new ArrayList<>(set); |
| } |
| |
| /** |
| * Find the common path prefix for all result files. This is returned as the length of the |
| * common prefix. |
| * |
| * @param results Results. |
| * @return the length |
| */ |
| private static int findCommonPathPrefixLength(List<TestResult> results) { |
| if (results.isEmpty()) { |
| return 0; |
| } |
| // Find the first prefix |
| final String prefix1 = getPathPrefix(results.get(0)); |
| int length = prefix1.length(); |
| for (int i = 1; i < results.size() && length != 0; i++) { |
| final String prefix2 = getPathPrefix(results.get(i)); |
| // Update |
| final int size = Math.min(prefix2.length(), length); |
| length = 0; |
| while (length < size && prefix1.charAt(length) == prefix2.charAt(length)) { |
| length++; |
| } |
| } |
| return length; |
| } |
| |
| /** |
| * Gets the path prefix. |
| * |
| * @param testResult Test result. |
| * @return the path prefix (or the empty string) |
| */ |
| private static String getPathPrefix(TestResult testResult) { |
| final String parent = testResult.getResultFile().getParent(); |
| return parent == null ? "" : parent; |
| } |
| |
| /** |
| * Creates the APT header. |
| * |
| * @param showBitReversedColumn Set to true to the show bit reversed column. |
| * @param testNames Test names. |
| * @return the header |
| */ |
| private static String createAPTHeader(boolean showBitReversedColumn, |
| List<String> testNames) { |
| final StringBuilder sb = new StringBuilder(100).append("|| RNG identifier ||"); |
| if (showBitReversedColumn) { |
| sb.append(" Bit-reversed ||"); |
| } |
| for (final String name : testNames) { |
| sb.append(' ').append(name).append(" ||"); |
| } |
| return sb.toString(); |
| } |
| |
| /** |
| * Creates the APT separator for each table row. |
| * |
| * <p>The separator is created using the '+' character to left align the columns. |
| * |
| * @param header Header. |
| * @return the separator |
| */ |
| private static String createAPTSeparator(String header) { |
| // Replace everything with '-' except '|' which is replaced with "*-" for the first |
| // character, "+-" for all other occurrences except "-+" at the end |
| final StringBuilder sb = new StringBuilder(header); |
| for (int i = 0; i < header.length(); i++) { |
| if (sb.charAt(i) == PIPE) { |
| sb.setCharAt(i, i == 0 ? '*' : '+'); |
| sb.setCharAt(i + 1, '-'); |
| } else { |
| sb.setCharAt(i, '-'); |
| } |
| } |
| // Fix the end |
| sb.setCharAt(header.length() - 2, '-'); |
| sb.setCharAt(header.length() - 1, '+'); |
| sb.append(System.lineSeparator()); |
| return sb.toString(); |
| } |
| |
| /** |
| * Write the column text to the output. |
| * |
| * @param output Output. |
| * @param text Text. |
| * @param highlight If {@code true} highlight the text in bold. |
| * @throws IOException Signals that an I/O exception has occurred. |
| */ |
| private static void writeAPTColumn(Appendable output, |
| String text, |
| boolean highlight) throws IOException { |
| output.append(' '); |
| if (highlight) { |
| output.append("<<"); |
| } |
| output.append(text); |
| if (highlight) { |
| output.append(">>"); |
| } |
| output.append(" |"); |
| } |
| |
| /** |
| * Gets the test results that match the arguments. |
| * |
| * @param results Results. |
| * @param randomSource Random source. |
| * @param bitReversed Bit reversed flag. |
| * @param testName Test name. |
| * @return the matching results |
| */ |
| private static List<TestResult> getTestResults(List<TestResult> results, |
| RandomSource randomSource, |
| boolean bitReversed, |
| String testName) { |
| final ArrayList<TestResult> list = new ArrayList<>(); |
| for (final TestResult result : results) { |
| if (result.getRandomSource() == randomSource && |
| result.bitReversed == bitReversed && |
| result.getTestApplicationName().equals(testName)) { |
| list.add(result); |
| } |
| } |
| return list; |
| } |
| |
| /** |
| * Gets the systematic failures (tests that fail in every test result). |
| * |
| * @param results Results. |
| * @return the systematic failures |
| */ |
| private static List<String> getSystematicFailures(List<TestResult> results) { |
| final HashMap<String, Integer> map = new HashMap<>(); |
| for (final TestResult result : results) { |
| // Ignore partial results |
| if (!result.isComplete()) { |
| continue; |
| } |
| // Some named tests can fail more than once on different statistics. |
| // For example TestU01 BigCrush LongestHeadRun can output in the summary: |
| // 86 LongestHeadRun, r = 0 eps |
| // 86 LongestHeadRun, r = 0 1 - eps1 |
| // This will be counted as 2 failed tests. For the purpose of systematic |
| // failures the name of the test is the same and should be counted once. |
| final HashSet<String> unique = new HashSet<>(result.getFailedTests()); |
| for (String test : unique) { |
| map.merge(test, 1, (i, j) -> i + j); |
| } |
| } |
| final int completeCount = (int) results.stream().filter(TestResult::isComplete).count(); |
| final List<String> list = map.entrySet().stream() |
| .filter(e -> e.getValue() == completeCount) |
| .map(Entry::getKey) |
| .collect(Collectors.toCollection( |
| (Supplier<List<String>>) ArrayList::new)); |
| // Special case for PractRand. Add the maximum RNG output length before failure. |
| // This is because some PractRand tests may not be counted as systematic failures |
| // as they have not been run to the same output length due to earlier failure of |
| // another test. |
| final int max = getMaxLengthExponent(results); |
| if (max != 0) { |
| list.add(bytesToString(max)); |
| } |
| return list; |
| } |
| |
| /** |
| * Gets the maximum length exponent from the PractRand results if <strong>all</strong> failed. |
| * Otherwise return zero (i.e. some passed the full length of the test). |
| * |
| * <p>This method excludes those results that are not complete. It assumes all complete |
| * tests are for the same length of RNG output. Thus if all failed then the max exponent |
| * is the systematic failure length.</p> |
| * |
| * @param results Results. |
| * @return the maximum length exponent (or zero) |
| */ |
| private static int getMaxLengthExponent(List<TestResult> results) { |
| if (results.isEmpty()) { |
| return 0; |
| } |
| // [0] = count of zeros |
| // [1] = max non-zero |
| final int[] data = new int[2]; |
| results.stream() |
| .filter(TestResult::isComplete) |
| .filter(r -> r instanceof PractRandTestResult) |
| .mapToInt(r -> ((PractRandTestResult) r).getLengthExponent()) |
| .forEach(i -> { |
| if (i == 0) { |
| // Count results that passed |
| data[0]++; |
| } else { |
| // Find the max of the failures |
| data[1] = Math.max(i, data[1]); |
| } |
| }); |
| // If all failed (i.e. no zeros) then return the max, otherwise zero. |
| return data[0] == 0 ? data[1] : 0; |
| } |
| |
| /** |
| * Gets a summary of the failures across all results. The text is empty if there are no |
| * failures to report. |
| * |
| * <p>For Dieharder and TestU01 this is the number of systematic failures (tests that fail |
| * in every test result). For PractRand it is the maximum byte output size that was reached |
| * before failure. |
| * |
| * <p>It is assumed all the results are for the same test suite.</p> |
| * |
| * @param results Results. |
| * @return the failures summary |
| */ |
| private static String getFailuresSummary(List<TestResult> results) { |
| if (results.isEmpty()) { |
| return ""; |
| } |
| if (results.get(0).getTestFormat() == TestFormat.PRACTRAND) { |
| final int max = getMaxLengthExponent(results); |
| return max == 0 ? "" : bytesToString(max); |
| } |
| final int count = getSystematicFailures(results).size(); |
| return count == 0 ? "" : Integer.toString(count); |
| } |
| |
| /** |
| * Write the results as a text table. |
| * |
| * @param out Output stream. |
| * @param results Results. |
| * @throws IOException Signals that an I/O exception has occurred. |
| */ |
| private static void writeTXT(OutputStream out, |
| List<TestResult> results) throws IOException { |
| // Identify all: |
| // RandomSources, bit-reversed, test names, |
| final List<RandomSource> randomSources = getRandomSources(results); |
| final List<Boolean> bitReversed = getBitReversed(results); |
| final List<String> testNames = getTestNames(results); |
| |
| // Create columns for RandomSource, bit-reversed, each test name. |
| // Make bit-reversed column optional if no generators are bit reversed. |
| final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE); |
| |
| final List<List<String>> columns = createTXTColumns(testNames, showBitReversedColumn); |
| |
| // Add all data |
| // This will collate results for each combination of 'RandomSource + bitReversed' |
| for (final RandomSource randomSource : randomSources) { |
| for (final boolean reversed : bitReversed) { |
| int i = 0; |
| columns.get(i++).add(randomSource.toString()); |
| if (showBitReversedColumn) { |
| columns.get(i++).add(Boolean.toString(reversed)); |
| } |
| for (final String testName : testNames) { |
| final List<TestResult> testResults = getTestResults(results, randomSource, |
| reversed, testName); |
| columns.get(i++).add(testResults.stream() |
| .map(TestResult::getFailureSummaryString) |
| .collect(Collectors.joining(","))); |
| columns.get(i++).add(getFailuresSummary(testResults)); |
| } |
| } |
| } |
| |
| writeColumns(out, columns); |
| } |
| |
| /** |
| * Creates the columns for the text output. |
| * |
| * @param testNames the test names |
| * @param showBitReversedColumn Set to true to show the bit reversed column |
| * @return the list of columns |
| */ |
| private static List<List<String>> createTXTColumns(final List<String> testNames, |
| final boolean showBitReversedColumn) { |
| final ArrayList<List<String>> columns = new ArrayList<>(); |
| columns.add(createColumn(COLUMN_RNG)); |
| if (showBitReversedColumn) { |
| columns.add(createColumn(BIT_REVERSED)); |
| } |
| for (final String testName : testNames) { |
| columns.add(createColumn(testName)); |
| columns.add(createColumn("∩")); |
| } |
| return columns; |
| } |
| |
| /** |
| * Creates the column. |
| * |
| * @param columnName Column name. |
| * @return the list |
| */ |
| private static List<String> createColumn(String columnName) { |
| final ArrayList<String> list = new ArrayList<>(); |
| list.add(columnName); |
| return list; |
| } |
| |
| |
| /** |
| * Creates the text format from column widths. |
| * |
| * @param columns Columns. |
| * @return the text format string |
| */ |
| private static String createTextFormatFromColumnWidths(final List<List<String>> columns) { |
| final StringBuilder sb = new StringBuilder(); |
| try (Formatter formatter = new Formatter(sb)) { |
| for (int i = 0; i < columns.size(); i++) { |
| if (i != 0) { |
| sb.append('\t'); |
| } |
| formatter.format("%%-%ds", getColumnWidth(columns.get(i))); |
| } |
| } |
| sb.append(System.lineSeparator()); |
| return sb.toString(); |
| } |
| |
| /** |
| * Gets the column width using the maximum length of the column items. |
| * |
| * @param column Column. |
| * @return the column width |
| */ |
| private static int getColumnWidth(List<String> column) { |
| int width = 0; |
| for (final String text : column) { |
| width = Math.max(width, text.length()); |
| } |
| return width; |
| } |
| |
| /** |
| * Write the columns as fixed width text to the output. |
| * |
| * @param out Output stream. |
| * @param columns Columns |
| * @throws IOException Signals that an I/O exception has occurred. |
| */ |
| private static void writeColumns(OutputStream out, |
| List<List<String>> columns) throws IOException { |
| // Create format using the column widths |
| final String format = createTextFormatFromColumnWidths(columns); |
| |
| // Output |
| try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8)); |
| Formatter formatter = new Formatter(output)) { |
| final int rows = columns.get(0).size(); |
| final Object[] args = new Object[columns.size()]; |
| for (int row = 0; row < rows; row++) { |
| for (int i = 0; i < args.length; i++) { |
| args[i] = columns.get(i).get(row); |
| } |
| formatter.format(format, args); |
| } |
| } |
| } |
| |
| /** |
| * Write the systematic failures as a text table. |
| * |
| * @param out Output stream. |
| * @param results Results. |
| * @throws IOException Signals that an I/O exception has occurred. |
| */ |
| private static void writeFailures(OutputStream out, |
| List<TestResult> results) throws IOException { |
| // Identify all: |
| // RandomSources, bit-reversed, test names, |
| final List<RandomSource> randomSources = getRandomSources(results); |
| final List<Boolean> bitReversed = getBitReversed(results); |
| final List<String> testNames = getTestNames(results); |
| |
| // Create columns for RandomSource, bit-reversed, each test name. |
| // Make bit-reversed column optional if no generators are bit reversed. |
| final boolean showBitReversedColumn = bitReversed.contains(Boolean.TRUE); |
| |
| final List<List<String>> columns = createFailuresColumns(testNames, showBitReversedColumn); |
| |
| final AlphaNumericComparator cmp = new AlphaNumericComparator(); |
| |
| // Add all data for each combination of 'RandomSource + bitReversed' |
| for (final RandomSource randomSource : randomSources) { |
| for (final boolean reversed : bitReversed) { |
| for (final String testName : testNames) { |
| final List<TestResult> testResults = getTestResults(results, randomSource, |
| reversed, testName); |
| final List<String> failures = getSystematicFailures(testResults); |
| if (failures.isEmpty()) { |
| continue; |
| } |
| Collections.sort(failures, cmp); |
| for (String failed : failures) { |
| int i = 0; |
| columns.get(i++).add(randomSource.toString()); |
| if (showBitReversedColumn) { |
| columns.get(i++).add(Boolean.toString(reversed)); |
| } |
| columns.get(i++).add(testName); |
| columns.get(i).add(failed); |
| } |
| } |
| } |
| } |
| |
| writeColumns(out, columns); |
| } |
| |
| /** |
| * Creates the columns for the failures output. |
| * |
| * @param testNames the test names |
| * @param showBitReversedColumn Set to true to show the bit reversed column |
| * @return the list of columns |
| */ |
| private static List<List<String>> createFailuresColumns(final List<String> testNames, |
| final boolean showBitReversedColumn) { |
| final ArrayList<List<String>> columns = new ArrayList<>(); |
| columns.add(createColumn(COLUMN_RNG)); |
| if (showBitReversedColumn) { |
| columns.add(createColumn(BIT_REVERSED)); |
| } |
| columns.add(createColumn("Test Suite")); |
| columns.add(createColumn("Test")); |
| return columns; |
| } |
| |
| /** |
| * Convert bytes to a human readable string. The byte size is expressed in powers of 2. |
| * The output units use the ISO binary prefix for increments of 2<sup>10</sup> or 1024. |
| * |
| * <p>This is a utility function used for reporting PractRand output sizes. |
| * Example output: |
| * |
| * <pre> |
| * exponent Binary |
| * 0 0 B |
| * 10 1 KiB |
| * 13 8 KiB |
| * 20 1 MiB |
| * 27 128 MiB |
| * 30 1 GiB |
| * 40 1 TiB |
| * 50 1 PiB |
| * 60 1 EiB |
| * </pre> |
| * |
| * @param exponent Exponent of the byte size (i.e. 2^exponent). |
| * @return the string |
| */ |
| static String bytesToString(int exponent) { |
| final int unit = exponent / 10; |
| final int size = 1 << (exponent - 10 * unit); |
| return size + BINARY_UNITS[unit]; |
| } |
| } |
