| #use strict; |
| |
| # usage perl harness.pl [testNum] |
| # where testNum is an integer from 1 to 41 |
| # testNum may also be a list of integers denoting tests to be run |
| # ommitting testNum will lead to all tests being run |
| # |
| # this harness outputs the results to the file results.html as well |
| # as reporting them to stderr. |
| # |
| # note that the XInclude sample application must be discoverable on |
| # the current PATH. |
| |
| # |
| # Find out the platform from 'uname -s' |
| # |
| open(PLATFORM, "uname -s|"); |
| $platform = <PLATFORM>; |
| chomp($platform); |
| close (PLATFORM); |
| |
| if ($platform =~ m/Windows/i || $platform =~ m/CYGWIN/i) { |
| $pathsep = "\\"; |
| } |
| else { |
| $pathsep = "/"; |
| } |
| |
| @CORRECT_TEST_RESULTS = ( |
| -3, |
| 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, |
| 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, |
| 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, |
| 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, |
| 1 |
| ); |
| |
| # private static boolean[] TEST_RESULTS = new boolean[] { |
| # // one value for each test |
| # true, true, true, true, true, true, false, true, false, true, // 10 |
| # false, false, false, false, true, true, true, false, true, true, // 20 |
| # true, false, true, false, false, false, true, true, false, true, // 30 |
| # true, false, true, true, true, true, true, true, false, false, // 40 |
| # true, }; |
| |
| #no need to add one to this value since there is a dummy value in the tables |
| $NUM_TESTS = $#CORRECT_TEST_RESULTS; |
| |
| @ACTUAL_TEST_RESULTS = ( |
| -3, |
| -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, |
| -1 |
| ); |
| |
| print "Running XInclude Tests\n"; |
| |
| $dateAndTime = localtime(); |
| |
| open(outfile, ">results.html"); |
| print outfile "<html><head><title>XInclude results generated at $dateAndTime </title>"; |
| print outfile "</head><body bgcolor=\"black\" text=\"white\"><center>"; |
| |
| print outfile "<table border cellspacing=\"0\" cellpadding=\"5\"><caption align=\"bottom\">Results of XI Test Run</caption>"; |
| print outfile "<tr><th>Test Number</th><th>Test Input File</th><th>Generated Output File</th>"; |
| print outfile "<th>Reference Result File</th><th>Result</th><th>Expected Result</th><th>Test Status</th></tr>"; |
| |
| $successes = 0; |
| $correct = 0; |
| $numTestsToRun = $#ARGV + 1; |
| while (<@ARGV>){ |
| $i = $_; |
| if ($i < 10) { |
| $testNum = "0$i"; |
| } else { |
| $testNum = "$i"; |
| } |
| $input_file = "tests".$pathsep."xinclude".$pathsep."tests".$pathsep."test$testNum.xml"; |
| if ($CORRECT_TEST_RESULTS[$i] == 1) { |
| $output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.xml"; |
| $expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.xml"; |
| } else { |
| $output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.txt"; |
| $expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.txt"; |
| } |
| |
| # run the tests and apture the output |
| $command = "XInclude $input_file $output_file"; |
| print "R[$command]\n"; |
| system("$command"); |
| |
| $outcome = 0; |
| if (! -e $output_file) { |
| $ACTUAL_TEST_RESULTS[$i] = 0; |
| # we didn't generate an output, we are correct if the test was expected to fail |
| if($CORRECT_TEST_RESULTS[$i] eq 0) { |
| $outcome = 1; |
| } |
| } else { |
| $ACTUAL_TEST_RESULTS[$i] = 1; |
| # we generated an output, we are correct if our result matches |
| if($CORRECT_TEST_RESULTS[$i] eq 1) { |
| $outcome = &compareFiles($expected_result_file, $output_file); |
| } |
| } |
| |
| print outfile "<tr bgcolor="; |
| if ($outcome eq 1) { |
| print outfile "green"; |
| } else { |
| print outfile "red"; |
| } |
| |
| print outfile ">"; |
| #test number |
| print outfile "<td>$testNum</td>"; |
| print outfile "<td>"; |
| $anchor = &createHTMLAnchor("$input_file"); |
| print outfile "$anchor"; |
| print outfile "</td><td>"; |
| $anchor = &createHTMLAnchor("$output_file"); |
| print outfile "$anchor"; |
| print outfile "</td><td>"; |
| $anchor = &createHTMLAnchor("$expected_result_file"); |
| print outfile "$anchor"; |
| print outfile "</td>"; |
| # actual result |
| $result = (($ACTUAL_TEST_RESULTS[$i])?"true":"false"); |
| print outfile "<td>$result</td>"; |
| |
| # correct result |
| $result = (($CORRECT_TEST_RESULTS[$i])?"true":"false"); |
| print outfile "<td>$result</td>"; |
| |
| if ($outcome eq 1) { |
| $successes++; |
| $result = "Passed"; |
| print "[test $testNum PASSED]\n"; |
| $correct++; |
| } else { |
| $result = "Failed"; |
| print "[test $testNum FAILED]\n"; |
| } |
| print outfile "<td>$result</td></tr>"; |
| } |
| |
| print outfile "</table>"; |
| |
| print "Tests Passed: $correct\n"; |
| $percentSucceeding = $correct / $numTestsToRun * 100.0; |
| print outfile "<hr/><h2>$percentSucceeding% of tests that were run passed ($correct out of $numTestsToRun).</h2>"; |
| |
| $shPass = 0; |
| $shFail = 0; |
| for ($i = 1; $i < $NUM_TESTS+1; $i++){ |
| if ($CORRECT_TEST_RESULTS[$i]){ |
| $shPass++; |
| } else { |
| $shFail++; |
| } |
| } |
| |
| print outfile "<h2>$shPass should pass ($successes) did</h2>"; |
| $failures = $NUM_TESTS - $successes; |
| print outfile "<h2>$shFail should fail ($failures) did</h2>"; |
| |
| $percentSucceeding = $correct / $NUM_TESTS * 100.0; |
| print outfile "<h2>$percentSucceeding% of all available tests passed ($correct out of $NUM_TESTS).</h2>"; |
| |
| $percentSucceeding = $numTestsToRun / $NUM_TESTS * 100.0; |
| print outfile "<h2>$percentSucceeding% of all available tests were run ($numTestsToRun out of $NUM_TESTS).</h2>"; |
| |
| print outfile "</center><hr/></body></html>"; |
| close(outfile); |
| |
| # quick and dirty but functional |
| sub compareFiles |
| { |
| $result = 1; |
| print "C[$_[0]]:[$_[1]]\n"; |
| |
| if (! -e $_[0]) { |
| print "No such file as $_[0] - cannot compare\n"; |
| return 0; |
| } |
| |
| if (! -e $_[1]) { |
| print "No such file as $_[1] - cannot compare\n"; |
| return 0; |
| } |
| |
| open(expected, "<$_[0]"); |
| open(actual, "<$_[1]"); |
| |
| #compare the files |
| @expectedData = <expected>; |
| @actualData = <actual>; |
| |
| close(expected); |
| close(actual); |
| |
| for ($x = 0, $a = 0; $x < $#expectedData && $a < $#actualData; $x++, $a++){ |
| $dataLineEx = $expectedData[$x]; |
| $dataLineAc = $actualData[$a]; |
| chomp($dataLineEx); |
| chomp($dataLineAc); |
| if ($dataLineEx ne $dataLineAc) { |
| # check if its a warning and can be ignored at this stage |
| if ($dataLineEx =~ /Warning/) { |
| print "probably just a warning line:\n"; |
| print "[$dataLineEx]\n"; |
| $a-- |
| } elsif ( $dataLineEx =~ m/^\s*$/ && $dataLineAc =~ m/^\s*$/ ) { |
| $line_num = $x + 1; |
| print "Line $line_num ws difference: \n[$dataLineEx] != \n[$dataLineAc]\n"; |
| print "probably just white space on both lines\n"; |
| $a-- |
| } else { |
| $line_num = $x + 1; |
| print "Line $line_num difference: \n[$dataLineEx] != \n[$dataLineAc]\n"; |
| print "From files: $_[0] $_[1]\n"; |
| return 0; |
| } |
| } |
| } |
| |
| return 1; |
| } |
| |
| sub createHTMLAnchor { |
| "<a target=\"_blank\" href=\"$_[0]\">$_[0]</a>"; |
| } |
