#use strict; | |
# usage perl harness.pl [testNum] | |
# where testNum is an integer from 1 to 41 | |
# testNum may also be a list of integers denoting tests to be run | |
# ommitting testNum will lead to all tests being run | |
# | |
# this harness outputs the results to the file results.html as well | |
# as reporting them to stderr. | |
# | |
# note that the XInclude sample application must be discoverable on | |
# the current PATH. | |
# | |
# Find out the platform from 'uname -s' | |
# | |
open(PLATFORM, "uname -s|"); | |
$platform = <PLATFORM>; | |
chomp($platform); | |
close (PLATFORM); | |
if ($platform =~ m/Windows/i || $platform =~ m/CYGWIN/i) { | |
$pathsep = "\\"; | |
} | |
else { | |
$pathsep = "/"; | |
} | |
@CORRECT_TEST_RESULTS = ( | |
-3, | |
1, 1, 1, 1, 1, 1, 0, 1, 0, 1, | |
0, 0, 0, 0, 1, 1, 1, 0, 1, 1, | |
1, 0, 1, 0, 0, 0, 1, 1, 0, 1, | |
1, 0, 1, 1, 1, 1, 1, 1, 0, 0, | |
1 | |
); | |
# private static boolean[] TEST_RESULTS = new boolean[] { | |
# // one value for each test | |
# true, true, true, true, true, true, false, true, false, true, // 10 | |
# false, false, false, false, true, true, true, false, true, true, // 20 | |
# true, false, true, false, false, false, true, true, false, true, // 30 | |
# true, false, true, true, true, true, true, true, false, false, // 40 | |
# true, }; | |
#no need to add one to this value since there is a dummy value in the tables | |
$NUM_TESTS = $#CORRECT_TEST_RESULTS; | |
@ACTUAL_TEST_RESULTS = ( | |
-3, | |
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | |
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | |
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | |
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | |
-1 | |
); | |
print "Running XInclude Tests\n"; | |
$dateAndTime = localtime(); | |
open(outfile, ">results.html"); | |
print outfile "<html><head><title>XInclude results generated at $dateAndTime </title>"; | |
print outfile "</head><body bgcolor=\"black\" text=\"white\"><center>"; | |
print outfile "<table border cellspacing=\"0\" cellpadding=\"5\"><caption align=\"bottom\">Results of XI Test Run</caption>"; | |
print outfile "<tr><th>Test Number</th><th>Test Input File</th><th>Generated Output File</th>"; | |
print outfile "<th>Reference Result File</th><th>Result</th><th>Expected Result</th><th>Test Status</th></tr>"; | |
$successes = 0; | |
$correct = 0; | |
$numTestsToRun = $#ARGV + 1; | |
while (<@ARGV>){ | |
$i = $_; | |
if ($i < 10) { | |
$testNum = "0$i"; | |
} else { | |
$testNum = "$i"; | |
} | |
$input_file = "tests".$pathsep."xinclude".$pathsep."tests".$pathsep."test$testNum.xml"; | |
if ($CORRECT_TEST_RESULTS[$i] == 1) { | |
$output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.xml"; | |
$expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.xml"; | |
} else { | |
$output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.txt"; | |
$expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.txt"; | |
} | |
# run the tests and apture the output | |
$command = "XInclude $input_file $output_file"; | |
print "R[$command]\n"; | |
system("$command"); | |
$outcome = 0; | |
if (! -e $output_file) { | |
$ACTUAL_TEST_RESULTS[$i] = 0; | |
# we didn't generate an output, we are correct if the test was expected to fail | |
if($CORRECT_TEST_RESULTS[$i] eq 0) { | |
$outcome = 1; | |
} | |
} else { | |
$ACTUAL_TEST_RESULTS[$i] = 1; | |
# we generated an output, we are correct if our result matches | |
if($CORRECT_TEST_RESULTS[$i] eq 1) { | |
$outcome = &compareFiles($expected_result_file, $output_file); | |
} | |
} | |
print outfile "<tr bgcolor="; | |
if ($outcome eq 1) { | |
print outfile "green"; | |
} else { | |
print outfile "red"; | |
} | |
print outfile ">"; | |
#test number | |
print outfile "<td>$testNum</td>"; | |
print outfile "<td>"; | |
$anchor = &createHTMLAnchor("$input_file"); | |
print outfile "$anchor"; | |
print outfile "</td><td>"; | |
$anchor = &createHTMLAnchor("$output_file"); | |
print outfile "$anchor"; | |
print outfile "</td><td>"; | |
$anchor = &createHTMLAnchor("$expected_result_file"); | |
print outfile "$anchor"; | |
print outfile "</td>"; | |
# actual result | |
$result = (($ACTUAL_TEST_RESULTS[$i])?"true":"false"); | |
print outfile "<td>$result</td>"; | |
# correct result | |
$result = (($CORRECT_TEST_RESULTS[$i])?"true":"false"); | |
print outfile "<td>$result</td>"; | |
if ($outcome eq 1) { | |
$successes++; | |
$result = "Passed"; | |
print "[test $testNum PASSED]\n"; | |
$correct++; | |
} else { | |
$result = "Failed"; | |
print "[test $testNum FAILED]\n"; | |
} | |
print outfile "<td>$result</td></tr>"; | |
} | |
print outfile "</table>"; | |
print "Tests Passed: $correct\n"; | |
$percentSucceeding = $correct / $numTestsToRun * 100.0; | |
print outfile "<hr/><h2>$percentSucceeding% of tests that were run passed ($correct out of $numTestsToRun).</h2>"; | |
$shPass = 0; | |
$shFail = 0; | |
for ($i = 1; $i < $NUM_TESTS+1; $i++){ | |
if ($CORRECT_TEST_RESULTS[$i]){ | |
$shPass++; | |
} else { | |
$shFail++; | |
} | |
} | |
print outfile "<h2>$shPass should pass ($successes) did</h2>"; | |
$failures = $NUM_TESTS - $successes; | |
print outfile "<h2>$shFail should fail ($failures) did</h2>"; | |
$percentSucceeding = $correct / $NUM_TESTS * 100.0; | |
print outfile "<h2>$percentSucceeding% of all available tests passed ($correct out of $NUM_TESTS).</h2>"; | |
$percentSucceeding = $numTestsToRun / $NUM_TESTS * 100.0; | |
print outfile "<h2>$percentSucceeding% of all available tests were run ($numTestsToRun out of $NUM_TESTS).</h2>"; | |
print outfile "</center><hr/></body></html>"; | |
close(outfile); | |
# quick and dirty but functional | |
sub compareFiles | |
{ | |
$result = 1; | |
print "C[$_[0]]:[$_[1]]\n"; | |
if (! -e $_[0]) { | |
print "No such file as $_[0] - cannot compare\n"; | |
return 0; | |
} | |
if (! -e $_[1]) { | |
print "No such file as $_[1] - cannot compare\n"; | |
return 0; | |
} | |
open(expected, "<$_[0]"); | |
open(actual, "<$_[1]"); | |
#compare the files | |
@expectedData = <expected>; | |
@actualData = <actual>; | |
close(expected); | |
close(actual); | |
for ($x = 0, $a = 0; $x < $#expectedData && $a < $#actualData; $x++, $a++){ | |
$dataLineEx = $expectedData[$x]; | |
$dataLineAc = $actualData[$a]; | |
chomp($dataLineEx); | |
chomp($dataLineAc); | |
if ($dataLineEx ne $dataLineAc) { | |
# check if its a warning and can be ignored at this stage | |
if ($dataLineEx =~ /Warning/) { | |
print "probably just a warning line:\n"; | |
print "[$dataLineEx]\n"; | |
$a-- | |
} elsif ( $dataLineEx =~ m/^\s*$/ && $dataLineAc =~ m/^\s*$/ ) { | |
$line_num = $x + 1; | |
print "Line $line_num ws difference: \n[$dataLineEx] != \n[$dataLineAc]\n"; | |
print "probably just white space on both lines\n"; | |
$a-- | |
} else { | |
$line_num = $x + 1; | |
print "Line $line_num difference: \n[$dataLineEx] != \n[$dataLineAc]\n"; | |
print "From files: $_[0] $_[1]\n"; | |
return 0; | |
} | |
} | |
} | |
return 1; | |
} | |
sub createHTMLAnchor { | |
"<a target=\"_blank\" href=\"$_[0]\">$_[0]</a>"; | |
} |