diff options
author | Ben Maurer <benm@mono-cvs.ximian.com> | 2004-03-25 07:55:54 +0300 |
---|---|---|
committer | Ben Maurer <benm@mono-cvs.ximian.com> | 2004-03-25 07:55:54 +0300 |
commit | f211f5d3c4b68d18e76d4893c71bd255a6c0405d (patch) | |
tree | 0c16e07e3db9bab69fb69e81a42f261d7912323b /mcs/errors/do-tests.pl | |
parent | 63cf8b1c33faf03f0ce3f93f05eaa5c1fe4bec10 (diff) |
2004-03-24 Ben Maurer <bmaurer@users.sourceforge.net>
* Makefile, do-tests.pl: A new system for testing.
* mcs-expect-no-errors, mcs-expect-wrong-error: a list
of tests that we fail on.
* CONTRIBUTORS_README: update docs.
svn path=/trunk/mcs/; revision=24562
Diffstat (limited to 'mcs/errors/do-tests.pl')
-rwxr-xr-x | mcs/errors/do-tests.pl | 129 |
1 files changed, 129 insertions, 0 deletions
diff --git a/mcs/errors/do-tests.pl b/mcs/errors/do-tests.pl new file mode 100755 index 00000000000..94e878bd31b --- /dev/null +++ b/mcs/errors/do-tests.pl @@ -0,0 +1,129 @@ +#!/usr/bin/perl -w + +unless ($#ARGV == 2) { + print STDERR "Usage: $0 profile compiler glob-pattern\n"; + exit 1; +} + +# +# Expected value constants +# +my $EXPECTING_WRONG_ERROR = 1; +my $EXPECTING_NO_ERROR = 2; +my %expecting_map = (); + +my $profile = $ARGV [0]; +my $compile = $ARGV [1]; +my $files = $ARGV [2]; + +if (open (EXPECT_WRONG, "<$profile-expect-wrong-error")) { + $expecting_map{$_} = $EXPECTING_WRONG_ERROR + foreach map { + chomp, # remove trailing \n + s/\#.*//g, # remove # style comments + s/ //g; # remove whitespace + $_ eq "" ? () : $_; # now copy over non empty stuff + } <EXPECT_WRONG>; + + close EXPECT_WRONG; +} + +if (open (EXPECT_NO, "<$profile-expect-no-error")) { + $expecting_map{$_} = $EXPECTING_NO_ERROR + foreach map { + chomp, # remove trailing \n + s/\#.*//g, # remove # style comments + s/ //g; # remove whitespace + $_ eq "" ? () : $_; # now copy over non empty stuff + } <EXPECT_NO>; + + close EXPECT_NO; +} +my $RESULT_UNEXPECTED_CORRECT_ERROR = 1; +my $RESULT_CORRECT_ERROR = 2; +my $RESULT_UNEXPECTED_INCORRECT_ERROR = 3; +my $RESULT_EXPECTED_INCORRECT_ERROR = 4; +my $RESULT_UNEXPECTED_NO_ERROR = 5; +my $RESULT_EXPECTED_NO_ERROR = 6; + +my @statuses = ( + "UNEXPECTED SUCCESS", + "SUCCESS", + "UNEXPECTED INCORRECT ERROR", + "INCORRECT ERROR", + "UNEXPECTED NO ERROR", + "NO ERROR", +); + +my @status_items = ( + [], + [], + [], + [], + [], + [], +); + +my %results_map = (); + +foreach (glob ($files)) { + + print "$_..."; + my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/); + my $options = `fgrep "// Compiler options:" $_ | sed -e 's/\\/\\/ Compiler options://'`; + system "$compile $_ --expect-error $error_number $options > /dev/null"; + + exit 1 if $? & 127; + + my $exit_value = $? >> 8; + + die "unexpected return from mcs" if $exit_value > 2; + + my $status; + + if ($exit_value == 0) { + $status = $RESULT_UNEXPECTED_CORRECT_ERROR if exists $expecting_map {$_}; + $status = $RESULT_CORRECT_ERROR unless exists $expecting_map {$_}; + } + + if ($exit_value == 1) { + $status = $RESULT_UNEXPECTED_INCORRECT_ERROR unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR; + $status = $RESULT_EXPECTED_INCORRECT_ERROR if exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR; + } + + if ($exit_value == 2) { + $status = $RESULT_UNEXPECTED_NO_ERROR unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR; + $status = $RESULT_EXPECTED_NO_ERROR if exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR; + } + + + push @{@status_items [($status - 1)]}, $_; + print "@statuses[($status - 1)]\n"; + $results_map{$_} = $status; +} +print scalar @{@status_items [($RESULT_CORRECT_ERROR - 1)]}, " Correct errors\n"; +print scalar @{@status_items [($RESULT_EXPECTED_INCORRECT_ERROR - 1)]}, " Incorrect errors\n"; +print scalar @{@status_items [($RESULT_EXPECTED_NO_ERROR - 1)]}, " No errors\n"; + +print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]}, " Unexpected correct errors\n"; +print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]}; + +print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}, " Unexpected incorrect errors\n"; +print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}; + +print scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}, " Unexpected no errors\n"; +print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}; + +print "\n"; + +print "OVERALL:"; +print scalar @{@status_items [($RESULT_CORRECT_ERROR - 1)]}, " tests succeeded\n"; +print scalar @{@status_items [($RESULT_EXPECTED_INCORRECT_ERROR - 1)]} + scalar @{@status_items [($RESULT_EXPECTED_NO_ERROR - 1)]}, " known errors\n"; +print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} + scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}, " errors\n"; +print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]}, " new tests passing\n"; + +exit ( + scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} + + scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]} + + scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]} +) == 0 ? 0 : 1; |