|
From: Josef W. <Jos...@gm...> - 2006-03-31 20:52:14
|
Hi,
while thinking about regression tests for callgrind, I found
out that there seems to be no way to post-process any further
files generated by a test, and eventually trigger a failure.
This would be handy to check for the expected call graph in
callgrind dump files.
What do you think about adding a post-test check which could
check for expected behaviour of the test and optionally,
by filtering the files and write them to stdout, be able to
check against expected results of generated files?
I think this could be useful for cachegrind or massif, too.
Patch attached for discussion.
Josef
Index: vg_regtest.in
===================================================================
--- vg_regtest.in (Revision 5807)
+++ vg_regtest.in (Arbeitskopie)
@@ -50,6 +50,7 @@
# - stdout_filter: <filter to run stdout through> (default: none)
# - stderr_filter: <filter to run stderr through> (default: ./filter_stderr)
# - prereq: <prerequisite command> (default: none)
+# - posttest: <post-test check command> (default: none)
# - cleanup: <post-test cleanup cmd to run> (default: none)
#
# Note that filters are necessary for stderr results to filter out things that
@@ -60,7 +61,8 @@
# stderr (filtered) is kept in <test>.stderr.exp[0-9]*.
#
# The prerequisite command, if present, must return 0 otherwise the test is
-# skipped.
+# skipped. The post-test command, if present, must return 0 and the stdout
+# match the expected stdout which is kept in <test>.posttest.exp[0-9]*.
#
# If results don't match, the output can be found in <test>.std<strm>.out,
# and the diff between expected and actual in <test>.std<strm>.diff[0-9]*.
@@ -86,12 +88,13 @@
my $stdout_filter; # filter program to run stdout results file through
my $stderr_filter; # filter program to run stderr results file through
my $prereq; # prerequisite test to satisfy before running test
+my $posttest; # check command after running test
my $cleanup; # cleanup command to run
my @failures; # List of failed tests
my $num_tests_done = 0;
-my %num_failures = (stderr => 0, stdout => 0);
+my %num_failures = (stderr => 0, stdout => 0, posttest => 0);
# Default valgrind to use is this build tree's (uninstalled) one
my $valgrind = "./coregrind/valgrind";
@@ -172,8 +175,8 @@
my ($f) = @_;
# Defaults.
- ($vgopts, $prog, $args, $stdout_filter, $stderr_filter, $prereq, $cleanup)
- = ("", undef, "", undef, undef, undef, undef);
+ ($vgopts, $prog, $args, $stdout_filter, $stderr_filter, $prereq, $posttest, $cleanup)
+ = ("", undef, "", undef, undef, undef, undef, undef);
# Every test directory must have a "filter_stderr"
$stderr_filter = validate_program(".", $default_stderr_filter, 1, 1);
@@ -195,6 +198,8 @@
$stderr_filter = validate_program(".", $1, 1, 1);
} elsif ($line =~ /^\s*prereq:\s*(.*)$/) {
$prereq = $1;
+ } elsif ($line =~ /^\s*posttest:\s*(.*)$/) {
+ $posttest = $1;
} elsif ($line =~ /^\s*cleanup:\s*(.*)$/) {
$cleanup = $1;
} else {
@@ -295,7 +300,20 @@
mysystem("$stderr_filter < $name.stderr.out > $tmp");
rename($tmp, "$name.stderr.out");
+ if (defined $posttest) {
+ if (mysystem("$posttest > $name.posttest.out") != 0) {
+ print("posttest failed: $posttest\n");
+ $num_failures{"posttest"}++;
+ }
+ else {
+ # Find all the .posttest.exp files. If none, use /dev/null.
+ my @posttest_exps = <$name.posttest.exp*>;
+ @posttest_exps = ( "/dev/null" ) if (0 == scalar @posttest_exps);
+ do_diffs($fullname, $name, "posttest", \@posttest_exps);
+ }
+ }
+
# Find all the .stdout.exp files. If none, use /dev/null.
my @stdout_exps = <$name.stdout.exp*>;
@stdout_exps = ( "/dev/null" ) if (0 == scalar @stdout_exps);
@@ -375,10 +393,11 @@
{
my $x = ( $num_tests_done == 1 ? "test" : "tests" );
- printf("\n== %d test%s, %d stderr failure%s, %d stdout failure%s =================\n",
+ printf("\n== %d test%s, %d stderr failure%s, %d stdout failure%s, %d posttest failure%s ==\n",
$num_tests_done, plural($num_tests_done),
$num_failures{"stderr"}, plural($num_failures{"stderr"}),
- $num_failures{"stdout"}, plural($num_failures{"stdout"}));
+ $num_failures{"stdout"}, plural($num_failures{"stdout"}),
+ $num_failures{"posttest"}, plural($num_failures{"posttest"}));
foreach my $failure (@failures) {
print "$failure\n";
@@ -415,7 +434,9 @@
}
summarise_results();
-if (0 == $num_failures{"stdout"} && 0 == $num_failures{"stderr"}) {
+if (0 == $num_failures{"stdout"} &&
+ 0 == $num_failures{"stderr"} &&
+ 0 == $num_failures{"posttest"}) {
exit 0;
} else {
exit 1;
|