commit 9c8a0916ad7697e4a5e36371593f0c2ea4840efb from: Sven M. Hallberg date: Sun May 25 13:18:06 2025 UTC separate run and total counters Total is the number of tests we intend to execute. It is semantically the counter that was called 'run' before. It may increase while running test suites but is otherwise set up-front. Run is now the number of tests we have so far (at least tried) to execute and increments as we go. It will normally be the sum of ok, fail, and error. If and when we add parallel execution of tests, the sum might lag behind because 'run' gets incremented before the test is finished. Counter consistency gains the requirement that run not exceed total. At the same time, it is relaxed such that run may end lower than total. Thus test suites (or exercise itself) may abort early (after the first failure, say) without such a run being considered invalid. This commit includes two new tests and fixes some bugs they exposed. commit - d020afe4a99674cac985dfc693d4f3a4d0bd5823 commit + 9c8a0916ad7697e4a5e36371593f0c2ea4840efb blob - 87bba5bd2068ef0369f5ef9408b91cc6a86ddefd blob + 5683696daa291398786336627041302aeb81e24c --- exercise +++ exercise @@ -36,13 +36,16 @@ proc execute_test {countervar path} { set errfile "/tmp/exercise.testerr.[pid]" } + # Count this test as an (attempted) run. + dict incr counters run + + set scounters {} if {[catch { # Execute the test, attaching pipes to stdin and stdout. set pipe [open "|$exe 2>$errfile" r+] # Consume the test's stdout. Ignore it for normal tests. # Continually update the progress report for test suites. - set scounters {} while {[gets $pipe out] >= 0} { if {$suite} { set scounters [sanitize_suite_counters $out] @@ -50,11 +53,11 @@ proc execute_test {countervar path} { } } - # Wait for the subprocess to exit. - close $pipe - # Sanity check on final test suite subcounters. if {$suite} {check_suite_counters $scounters} + + # Wait for the subprocess to exit. + close $pipe } res opt] } { set details [dict get $opt -errorcode] @@ -141,9 +144,9 @@ proc find_files_iter {resultvar dirsvar paths} { if {[dict exists $dirs $normpath]} continue dict set dirs $normpath {} - set subs [glob -type d -nocomplain -directory $path $opts(-pd)] - set suites [glob -type f -nocomplain -directory $path $opts(-ps)] - set tests [glob -type f -nocomplain -directory $path $opts(-pt)] + set subs [glob -type d -nocomplain -directory $path -- $opts(-pd)] + set suites [glob -type f -nocomplain -directory $path -- $opts(-ps)] + set tests [glob -type f -nocomplain -directory $path -- $opts(-pt)] find_files_iter result dirs [lsort $subs] lappend result {*}[lsort $suites] @@ -176,7 +179,7 @@ proc color {c s} { proc inconsistent {counters} { dict with counters { - expr {$run != $ok + $fail + $error} + expr {$run > $total || $run != $ok + $fail + $error} } } @@ -188,6 +191,7 @@ proc notok {counters} { proc summary {counters} { dict with counters { + set t "total $total" set r "run $run" set o "ok $ok" set f "fail $fail" @@ -197,13 +201,14 @@ proc summary {counters} { if {[isatty stdout]} { set good [expr {$ok>0 && ![notok $counters]}] - if {$run==0} {set r [color {31} $r]} ;# dark red - if {$good} {set o [color {1;32} $o]} ;# bright green - if {$fail>0} {set f [color {1;31} $f]} ;# bright red - if {$error>0} {set e [color {31} $e]} ;# dark red + if {$total == 0} {set t [color {31} $t]} ;# dark red + if {$run < $total} {set r [color {1} $r]} ;# bold + if {$good} {set o [color {1;32} $o]} ;# bright green + if {$fail > 0} {set f [color {1;31} $f]} ;# bright red + if {$error > 0} {set e [color {31} $e]} ;# dark red } - return "$r $o $f $e" + return "$t $r $o $f $e" } } @@ -230,15 +235,18 @@ proc add_suite_counters {counters scounters} { dict incr counters $key $cnt } } - dict incr counters run -1 ;# remove the suite itself from the total count + + # remove the suite itself from the "run" and "total" counts + dict incr counters run -1 + dict incr counters total -1 + return $counters } # Make sure counters is a dict of non-negative integers. proc sanitize_suite_counters {counters} { if {[catch {dict create {*}$counters} res]} { - return -code error -errorcode EXER_SUITE \ - "malformed summary: $counters" + return -code error -errorcode EXER_SUITE "malformed summary: $counters" } dict for {key val} $counters { @@ -256,18 +264,17 @@ proc check_suite_counters {counters} { dict with counters {} if {[dict size $counters] == 0} { - return -code error -errorcode EXER_SUITE "no summary on stdout" + return -code error -errorcode EXER_SUITE "empty summary on stdout" } - foreach key {run ok fail error} { + foreach key {total run ok fail error} { if {![info exists $key]} { return -code error -errorcode EXER_SUITE "missing counter: $key" } } if {[inconsistent $counters]} { - return -code error -errorcode EXER_SUITE \ - "counters do not sum up: $counters" + return -code error -errorcode EXER_SUITE "counters inconsistent" } } @@ -320,7 +327,7 @@ proc getopt {argvvar optstring optvar argvar} { proc main {argc argv} { global opts - set counters {run 0 ok 0 fail 0 error 0} + set counters {total 0 run 0 ok 0 fail 0 error 0} # handle command line options set u {[-q] [-p k=pattern] [path ...]} @@ -352,8 +359,8 @@ proc main {argc argv} { set paths [expr {$argc == 0 ? "." : $argv}] set files [find_files $paths] - dict set counters run [llength $files] ;# tentative total run count - ;# might adjust when suites run + dict set counters total [llength $files] ;# tentative total test count + ;# might adjust when suites run foreach file $files { print_progress $counters execute_test counters $file blob - b44ac0cda14e662068ce190f013512a8b1ba4979 blob + 020d6bbd2fdad8d8e61b070f5a2e30585a7fabaf --- tests/test-error.t +++ tests/test-error.t @@ -2,7 +2,7 @@ d=`dirname $0` exec $d/assert-exercise $0 \ - 'run 1 ok 0 fail 0 error 1\n' \ + 'total 1 run 1 ok 0 fail 0 error 1\n' \ "error $0.tst\\n\\n" < $expout +printf 'total 1 run 1 ok 0 fail 0 error 1\n' > $expout printf "$0.tst: permission denied\\nerror $0.tst\\n\\n" > $experr # test to run blob - 98a912e46b1402cdd6c7dc8b001083d6e96cab14 blob + 56684357313b4d2620f58a5b80dd6466467232b7 --- tests/test-ok.t +++ tests/test-ok.t @@ -1,7 +1,7 @@ #!/bin/sh d=`dirname $0` -exec $d/assert-exercise $0 'run 1 ok 1 fail 0 error 0\n' '' <&2 blob - 17db9d761a8d3614cc0c40489023ed97c8c60087 blob + 47515912398bb9d2996724b46bf5c58d9516eefd --- tests/test-stderr-fail.t +++ tests/test-stderr-fail.t @@ -5,7 +5,7 @@ d=`dirname $0` exec $d/assert-exercise $0 \ - 'run 1 ok 0 fail 1 error 0\n' \ + 'total 1 run 1 ok 0 fail 1 error 0\n' \ "test test test\\nfail $0.tst\\n\\n" <&2 blob - a4fdaa668b35e2ae26dca4ee48af1cae0877b1d1 blob + 9c68bd1b5f13af51a957bbdb527714650129ec68 --- tests/test-stderr-ok.t +++ tests/test-stderr-ok.t @@ -5,7 +5,7 @@ d=`dirname $0` exec $d/assert-exercise $0 \ - 'run 1 ok 0 fail 1 error 0\n' \ + 'total 1 run 1 ok 0 fail 1 error 0\n' \ "test test test\\nfail $0.tst\\n\\n" <&2 blob - 6f61b5c86339c1c37cb5cb1aa8ba21e53d79b1f8 blob + a517ad3f792a706c406b0b9835d2b90a9b95c359 --- tests/testsuite-error.t +++ tests/testsuite-error.t @@ -6,9 +6,10 @@ d=`dirname $0` export tst=$0.tests exec $d/assert-exercise $0 \ - 'run 1 ok 0 fail 0 error 1\n' \ + 'total 1 run 1 ok 0 fail 0 error 1\n' \ "test test test\\nerror $tst\\n\\n" <&2 + echo 'total 1 run 1 ok 0 fail 0 error 1' exit 127 EOF blob - 08cf9d4de83219119c701c0444559996f28b6b64 blob + b20e999df43e5c70b51c0d8d1ae30b2cfb37e251 --- tests/testsuite-progress.t +++ tests/testsuite-progress.t @@ -6,15 +6,16 @@ d=`dirname $0` export tst=$0.tests exec $d/assert-exercise $0 \ - 'run 6 ok 1 fail 2 error 3\n' \ + 'total 6 run 6 ok 1 fail 2 error 3\n' \ '' <&2 + echo 'total 10 run 23 ok 22 fail 1 error 0' + exit 1 +EOF blob - 704847c58a80187b9ee6170c4608dc08a5ce495b blob + 0b4180bc91b436dbf871c0e6fdf5331453034e55 --- tests/testsuite-stderr-fd.t +++ tests/testsuite-stderr-fd.t @@ -11,10 +11,10 @@ export tst=$0.tests >$0.err # create the file assert-exercise will use err=`$STAT <$0.err` # stat it exec $d/assert-exercise $0 \ - 'run 6 ok 1 fail 2 error 3\n' \ + 'total 6 run 6 ok 1 fail 2 error 3\n' \ "$err\\n" <&2 # stat stderr, print result to stderr - echo 'run 6 ok 1 fail 2 error 3' + echo 'total 6 run 6 ok 1 fail 2 error 3' exit 1 EOF blob - /dev/null blob + 643e12c3a518322660f34abe8cc8c55aa7a3b4ec (mode 755) --- /dev/null +++ tests/testsuite-inconsistent-2.t @@ -0,0 +1,19 @@ +#!/bin/sh + +# if a test suite reports inconsistent counters it should be counted as a +# single test error, the same as if the suite had exited with 127. +# stderr should be passed through and followed by an error message and a +# generic error report. + +# this tests the case where run is not exactly the sum of ok, fail, and error. + +d=`dirname $0` +export tst=$0.tests +exec $d/assert-exercise $0 \ + 'total 1 run 1 ok 0 fail 0 error 1\n' \ + "test test test\\n$tst: counters inconsistent\\nerror $tst\\n\\n" <&2 + echo 'total 23 run 22 ok 22 fail 1 error 0' + exit 1 +EOF blob - f27d7eb9771fc58f966de15835509fc64a1cd269 blob + a779442e20f957b65558e40201546a8d13a64d48 --- tests/testsuite.t +++ tests/testsuite.t @@ -8,10 +8,10 @@ d=`dirname $0` export tst=$0.tests exec $d/assert-exercise $0 \ - 'run 6 ok 1 fail 2 error 3\n' \ + 'total 6 run 6 ok 1 fail 2 error 3\n' \ 'test test test\n' <&2 - echo 'run 6 ok 1 fail 2 error 3' + echo 'total 6 run 6 ok 1 fail 2 error 3' exit 1 EOF