commit 3a5d9be4125c991c05964e2ce88b66fcba21793e from: Sven M. Hallberg date: Sun Jun 29 14:02:28 2025 UTC back out separation of run and total counters (b7f3e3bf) The "run" counter is again the total number we intend to execute. The number of tests started so far is no longer counted. A separate counter for the number of tests that have been started is a distraction and serves very little use. I want the user to look at the (one) growing number and for that number to turn green upon success. That is the "ok" counter. Having another count up next to it (in bold no less) invites the user to look at the wrong one. (In fact, I might have made this design decision consciously before). commit - 9e8613cdd656dae8a2147f346896043ce7bca20f commit + 3a5d9be4125c991c05964e2ce88b66fcba21793e blob - b9ee736e17b8134c69d8125fa4efe3996c7c620d blob + 7386004da2c90a01a7f1488cdfd2f7090b141b7f --- exercise +++ exercise @@ -36,9 +36,6 @@ proc execute_test {countervar path} { set errfile "/tmp/exercise.testerr.[pid]" } - # Count this test as an (attempted) run. - dict incr counters run - set scounters {} if {[catch { # Execute the test, attaching pipes to stdin and stdout. @@ -186,7 +183,7 @@ proc color {c s} { proc inconsistent {counters} { dict with counters { - expr {$run > $total || $run != $ok + $fail + $error} + expr {$run < $ok + $fail + $error} } } @@ -198,7 +195,6 @@ proc notok {counters} { proc summary {counters} { dict with counters { - set t "total $total" set r "run $run" set o "ok $ok" set f "fail $fail" @@ -208,14 +204,13 @@ proc summary {counters} { if {[isatty stdout]} { set good [expr {$ok>0 && ![notok $counters]}] - if {$total == 0} {set t [color {31} $t]} ;# dark red - if {$run < $total} {set r [color {1} $r]} ;# bold + if {$run == 0} {set r [color {31} $r]} ;# dark red if {$good} {set o [color {1;32} $o]} ;# bright green if {$fail > 0} {set f [color {1;31} $f]} ;# bright red if {$error > 0} {set e [color {31} $e]} ;# dark red } - return "$t $r $o $f $e" + return "$r $o $f $e" } } @@ -243,9 +238,8 @@ proc add_suite_counters {counters scounters} { } } - # remove the suite itself from the "run" and "total" counts + # remove the suite itself from the "run" count dict incr counters run -1 - dict incr counters total -1 return $counters } @@ -274,7 +268,7 @@ proc check_suite_counters {counters} { return -code error -errorcode EXER_SUITE "empty summary on stdout" } - foreach key {total run ok fail error} { + foreach key {run ok fail error} { if {![info exists $key]} { return -code error -errorcode EXER_SUITE "missing counter: $key" } @@ -334,7 +328,7 @@ proc getopt {argvvar optstring optvar argvar} { proc main {argc argv} { global opts - set counters {total 0 run 0 ok 0 fail 0 error 0} + set counters {run 0 ok 0 fail 0 error 0} # handle command line options set u {[-q] [-B dir] [-C dir] [-p k=pattern] [path ...]} @@ -371,8 +365,8 @@ proc main {argc argv} { set files [find_files $paths] # run the tests - dict set counters total [llength $files] ;# tentative total test count - ;# might adjust when suites run + dict set counters run [llength $files] ;# tentative total run count + ;# might adjust when suites run foreach file $files { print_progress $counters execute_test counters $file blob - 17f23adf74f0d3a17bcc3c14884f409dcaa56aa7 blob + 03e968ae46b991f23dbddefa0ffa8b0721edd154 --- exercise.1 +++ exercise.1 @@ -89,7 +89,7 @@ status counters as detailed below. .Pp The standard output from a test suite executable should consist of lines of the form -.Sq total Va o No run Va n No ok Va m No fail Va l No error Va k , +.Sq run Va n No ok Va m No fail Va l No error Va k , representing running counts as printed by .Nm itself. @@ -110,24 +110,15 @@ option, .Nm prints the following running counters to its standard output: .Bl -tag -width Ds -.It Sy total +.It Sy run The number of tests that are planned to execute. This is first set to the number of test files found, including test suites. The latter are initially counted as single tests because the number of tests they contain is not known beforehand. The -.Sy total -counter will increase at run time as test suites report their totals. -.It Sy run -The number of tests that have been started so far. -This is normally the sum of -.Sy ok , fail , -and -.Sy error -plus any tests that are currently in progress. -Test suites may increment this counter before or after a test finishes, -so it may or may not include all tests in progress. +.Sy run +counter will increase dynamically as test suites report their totals. .It Sy ok The number of tests that have passed successfully, i.e. exited 0 without any output to standard error. @@ -140,31 +131,25 @@ The number of tests that could not be executed despite or that exited with the special status 127. .El .Pp -The following conditions should hold at the end of a run: -.Bl -enum -offset Ds -.It +The following should hold at the end of a run: +.Bd -filled -offset Ds .Sy run -\(<= -.Sy total -.It -.Sy run = .Sy ok + .Sy fail + .Sy error -.El +.Ed .Pp -The second condition is relaxed to an inequality at run time to account -for tests in progress, as explained above. -The +In practice, the condition is relaxed to an inequality +to account for the fact that +the sum may remain short of .Sy run -count is permitted to ultimately remain short of -.Sy total -as the result of test suites or +if +test suites or .Nm -terminating early. +itself terminate early. . .Sh DEPENDENCIES The blob - 6ea68beae5c9081fd1c50b8291b2f67efa309ede blob + 4f8b969e67d7522cc2655bd972c20c029fd0aaa6 --- tests/exercise/flag/B.t +++ tests/exercise/flag/B.t @@ -10,7 +10,7 @@ trap "rmdir $wd" EXIT mkdir $wd $h/assert-exercise $wd/foo \ - "total 1 run 1 ok 1 fail 0 error 0\n" \ + "run 1 ok 1 fail 0 error 0\n" \ '' \ < $expout +printf 'run 1 ok 0 fail 0 error 1\n' > $expout printf "$0.tst: permission denied\\nerror $0.tst\\n\\n" > $experr # test to run blob - 134cfa457bd30ef4757d0d00e7fb40d4b1a21287 blob + 12554f7b404459be430a5f0a940fef6868d19c79 --- tests/exercise/test/ok.t +++ tests/exercise/test/ok.t @@ -1,6 +1,6 @@ #!/bin/sh -exec ./assert-exercise $0 'total 1 run 1 ok 1 fail 0 error 0\n' '' <&2 blob - 869e26672939d941ae9749e15c5135b26a3d3f28 blob + 8479e06e3236600e965d7002f29cd002facb48bd --- tests/exercise/test/stderr/fail.t +++ tests/exercise/test/stderr/fail.t @@ -4,7 +4,7 @@ # should count as failure exec ./assert-exercise $0 \ - 'total 1 run 1 ok 0 fail 1 error 0\n' \ + 'run 1 ok 0 fail 1 error 0\n' \ "test test test\\nfail $0.tst\\n\\n" <&2 blob - b967245ae2d2bfe979593d7d56c1c97140b2261b blob + dd1491c8623249fad9694b680b0d5856077024e0 --- tests/exercise/test/stderr/ok.t +++ tests/exercise/test/stderr/ok.t @@ -4,7 +4,7 @@ # should count as failure exec ./assert-exercise $0 \ - 'total 1 run 1 ok 0 fail 1 error 0\n' \ + 'run 1 ok 0 fail 1 error 0\n' \ "test test test\\nfail $0.tst\\n\\n" <&2 blob - d5dadb5dd9a6cbdea1584de4e04583ca386635c5 blob + 7d44d2dff81891d51401a3f779110f44762c39d6 --- tests/exercise/testsuite/error.t +++ tests/exercise/testsuite/error.t @@ -5,10 +5,10 @@ export tst=$0.tests exec ./assert-exercise $0 \ - 'total 1 run 1 ok 0 fail 0 error 1\n' \ + 'run 1 ok 0 fail 0 error 1\n' \ "test test test\\nerror $tst\\n\\n" <&2 - echo 'total 1 run 1 ok 0 fail 0 error 1' + echo 'run 1 ok 0 fail 0 error 1' exit 127 EOF blob - 1347c4ab3021bafb10c0b6b7fee8e8aefcf5688f (mode 755) blob + /dev/null --- tests/exercise/testsuite/inconsistent-1.t +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# if a test suite reports inconsistent counters it should be counted as a -# single test error, the same as if the suite had exited with 127. -# stderr should be passed through and followed by an error message and a -# generic error report. - -# this tests the case where run exceeds total. - -export tst=$0.tests -exec ./assert-exercise $0 \ - 'total 1 run 1 ok 0 fail 0 error 1\n' \ - "test test test\\n$tst: counters inconsistent\\nerror $tst\\n\\n" <&2 - echo 'total 10 run 23 ok 22 fail 1 error 0' - exit 1 -EOF blob - /dev/null blob + 9790d53e7c84df2b62e136164c011d69b87f5ad8 (mode 755) --- /dev/null +++ tests/exercise/testsuite/inconsistent.t @@ -0,0 +1,18 @@ +#!/bin/sh + +# if a test suite reports inconsistent counters it should be counted as a +# single test error, the same as if the suite had exited with 127. +# stderr should be passed through and followed by an error message and a +# generic error report. + +# this tests the case where run is less than the sum of ok, fail, and error. + +export tst=$0.tests +exec ./assert-exercise $0 \ + 'run 1 ok 0 fail 0 error 1\n' \ + "test test test\\n$tst: counters inconsistent\\nerror $tst\\n\\n" <&2 + echo 'run 22 ok 22 fail 1 error 0' + exit 1 +EOF blob - 002e40d5bdf054b6c0121aee77b9ebea61fbf883 (mode 755) blob + /dev/null --- tests/exercise/testsuite/inconsistent-2.t +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# if a test suite reports inconsistent counters it should be counted as a -# single test error, the same as if the suite had exited with 127. -# stderr should be passed through and followed by an error message and a -# generic error report. - -# this tests the case where run is not exactly the sum of ok, fail, and error. - -export tst=$0.tests -exec ./assert-exercise $0 \ - 'total 1 run 1 ok 0 fail 0 error 1\n' \ - "test test test\\n$tst: counters inconsistent\\nerror $tst\\n\\n" <&2 - echo 'total 23 run 22 ok 22 fail 1 error 0' - exit 1 -EOF blob - 5e717acc85d5feb61e1dad81884d94c0f612ddae blob + b324c36aec5da0bf2ed708442d7aa9d999fcc7d7 --- tests/exercise/testsuite/progress.t +++ tests/exercise/testsuite/progress.t @@ -5,16 +5,15 @@ export tst=$0.tests exec ./assert-exercise $0 \ - 'total 6 run 6 ok 1 fail 2 error 3\n' \ + 'run 6 ok 1 fail 2 error 3\n' \ '' <$0.err # create the file assert-exercise will use err=`$STAT <$0.err` # stat it exec ./assert-exercise $0 \ - 'total 6 run 6 ok 1 fail 2 error 3\n' \ + 'run 6 ok 1 fail 2 error 3\n' \ "$err\\n" <&2 # stat stderr, print result to stderr - echo 'total 6 run 6 ok 1 fail 2 error 3' + echo 'run 6 ok 1 fail 2 error 3' exit 1 EOF blob - 0601a7382e26729f6333bf4a51c8ed54f305545b blob + d0ec8887581392b282fffa4f189280b174d90d49 --- tests/exercise/testsuite.t +++ tests/exercise/testsuite.t @@ -7,10 +7,10 @@ export tst=$0.tests exec ./assert-exercise $0 \ - 'total 6 run 6 ok 1 fail 2 error 3\n' \ + 'run 6 ok 1 fail 2 error 3\n' \ 'test test test\n' <&2 - echo 'total 6 run 6 ok 1 fail 2 error 3' + echo 'run 6 ok 1 fail 2 error 3' exit 1 EOF