commit - d020afe4a99674cac985dfc693d4f3a4d0bd5823
commit + 9c8a0916ad7697e4a5e36371593f0c2ea4840efb
blob - 87bba5bd2068ef0369f5ef9408b91cc6a86ddefd
blob + 5683696daa291398786336627041302aeb81e24c
--- exercise
+++ exercise
set errfile "/tmp/exercise.testerr.[pid]"
}
+ # Count this test as an (attempted) run.
+ dict incr counters run
+
+ set scounters {}
if {[catch {
# Execute the test, attaching pipes to stdin and stdout.
set pipe [open "|$exe 2>$errfile" r+]
# Consume the test's stdout. Ignore it for normal tests.
# Continually update the progress report for test suites.
- set scounters {}
while {[gets $pipe out] >= 0} {
if {$suite} {
set scounters [sanitize_suite_counters $out]
}
}
- # Wait for the subprocess to exit.
- close $pipe
-
# Sanity check on final test suite subcounters.
if {$suite} {check_suite_counters $scounters}
+
+ # Wait for the subprocess to exit.
+ close $pipe
} res opt]
} {
set details [dict get $opt -errorcode]
if {[dict exists $dirs $normpath]} continue
dict set dirs $normpath {}
- set subs [glob -type d -nocomplain -directory $path $opts(-pd)]
- set suites [glob -type f -nocomplain -directory $path $opts(-ps)]
- set tests [glob -type f -nocomplain -directory $path $opts(-pt)]
+ set subs [glob -type d -nocomplain -directory $path -- $opts(-pd)]
+ set suites [glob -type f -nocomplain -directory $path -- $opts(-ps)]
+ set tests [glob -type f -nocomplain -directory $path -- $opts(-pt)]
find_files_iter result dirs [lsort $subs]
lappend result {*}[lsort $suites]
proc inconsistent {counters} {
dict with counters {
- expr {$run != $ok + $fail + $error}
+ expr {$run > $total || $run != $ok + $fail + $error}
}
}
proc summary {counters} {
dict with counters {
+ set t "total $total"
set r "run $run"
set o "ok $ok"
set f "fail $fail"
if {[isatty stdout]} {
set good [expr {$ok>0 && ![notok $counters]}]
- if {$run==0} {set r [color {31} $r]} ;# dark red
- if {$good} {set o [color {1;32} $o]} ;# bright green
- if {$fail>0} {set f [color {1;31} $f]} ;# bright red
- if {$error>0} {set e [color {31} $e]} ;# dark red
+ if {$total == 0} {set t [color {31} $t]} ;# dark red
+ if {$run < $total} {set r [color {1} $r]} ;# bold
+ if {$good} {set o [color {1;32} $o]} ;# bright green
+ if {$fail > 0} {set f [color {1;31} $f]} ;# bright red
+ if {$error > 0} {set e [color {31} $e]} ;# dark red
}
- return "$r $o $f $e"
+ return "$t $r $o $f $e"
}
}
dict incr counters $key $cnt
}
}
- dict incr counters run -1 ;# remove the suite itself from the total count
+
+ # remove the suite itself from the "run" and "total" counts
+ dict incr counters run -1
+ dict incr counters total -1
+
return $counters
}
# Make sure counters is a dict of non-negative integers.
proc sanitize_suite_counters {counters} {
if {[catch {dict create {*}$counters} res]} {
- return -code error -errorcode EXER_SUITE \
- "malformed summary: $counters"
+ return -code error -errorcode EXER_SUITE "malformed summary: $counters"
}
dict for {key val} $counters {
dict with counters {}
if {[dict size $counters] == 0} {
- return -code error -errorcode EXER_SUITE "no summary on stdout"
+ return -code error -errorcode EXER_SUITE "empty summary on stdout"
}
- foreach key {run ok fail error} {
+ foreach key {total run ok fail error} {
if {![info exists $key]} {
return -code error -errorcode EXER_SUITE "missing counter: $key"
}
}
if {[inconsistent $counters]} {
- return -code error -errorcode EXER_SUITE \
- "counters do not sum up: $counters"
+ return -code error -errorcode EXER_SUITE "counters inconsistent"
}
}
proc main {argc argv} {
global opts
- set counters {run 0 ok 0 fail 0 error 0}
+ set counters {total 0 run 0 ok 0 fail 0 error 0}
# handle command line options
set u {[-q] [-p k=pattern] [path ...]}
set paths [expr {$argc == 0 ? "." : $argv}]
set files [find_files $paths]
- dict set counters run [llength $files] ;# tentative total run count
- ;# might adjust when suites run
+ dict set counters total [llength $files] ;# tentative total test count
+ ;# might adjust when suites run
foreach file $files {
print_progress $counters
execute_test counters $file
blob - b44ac0cda14e662068ce190f013512a8b1ba4979
blob + 020d6bbd2fdad8d8e61b070f5a2e30585a7fabaf
--- tests/test-error.t
+++ tests/test-error.t
d=`dirname $0`
exec $d/assert-exercise $0 \
- 'run 1 ok 0 fail 0 error 1\n' \
+ 'total 1 run 1 ok 0 fail 0 error 1\n' \
"error $0.tst\\n\\n" <<EOF
#!/bin/sh
exit 127
blob - 13bbb3a124f0f5e6534b7bff5c883a40dcaed4a2
blob + 02d199529159a9f42c1459e53920c82418956481
--- tests/test-fail.t
+++ tests/test-fail.t
d=`dirname $0`
exec $d/assert-exercise $0 \
- 'run 1 ok 0 fail 1 error 0\n' \
+ 'total 1 run 1 ok 0 fail 1 error 0\n' \
"fail $0.tst\\n\\n" <<EOF
#!/bin/sh
exit 1
blob - 2aff8237a6ece1a57194e359cb84d2c4d715be2c
blob + ef5add01a9d12d5e9a8e2a2bc56a183feee19219
--- tests/test-noexec.t
+++ tests/test-noexec.t
rm -f $tst $out $err $expout $experr
# expected output
-printf 'run 1 ok 0 fail 0 error 1\n' > $expout
+printf 'total 1 run 1 ok 0 fail 0 error 1\n' > $expout
printf "$0.tst: permission denied\\nerror $0.tst\\n\\n" > $experr
# test to run
blob - 98a912e46b1402cdd6c7dc8b001083d6e96cab14
blob + 56684357313b4d2620f58a5b80dd6466467232b7
--- tests/test-ok.t
+++ tests/test-ok.t
#!/bin/sh
d=`dirname $0`
-exec $d/assert-exercise $0 'run 1 ok 1 fail 0 error 0\n' '' <<EOF
+exec $d/assert-exercise $0 'total 1 run 1 ok 1 fail 0 error 0\n' '' <<EOF
#!/bin/sh
exit 0
EOF
blob - e3c95fd03ed5d4a958041aafea655f821303278d
blob + dd69eae495857c0d8c5207a1224ee566e96ac60c
--- tests/test-stderr-error.t
+++ tests/test-stderr-error.t
d=`dirname $0`
exec $d/assert-exercise $0 \
- 'run 1 ok 0 fail 0 error 1\n' \
+ 'total 1 run 1 ok 0 fail 0 error 1\n' \
"test test test\\nerror $0.tst\\n\\n" <<EOF
#!/bin/sh
echo test test test >&2
blob - 17db9d761a8d3614cc0c40489023ed97c8c60087
blob + 47515912398bb9d2996724b46bf5c58d9516eefd
--- tests/test-stderr-fail.t
+++ tests/test-stderr-fail.t
d=`dirname $0`
exec $d/assert-exercise $0 \
- 'run 1 ok 0 fail 1 error 0\n' \
+ 'total 1 run 1 ok 0 fail 1 error 0\n' \
"test test test\\nfail $0.tst\\n\\n" <<EOF
#!/bin/sh
echo test test test >&2
blob - a4fdaa668b35e2ae26dca4ee48af1cae0877b1d1
blob + 9c68bd1b5f13af51a957bbdb527714650129ec68
--- tests/test-stderr-ok.t
+++ tests/test-stderr-ok.t
d=`dirname $0`
exec $d/assert-exercise $0 \
- 'run 1 ok 0 fail 1 error 0\n' \
+ 'total 1 run 1 ok 0 fail 1 error 0\n' \
"test test test\\nfail $0.tst\\n\\n" <<EOF
#!/bin/sh
echo test test test >&2
blob - 6f61b5c86339c1c37cb5cb1aa8ba21e53d79b1f8
blob + a517ad3f792a706c406b0b9835d2b90a9b95c359
--- tests/testsuite-error.t
+++ tests/testsuite-error.t
d=`dirname $0`
export tst=$0.tests
exec $d/assert-exercise $0 \
- 'run 1 ok 0 fail 0 error 1\n' \
+ 'total 1 run 1 ok 0 fail 0 error 1\n' \
"test test test\\nerror $tst\\n\\n" <<EOF
#!/bin/sh
echo 'test test test' >&2
+ echo 'total 1 run 1 ok 0 fail 0 error 1'
exit 127
EOF
blob - 08cf9d4de83219119c701c0444559996f28b6b64
blob + b20e999df43e5c70b51c0d8d1ae30b2cfb37e251
--- tests/testsuite-progress.t
+++ tests/testsuite-progress.t
d=`dirname $0`
export tst=$0.tests
exec $d/assert-exercise $0 \
- 'run 6 ok 1 fail 2 error 3\n' \
+ 'total 6 run 6 ok 1 fail 2 error 3\n' \
'' <<EOF
#!/bin/sh
- printf 'run 6\r'
- printf 'run 6 ok 1 fail 0 error 0\r'
- printf 'run 6 ok 1 fail 1 error 0\r'
- printf 'run 6 ok 1 fail 2 error 0\r'
- printf 'run 6 ok 1 fail 2 error 1\r'
- printf 'run 6 ok 1 fail 2 error 2\r'
- printf 'run 6 ok 1 fail 2 error 3\n'
+ printf 'total 6\r'
+ printf 'total 6 run 1\r'
+ printf 'total 6 run 1 ok 1 fail 0 error 0\r'
+ printf 'total 6 run 2 ok 1 fail 1 error 0\r'
+ printf 'total 6 run 5 ok 1 fail 2 error 0\r'
+ printf 'total 6 run 6 ok 1 fail 2 error 1\r'
+ printf 'total 6 run 6 ok 1 fail 2 error 2\r'
+ printf 'total 6 run 6 ok 1 fail 2 error 3\n'
exit 1
EOF
blob - /dev/null
blob + f7369623b5d2493555abe0a0eb21cec9a4d76f74 (mode 755)
--- /dev/null
+++ tests/testsuite-inconsistent-1.t
+#!/bin/sh
+
+# if a test suite reports inconsistent counters it should be counted as a
+# single test error, the same as if the suite had exited with 127.
+# stderr should be passed through and followed by an error message and a
+# generic error report.
+
+# this tests the case where run exceeds total.
+
+d=`dirname $0`
+export tst=$0.tests
+exec $d/assert-exercise $0 \
+ 'total 1 run 1 ok 0 fail 0 error 1\n' \
+ "test test test\\n$tst: counters inconsistent\\nerror $tst\\n\\n" <<EOF
+#!/bin/sh
+ echo 'test test test' >&2
+ echo 'total 10 run 23 ok 22 fail 1 error 0'
+ exit 1
+EOF
blob - 704847c58a80187b9ee6170c4608dc08a5ce495b
blob + 0b4180bc91b436dbf871c0e6fdf5331453034e55
--- tests/testsuite-stderr-fd.t
+++ tests/testsuite-stderr-fd.t
>$0.err # create the file assert-exercise will use
err=`$STAT <$0.err` # stat it
exec $d/assert-exercise $0 \
- 'run 6 ok 1 fail 2 error 3\n' \
+ 'total 6 run 6 ok 1 fail 2 error 3\n' \
"$err\\n" <<EOF
#!/bin/sh
$STAT <&2 >&2 # stat stderr, print result to stderr
- echo 'run 6 ok 1 fail 2 error 3'
+ echo 'total 6 run 6 ok 1 fail 2 error 3'
exit 1
EOF
blob - /dev/null
blob + 643e12c3a518322660f34abe8cc8c55aa7a3b4ec (mode 755)
--- /dev/null
+++ tests/testsuite-inconsistent-2.t
+#!/bin/sh
+
+# if a test suite reports inconsistent counters it should be counted as a
+# single test error, the same as if the suite had exited with 127.
+# stderr should be passed through and followed by an error message and a
+# generic error report.
+
+# this tests the case where run is not exactly the sum of ok, fail, and error.
+
+d=`dirname $0`
+export tst=$0.tests
+exec $d/assert-exercise $0 \
+ 'total 1 run 1 ok 0 fail 0 error 1\n' \
+ "test test test\\n$tst: counters inconsistent\\nerror $tst\\n\\n" <<EOF
+#!/bin/sh
+ echo 'test test test' >&2
+ echo 'total 23 run 22 ok 22 fail 1 error 0'
+ exit 1
+EOF
blob - f27d7eb9771fc58f966de15835509fc64a1cd269
blob + a779442e20f957b65558e40201546a8d13a64d48
--- tests/testsuite.t
+++ tests/testsuite.t
d=`dirname $0`
export tst=$0.tests
exec $d/assert-exercise $0 \
- 'run 6 ok 1 fail 2 error 3\n' \
+ 'total 6 run 6 ok 1 fail 2 error 3\n' \
'test test test\n' <<EOF
#!/bin/sh
echo 'test test test' >&2
- echo 'run 6 ok 1 fail 2 error 3'
+ echo 'total 6 run 6 ok 1 fail 2 error 3'
exit 1
EOF
