commit a6b35f66241ad24fdd4e12e934bbfc5c7e7e985e from: Sven M. Hallberg date: Mon Jul 21 17:10:02 2025 UTC add -k option (keep going on test failure) commit - 991d115e68a80435a0bfeee4268cb569e606c259 commit + a6b35f66241ad24fdd4e12e934bbfc5c7e7e985e blob - 3add364e7c31d76d06a05ee5f42d84b7fe8b7d62 blob + bd8955ed14f940bddd9d0e9c3e9d0ac6507faa13 --- exercise +++ exercise @@ -10,6 +10,7 @@ package require Tcl 8.4 # config options set opts(-B) . ;# basedir set opts(-j) 1 ;# number of concurrent jobs +set opts(-k) {} ;# for test suites: keep going on test failure set opts(-n) 0 ;# dry run set opts(-pd) {*} ;# pattern for test search directories set opts(-ps) {*.ts} ;# pattern for test suite executable files @@ -55,8 +56,9 @@ proc execute_test {job basedir path} { # Pass stderr through when calling test suites. set errfile @stderr - # Pass -n, -v(v), and -x(x) on to test suites. - if {$opts(-n)} {set args "-n $args"} + # Pass -k, -n, -v(v), and -x(x) on to test suites. + if {$opts(-k) ne ""} {set args "$args -k"} + if {$opts(-n)} {set args "$args -n"} if {$opts(-v) > 0} {set args "$args -[string repeat v $opts(-v)]"} if {$opts(-x) > 0} {set args "$args -[string repeat x $opts(-x)]"} } elseif {$opts(-n)} { @@ -458,7 +460,7 @@ proc main {argc argv} { # handle command line options set u {[-nqvx] [-B dir] [-C dir] [-D dir] [-j n] [-p k=pattern] [path ...]} set prog [file tail $::argv0] - while {[getopt argv {B:C:D:j:np:qvx} opt arg]} { + while {[getopt argv {B:C:D:j:knp:qvx} opt arg]} { if {$opt eq "?"} { puts stderr "usage: $prog $u" exit 1 @@ -469,6 +471,9 @@ proc main {argc argv} { exit 1 } } + if {$opt eq "k"} { + set opts(-x) 0 ;# negate -x + } if {$opt eq "p"} { # handle key-value options -pd=, -ps=, -pt= if {[set i [string first "=" $arg]] == -1} { @@ -484,9 +489,13 @@ proc main {argc argv} { set opt p$k set arg $v } - if {$opt eq "v" || $opt eq "x"} { + if {$opt eq "v"} { set arg [expr $opts(-$opt) + 1] ;# increment } + if {$opt eq "x"} { + set arg [expr $opts(-$opt) + 1] ;# increment + set opts(-k) {} ;# negate -k + } set opts(-$opt) $arg } set argc [llength $argv] blob - c298a3d1a1fe836f8baa0723a17e0829349ebe3c blob + 4387b4bc96d75f771e7e6cc1220532f315c101e7 --- exercise.1 +++ exercise.1 @@ -8,7 +8,7 @@ . .Sh SYNOPSIS .Nm exercise -.Op Fl nqvx +.Op Fl knqvx .Op Fl B Ar dir .Op Fl C Ar dir .Op Fl D Ar dir @@ -70,6 +70,14 @@ contains common helper or data files for the tests to Execute up to .Ar n test jobs concurrently. +.It Fl k +Keep going after a failed test. +This is the default, +but the option is passed on to test suites +which may have different behavior. +Negates any previous +.Fl x +option. .It Fl n Dry run. Do not execute the tests. @@ -107,7 +115,10 @@ with an additional level of quoting using This flag is passed on to test suites. .It Fl x Exit immediately after a test failure or error. -This flag is passed on to test suites. +This flag is passed on to test suites and +negates any previous +.Fl k +option. .El . .Ss Test Files blob - 9e6c71d8e143191a39ebc3acfa8309aa69d8ae54 blob + e85cc8b73d9e95efa8244bafc513506040ebc61b --- support/adapt-tst +++ support/adapt-tst @@ -16,12 +16,14 @@ cmd="$1" shift # handle command line flags +kflag=0 nflag=0 vflag=0 xflag=0 -while getopts nvx ch +while getopts knvx ch do case $ch in + k) kflag=1 ;; n) nflag=1 ;; v) vflag=$(($vflag + 1)) ;; x) xflag=$(($xflag + 1)) ;; @@ -66,6 +68,8 @@ fi [ $vflag -ge 2 ] && export V=1 # let -xx set X (-x is already our normal behavior) [ $xflag -ge 2 ] && export X=1 +# let -k set K +[ $kflag -eq 1 ] && export K=1 # run the tests in the background, redirecting output to a fifo rm -f $tmp blob - /dev/null blob + d9df42da81b92a846e9c2e4e8dc1e7b1ed6aa3e4 (mode 755) --- /dev/null +++ tests/exercise/flag/k.t @@ -0,0 +1,53 @@ +#!/bin/sh + +set -e + +./assert-exercise $0 \ + "run 1 ok 1 fail 0 error 0\n" \ + '' \ + -k <$tmp <<-"EOF" echo "test_1()" echo "test_2()" echo "test_3()" + echo "test_4()" else [ -n "$V" ] && echo "foo() -" echo "test_1() OK" @@ -18,6 +19,7 @@ cat >$tmp <<-"EOF" echo "something something unexpected" >&2 [ -n "$X" ] && exit 1 echo "test_3() FAIL" + [ -n "$K" ] && echo "test_4() OK" exit 1 fi EOF @@ -26,7 +28,7 @@ chmod +x $tmp export tst=$0.ts ./assert-exercise $0 \ - "run 3 ok 2 fail 1 error 0\n" \ + "run 4 ok 2 fail 1 error 0\n" \ "something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \ <<-EOF #!/bin/sh @@ -35,7 +37,7 @@ export tst=$0.ts ./assert-exercise $0 \ ". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n\ -run 3 ok 2 fail 1 error 0\n" \ +run 4 ok 2 fail 1 error 0\n" \ "something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \ -v \ <<-EOF @@ -45,7 +47,7 @@ run 3 ok 2 fail 1 error 0\n" \ ./assert-exercise $0 \ ". $tst\n. . foo()\tskipped\n. . test_1()\n. . >\tbemerke auch\n\ -. . test_2()\n. . test_3()\nrun 3 ok 2 fail 1 error 0\n" \ +. . test_2()\n. . test_3()\nrun 4 ok 2 fail 1 error 0\n" \ "something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \ -vv \ <<-EOF @@ -54,7 +56,7 @@ run 3 ok 2 fail 1 error 0\n" \ EOF ./assert-exercise $0 \ - "run 3 ok 0 fail 0 error 0\n" \ + "run 4 ok 0 fail 0 error 0\n" \ '' \ -n \ <<-EOF @@ -63,8 +65,8 @@ run 3 ok 2 fail 1 error 0\n" \ EOF ./assert-exercise $0 \ - ". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n\ -run 3 ok 0 fail 0 error 0\n" \ + ". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n. . test_4()\n\ +run 4 ok 0 fail 0 error 0\n" \ '' \ -nv \ <<-EOF @@ -73,8 +75,8 @@ run 3 ok 0 fail 0 error 0\n" \ EOF ./assert-exercise $0 \ - ". $tst\n. . test_1()\n\ -. . test_2()\n. . test_3()\nrun 3 ok 0 fail 0 error 0\n" \ + ". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n. . test_4()\n\ +run 4 ok 0 fail 0 error 0\n" \ '' \ -nvv \ <<-EOF @@ -83,9 +85,26 @@ run 3 ok 0 fail 0 error 0\n" \ EOF ./assert-exercise $0 \ - "run 3 ok 2 fail 0 error 0\n" \ + "run 4 ok 2 fail 0 error 0\n" \ "something something unexpected\nfail $tst\n\n" \ -xx <<-EOF #!/bin/sh exec ../../support/adapt-tst $tmp "\$@" EOF + +./assert-exercise $0 \ + "run 4 ok 3 fail 1 error 0\n" \ + "something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \ + -k <<-EOF + #!/bin/sh + exec ../../support/adapt-tst $tmp "\$@" + EOF + +./assert-exercise $0 \ + ". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n. . test_4()\n\ +run 4 ok 3 fail 1 error 0\n" \ + "something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \ + -kv <<-EOF + #!/bin/sh + exec ../../support/adapt-tst $tmp "\$@" + EOF