KHJK gotweb

Commit Diff

Commit:: a6b35f66241ad24fdd4e12e934bbfc5c7e7e985e
From:: Sven M. Hallberg <pesco@khjk.org>
Date:: Mon Jul 21 17:10:02 2025 UTC
Message:: add -k option (keep going on test failure)
Actions:: Patch | Tree
commit - 991d115e68a80435a0bfeee4268cb569e606c259
commit + a6b35f66241ad24fdd4e12e934bbfc5c7e7e985e
blob - 3add364e7c31d76d06a05ee5f42d84b7fe8b7d62
blob + bd8955ed14f940bddd9d0e9c3e9d0ac6507faa13
--- exercise
+++ exercise
@@ -10,6 +10,7 @@ package require Tcl 8.4
 # config options
 set opts(-B) .                  ;# basedir
 set opts(-j) 1                  ;# number of concurrent jobs
+set opts(-k) {}                 ;# for test suites: keep going on test failure
 set opts(-n) 0                  ;# dry run
 set opts(-pd) {*}               ;# pattern for test search directories
 set opts(-ps) {*.ts}            ;# pattern for test suite executable files
@@ -55,8 +56,9 @@ proc execute_test {job basedir path} {
         # Pass stderr through when calling test suites.
         set errfile @stderr
 
-        # Pass -n, -v(v), and -x(x) on to test suites.
-        if {$opts(-n)} {set args "-n $args"}
+        # Pass -k, -n, -v(v), and -x(x) on to test suites.
+        if {$opts(-k) ne ""} {set args "$args -k"}
+        if {$opts(-n)} {set args "$args -n"}
         if {$opts(-v) > 0} {set args "$args -[string repeat v $opts(-v)]"}
         if {$opts(-x) > 0} {set args "$args -[string repeat x $opts(-x)]"}
     } elseif {$opts(-n)} {
@@ -458,7 +460,7 @@ proc main {argc argv} {
     # handle command line options
     set u {[-nqvx] [-B dir] [-C dir] [-D dir] [-j n] [-p k=pattern] [path ...]}
     set prog [file tail $::argv0]
-    while {[getopt argv {B:C:D:j:np:qvx} opt arg]} {
+    while {[getopt argv {B:C:D:j:knp:qvx} opt arg]} {
         if {$opt eq "?"} {
             puts stderr "usage: $prog $u"
             exit 1
@@ -469,6 +471,9 @@ proc main {argc argv} {
                 exit 1
             }
         }
+        if {$opt eq "k"} {
+            set opts(-x) 0                      ;# negate -x
+        }
         if {$opt eq "p"} {
             # handle key-value options -pd=, -ps=, -pt=
             if {[set i [string first "=" $arg]] == -1} {
@@ -484,9 +489,13 @@ proc main {argc argv} {
             set opt p$k
             set arg $v
         }
-        if {$opt eq "v" || $opt eq "x"} {
+        if {$opt eq "v"} {
             set arg [expr $opts(-$opt) + 1]     ;# increment
         }
+        if {$opt eq "x"} {
+            set arg [expr $opts(-$opt) + 1]     ;# increment
+            set opts(-k) {}                     ;# negate -k
+        }
         set opts(-$opt) $arg
     }
     set argc [llength $argv]
blob - c298a3d1a1fe836f8baa0723a17e0829349ebe3c
blob + 4387b4bc96d75f771e7e6cc1220532f315c101e7
--- exercise.1
+++ exercise.1
@@ -8,7 +8,7 @@
 .
 .Sh SYNOPSIS
 .Nm exercise
-.Op Fl nqvx
+.Op Fl knqvx
 .Op Fl B Ar dir
 .Op Fl C Ar dir
 .Op Fl D Ar dir
@@ -70,6 +70,14 @@ contains common helper or data files for the tests to 
 Execute up to
 .Ar n
 test jobs concurrently.
+.It Fl k
+Keep going after a failed test.
+This is the default,
+but the option is passed on to test suites
+which may have different behavior.
+Negates any previous
+.Fl x
+option.
 .It Fl n
 Dry run.
 Do not execute the tests.
@@ -107,7 +115,10 @@ with an additional level of quoting using
 This flag is passed on to test suites.
 .It Fl x
 Exit immediately after a test failure or error.
-This flag is passed on to test suites.
+This flag is passed on to test suites and
+negates any previous
+.Fl k
+option.
 .El
 .
 .Ss Test Files
blob - 9e6c71d8e143191a39ebc3acfa8309aa69d8ae54
blob + e85cc8b73d9e95efa8244bafc513506040ebc61b
--- support/adapt-tst
+++ support/adapt-tst
@@ -16,12 +16,14 @@ cmd="$1"
 shift
 
 # handle command line flags
+kflag=0
 nflag=0
 vflag=0
 xflag=0
-while getopts nvx ch
+while getopts knvx ch
 do
 	case $ch in
+	k)	kflag=1 ;;
 	n)	nflag=1 ;;
 	v)	vflag=$(($vflag + 1)) ;;
 	x)	xflag=$(($xflag + 1)) ;;
@@ -66,6 +68,8 @@ fi
 [ $vflag -ge 2 ] && export V=1
 # let -xx set X (-x is already our normal behavior)
 [ $xflag -ge 2 ] && export X=1
+# let -k set K
+[ $kflag -eq 1 ] && export K=1
 
 # run the tests in the background, redirecting output to a fifo
 rm -f $tmp
blob - /dev/null
blob + d9df42da81b92a846e9c2e4e8dc1e7b1ed6aa3e4 (mode 755)
--- /dev/null
+++ tests/exercise/flag/k.t
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+set -e
+
+./assert-exercise $0 \
+	"run 1  ok 1  fail 0  error 0\n" \
+	'' \
+	-k <<EOF
+#!/bin/sh
+    exit 0
+EOF
+
+./assert-exercise $0 \
+	"run 1  ok 0  fail 1  error 0\n" \
+	"fail $0.tst\n\n" \
+	-k <<EOF
+#!/bin/sh
+    exit 1
+EOF
+
+./assert-exercise $0 \
+	"run 2  ok 0  fail 2  error 0\n" \
+	"fail $0.tst\n\nfail $0.tst\n\n" \
+	-k $0.tst <<EOF
+#!/bin/sh
+    exit 1
+EOF
+
+export tst=$0.ts
+./assert-exercise $0 \
+	"run 2  ok 1  fail 1  error 0\n" \
+	"fail $tst\n\n" \
+	-k <<EOF
+#!/bin/sh
+    test "\$1" = "-k" || exit 127
+    echo "run 2  ok 0  fail 0  error 0"
+    echo "run 2  ok 0  fail 1  error 0"
+    echo "run 2  ok 1  fail 1  error 0"
+    exit 1
+EOF
+
+export tst=$0.ts
+./assert-exercise $0 \
+	"run 2  ok 1  fail 1  error 0\n" \
+	"fail $tst\n\n" \
+	-xk <<EOF
+#!/bin/sh
+    test "\$1" = "-k" || exit 127
+    echo "run 2  ok 0  fail 0  error 0"
+    echo "run 2  ok 0  fail 1  error 0"
+    echo "run 2  ok 1  fail 1  error 0"
+    exit 1
+EOF
blob - e51d7c2a07315745b722cbac46c0ddbfbf3df00d
blob + 66def72ab4cfd678109314559cbda7a59f757aae
--- tests/support/adapt-tst.t
+++ tests/support/adapt-tst.t
@@ -10,6 +10,7 @@ cat >$tmp <<-"EOF"
 	    echo "test_1()"
 	    echo "test_2()"
 	    echo "test_3()"
+	    echo "test_4()"
 	else
 	    [ -n "$V" ] && echo "foo()	-"
 	    echo "test_1()	OK"
@@ -18,6 +19,7 @@ cat >$tmp <<-"EOF"
 	    echo "something something unexpected" >&2
 	    [ -n "$X" ] && exit 1
 	    echo "test_3()	FAIL"
+	    [ -n "$K" ] && echo "test_4()	OK"
 	    exit 1
 	fi
 EOF
@@ -26,7 +28,7 @@ chmod +x $tmp
 export tst=$0.ts
 
 ./assert-exercise $0 \
-	"run 3  ok 2  fail 1  error 0\n" \
+	"run 4  ok 2  fail 1  error 0\n" \
 	"something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \
 	<<-EOF
 		#!/bin/sh
@@ -35,7 +37,7 @@ export tst=$0.ts
 
 ./assert-exercise $0 \
 	". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n\
-run 3  ok 2  fail 1  error 0\n" \
+run 4  ok 2  fail 1  error 0\n" \
 	"something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \
 	-v \
 	<<-EOF
@@ -45,7 +47,7 @@ run 3  ok 2  fail 1  error 0\n" \
 
 ./assert-exercise $0 \
 	". $tst\n. . foo()\tskipped\n. . test_1()\n. . >\tbemerke auch\n\
-. . test_2()\n. . test_3()\nrun 3  ok 2  fail 1  error 0\n" \
+. . test_2()\n. . test_3()\nrun 4  ok 2  fail 1  error 0\n" \
 	"something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \
 	-vv \
 	<<-EOF
@@ -54,7 +56,7 @@ run 3  ok 2  fail 1  error 0\n" \
 	EOF
 
 ./assert-exercise $0 \
-	"run 3  ok 0  fail 0  error 0\n" \
+	"run 4  ok 0  fail 0  error 0\n" \
 	'' \
 	-n \
 	<<-EOF
@@ -63,8 +65,8 @@ run 3  ok 2  fail 1  error 0\n" \
 	EOF
 
 ./assert-exercise $0 \
-	". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n\
-run 3  ok 0  fail 0  error 0\n" \
+	". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n. . test_4()\n\
+run 4  ok 0  fail 0  error 0\n" \
 	'' \
 	-nv \
 	<<-EOF
@@ -73,8 +75,8 @@ run 3  ok 0  fail 0  error 0\n" \
 	EOF
 
 ./assert-exercise $0 \
-	". $tst\n. . test_1()\n\
-. . test_2()\n. . test_3()\nrun 3  ok 0  fail 0  error 0\n" \
+	". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n. . test_4()\n\
+run 4  ok 0  fail 0  error 0\n" \
 	'' \
 	-nvv \
 	<<-EOF
@@ -83,9 +85,26 @@ run 3  ok 0  fail 0  error 0\n" \
 	EOF
 
 ./assert-exercise $0 \
-	"run 3  ok 2  fail 0  error 0\n" \
+	"run 4  ok 2  fail 0  error 0\n" \
 	"something something unexpected\nfail $tst\n\n" \
 	-xx <<-EOF
 		#!/bin/sh
 		exec ../../support/adapt-tst $tmp "\$@"
 	EOF
+
+./assert-exercise $0 \
+	"run 4  ok 3  fail 1  error 0\n" \
+	"something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \
+	-k <<-EOF
+		#!/bin/sh
+		exec ../../support/adapt-tst $tmp "\$@"
+	EOF
+
+./assert-exercise $0 \
+	". $tst\n. . test_1()\n. . test_2()\n. . test_3()\n. . test_4()\n\
+run 4  ok 3  fail 1  error 0\n" \
+	"something something unexpected\ntest_3()\tFAIL\nfail $tst\n\n" \
+	-kv <<-EOF
+		#!/bin/sh
+		exec ../../support/adapt-tst $tmp "\$@"
+	EOF