commit - 8256f5dffdca364cde5c43fcaa001bfe255f4fcd
commit + 991d115e68a80435a0bfeee4268cb569e606c259
blob - 9446a3b3d20d03a46d354f149a677ed155005458
blob + 3add364e7c31d76d06a05ee5f42d84b7fe8b7d62
--- exercise
+++ exercise
set opts(-pt) {*.t} ;# pattern for test executable files
set opts(-q) 0 ;# quiet mode
set opts(-v) 0 ;# verbose mode
+set opts(-x) 0 ;# exit on failed test
# executing tests
# Pass stderr through when calling test suites.
set errfile @stderr
- # Pass -v(v) and -n on to test suites.
- if {$opts(-v) > 0} {set args "-[string repeat v $opts(-v)] $args"}
+ # Pass -n, -v(v), and -x(x) on to test suites.
if {$opts(-n)} {set args "-n $args"}
+ if {$opts(-v) > 0} {set args "$args -[string repeat v $opts(-v)]"}
+ if {$opts(-x) > 0} {set args "$args -[string repeat x $opts(-x)]"}
} elseif {$opts(-n)} {
# Dry run requested; stop here.
return error ;# dummy to signal we did not create a channel
}
}
+proc failed {counters} {
+ dict with counters {
+ expr {$fail > 0 || $error > 0}
+ }
+}
+
proc notok {counters} {
dict with counters {
expr {[inconsistent $counters] || $ok != $run}
if {[info exists env(EXERCISEDIR)]} {set opts(-D) $env(EXERCISEDIR)}
# handle command line options
- set u {[-nqv] [-B dir] [-C dir] [-D dir] [-j n] [-p k=pattern] [path ...]}
+ set u {[-nqvx] [-B dir] [-C dir] [-D dir] [-j n] [-p k=pattern] [path ...]}
set prog [file tail $::argv0]
- while {[getopt argv {B:C:D:j:np:qv} opt arg]} {
+ while {[getopt argv {B:C:D:j:np:qvx} opt arg]} {
if {$opt eq "?"} {
puts stderr "usage: $prog $u"
exit 1
set opt p$k
set arg $v
}
- if {$opt eq "v"} {
- set arg [expr $opts(-v) + 1] ;# increment
+ if {$opt eq "v" || $opt eq "x"} {
+ set arg [expr $opts(-$opt) + 1] ;# increment
}
set opts(-$opt) $arg
}
# wait for a slot to become available
while {$J >= $opts(-j)} {vwait J}
+ # exit after a test failure if requested with -x
+ if {$opts(-x) && [failed $counters]} break
+
# start job
set job job[incr i]
set chan [coroutine $job execute_test $job $basedir $file]
blob - ec62f029162839840dd8b1b2f2416f79e5bc8650
blob + c298a3d1a1fe836f8baa0723a17e0829349ebe3c
--- exercise.1
+++ exercise.1
.
.Sh SYNOPSIS
.Nm exercise
-.Op Fl nqv
+.Op Fl nqvx
.Op Fl B Ar dir
.Op Fl C Ar dir
.Op Fl D Ar dir
If given twice, also print any test standard output
with an additional level of quoting using
.Dq "> " .
+This flag is passed on to test suites.
+.It Fl x
+Exit immediately after a test failure or error.
+This flag is passed on to test suites.
.El
.
.Ss Test Files
.Nm
utility can be used as a (nested) test suite runner if desired.
.Pp
-A test suite must admit the
-.Fl n
+A test suite must accept the
+.Fl n ,
+.Fl v
and
-.Fl v
+.Fl x
options.
It must honor
.Fl n
by convention, should be prefixed with
.Dq ". >"
(period, space, greater-than).
+The
+.Fl x
+option should cause the test suite to exit after a test failure
+if it would normally continue.
+Given
+.Fl xx ,
+it may exit even earlier,
+i.e. on the first failed condition within a test.
.Pp
After running a test suite, its final report is checked for consistency.
If any of the conditions (listed below) are not satisfied,
blob - 63a4c86af485f4ff8efa935e36dadf3eeea2aae7
blob + 9e6c71d8e143191a39ebc3acfa8309aa69d8ae54
--- support/adapt-tst
+++ support/adapt-tst
# handle command line flags
nflag=0
vflag=0
-while getopts nv ch
+xflag=0
+while getopts nvx ch
do
case $ch in
n) nflag=1 ;;
v) vflag=$(($vflag + 1)) ;;
- ?) echo "usage: adapt-tst cmd [-v] [arg ...]"
+ x) xflag=$(($xflag + 1)) ;;
+ ?) echo "usage: adapt-tst cmd [-nvx] [arg ...]"
esac
done
shift $(($OPTIND - 1))
exit 0
fi
-# let -vv set V, but not for dry runs
+# let -vv set V (but not for dry runs)
[ $vflag -ge 2 ] && export V=1
+# let -xx set X (-x is already our normal behavior)
+[ $xflag -ge 2 ] && export X=1
# run the tests in the background, redirecting output to a fifo
rm -f $tmp
blob - 3d4dffafc2e6a23ee1786320e3ac01e1f57301c8
blob + ba29b3f3e0985b91b3d29d15e6c346d1b489df7f
--- tests/exercise/flag/n.t
+++ tests/exercise/flag/n.t
set -e
./assert-exercise $0 \
- "run 1 ok 0 fail 0 error 0\n" \
- '' \
+ "run 1 ok 0 fail 0 error 0\n" \
+ '' \
-n <<EOF
#!/bin/sh
exit 0
./assert-exercise $0 \
". $0.tst\nrun 1 ok 0 fail 0 error 0\n" \
- '' \
+ '' \
-nv <<EOF
#!/bin/sh
exit 0
export tst=$0.ts
./assert-exercise $0 \
- "run 2 ok 0 fail 0 error 0\n" \
- '' \
+ "run 2 ok 0 fail 0 error 0\n" \
+ '' \
-n <<EOF
#!/bin/sh
echo "run 2 ok 0 fail 0 error 0"
export tst=$0.ts
./assert-exercise $0 \
- ". $0.ts\n. . test1\n. . test2\nrun 2 ok 0 fail 0 error 0\n" \
- '' \
+ ". $0.ts\n. . test1\n. . test2\nrun 2 ok 0 fail 0 error 0\n" \
+ '' \
-nv <<EOF
#!/bin/sh
echo "run 2 ok 0 fail 0 error 0"
export tst=$0.ts
./assert-exercise $0 \
- ". $0.ts\n. . test1\n. . > hallo\nrun 1 ok 0 fail 0 error 0\n" \
- '' \
+ ". $0.ts\n. . test1\n. . > hallo\nrun 1 ok 0 fail 0 error 0\n" \
+ '' \
-nvv <<EOF
#!/bin/sh
echo "run 1 ok 0 fail 0 error 0"
blob - /dev/null
blob + 1d533335f309f0859393d34556045cd03bd7a6d1 (mode 755)
--- /dev/null
+++ tests/exercise/flag/x.t
+#!/bin/sh
+
+set -e
+
+./assert-exercise $0 \
+ "run 1 ok 1 fail 0 error 0\n" \
+ '' \
+ -x <<EOF
+#!/bin/sh
+ exit 0
+EOF
+
+./assert-exercise $0 \
+ "run 1 ok 0 fail 1 error 0\n" \
+ "fail $0.tst\n\n" \
+ -x <<EOF
+#!/bin/sh
+ exit 1
+EOF
+
+./assert-exercise $0 \
+ "run 2 ok 0 fail 1 error 0\n" \
+ "fail $0.tst\n\n" \
+ -x $0.tst <<EOF
+#!/bin/sh
+ exit 1
+EOF
+
+export tst=$0.ts
+./assert-exercise $0 \
+ "run 2 ok 1 fail 0 error 0\n" \
+ "fail $tst\n\n" \
+ -x <<EOF
+#!/bin/sh
+ test "\$1" = "-x" || exit 127
+ echo "run 2 ok 0 fail 0 error 0"
+ echo "run 2 ok 1 fail 0 error 0"
+ exit 1
+EOF
+
+export tst=$0.ts
+./assert-exercise $0 \
+ "run 2 ok 1 fail 0 error 0\n" \
+ "fail $tst\n\n" \
+ -xx <<EOF
+#!/bin/sh
+ test "\$1" = "-xx" || exit 127
+ echo "run 2 ok 0 fail 0 error 0"
+ echo "run 2 ok 1 fail 0 error 0"
+ exit 1
+EOF
blob - ecdb3353b3508b43c2d7028dbf7a266ec4e00433
blob + e51d7c2a07315745b722cbac46c0ddbfbf3df00d
--- tests/support/adapt-tst.t
+++ tests/support/adapt-tst.t
[ -n "$V" ] && echo " bemerke auch"
echo "test_2() OK"
echo "something something unexpected" >&2
+ [ -n "$X" ] && exit 1
echo "test_3() FAIL"
exit 1
fi
#!/bin/sh
exec ../../support/adapt-tst $tmp "\$@"
EOF
+
+./assert-exercise $0 \
+ "run 3 ok 2 fail 0 error 0\n" \
+ "something something unexpected\nfail $tst\n\n" \
+ -xx <<-EOF
+ #!/bin/sh
+ exec ../../support/adapt-tst $tmp "\$@"
+ EOF
