diff options
| author | Ian Jackson <ijackson@chiark.greenend.org.uk> | 2018-01-07 15:50:36 +0000 |
|---|---|---|
| committer | Ian Jackson <ijackson@chiark.greenend.org.uk> | 2018-01-07 16:18:53 +0000 |
| commit | 88c8440dc1ac4e6de35615ac18fdb3ec281e1d96 (patch) | |
| tree | 9eb521ef6e01d60d955e54ffd0561bebba35aa43 | |
| parent | 13a350a650fa9847bfbfadea40262e21c0864c39 (diff) | |
test suite: In ad-hoc runs, honour DGIT_TEST_RETRY_COUNT
This lets us retry each failed test.
This is beneficial because the gnupg race is sort-of independent.
When we want to test every commit, the success probability before
is
( 1 - P(gnupg causes test fail)) ) ^ (number of tests)
for each commit. With this change it's
( 1 - P(gnupg causes test fail)^retries ) ^ (number of tests)
which is much much better.
Signed-off-by: Ian Jackson <ijackson@chiark.greenend.org.uk>
| -rw-r--r-- | tests/Makefile | 19 |
1 files changed, 18 insertions, 1 deletions
diff --git a/tests/Makefile b/tests/Makefile index 5bd3eee..e6ce153 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,7 @@ # usage: tests/using-intree make -f tests/Makefile -# (optionally setting TESTSCRIPTS='tests/tests/foo tests/tests/bar') +# optionally setting +# TESTSCRIPTS='tests/tests/foo tests/tests/bar') +# DGIT_TEST_RETRY_COUNT=<a smallish integer> TESTSCRIPTS ?= $(shell tests/enumerate-tests) TESTNAMES := $(notdir $(TESTSCRIPTS)) @@ -10,5 +12,20 @@ all: $(foreach t,$(TESTNAMES),tests/tmp/$t.ok) tests/tmp: mkdir -p $@ +ifeq ($(DGIT_TEST_RETRY_COUNT),) + tests/tmp/%.ok: tests/tmp tests/tests/$* >tests/tmp/$*.log 2>&1 + +else + +tests/tmp/%.ok: tests/tmp + @for retry in $$( seq 1 $(DGIT_TEST_RETRY_COUNT) ); do \ + echo "[$$retry] $*"; \ + tests/tests/$* >tests/tmp/$*.$$retry.log 2>&1; \ + rc=$$?; \ + if [ $$rc = 0 ]; then exit 0; fi; \ + echo >&2 "[$$retry] $* TEST FAILED $$rc"; \ + done; exit $$rc + +endif |