diff options
author | Kyle McMartin <kyle@debian.org> | 2004-06-06 01:23:28 +0200 |
---|---|---|
committer | Kyle McMartin <kyle@debian.org> | 2004-06-06 01:23:28 +0200 |
commit | 194c0bbde98196ce6871bd21a9e3d52a771f8dfd (patch) | |
tree | f77af65ebcb4c1fa9d33dce45d206407a98c646d /abxtest |
Import madplay_0.15.2b.orig.tar.gz
[dgit import orig madplay_0.15.2b.orig.tar.gz]
Diffstat (limited to 'abxtest')
-rw-r--r-- | abxtest | 291 |
1 files changed, 291 insertions, 0 deletions
@@ -0,0 +1,291 @@ +#! /bin/sh +# +# abxtest - simple ABX double-blind testing script +# Copyright (C) 2000-2004 Robert Leslie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# $Id: abxtest,v 1.18 2004/02/23 21:34:53 rob Exp $ +# + +min=10 +max=20 +goal=.05 + +version="0.15.2 (beta)" +publishyear="2000-2004" +author="Robert Leslie" + +usage() { + echo >&2 "Usage: $0 [-n min] [-m max] [-g goal] A-cmd B-cmd" + exit $1 +} + +banner() { + echo >&2 \ + "ABX Double-Blind Test $version - Copyright (C) $publishyear $author" +} + +while [ $# -gt 0 ] +do + case "$1" in + --help) + usage 0 + ;; + + --version) + banner + exit 0 + ;; + + -n) + test $# -gt 1 || usage 1 + min="$2" + shift 2 + ;; + + -m) + test $# -gt 1 || usage 1 + max="$2" + shift 2 + ;; + + -g) + test $# -gt 1 || usage 1 + goal="$2" + shift 2 + ;; + + --) + shift + break + ;; + + -*) + usage 1 + ;; + + *) + break + ;; + esac +done + +test $# -eq 2 || usage 1 + +banner +echo "minimum $min, maximum $max trials" +echo "statistical goal to disprove null hypothesis is p <= $goal" + +A="$1" +B="$2" + +echo "randomizing ..." + +tmp="/tmp/abx.$$" +trap "rm -f $tmp" 0 + +rand="${RANDOM_FILE:-/dev/random}" + +od -t o1 -N "$max" "$rand" >$tmp || exit 2 +exec 3<$tmp + +actual="" + +trial=1 +while read <&3 line +do + set -- $line + shift + + while [ $# -gt 0 ] + do + case $1 in + *[0246]) x="A" ;; + *[1357]) x="B" ;; + + *) + echo >&2 "bad output from od" + exit 3 + ;; + esac + shift + + eval x$trial=$x + actual="$actual$x" + + trial=`expr $trial + 1` + done +done + +exec 3<&- +rm -f $tmp + +probability() { + bc <<EOF + + define f(x) { + auto i; + + if (x == 0) return (1); + + i = x; + while (--i > 1) x *= i; + + return (x); + } + + define c(n, r) { + return (f(n) / (f(n - r) * f(r))); + } + + define p(r, n, p) { + return (c(n, r) * (p ^ r) * ((1 - p) ^ (n - r))); + } + + define g(r, n) { + auto p; + + while (r <= n) p += p(r++, n, 0.5); + + return (p); + } + + scale = 7; + g($1, $2) +EOF +} + +notdisproved() { + return `bc <<EOF + if ($1 <= $2) 1 + if ($1 > $2) 0 +EOF` +} + +if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null +then + if (echo -n testing; echo 1,2,3) | sed s/-n/xn/ | grep xn >/dev/null + then + n="" c=' +' + else + n="-n" c="" + fi +else + n="" c='\c' +fi + +input="" +votes="" +trials=0 +correct=0 +prob=1 + +echo + +trial=1 +while [ $trial -le "$min" ] || { + [ $trial -le "$max" ] && notdisproved $prob $goal +} +do + echo $n "trial $trial: $c" + X=`eval eval echo \\\\\$\\\$x$trial` + + if [ -z "$input" ] + then + echo "play [a] / play [b] / play [x] / vote [A] / vote [B] / [stop]" + echo $n "> $c" + if read input + then + continue + else + break + fi + fi + + case "$input" in + a|b|x) + input=`echo $input | tr abx ABX` + echo "playing $input ..." + cmd=`eval echo \\\$$input` + trap "" 2 + sh -c "$cmd" 1>/dev/null 2>$tmp + status=$? + if [ $status -eq 0 -o $status -ge 128 ] + then + rm -f $tmp + trap - 2 + else + cat >&2 $tmp + exit $status + fi + input="" + ;; + + A|B) + echo "voting for $input" + eval vote$trial="$input" + votes="$votes$input" + if [ $input = "`eval echo \\\$x$trial`" ] + then + correct=`expr $correct + 1` + fi + trials=$trial + prob=`probability $correct $trials` + trial=`expr $trial + 1` + input="x" + ;; + + stop) + echo "stopping" + break + ;; + + *) + echo "invalid input" + input="" + ;; + esac +done + +echo +echo "$trials trials completed" +echo "A = $A" +echo "B = $B" + +echo +echo " votes: $votes" +echo " actual: $actual" +echo $n "correct: $correct/$trials$c" + +if [ $trials -gt 0 ] +then + echo $n " (`expr \( $correct \* 1000 / $trials + 5 \) / 10`%)$c" +fi + +echo + +echo +echo "probability (p) of result being the same as random guesses = $prob" + +if notdisproved $prob $goal +then + echo "failed to disprove null hypothesis (p > $goal)" + exit 9 +else + echo "null hypothesis disproved (p <= $goal)" + exit 0 +fi |