run.stress-tests.bash 4.97 KB
Newer Older
Leif Walsh's avatar
Leif Walsh committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
#!/bin/bash
# $Id$

DOC=<<EOF

  PARAMETERS

    table size: small (2 000), medium (200 000), large (50 000 000)

    cachetable size: small (num_elements * 50), large (1 000 000 000)

    update threads: 1, random number <= 20

    point query threads: 1, random number <= 20

    recover-test_stress1, recover-test_stress2

  DATA

    currently running tests

    log of success/failure ("./recover-test_stress1.tdb --num_elements blah blah blah      PASS")

    if failed:

      parameters

      corefile

      stdout/stderr

      data directory

EOF

set -e

scriptname=$(basename "$0")
toku_toplevel=$(dirname $(dirname $(readlink -f "$PWD/$0")))
log=/tmp/run.stress-tests.log
savedir=/tmp/run.stress-tests.failures

usage() {
    echo "Usage: $scriptname" 1>&2
    echo "  [--toku_toplevel=<dir>]" 1>&2
    echo "  [--log=<file>]" 1>&2
    echo "  [--savedir=<dir>]" 1>&2
}

# parse the command line
while [ $# -gt 0 ] ; do
    arg=$1; shift
    if [[ $arg =~ --(.*)=(.*) ]] ; then
        ok=no
        for opt in toku_toplevel log savedir
        do
            if [[ ${BASH_REMATCH[1]} = $opt ]]
            then
                ok=yes
            fi
        done
        if [[ $ok = no ]]
        then
            usage; exit 1
        fi
        eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
    else
        usage; exit 1
    fi
done

src_tests="${toku_toplevel}/src/tests"
testnames=(recover-test_stress1.tdb \
           recover-test_stress2.tdb)

save_failure() {
    dir="$1"; shift
    out="$1"; shift
    envdir="$1"; shift
    exec="$1"; shift
    table_size=$1; shift
    cachetable_size=$1; shift
    num_ptquery=$1; shift
    num_update=$1; shift
    phase=$1; shift
    dest="${dir}/${exec}-${table_size}-${cachetable_size}-${num_ptquery}-${num_update}-${phase}-$$"
87
    mkdir -p "$dest"
Leif Walsh's avatar
Leif Walsh committed
88 89 90 91 92 93 94 95 96 97 98 99 100 101
    mv $out "${dest}/output.txt"
    mv core* "${dest}/"
    mv $envdir "${dest}/"
}

run_test() {
    exec="$1"; shift
    table_size="$1"; shift
    cachetable_size="$1"; shift
    num_ptquery="$1"; shift
    num_update="$1"; shift
    mylog="$1"; shift
    mysavedir="$1"; shift

102
    tmplog=$(mktemp)
Leif Walsh's avatar
Leif Walsh committed
103 104 105 106 107
    ulimit -c unlimited
    t0="$(date)"
    t1=""
    t2=""
    envdir="./${exec}-${table_size}-${cachetable_size}-${num_ptquery}-${num_update}-$$.dir"
108
    if ! ./$exec -v --test --num_seconds 180 --envdir "$envdir" \
Leif Walsh's avatar
Leif Walsh committed
109 110 111
        --num_elements $table_size \
        --cachetable_size $cachetable_size \
        --num_ptquery_threads $num_ptquery \
112
        --num_update_threads $num_update > $tmplog
Leif Walsh's avatar
Leif Walsh committed
113
    then
114
        rm -f $tmplog
Leif Walsh's avatar
Leif Walsh committed
115 116 117
        t1="$(date)"
        if ./$exec -v --recover --envdir "$envdir" \
            --num_elements $table_size \
118
            --cachetable_size $cachetable_size > $tmplog
Leif Walsh's avatar
Leif Walsh committed
119
        then
120
            rm -f $tmplog
Leif Walsh's avatar
Leif Walsh committed
121 122 123
            t2="$(date)"
            echo "\"$exec\",$table_size,$cachetable_size,$num_ptquery,$num_update,$t0,$t1,$t2,PASS" > "$mylog"
        else
124
            save_failure "$mysavedir" $tmplog $envdir $exec $table_size $cachetable_size $num_ptquery $num_update recover
Leif Walsh's avatar
Leif Walsh committed
125 126 127
            echo "\"$exec\",$table_size,$cachetable_size,$num_ptquery,$num_update,$t0,$t1,$t2,FAIL" > "$mylog"
        fi
    else
128
        save_failure "$mysavedir" $tmplog $envdir $exec $table_size $cachetable_size $num_ptquery $num_update test
Leif Walsh's avatar
Leif Walsh committed
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
        echo "\"$exec\",$table_size,$cachetable_size,$num_ptquery,$num_update,$t0,$t1,$t2,FAIL" > "$mylog"
    fi
}

loop_test() {
    exec="$1"; shift
    table_size="$1"; shift
    cachetable_size="$1"; shift
    mylog="$1"; shift
    mysavedir="$1"; shift

    ptquery_rand=0
    update_rand=0
    while true
    do
        num_ptquery=1
        num_update=1
        if [[ $ptquery_rand -gt 1 ]]
        then
            (( num_ptquery = $RANDOM % 16 ))
        fi
        if [[ $update_rand -gt 0 ]]
        then
            (( num_update = $RANDOM % 16 ))
        fi
        (( ptquery_rand = (ptquery_rand + 1) % 4 ))
        (( update_rand = (update_rand + 1) % 2 ))
        run_test $exec $table_size $cachetable_size $num_ptquery $num_update $mylog $mysavedir
    done
}

cd $src_tests

for exec in ${testnames[@]}
do
    if [[ ! -x $exec ]]
    then
        echo "Please build $exec" 1>&2
        exit 1
    fi
done

mkdir -p $log
mkdir -p $savedir

Leif Walsh's avatar
Leif Walsh committed
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
declare -a pids
i=0

savepid() {
    pids[i]=$1
    (( i = i + 1 ))
}

killchildren() {
    for pid in ${pids[@]}
    do
        kill $pid
    done
}

Leif Walsh's avatar
Leif Walsh committed
189 190 191 192 193 194
for exec in ${testnames[@]}
do
    for table_size in 2000 200000 50000000
    do
        (( small_cachetable = table_size * 50 ))
        suffix="${exec}-${table_size}-${small_cachetable}-$$"
195
        touch "${log}/${suffix}"
Leif Walsh's avatar
Leif Walsh committed
196 197
        loop_test $exec $table_size $small_cachetable "${log}/${suffix}" "${savedir}/${suffix}" & savepid $!
        tail -f "${log}/${suffix}" & savepid $!
Leif Walsh's avatar
Leif Walsh committed
198
        suffix="${exec}-${table_size}-1000000000-$$"
Leif Walsh's avatar
Leif Walsh committed
199 200 201
        touch "${log}/${suffix}"
        loop_test $exec $table_size 1000000000 "${log}/${suffix}" "${savedir}/${suffix}" & savepid $!
        tail -f "${log}/${suffix}" & savepid $!
Leif Walsh's avatar
Leif Walsh committed
202 203
    done
done
Leif Walsh's avatar
Leif Walsh committed
204 205 206 207 208 209 210

trap killchildren INT TERM EXIT

for pid in ${pids[@]}
do
    wait $pid
done