#! /bin/bash

# file: quiz			G. Moody	21 December 2013
#				Last revised:	1 February 2017
# Challenge 2017 evaluation, stage 2:
# Run a Challenge 2017 entry on the training set in a virtual machine,
# and check that its output is as expected.

set -e

# Check the environment variables.
if [ ! -d "$CHALLENGE" ]; then
    echo "CHALLENGE not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ROOT_IMG" ]; then
    echo "CHALLENGE_ROOT_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_HOME_IMG" ]; then
    echo "CHALLENGE_HOME_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ENTRY" ]; then
    echo "CHALLENGE_ENTRY not set"
    exit 1
fi

DATA=$CHALLENGE/data
DB=validation

cat >quiz.sh <<EOF
#! /bin/bash

# Work in the user's home directory, because the root file system is read-only.
cd

[ -f /challenge/mlconf.sh ] && . /challenge/mlconf.sh

mv answers.txt /tmp/answers.orig

# Annotate the training set, compare results with submitted answers.txt
for F in /challenge/*.hea
do
    R=\`basename \$F .hea\`
    rm -f answers.txt
    ln -sf /challenge/\$R.* .
    echo \$R >&2
    if /challenge/ptimeout -t $TQUOTA -n $IQUOTA \
        -o /tmp/eval/\$R.log -e /tmp/eval/\$R.log \
        ./next.sh \$R &>> /tmp/eval/perf
    then
        tr -d '\\\\r' < answers.txt > /tmp/eval/\$R.txt
        S1=\`grep -m1 -x "\$R,.*" /tmp/answers.orig\`
        S2=\`cat /tmp/eval/\$R.txt\`
        if [ "\$S1" != "\$S2" ]; then
           if [ -z "\$S2" ]; then
               echo "Error: \"./next.sh \$R\" did not produce output"
           else
               echo "Error: ./next.sh \$R: output (\$S2) does not match expected result"
           fi
           echo \$R > /tmp/eval/00_failed_record
           exit 1  # quit if mismatch
        fi
    else
	echo -n "Error: "; tail -1 /tmp/eval/perf
	echo \$R > /tmp/eval/00_failed_record
	touch /tmp/eval/_cvmrun_retry
	exit 1  # quit if error or timeout in next.sh
    fi
    rm -f \$R.*
done

uptime > /tmp/eval/uptime
touch /tmp/eval/00_success
EOF

chmod +x quiz.sh

# Run the script generated by quiz in the VM.

opt=
if [ -f results/prep/00_matlab ]; then
    opt="--net --opt $CHALLENGE/matlab.img $CHALLENGE/mlconf.sh"
fi

parallel -X -u -j$NJOBS -n$RBATCH --halt=1 \
    cvmrun $opt --overlay -T $((TQUOTA*RBATCH+300)) \
    -c \''./quiz.sh >/tmp/eval/00_log'\' \
    quiz.sh `which ptimeout` "$DATA/$DB/{}.hea" "$DATA/$DB/{}.mat" \
    -i /tmp/eval -o "results/quiz{#}" -t 00_success \
    < $DATA/$DB/RECORDS

[ ! -e results/prep/00_dry_run ]