#! /bin/bash

# file: quiz			G. Moody	21 December 2013
#				Last revised:	13 March 2015
# Challenge 2015 evaluation, stage 2:
# Run a Challenge 2015 entry on the training set in a virtual machine,
# and check that its output is as expected.
#
# This script must be run in the VM since it executes user code!

set -e

# Check the environment variables.
if [ ! -d "$CHALLENGE" ]; then
    echo "CHALLENGE not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ROOT_IMG" ]; then
    echo "CHALLENGE_ROOT_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_HOME_IMG" ]; then
    echo "CHALLENGE_HOME_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ENTRY" ]; then
    echo "CHALLENGE_ENTRY not set"
    exit 1
fi

DATA=$CHALLENGE/data
DB=quiz

# Randomly choose up to 10 "true" and 10 "false" records of each type
# to quiz (note that 'results/prep/answers' has already been checked
# by prep; it should contain exactly one answer for each record in the
# training set)
rm -f RECORDS
for T in a b t v f; do
    for S in 0 1; do
        grep "^$T.*$S\$" results/prep/answers | cut -d, -f1 | shuf | head -10 >> RECORDS
    done
done
sed -n 'p;n' < RECORDS > RECORDS1
sed -n 'n;p' < RECORDS > RECORDS2
NR1=`wc -l < RECORDS1`
NR2=`wc -l < RECORDS2`

cat >quiz.sh <<EOF
#! /bin/bash

# Work in the user's home directory, because the root file system is read-only.
cd

[ -f /challenge/mlconf.sh ] && . /challenge/mlconf.sh

mv answers.txt /tmp/answers.orig

# Annotate the training set, compare results with submitted answers.txt
for R in \`cat /challenge/RECORDS\$1\`
do
    rm -f answers.txt
    ln -sf /challenge/$DB/\$R.* .
    echo \$R >&2
    if /challenge/ptimeout -t $TQUOTA -n $IQUOTA \
        -o /tmp/eval/\$R.log -e /tmp/eval/\$R.log \
        ./next.sh \$R &>> /tmp/eval/perf
    then
        tr -d '\\\\r' < answers.txt > /tmp/eval/\$R.txt
        S1=\`grep -m1 -x "\$R,[01]" /tmp/answers.orig\`
        S2=\`cat /tmp/eval/\$R.txt\`
        if [ "\$S1" != "\$S2" ]; then
           if [ -z "\$S2" ]; then
               echo "Error: \"./next.sh \$R\" did not produce output"
           else
               echo "Error: ./next.sh \$R: output (\$S2) does not match expected result"
           fi
           echo \$R > /tmp/eval/00_failed_record
           exit 1  # quit if mismatch
        fi   
    else
	echo -n "Error: "; tail -1 /tmp/eval/perf
	echo \$R > /tmp/eval/00_failed_record
	exit 1  # quit if error or timeout in next.sh
    fi
    rm -f \$R.*
done

if [ -f DRYRUN ]; then
    touch /tmp/eval/00_dry_run
else
    touch /tmp/eval/00_success
fi
EOF

chmod +x quiz.sh

# Run the script generated by quiz in the VM.
if [ -f results/prep/00_matlab ]; then
    cvmrun --net --opt $CHALLENGE/matlab.img \
        -T $((TQUOTA*NR1+300)) -c './quiz.sh 1 >/tmp/eval/00_log' quiz.sh \
	$DATA/$DB RECORDS1 `which ptimeout` \
	$CHALLENGE/mlconf.sh -i /tmp/eval -o tainted-quiz1 &
    cvmrun --net --opt $CHALLENGE/matlab.img \
        -T $((TQUOTA*NR2+300)) -c './quiz.sh 2 >/tmp/eval/00_log' quiz.sh \
	$DATA/$DB RECORDS2 `which ptimeout` \
	$CHALLENGE/mlconf.sh -i /tmp/eval -o tainted-quiz2 &
    wait
    $CHALLENGE/scrub tainted-quiz1 results/quiz1
    $CHALLENGE/scrub tainted-quiz2 results/quiz2
else
    cvmrun -T $((TQUOTA*NR1+300)) -c './quiz.sh 1 >/tmp/eval/00_log' quiz.sh \
	$DATA/$DB RECORDS1 `which ptimeout` \
	-i /tmp/eval -o results/quiz1 &
    cvmrun -T $((TQUOTA*NR2+300)) -c './quiz.sh 2 >/tmp/eval/00_log' quiz.sh \
	$DATA/$DB RECORDS2 `which ptimeout` \
	-i /tmp/eval -o results/quiz2 &
    wait
fi

[ -f results/quiz1/00_success ] && [ -f results/quiz2/00_success ]