#! /bin/bash

# file: exam			G. Moody	21 December 2013
#				Last revised:	 9 March 2015
# Challenge 2015 evaluation, stage 3:
# Run a Challenge 2015 entry on the test set in a virtual machine,
# and save its output for scoring.
#
# This script must be run in the VM since it executes user code!

set -e

# Check the environment variables.
if [ ! -d "$CHALLENGE" ]; then
    echo "CHALLENGE not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ROOT_IMG" ]; then
    echo "CHALLENGE_ROOT_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_HOME_IMG" ]; then
    echo "CHALLENGE_HOME_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ENTRY" ]; then
    echo "CHALLENGE_ENTRY not set"
    exit 1
fi

DATA=$CHALLENGE/data
DB=exam
NR=`wc -l < $DATA/$DB/RECORDS`
head -$((NR/2)) < $DATA/$DB/RECORDS > RECORDS1
tail -$(((NR+1)/2)) < $DATA/$DB/RECORDS > RECORDS2
NR1=`wc -l < RECORDS1`
NR2=`wc -l < RECORDS2`

cat >exam.sh <<EOF
#! /bin/bash

# Work in the user's home directory, because the root file system is read-only.
cd

[ -f /challenge/mlconf.sh ] && . /challenge/mlconf.sh

# Annotate the test set.
for R in \`cat /challenge/RECORDS\$1\`
do
    rm -f answers.txt
    ln -sf /challenge/$DB/\$R.* .
    echo \$R >&2
    if /challenge/ptimeout -t $TQUOTA -n $IQUOTA -N $IHQUOTA \
        -o /tmp/eval/\$R.log -e /tmp/eval/\$R.log \
        ./next.sh \$R &>> /tmp/eval/perf
    then
        tr -d '\\r' < answers.txt > /tmp/eval/\$R.txt
    fi
    rm -f \$R.*
done
EOF

chmod +x exam.sh

# Run the script generated by exam in the VM.
if [ -f results/prep/00_matlab ]; then
    cvmrun --net --opt $CHALLENGE/matlab.img \
        -T $((TQUOTA*NR1+300)) -c './exam.sh 1' exam.sh \
        $DATA/$DB RECORDS1 `which ptimeout` \
        $CHALLENGE/mlconf.sh -i /tmp/eval -o tainted-exam1 &
    cvmrun --net --opt $CHALLENGE/matlab.img \
        -T $((TQUOTA*NR2+300)) -c './exam.sh 2' exam.sh \
        $DATA/$DB RECORDS2 `which ptimeout` \
        $CHALLENGE/mlconf.sh -i /tmp/eval -o tainted-exam2 &
    wait
    $CHALLENGE/scrub tainted-exam1 results/exam1
    $CHALLENGE/scrub tainted-exam2 results/exam2
else
    cvmrun -T $((TQUOTA*NR1+300)) -c './exam.sh 1' exam.sh \
        $DATA/$DB RECORDS1 `which ptimeout` \
        -i /tmp/eval -o results/exam1 &
    cvmrun -T $((TQUOTA*NR2+300)) -c './exam.sh 2' exam.sh \
        $DATA/$DB RECORDS2 `which ptimeout` \
        -i /tmp/eval -o results/exam2 &
    wait
fi
exit 0