#! /bin/bash

# file: exam			G. Moody	21 December 2013
#				Last revised:	1 February 2017
# Challenge 2017 evaluation, stage 3:
# Run a Challenge 2017 entry on the test set in a virtual machine,
# and save its output for scoring.

set -e

# Check the environment variables.
if [ ! -d "$CHALLENGE" ]; then
    echo "CHALLENGE not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ROOT_IMG" ]; then
    echo "CHALLENGE_ROOT_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_HOME_IMG" ]; then
    echo "CHALLENGE_HOME_IMG not set"
    exit 1
fi
if [ ! -f "$CHALLENGE_ENTRY" ]; then
    echo "CHALLENGE_ENTRY not set"
    exit 1
fi

DATA=$CHALLENGE/data
DB=test

cat >exam.sh <<EOF
#! /bin/bash

# Work in the user's home directory, because the root file system is read-only.
cd

[ -f /challenge/mlconf.sh ] && . /challenge/mlconf.sh

# Annotate the test set.
for F in /challenge/*.hea
do
    R=\`basename \$F .hea\`
    rm -f answers.txt
    ln -sf /challenge/\$R.* .
    echo \$R >&2
    if /challenge/ptimeout -t $TQUOTA -n $IQUOTA -N $IHQUOTA \
        -o /tmp/eval/\$R.log -e /tmp/eval/\$R.log \
        ./next.sh \$R &>> /tmp/eval/perf
    then
        tr -d '\\r' < answers.txt > /tmp/eval/\$R.txt
    else
        touch /tmp/eval/_cvmrun_retry
    fi
    rm -f \$R.*
done

uptime > /tmp/eval/uptime
EOF

chmod +x exam.sh

# Run the script generated by exam in the VM.

opt=
if [ -f results/prep/00_matlab ]; then
    opt="--net --opt $CHALLENGE/matlab.img $CHALLENGE/mlconf.sh"
fi

parallel -X -u -j$NJOBS -n$RBATCH --halt=0 \
    cvmrun $opt --overlay -T $((TQUOTA*RBATCH+300)) \
    -c './exam.sh' \
    exam.sh `which ptimeout` "$DATA/$DB/{}.hea" "$DATA/$DB/{}.mat" \
    -i /tmp/eval -o "results/exam{#}" \
    < $DATA/$DB/RECORDS