#! /usr/bin/perl -w

# file: score			B. Moody	8 February 2015
#				Last revised:	15 April 2015

# Challenge 2015 evaluation, stage 4:  calculate scores

use strict;

chomp (my $CHALLENGE = qx(dirname `which $0`));
my $DATA = "$CHALLENGE/data";
my $ALARMS = "$DATA/exam-ALARMS";

my %count_by_type;
my %count_by_length;
my %count_overall;
my $n_missing = 0;

open A, $ALARMS or die "cannot read $ALARMS: $!";
while (<A>) {
  s/[\r\n]//g;
  my ($record, $type, $real_answer) = split /,/;
  my $user_answer = '';

  if (open B, "results/exam1/$record.txt"
      or open B, "results/exam2/$record.txt") {
    $_ = <B>;
    s/[\r\n]//g;
    if (/^[^,]*,([01])$/) {
      $user_answer = $1;
    }
    close B;
  }

  if ($user_answer eq '') {
    $user_answer = 1;
    $n_missing++;
  }
  $count_by_type{$type}->{"$real_answer$user_answer"}++;
  my ($l) = ($record =~ /([sl])$/);
  $count_by_length{$l}->{"$real_answer$user_answer"}++;
  $count_overall{"$real_answer$user_answer"}++;
}
close A;

sub print_scores {
  my ($label, $table) = @_;

  # TP = true alarms classified as true
  my $tp = $table->{'11'} // 0;
  # FP = false alarms classified as true
  my $fp = $table->{'01'} // 0;
  # FN = true alarms classified as false
  my $fn = $table->{'10'} // 0;
  # TN = false alarms classified as false
  my $tn = $table->{'00'} // 0;

  my $tpr = $tp / ($tp + $fn);
  my $tnr = $tn / ($tn + $fp);

  #my $b = 10;
  #my $score = (((1+$b) * $tp) /
  #(((1+$b) * $tp) + ($b**2 * $fn) + $fp));
  my $score = (($tp + $tn) / ($tp + $tn + $fp + 5*$fn));

  printf '%-24s %3.0f%% %3.0f%% %6.2f',
    $label, ($tpr * 100), ($tnr * 100), ($score * 100);
  print "\n";
}

if ($n_missing > 0) {
  print "Warning: Program did not produce output for $n_missing records\n";
  print "  (these will be scored as TRUE alarms.)\n";
  print "\n";
}

print "                          TPR  TNR  Score\n";
print "-----------------------------------------\n";
foreach my $type (sort keys %count_by_type) {
  print_scores($type, $count_by_type{$type});
}
print "-----------------------------------------\n";
print_scores('Real-time', $count_by_length{s});
print_scores('Retrospective', $count_by_length{l});
# print "-----------------------------------------\n";
# print_scores('Overall', \%count_overall);