All files / src evaluate.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
 
 
 
 
 
 
 
 
 
 
 
 
1x
1x
 
4x
10x
 
 
1x
 
 
1x
 
1x
 
4x
10x
 
10x
 
 
48x
 
 
48x
38x
 
38x
 
4x
4x
4x
 
4x
4x
4x
4x
4x
4x
4x
4x
4x
4x
4x
4x
4x
 
4x
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1x
1x
 
4x
 
84x
84x
 
80x
 
 
 
 
1x
 
 
 
 
 
9x
 
"use strict";
 
/**
 * Evaluate an IR system.
 *
 * @param  {System} system description
 * @param  {QueryParser} parser  description
 * @param  {object} cases  description
 * @return {object}        description
 */
async function evaluate(system, parser, expected)
{
  // find actual results
  const actual = {};
  for (const query of Object.keys(expected))
  {
    actual[query] = (await system.retrieveDocuments(parser.parse(query)));
    actual[query] = actual[query].results.map(x => x.id);
  }
 
  let count = Object.keys(expected)
    .length;
 
  const individual = {};
 
  for (let [query, T] of Object.entries(expected))
  {
    let P = actual[query];
    const TP = P.filter(a => T.indexOf(a) !== -1)
      .length;
    const FP = P.filter(a => T.indexOf(a) === -1)
      .length;
 
    let F = system.ids.filter(n => T.indexOf(n) === -1)
      .length;
 
    let N = system.ids.filter(n => P.indexOf(n) === -1);
    const TN = N.filter(a => T.indexOf(a) === -1)
      .length;
    const FN = N.filter(a => T.indexOf(a) !== -1)
      .length;
    N = N.length;
    P = P.length;
    T = T.length;
 
    let TPR = (TP + FN) !== 0 ? TP / (TP + FN) : 0;
    let TNR = (TN + FP) !== 0 ? TN / (TN + FP) : 0;
    let PPV = P !== 0 ? TP / (TP + FP) : 0;
    let NPV = N !== 0 ? TN / (TN + FN) : 0;
    let FNR = 1 - TPR;
    let FPR = 1 - TNR;
    let FDR = 1 - PPV;
    let FOR = 1 - NPV;
    let ACC = (TN + TP) / (P + N);
    let F1 = 2 * PPV * TPR / (PPV + TPR);
    let MCC = (TP * TN - FP * FN) / Math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN));
    let BM = TPR + TNR - 1;
    let MK = PPV + NPV - 1;
 
    individual[query] = {
      T,
      F,
      P,
      N,
      TP,
      TN,
      FP,
      FN,
      TPR,
      TNR,
      PPV,
      NPV,
      FNR,
      FPR,
      FDR,
      FOR,
      ACC,
      F1,
      MCC,
      BM,
      MK
    }
 
  }
 
  const overall = {};
  for (let ind of Object.values(individual))
  {
    for (let [marker, value] of Object.entries(ind))
    {
      overall[marker] = overall[marker] || 0;
      if (Number.isFinite(value))
      {
        overall[marker] += value / count;
      }
    }
  }
 
  return {
    individual,
    overall
  };
}
 
module.exports = evaluate;