// Path: src/edu/stanford/nlp/semparse/open/dataset/Example.java
// public class Example {
//   public String displayId;  // For debugging
//   
//   public final String phrase;
//   public final ExpectedAnswer expectedAnswer;
//   public KNode tree;  // Deterministic function of the phrase
//   public List<CandidateGroup> candidateGroups;
//   public List<Candidate> candidates;  // Candidate predictions
//   public AveragedWordVector averagedWordVector;
// 
//   public Example(String phrase) {
//     this(phrase, null);
//   }
//   
//   public Example(String phrase, ExpectedAnswer expectedAnswer) {
//     this.phrase = phrase;
//     this.expectedAnswer = expectedAnswer;
//   }
//   
//   @Override public String toString() {
//     return "[" + phrase + "]";
//   }
//   
//   public void initAveragedWordVector() {
//     if (averagedWordVector == null)
//       averagedWordVector = new AveragedWordVector(phrase);
//   }
// }
// 
// Path: src/edu/stanford/nlp/semparse/open/model/Learner.java
// public interface Learner {
//   
//   // ============================================================
//   // Log
//   // ============================================================
// 
//   public void logParam();
//   public void logFeatureWeights(Candidate candidate);
//   public void logFeatureDiff(Candidate trueCandidate, Candidate predCandidate);
//   public void shutUp();
//   
//   // ============================================================
//   // Predict
//   // ============================================================
//   
//   public List<Pair<Candidate, Double>> getRankedCandidates(Example example);
//   
//   // ============================================================
//   // Learn
//   // ============================================================
//   
//   public void learn(Dataset dataset, FeatureMatcher additionalFeatureMatcher);
//   public void setIterativeTester(IterativeTester tester);
//   
//   // ============================================================
//   // Persistence
//   // ============================================================
//   
//   public void saveModel(String path);
//   public void loadModel(String path);
//   
// }
// 
// Path: src/edu/stanford/nlp/semparse/open/util/Multiset.java
// public class Multiset<T> {
//   protected final HashMap<T, Integer> map = new HashMap<>();
//   protected int size = 0;
// 
//   public void add(T entry) {
//     Integer count = map.get(entry);
//     if (count == null)
//       count = 0;
//     map.put(entry, count + 1);
//     size++;
//   }
//   
//   public void add(T entry, int incr) {
//     Integer count = map.get(entry);
//     if (count == null)
//       count = 0;
//     map.put(entry, count + incr);
//     size += incr;
//   }
// 
//   public boolean contains(T entry) {
//     return map.containsKey(entry);
//   }
// 
//   public int count(T entry) {
//     Integer count = map.get(entry);
//     if (count == null)
//       count = 0;
//     return count;
//   }
// 
//   public Set<T> elementSet() {
//     return map.keySet();
//   }
// 
//   public Set<Map.Entry<T, Integer>> entrySet() {
//     return map.entrySet();
//   }
// 
//   public int size() {
//     return size;
//   }
// 
//   public boolean isEmpty() {
//     return size == 0;
//   }
//   
//   public Multiset<T> getPrunedByCount(int minCount) {
//     Multiset<T> pruned = new Multiset<>();
//     for (Map.Entry<T, Integer> entry : map.entrySet()) {
//       if (entry.getValue() >= minCount)
//         pruned.add(entry.getKey(), entry.getValue());
//     }
//     return pruned;
//   }
// }


import java.util.*;
import edu.stanford.nlp.semparse.open.dataset.Example;
import edu.stanford.nlp.semparse.open.model.Learner;
import edu.stanford.nlp.semparse.open.util.Multiset;
import fig.basic.Fmt;
import fig.basic.ListUtils;
import fig.basic.LogInfo;
import fig.exec.Execution;
package edu.stanford.nlp.semparse.open.core.eval;



/**
 * Deal with evaluation.
 * With zero-one loss, "correct" means matching all criteria
 * - Wiki : match all target entities
 * - Web : match first, second, and last
 * 
 * pred = predicted candidate (first candidate in rankedCandidates)
 * true = first correct candidate in rankedCandidates
 * best = the best thing we can ever select from the web page (may not match all target entities)
 * 
 * SUPER FAIL = (firstTrue == null)
 * NORMAL FAIL = (firstTrue.rank != 1)
 * SUCCESS = (firstTrue.rank == 1)
 */
public class Evaluator {
  String testSuiteName;
  Learner learner;
  int numExamples = 0, numSuccess = 0, numNormalFail = 0, numSuperFail = 0, numFound = 0;
  List<EvaluationCase> successes, normalFails, superFails;
  double sumF1onExpectedEntities = 0, sumF1onBest = 0;
  Multiset<Integer> firstTrueUniqueRanks;
  
  public Evaluator(String testSuiteName, Learner learner) {
    this.testSuiteName = testSuiteName;
    this.learner = learner;
    successes = new ArrayList<>();
    normalFails = new ArrayList<>();
    superFails = new ArrayList<>();
    firstTrueUniqueRanks = new Multiset<>();
  }
  
// Path: src/edu/stanford/nlp/semparse/open/dataset/Example.java
// public class Example {
//   public String displayId;  // For debugging
//   
//   public final String phrase;
//   public final ExpectedAnswer expectedAnswer;
//   public KNode tree;  // Deterministic function of the phrase
//   public List<CandidateGroup> candidateGroups;
//   public List<Candidate> candidates;  // Candidate predictions
//   public AveragedWordVector averagedWordVector;
// 
//   public Example(String phrase) {
//     this(phrase, null);
//   }
//   
//   public Example(String phrase, ExpectedAnswer expectedAnswer) {
//     this.phrase = phrase;
//     this.expectedAnswer = expectedAnswer;
//   }
//   
//   @Override public String toString() {
//     return "[" + phrase + "]";
//   }
//   
//   public void initAveragedWordVector() {
//     if (averagedWordVector == null)
//       averagedWordVector = new AveragedWordVector(phrase);
//   }
// }
// 
// Path: src/edu/stanford/nlp/semparse/open/model/Learner.java
// public interface Learner {
//   
//   // ============================================================
//   // Log
//   // ============================================================
// 
//   public void logParam();
//   public void logFeatureWeights(Candidate candidate);
//   public void logFeatureDiff(Candidate trueCandidate, Candidate predCandidate);
//   public void shutUp();
//   
//   // ============================================================
//   // Predict
//   // ============================================================
//   
//   public List<Pair<Candidate, Double>> getRankedCandidates(Example example);
//   
//   // ============================================================
//   // Learn
//   // ============================================================
//   
//   public void learn(Dataset dataset, FeatureMatcher additionalFeatureMatcher);
//   public void setIterativeTester(IterativeTester tester);
//   
//   // ============================================================
//   // Persistence
//   // ============================================================
//   
//   public void saveModel(String path);
//   public void loadModel(String path);
//   
// }
// 
// Path: src/edu/stanford/nlp/semparse/open/util/Multiset.java
// public class Multiset<T> {
//   protected final HashMap<T, Integer> map = new HashMap<>();
//   protected int size = 0;
// 
//   public void add(T entry) {
//     Integer count = map.get(entry);
//     if (count == null)
//       count = 0;
//     map.put(entry, count + 1);
//     size++;
//   }
//   
//   public void add(T entry, int incr) {
//     Integer count = map.get(entry);
//     if (count == null)
//       count = 0;
//     map.put(entry, count + incr);
//     size += incr;
//   }
// 
//   public boolean contains(T entry) {
//     return map.containsKey(entry);
//   }
// 
//   public int count(T entry) {
//     Integer count = map.get(entry);
//     if (count == null)
//       count = 0;
//     return count;
//   }
// 
//   public Set<T> elementSet() {
//     return map.keySet();
//   }
// 
//   public Set<Map.Entry<T, Integer>> entrySet() {
//     return map.entrySet();
//   }
// 
//   public int size() {
//     return size;
//   }
// 
//   public boolean isEmpty() {
//     return size == 0;
//   }
//   
//   public Multiset<T> getPrunedByCount(int minCount) {
//     Multiset<T> pruned = new Multiset<>();
//     for (Map.Entry<T, Integer> entry : map.entrySet()) {
//       if (entry.getValue() >= minCount)
//         pruned.add(entry.getKey(), entry.getValue());
//     }
//     return pruned;
//   }
// }


// Path: src/edu/stanford/nlp/semparse/open/core/eval/Evaluator.java
import java.util.*;
import edu.stanford.nlp.semparse.open.dataset.Example;
import edu.stanford.nlp.semparse.open.model.Learner;
import edu.stanford.nlp.semparse.open.util.Multiset;
import fig.basic.Fmt;
import fig.basic.ListUtils;
import fig.basic.LogInfo;
import fig.exec.Execution;

package edu.stanford.nlp.semparse.open.core.eval;



/**
 * Deal with evaluation.
 * With zero-one loss, "correct" means matching all criteria
 * - Wiki : match all target entities
 * - Web : match first, second, and last
 * 
 * pred = predicted candidate (first candidate in rankedCandidates)
 * true = first correct candidate in rankedCandidates
 * best = the best thing we can ever select from the web page (may not match all target entities)
 * 
 * SUPER FAIL = (firstTrue == null)
 * NORMAL FAIL = (firstTrue.rank != 1)
 * SUCCESS = (firstTrue.rank == 1)
 */
public class Evaluator {
  String testSuiteName;
  Learner learner;
  int numExamples = 0, numSuccess = 0, numNormalFail = 0, numSuperFail = 0, numFound = 0;
  List<EvaluationCase> successes, normalFails, superFails;
  double sumF1onExpectedEntities = 0, sumF1onBest = 0;
  Multiset<Integer> firstTrueUniqueRanks;
  
  public Evaluator(String testSuiteName, Learner learner) {
    this.testSuiteName = testSuiteName;
    this.learner = learner;
    successes = new ArrayList<>();
    normalFails = new ArrayList<>();
    superFails = new ArrayList<>();
    firstTrueUniqueRanks = new Multiset<>();
  }
  
  public EvaluationCase add(Example ex, CandidateStatistics pred, CandidateStatistics firstTrue, CandidateStatistics best) {