public class AllOptions {
  
  public static OptionsParser getOptionsParser() {
    OptionsParser parser = new OptionsParser();
    parser.registerAll(new Object[] {
        "main", Main.opts,
        "OpenSemanticParser", OpenSemanticParser.opts,
        "Parallelizer", Parallelizer.opts,
        // Dataset
        "AbstractJSONDatasetReader", JSONDatasetReader.opts,
        "CandidateGenerator", CandidateGenerator.opts,
        "CandidateGroup", CandidateGroup.opts,
        "ExpectedAnswer", ExpectedAnswer.opts,
        "ExpectedAnswerInjectiveMatch", ExpectedAnswerInjectiveMatch.opts,
        "ExpectedAnswerCriteriaMatch", ExpectedAnswerCriteriaMatch.opts,
        "KnowledgeTreeBuilder", KnowledgeTreeBuilder.opts,
        "TargetEntityNearMatch", TargetEntityNearMatch.opts,
        // Leaner
        "AdvancedWordVectorParams", AdvancedWordVectorParams.opts,
        "AdvancedWordVectorParamsLowRank", AdvancedWordVectorParamsLowRank.opts,
        "LearnerBaseline", LearnerBaseline.opts,
        "LearnerMaxEnt", LearnerMaxEnt.opts,
        "LearnerMaxEntWithBeamSearch", LearnerMaxEntWithBeamSearch.opts,
        "Params", Params.opts,
        // Linguistic resources
        "BrownClusterTable", BrownClusterTable.opts,
        "FrequencyTable", FrequencyTable.opts,
        "LingData", LingData.opts,
        "QueryTypeTable", QueryTypeTable.opts,
        "WordNetClusterTable", WordNetClusterTable.opts,
        "WordVectorTable", WordVectorTable.opts,
        // Features
        "FeatureType", FeatureType.opts,
        "FeatureTypeHoleBased", FeatureTypeHoleBased.opts,
        "FeatureTypeNaiveEntityBased", FeatureTypeNaiveEntityBased.opts,
        "FeatureTypeLinguisticsBased", FeatureTypeLinguisticsBased.opts,
        "FeatureTypeNodeBased", FeatureTypeNodeBased.opts,
        "FeatureTypePathBased", FeatureTypePathBased.opts,
        "FeaturePostProcessorConjoin", FeaturePostProcessorConjoin.opts
    });
    return parser;
  }
  
  // ============================================================
  // Persistence with model
  // ============================================================
  
  public static String getOptionsMapFilename(String modelDir) {
    return new File(modelDir, "options.map").getPath();
  }
  
  public static String getParamsFilename(String modelDir) {
    return new File(modelDir, "params").getPath();
  }
  
  public static OpenSemanticParser loadModel(String modelDir) {
    try {
      // Load options
      String optionsMapFilename = getOptionsMapFilename(modelDir);
      LogInfo.log("Loading options from " + optionsMapFilename);
      String tempFilename = prepareWhitelistedOptionsFile(optionsMapFilename);
      getOptionsParser().parseOptionsFile(tempFilename);
      // Load parameters
      String paramsFilename = getParamsFilename(modelDir);
      LogInfo.log("Loading parameters from " + paramsFilename);
      OpenSemanticParser.init();
      OpenSemanticParser parser = new OpenSemanticParser();
      parser.load(paramsFilename);
      return parser;
    } catch (IOException e) {
      LogInfo.fail(e);
    }
    return null;
  }
  
  private static String prepareWhitelistedOptionsFile(String optionsMapFilename) throws IOException {
    File tempFile = File.createTempFile("options", ".whitelisted.tmp");
    LogInfo.logs("Copying whitelisted options %s > %s", optionsMapFilename, tempFile.getPath());
    tempFile.deleteOnExit();
    try (BufferedReader reader = new BufferedReader(new FileReader(optionsMapFilename));
        PrintWriter writer = new PrintWriter(tempFile)) {
      String line;
      while ((line = reader.readLine()) != null) {
        line = line.trim();
        if (line.isEmpty() || line.startsWith("#")) continue;
        String[] tokens = line.split("\\s+", 2);
        if (isImportantOption(tokens[0])) {
          writer.println(line);
        }
      }
    }
    return tempFile.getPath();
  }
  
  public static void saveModel(String modelDir, OpenSemanticParser parser) {
    try {
      if (!new File(modelDir).mkdirs()) {
        throw new RuntimeException("Cannot create directory " + modelDir);
      }
      // Save options
      String optionsMapFilename = getOptionsMapFilename(modelDir);
      LogInfo.log("Saving options to " + optionsMapFilename);
      OrderedStringMap allOptions = getOptionsParser().getOptionPairs();
      OrderedStringMap importantOptions = new OrderedStringMap();
      for (String key : allOptions.keys()) {
        if (isImportantOption(key))
          importantOptions.put(key, allOptions.get(key));
      }
      importantOptions.printHard(optionsMapFilename);
      // Save parameters
      String paramsFilename = getParamsFilename(modelDir);
      LogInfo.log("Saving parameters to " + paramsFilename);
      parser.save(paramsFilename);
    } catch (RuntimeException e) {
      LogInfo.warning(e);
      LogInfo.warnings("Cannot save to %s/{options.map,params}, but will continue anyway.", modelDir);
    }
  }
  
  private static Set<String> importantClasses = new HashSet<>(Arrays.asList(
      // Dataset
      "CandidateGenerator",
      "CandidateGroup",
      "KnowledgeTreeBuilder",
      "TargetEntityNearMatch",
      // Leaner
      "AdvancedWordVectorParams",
      "AdvancedWordVectorParamsLowRank",
      "LearnerBaseline",
      "LearnerMaxEnt",
      "LearnerMaxEntWithBeamSearch",
      // Linguistic resources
      "BrownClusterTable",
      "FrequencyTable",
      "QueryTypeTable",
      "WordNetClusterTable",
      "WordVectorTable",
      // Features
      "FeatureType",
      "FeatureTypeHoleBased",
      "FeatureTypeNaiveEntityBased",
      "FeatureTypeLinguisticsBased",
      "FeatureTypeNodeBased",
      "FeatureTypePathBased",
      "FeaturePostProcessorConjoin"
      ));
  private static Set<String> importantOptions = new HashSet<>(Arrays.asList(
      "LingData.annotators",
      "LingData.useAnnotators",
      "LingData.caseSensitive",
      "OpenSemanticParser.learner"
      ));

  /**
   * Return true if the option is important for prediction (not for learning parameters).
   */
  public static boolean isImportantOption(String key) {
    return importantClasses.contains(key.split("\\.")[0]) || importantOptions.contains(key);
  }

} public class InteractiveDemo {
  
  public final OpenSemanticParser parser;
  
  public InteractiveDemo(OpenSemanticParser parser) {
    this.parser = parser;
  }

  public void run() {
    LogInfo.log("Starting interactive mode ...");
    try (BufferedReader in = new BufferedReader(new InputStreamReader(System.in))) {
      while (true) {
        System.out.println("============================================================");
        System.out.print("Query: list of ");
        String phrase = in.readLine();
        if (phrase == null) {System.out.println(); break;}
        if (phrase.isEmpty()) continue;
        System.out.print("Web Page URL (blank for Google Search): ");
        String url = in.readLine();
        if (url == null) {System.out.println(); break;}
        CandidateStatistics pred = url.isEmpty() ? parser.predict(phrase) : parser.predict(phrase, url);
        LogInfo.begin_track("PRED (top scoring candidate):");
        if (pred == null) {
          LogInfo.log("Rank 1 [Unique Rank 1]: NO CANDIDATE FOUND!");
        } else {
          LogInfo.logs("Rank 1 [Unique Rank 1]: (Total Feature Score = %s)", pred.score);
          Candidate candidate = pred.candidate;
          LogInfo.logs("Extraction Predicate: %s", candidate.pattern);
          LogInfo.log(candidate.sampleEntities());
        }
        LogInfo.end_track();
      }
    } catch (IOException e) {
      LogInfo.fail(e);
    }
  }
  
} public class OpenSemanticParser {
  public static class Options {
    @Option(gloss = "Whether to test on training and test data every training iteration")
    public boolean testEveryIteration = true;
    @Option(gloss = "Log parameters after training")
    public boolean logParams = true;
    /**
     * 0 = no log whatsoever
     * 1 = log only tree patterns and scores
     * 2 = add predicted entities and answer keys
     * 3 = add feature vectors for all examples
     */
    @Option(gloss = "Verbosity of the log")
    public int logVerbosity = 3;
    @Option(gloss = "Don't print feature weights for correct answers")
    public boolean ignoreCorrectAnswers = true;
    @Option(gloss = "Learner name (maxent / base / beam)")
    public String learner = "maxent";
    @Option(gloss = "Whether to cheat and use only the candidates that contain the seed answer")
    public boolean useSeed = false;
  }
  public static Options opts = new Options();
  
  private Learner learner;
  private IterativeTester iterativeTester;
  private KnowledgeTreeBuilder knowledgeTreeBuilder = new KnowledgeTreeBuilder();
  private CandidateGenerator candidateGenerator = new CandidateGenerator();
  private static boolean initialized = false;
  
  public static void init() {
    if (initialized) return;
    // Load JavaNLP and other linguistics stuff
    LingData.initModels();
    // Try to load cache
    LingData.loadCache();
    // Check the feature options
    FeatureType.checkFeatureTypeOptionsSanity();
    FeaturePostProcessor.checkFeaturePostProcessorOptionsSanity();
    initialized = true;
  }
  
  public static void cleanUp() {
    // Try to save cache
    LingData.saveCache();
  }
  
  private Learner getLearner() {
    switch (opts.learner) {
      case "maxent":
        //LogInfo.log("Using MaxEnt learner ...");
        return new LearnerMaxEnt();
      case "base":
      case "baseline":
        //LogInfo.log("Using baseline learner ...");
        return new LearnerBaseline();
      case "beam":
      case "beamsearch":
        //LogInfo.logs("Using MaxEnt learner with beam search (beam size = %d) ...",
        //    LearnerMaxEntWithBeamSearch.opts.beamSize);
        return new LearnerMaxEntWithBeamSearch();
    }
    LogInfo.fails("Unknown learner: %s", opts.learner);
    return null;
  }
  
  // ============================================================
  // Preprocessing
  // ============================================================
  
  /**
   * Extract the data (trees / candidates / features) in advance and cache them.
   * It is not necessary to call this function before calling train() or test(), though.
   */
  public void preTrain(Dataset dataset) {
    LogInfo.begin_track("preTrain: %d train, %d test", dataset.trainExamples.size(), dataset.testExamples.size());
    // Process examples in the dataset
    extractData(dataset.trainExamples);
    extractData(dataset.testExamples);
    LogInfo.end_track();
  }
  
  private void extractData(List<Example> examples) {
    List<Example> toExtract = new ArrayList<>();
    for (Example ex : examples)
      if (ex.tree == null || ex.candidates == null)
        toExtract.add(ex);
    for (int i = 0; i < toExtract.size(); i++)
      toExtract.get(i).displayId = i + "/" + toExtract.size();
    if (toExtract.isEmpty()) return;
    
    if (Parallelizer.opts.numThreads == 1) {
      for (Example ex : toExtract) extractData(ex);
    } else {
      try {
        extractParallel(toExtract);
      } catch (InterruptedException e) {
        LogInfo.fail("Data extraction was interrupted!");
      }
    }
  }
  
  private void extractData(Example ex) {
    LogInfo.begin_track("extractData (%s): %s", ex.displayId, ex.phrase);
    Execution.putOutput("currExample", ex.displayId);
    if (ex.tree == null)
      knowledgeTreeBuilder.buildKnowledgeTree(ex);
    if (ex.candidates == null)
      candidateGenerator.process(ex);
    LogInfo.end_track();
  }
  
  private void extractParallel(List<Example> examples) throws InterruptedException {
    List<Runnable> tasks = new ArrayList<>();
    for (final Example ex : examples) {
      tasks.add(new Runnable() {
        @Override public void run() {
          extractData(ex);
        }
      });
    }
    Parallelizer.run(tasks);
  }

  // ============================================================
  // Train
  // ============================================================
  
  public void train(Dataset dataset, boolean beVeryQuiet) {
    // Extract data
    extractData(dataset.trainExamples);
    // Learn a model
    learner = getLearner();
    if (beVeryQuiet) learner.shutUp();
    if (opts.testEveryIteration) {
      iterativeTester = new IterativeTester(this, dataset);
      learner.setIterativeTester(iterativeTester);
      if (beVeryQuiet) iterativeTester.beVeryQuiet = true;
    }
    learner.learn(dataset, null);
    if (opts.logParams && !beVeryQuiet) learner.logParam();
  }
  
  public void train(Dataset dataset) {
    train(dataset, false);
  }
  
  public IterativeTester getIterativeTester() {
    return iterativeTester;
  }
  
  public void load(String path) {
    learner = getLearner();
    learner.loadModel(path);
  }
  
  public void save(String path) {
    learner.saveModel(path);
  }
  
  // ============================================================
  // Predict + Test + Evaluate
  // ============================================================
  
  public CandidateStatistics predict(String phrase) {
    return predict(new Example(phrase));
  }
  
  public CandidateStatistics predict(String phrase, String url) {
    return predict(new ExampleCached(phrase, url));
  }
  
  public CandidateStatistics predict(Example ex) {
    extractData(ex);
    List<Pair<Candidate, Double>> rankedCandidates = learner.getRankedCandidates(ex);
    List<CandidateStatistics> rankedCandidateStats = CandidateStatistics.getRankedCandidateStats(rankedCandidates);
    CandidateStatistics pred = (rankedCandidateStats.isEmpty()) ? null : rankedCandidateStats.get(0);
    return pred;
  }
  
  public List<Pair<Candidate, Double>> getRankedCandidates(Example ex) {
    extractData(ex);
    return learner.getRankedCandidates(ex);
  }
  
  public Evaluator test(List<Example> examples, String testSuiteName) {
    Evaluator evaluator = new Evaluator(testSuiteName, learner);
    if (examples.isEmpty()) {
      LogInfo.warnings("Cannot test on an empty list %s", testSuiteName);
      return evaluator;
    }
    
    if (opts.logVerbosity > 0)
      LogInfo.begin_track("Testing on %s", testSuiteName);
    
    // Process examples in the dataset
    extractData(examples);
    
    for (Example ex : examples) {
      List<Pair<Candidate, Double>> rankedCandidates = learner.getRankedCandidates(ex);
      if (opts.useSeed) {
        // Only keep the candidates that contain the seed
        List<Pair<Candidate, Double>> filteredRankedCandidates = new ArrayList<>();
        TargetEntity seed = ex.expectedAnswer.targetEntities.get(1);
        for (Pair<Candidate, Double> pair : rankedCandidates) {
          List<String> predicted = pair.getFirst().predictedEntities;
          if (seed.matchAny(predicted))
            filteredRankedCandidates.add(pair);
        }
        rankedCandidates = filteredRankedCandidates;
      }
      List<CandidateStatistics> rankedCandidateStats = CandidateStatistics.getRankedCandidateStats(rankedCandidates);
      
      CandidateStatistics pred, firstTrue, best;
      pred = (rankedCandidateStats.isEmpty()) ? null : rankedCandidateStats.get(0);
      firstTrue = ex.expectedAnswer.findFirstTrueCandidate(rankedCandidateStats);
      best = ex.expectedAnswer.findBestCandidate(rankedCandidateStats);
      
      // Send to evaluator
      EvaluationCase evaluationCase = evaluator.add(ex, pred, firstTrue, best);
      // Log stuff
      if (opts.logVerbosity > 0) {
        LogInfo.begin_track("Found %d candidates for %s", ex.candidates.size(), ex);
        if (opts.ignoreCorrectAnswers && evaluationCase instanceof EvaluationSuccess) {
          LogInfo.logs("<%s SUCCESS> Rank 1 [Unique Rank 1]: (Total Feature Score = %s)",
              testSuiteName, pred.score);
        } else {
          // Log answer key
          if (opts.logVerbosity >= 2) {
            LogInfo.begin_track("Answer Key (%d entities):", ex.expectedAnswer.size());
            LogInfo.log(ex.expectedAnswer.sampleEntities());
            LogInfo.end_track();
          }
          // Log TRUE and PRED
          evaluationCase.logTrue();
          evaluationCase.logPred();
        }
        LogInfo.end_track();
      }
    }
    
    if (opts.logVerbosity > 0)
      LogInfo.end_track();
    return evaluator;
  }

} public class ParallelizedTrainer implements Callable<OpenSemanticParser> {
  Dataset dataset;
  boolean beVeryQuiet;

  public ParallelizedTrainer(Dataset dataset, boolean beVeryQuiet) {
    this.dataset = dataset;
    this.beVeryQuiet = beVeryQuiet;
  }

  @Override
  public OpenSemanticParser call() throws Exception {
    OpenSemanticParser parser = new OpenSemanticParser();
    parser.train(dataset, beVeryQuiet);
    return parser;
  }
} public class Evaluator {
  String testSuiteName;
  Learner learner;
  int numExamples = 0, numSuccess = 0, numNormalFail = 0, numSuperFail = 0, numFound = 0;
  List<EvaluationCase> successes, normalFails, superFails;
  double sumF1onExpectedEntities = 0, sumF1onBest = 0;
  Multiset<Integer> firstTrueUniqueRanks;
  
  public Evaluator(String testSuiteName, Learner learner) {
    this.testSuiteName = testSuiteName;
    this.learner = learner;
    successes = new ArrayList<>();
    normalFails = new ArrayList<>();
    superFails = new ArrayList<>();
    firstTrueUniqueRanks = new Multiset<>();
  }
  
  public EvaluationCase add(Example ex, CandidateStatistics pred, CandidateStatistics firstTrue, CandidateStatistics best) {
    numExamples++;
    EvaluationCase evaluationCase;
    if (firstTrue == null) {
      evaluationCase = new EvaluationSuperFail(this, ex, pred, firstTrue, best);
      superFails.add(evaluationCase);
      numSuperFail++;
      firstTrueUniqueRanks.add(Integer.MAX_VALUE);
    } else if (firstTrue.rank != 1) {
      evaluationCase = new EvaluationNormalFail(this, ex, pred, firstTrue, best);
      normalFails.add(evaluationCase);
      numNormalFail++;
      numFound++;
      firstTrueUniqueRanks.add(firstTrue.uniqueRank);
    } else {
      evaluationCase = new EvaluationSuccess(this, ex, pred, firstTrue, best);
      successes.add(evaluationCase);
      numSuccess++;
      numFound++;
      firstTrueUniqueRanks.add(firstTrue.uniqueRank);
    }
    if (evaluationCase.predIRScore != null)
      sumF1onExpectedEntities += evaluationCase.predIRScore.f1;
    if (evaluationCase.predIRScoreOnBest != null)
      sumF1onBest += evaluationCase.predIRScoreOnBest.f1;
    return evaluationCase;
  }
  
  public double[] getAccuracyAtK(int maxK) {
    double[] accuracyAtK = new double[maxK + 1];
    accuracyAtK[0] = 0.0;     // Accuracy at 0 is always 0 (used for padding)
    int sumCorrect = 0;
    for (int k = 1; k <= maxK; k++) {
      sumCorrect += firstTrueUniqueRanks.count(k);
      accuracyAtK[k] = sumCorrect * 1.0 / numExamples;
    }
    return accuracyAtK;
  }
  
  public void printDetails() {
    LogInfo.begin_track("### %s: %d Normal FAILS (correct candidate in other rank) ###", testSuiteName, numNormalFail);
    for (EvaluationCase fail : normalFails) fail.logFeatureDiff();
    LogInfo.end_track();
    LogInfo.begin_track("### %s: %d Super FAILS (correct candidate not found) ###", testSuiteName, numSuperFail);
    for (EvaluationCase fail : superFails) fail.logFeatureDiff();
    LogInfo.end_track();
  }

  public Evaluator putOutput(String prefix) {
    Execution.putOutput(prefix + ".numExamples", numExamples);
    Execution.putOutput(prefix + ".accuracy", 1.0 * numSuccess / numExamples);
    Execution.putOutput(prefix + ".accuracyFound", 1.0 * numSuccess / numFound);
    Execution.putOutput(prefix + ".oracle", 1.0 * numFound / numExamples);
    Execution.putOutput(prefix + ".averageF1onTargetEntities", sumF1onExpectedEntities * 1.0 / numExamples);
    Execution.putOutput(prefix + ".averageF1onBestCandidate", sumF1onBest * 1.0 / numExamples);
    Execution.putOutput(prefix + ".accuracyAtK", Fmt.D(ListUtils.subArray(getAccuracyAtK(10), 1)));
    return this;
  }
  
  public Evaluator printScores() {
    LogInfo.begin_track("%s Evaluation", testSuiteName);
    LogInfo.logs("Number of examples: %d", numExamples);
    LogInfo.logs("Correct candidate in rank #1: %d", numSuccess);
    LogInfo.logs("Correct candidate in other rank: %d", numNormalFail);
    LogInfo.logs("Correct candidate not found: %d", numSuperFail);
    LogInfo.logs("Oracle: %.3f%% ( %d / %d )", numFound * 100.0 / numExamples, numFound, numExamples);
    LogInfo.logs("Accuracy (vs all): %.3f%% ( %d / %d )", numSuccess * 100.0 / numExamples, numSuccess, numExamples);
    LogInfo.logs("Accuracy (vs found): %.3f%% ( %d / %d )", numSuccess * 100.0 / numFound, numSuccess, numFound);
    LogInfo.logs("Average F1 (vs target entities): %.3f%%", sumF1onExpectedEntities * 100.0 / numExamples);
    LogInfo.logs("Average F1 (vs best candidate): %.3f%%", sumF1onBest * 100.0 / numExamples);
    LogInfo.logs("Accuracy @ k : %s", Fmt.D(ListUtils.mult(100.0, ListUtils.subArray(getAccuracyAtK(10), 1))));
    LogInfo.end_track();
    return this;
  }
  
  public Evaluator printScores(boolean beVeryQuiet) {
    return beVeryQuiet ? this : printScores();
  }
  
} public class EvaluatorStatistics {
  final int numExamples, numSuccess, numNormalFail, numSuperFail, numFound;
  // These values are in [0, 1]
  final double oracle, accuracyAll, accuracyFound;
  final double avgF1onExpectedEntities, avgF1onBest;
  final double[] accuracyAtK;
  
  public EvaluatorStatistics(Evaluator evaluator) {
    numExamples = evaluator.numExamples;
    numSuccess = evaluator.numSuccess;
    numNormalFail = evaluator.numNormalFail;
    numSuperFail = evaluator.numSuperFail;
    numFound = evaluator.numFound;
    oracle = numFound * 1.0 / numExamples;
    accuracyAll = numSuccess * 1.0 / numExamples;
    accuracyFound = numSuccess * 1.0 / numFound;
    avgF1onExpectedEntities = evaluator.sumF1onExpectedEntities * 1.0 / numExamples;
    avgF1onBest = evaluator.sumF1onBest * 1.0 / numExamples;
    accuracyAtK = evaluator.getAccuracyAtK(IterativeTester.MAX_K);
  }
  
  private static double getAverage(double[] stuff) {
    return ListUtils.sum(stuff) / stuff.length;
  }
  
  // Divide by n instead of (n-1)
  private static double getVariance(double[] stuff) {
    double sumSq = 0, sum = 0, n = stuff.length;
    for (double x: stuff) { sumSq += x * x; sum += x; }
    return (sumSq / n) - (sum / n) * (sum / n);
  }
  
  // Divide by n instead of (n-1)
  private static double getSD(double[] stuff) {
    return Math.sqrt(getVariance(stuff));
  }
  
  public static void logAverage(List<EvaluatorStatistics> stats, String prefix) {
    int n = stats.size(), K = IterativeTester.MAX_K;
    double[]           oracleL = new double[n],
                  accuracyAllL = new double[n],
                accuracyFoundL = new double[n],
      avgF1onExpectedEntitiesL = new double[n],
                  avgF1onBestL = new double[n];
    double[][]    accuracyAtKL = new double[K + 1][n];
    
    // Compile each statistic into a list
    for (int i = 0; i < n; i++) {
      EvaluatorStatistics stat = stats.get(i);
      oracleL[i] = stat.oracle;
      accuracyAllL[i] = stat.accuracyAll;
      accuracyFoundL[i] = stat.accuracyFound;
      avgF1onExpectedEntitiesL[i] = stat.avgF1onExpectedEntities;
      avgF1onBestL[i] = stat.avgF1onBest;
      for (int j = 0; j <= K; j++)
        accuracyAtKL[j][i] = stat.accuracyAtK[j];
    }
    double[] accuracyAtK = new double[K + 1], accuracyAtKSD = new double[K + 1];
    for (int j = 0; j <= K; j++) {
      accuracyAtK[j] = getAverage(accuracyAtKL[j]);
      accuracyAtKSD[j] = getSD(accuracyAtKL[j]);
    }
    
    // Log the statistics
    LogInfo.begin_track("@@@@@ SUMMARY %s (%d folds) @@@@@", prefix, n);
    logAverageSingle(oracleL, prefix + ".oracle", "Oracle");
    logAverageSingle(accuracyAllL, prefix + ".accuracy", "Accuracy (vs all)");
    logAverageSingle(accuracyFoundL, prefix + ".accuracyFound", "Accuracy (vs found)");
    logAverageSingle(avgF1onExpectedEntitiesL, prefix + ".averageF1onTargetEntities", "Average F1 (vs target entities)");
    logAverageSingle(avgF1onBestL, prefix + ".averageF1onBestCandidate", "Average F1 (vs best candidate)");
    // Ignore accuracy at 0 (which is always 0)
    Execution.putOutput(prefix + ".accuracyAtK", Fmt.D(ListUtils.subArray(accuracyAtK, 1)));
    Execution.putOutput(prefix + ".accuracyAtKSD", Fmt.D(ListUtils.subArray(accuracyAtKSD, 1)));
    LogInfo.logs("Accuracy @ k : %s", Fmt.D(ListUtils.mult(100.0, ListUtils.subArray(accuracyAtK, 1))));
    LogInfo.logs("            +- %s", Fmt.D(ListUtils.mult(100.0, ListUtils.subArray(accuracyAtKSD, 1))));
    LogInfo.end_track();
  }
  
  private static void logAverageSingle(double[] stuff, String outputName, String logName) {
    Execution.putOutput(outputName, getAverage(stuff));
    Execution.putOutput(outputName + "SD", getSD(stuff));
    LogInfo.logs("%s: %.3f%% +- %.3f%%   [%s]", logName, getAverage(stuff) * 100, getSD(stuff) * 100,
        Fmt.D(ListUtils.mult(100.0, stuff)));
  }
} public class IterativeTester {
  private final OpenSemanticParser openSemanticParser;
  private final Dataset dataset;
  public String message = "";
  List<EvaluatorStatistics> trainStats, testStats;
  
  public boolean beVeryQuiet = false;
  public static final int MAX_K = 10;
  
  public IterativeTester(OpenSemanticParser openSemanticParser, Dataset dataset) {
    this.openSemanticParser = openSemanticParser;
    this.dataset = dataset;
    this.trainStats = new ArrayList<>();
    this.testStats = new ArrayList<>();
  }
  
  public void run() {
    int oldLogVerbosity = OpenSemanticParser.opts.logVerbosity;
    OpenSemanticParser.opts.logVerbosity = 0;
    trainStats.add(new EvaluatorStatistics(
        openSemanticParser.test(dataset.trainExamples, "[" + message + "] ITERATIVE TEST on TRAINING SET")
        .printScores(beVeryQuiet).putOutput("train")));
    testStats.add(new EvaluatorStatistics(
        openSemanticParser.test(dataset.testExamples, "[" + message + "] ITERATIVE TEST on TEST SET")
        .printScores(beVeryQuiet).putOutput("test")));
    OpenSemanticParser.opts.logVerbosity = oldLogVerbosity;
  }
  
  public void summarize() {
    LogInfo.begin_track("@@@ SUMMARY @@@");
    LogInfo.logs("%7s | %7s %7s %7s | %7s %7s %7s", "iter",
        "tracc", "trora", "traf1", "tsacc", "tsora", "tsaf1");
    for (int i = 0; i < trainStats.size(); i++) {
      EvaluatorStatistics t = trainStats.get(i), s = testStats.get(i);
      double tF1, sF1;
      if ("wiki".equals(Main.opts.dataset.split("[.]")[0])) {
        tF1 = t.avgF1onExpectedEntities;
        sF1 = s.avgF1onExpectedEntities;
      } else {
        tF1 = t.avgF1onBest;
        sF1 = s.avgF1onBest;
      }
      LogInfo.logs("%7s | %7.2f %7.2f %7.2f | %7.2f %7.2f %7.2f", i+1,
          t.accuracyAll, t.oracle, tF1,
          s.accuracyAll, s.oracle, sF1);
    }
    LogInfo.end_track();
  }

  public EvaluatorStatistics getLastTrainStat() {
    return trainStats.get(trainStats.size() - 1);
  }
  
  public EvaluatorStatistics getLastTestStat() {
    return testStats.get(testStats.size() - 1);
  }
  
  
  
} public class Dataset {
  public final List<Example> trainExamples = new ArrayList<>();
  public final List<Example> testExamples = new ArrayList<>();
  
  public Dataset() {
    // Do nothing
  }
  
  public Dataset(List<Example> train, List<Example> test) {
    trainExamples.addAll(train);
    testExamples.addAll(test);
  }

  public Dataset addTrainExample(Example ex) {
    trainExamples.add(ex);
    return this;
  }
  
  public Dataset addTestExample(Example ex) {
    testExamples.add(ex);
    return this;
  }
  
  public Dataset addFromDataset(Dataset that) {
    this.trainExamples.addAll(that.trainExamples);
    this.testExamples.addAll(that.testExamples);
    return this;
  }
  
  public Dataset addTrainFromDataset(Dataset that) {
    this.trainExamples.addAll(that.trainExamples);
    this.trainExamples.addAll(that.testExamples);
    return this;
  }
  
  public Dataset addTestFromDataset(Dataset that) {
    this.testExamples.addAll(that.trainExamples);
    this.testExamples.addAll(that.testExamples);
    return this;
  }
  
  /**
   * @return a new Dataset with the Examples shuffled up.
   * The train/test ratio remain the same.
   * The original Dataset is not modified.
   */
  public Dataset getNewShuffledDataset() {
    List<Example> allExamples = new ArrayList<>(trainExamples);
    allExamples.addAll(testExamples);
    Collections.shuffle(allExamples, new Random(42));
    List<Example> newTrain = allExamples.subList(0, trainExamples.size());
    List<Example> newTest = allExamples.subList(trainExamples.size(), allExamples.size());
    return new Dataset(newTrain, newTest);
  }
  
  /**
   * @return a new Dataset with the specified train/test ratio.
   */
  public Dataset getNewSplitDataset(double trainRatio) {
    List<Example> allExamples = new ArrayList<>(trainExamples);
    allExamples.addAll(testExamples);
    Collections.shuffle(allExamples, new Random(42));
    int trainEndIndex = (int) (allExamples.size() * trainRatio);
    List<Example> newTrain = allExamples.subList(0, trainEndIndex);
    List<Example> newTest = allExamples.subList(trainEndIndex, allExamples.size());
    return new Dataset(newTrain, newTest);
  }
  
  // ============================================================
  // Caching rewards
  // ============================================================
  
  public void cacheRewards() {
    List<Example> uncached = new ArrayList<>();
    for (Example ex : trainExamples)
      if (!ex.expectedAnswer.frozenReward) uncached.add(ex);
    for (Example ex : testExamples)
      if (!ex.expectedAnswer.frozenReward) uncached.add(ex);
    if (uncached.isEmpty()) return;
    LogInfo.begin_track("Cache rewards ...");
    for (Example ex : uncached) {
      LogInfo.begin_track("Computing rewards for example %s ...", ex);
      for (Candidate candidate : ex.candidates) {
        ex.expectedAnswer.reward(candidate);
      }
      ex.expectedAnswer.frozenReward = true;
      LogInfo.end_track();
    }
    LogInfo.end_track();
  }
  
  // ============================================================
  // Shorthands for creating datasets.
  // ============================================================
  
  public Example E(String phrase, ExpectedAnswer expectedAnswer) {
    return E(phrase, expectedAnswer, true);
  }
  
  public Example E(String phrase, ExpectedAnswer expectedAnswer, boolean isTrain) {
    Example ex = new Example(phrase, expectedAnswer);
    if (isTrain)
      addTrainExample(ex);
    else
      addTestExample(ex);
    return ex;
  }
  
  public ExpectedAnswer L(String... items) {
    return L(false, items);
  }
  
  public ExpectedAnswer L(boolean exact, String... items) {
    return new ExpectedAnswerInjectiveMatch(items);
  }
  
  public ExpectedAnswer LN(String... items) {
    return LN(false, items);
  }
  
  public ExpectedAnswer LN(boolean exact, String... items) {
    TargetEntity[] targetEntities = new TargetEntity[items.length];
    for (int i = 0; i < items.length; i++) targetEntities[i] = N(items[i]);
    return new ExpectedAnswerInjectiveMatch(items);
  }
  
  public TargetEntityPersonName N(String full) {
    String[] parts = full.split(" ");
    if (parts.length == 2)
      return new TargetEntityPersonName(parts[0], parts[1]);
    else if (parts.length == 3)
      return new TargetEntityPersonName(parts[0], parts[1], parts[2]);
    throw new RuntimeException("N(...) requires two or three words.");
  }

} public class DatasetLibrary {

  public static Dataset getDataset(String fullname) {
    if (fullname == null)
      return null;
    
    String[] parts = fullname.split("\\.");
    if (parts.length != 2)
      LogInfo.fails("Expected dataset format = family.name; got " + fullname);
    
    String family = parts[0], name = parts[1];
    if (family == null)
      LogInfo.fails("No dataset family specified.");
    
    // Special case : Unary family (the very old dataset)
    if ("unary".equals(family))
      return new UnaryDatasets().getDataset(name);
    
    // Load from the `datasets` directory
    try {
      return new JSONDatasetReader(family, name).getDataset();
    } catch (IOException e) {
      LogInfo.fail(e);
    }
    return null;
  }

} public class Parallelizer {
  public static class Options {
    @Option(gloss = "Number of threads for execution")
    public int numThreads = 1;
  }
  public static Options opts = new Options();
  
  public static int getNumThreads() {
    int numThreads = Runtime.getRuntime().availableProcessors();
    if (opts.numThreads > 0 && numThreads > opts.numThreads)
      numThreads = opts.numThreads;
    return numThreads;
  }
  
  public static void run(List<Runnable> tasks) {
    LogInfo.begin_threads();
    ExecutorService service = Executors.newFixedThreadPool(getNumThreads());
    try {
      for (Runnable task : tasks) {
        service.submit(task);
      }
      service.shutdown();
      service.awaitTermination(1, TimeUnit.DAYS);
    } catch (InterruptedException e) {
      LogInfo.fail(e);
    }
    LogInfo.end_threads();
  }
  
  public static <T, S extends Callable<T>> List<Future<T>> runAndReturnStuff(List<S> tasks) {
    LogInfo.begin_threads();
    List<Future<T>> results = null;
    ExecutorService service = Executors.newFixedThreadPool(getNumThreads());
    try {
      // Invoke all trainers
      results = service.invokeAll(tasks);
      service.shutdown();
      service.awaitTermination(1, TimeUnit.DAYS);
    } catch (InterruptedException e) {
      LogInfo.fail(e);
    }
    LogInfo.end_threads();
    return results;
  }
  
}
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import edu.stanford.nlp.semparse.open.core.AllOptions;
import edu.stanford.nlp.semparse.open.core.InteractiveDemo;
import edu.stanford.nlp.semparse.open.core.OpenSemanticParser;
import edu.stanford.nlp.semparse.open.core.ParallelizedTrainer;
import edu.stanford.nlp.semparse.open.core.eval.Evaluator;
import edu.stanford.nlp.semparse.open.core.eval.EvaluatorStatistics;
import edu.stanford.nlp.semparse.open.core.eval.IterativeTester;
import edu.stanford.nlp.semparse.open.dataset.Dataset;
import edu.stanford.nlp.semparse.open.dataset.library.DatasetLibrary;
import edu.stanford.nlp.semparse.open.util.Parallelizer;
import fig.basic.LogInfo;
import fig.basic.Option;
import fig.exec.Execution;
package edu.stanford.nlp.semparse.open;



/**
 * Main class for training and testing models.
 * 
 * <h1>Development Mode</h1>
 * <p><blockquote><code>
 *   java edu.stanford.nlp.semparse.open.Main -dataset FAMILY.NAME [-trainFrac 0.8] [-testFrac 0.2] [-saveModel FILENAME] [-folds 1]
 * </blockquote></code></p>
 * <p>Specify 1 dataset to be divided into train + test based on the specified ratio (trainFrac & testFrac).
 * <p>To use the whole dataset as the training data, use options trainFrac = 1 and testFrac = 0.
 * <p>The model can be saved by specifying the saveModel option.
 * <p>To run on multiple random splits in parallel, use the 'folds' option. (Note: saveModel cannot be used when folds > 1)
 * 
 * <h1>Train & Test Mode</h1>
 * <p><blockquote><code>
 *   java edu.stanford.nlp.semparse.open.Main -dataset FAMILY.TRAIN_NAME@TEST_NAME [-saveModel FILENAME]
 * </blockquote></code></p>
 * <p>Specify 2 datasets from the same family (one for train, one for test) separated by the "@" sign.
 * <p>The model can be saved by specifying the saveModel option.
 * 
 * <h1>Test Only Mode</h1>
 * <p><blockquote><code>
 *   java edu.stanford.nlp.semparse.open.Main -loadModel FILENAME -dataset FAMILY.NAME
 * </blockquote></code></p>
 * <p>Load model from file + Test on the specified dataset.
 *  
 * <h1>Interactive Mode</h1>
 * <p><blockquote><code>
 *   java edu.stanford.nlp.semparse.open.Main -loadModel FILENAME
 * </blockquote></code></p>
 * <p>Load model from file + Start interactive mode where the desired query and web page can be entered directly. 
 *   
 * <h1>Experiment Mode</h1>
 * <p><blockquote><code>
 *   java edu.stanford.nlp.semparse.open.experiment.Experiments -experiment NAME
 * </blockquote></code></p>
 * See {@link edu.stanford.nlp.semparse.open.experiment.Experiments Experiments}.
 * 
 */
public class Main implements Runnable {
  public static class Options {
    @Option(gloss="dataset name (format = family.name)") public String dataset = null;
    @Option(gloss="filename for saving a trained model") public String saveModel = null;
    @Option(gloss="filename for loading a trained model") public String loadModel = null;
    @Option(gloss="number of folds") public int folds = 1;
  }
  public static Options opts = new Options();
  
  public static void main(String args[]) {
    Execution.run(args, new Main(), AllOptions.getOptionsParser());
  }
    
  // ============================================================
  // Overall run script
  // ============================================================

  public void run() {
    if (opts.folds > 1 && (opts.saveModel != null || opts.loadModel != null))
      LogInfo.fail("Cannot save or load a model with folds > 1");
    if (opts.loadModel != null) {
      loadAndTestMode();
    } else {
      trainAndTestMode();
    }
  }
  
  public void loadAndTestMode() {
    loadAndTestMode(DatasetLibrary.getDataset(opts.dataset));
  }

  public void loadAndTestMode(Dataset dataset) {
    OpenSemanticParser parser = AllOptions.loadModel(opts.loadModel);