public class CoreLabelComparator implements Comparator<CoreLabel>, Serializable {

  /**
   * 
   */
  private static final long serialVersionUID = 1L;

  @Override
  public int compare(CoreLabel labelA, CoreLabel labelB) {
    String baseOfComparisonA = labelA.getString(NormalizerAnnotation.class);
    String baseOfComparisonB = labelB.getString((NormalizerAnnotation.class));
    int comparison = baseOfComparisonA.compareTo(baseOfComparisonB);
    String shortTagA = labelA.tag().toLowerCase().substring(0, Math.min(labelA.tag().length(), 2));
    String shortTagB = labelB.tag().toLowerCase().substring(0, Math.min(labelB.tag().length(), 2));
    return comparison != 0 ? comparison : shortTagA.compareTo(shortTagB);
  }

  public int compareForNGramEquality(CoreLabel labelA, CoreLabel labelB) {
    String baseOfComparisonA = labelA.word();
    String baseOfComparisonB = labelB.word();
    if (baseOfComparisonA.length() > 1)
      baseOfComparisonA = Character.toLowerCase(baseOfComparisonA.charAt(0)) + baseOfComparisonA.substring(1);
    if (baseOfComparisonB.length() > 1)
      baseOfComparisonB = Character.toLowerCase(baseOfComparisonB.charAt(0)) + baseOfComparisonB.substring(1);
    int comparison = baseOfComparisonA.compareTo(baseOfComparisonB);
    String shortTagA = labelA.tag().toLowerCase().substring(0, Math.min(labelA.tag().length(), 2));
    String shortTagB = labelB.tag().toLowerCase().substring(0, Math.min(labelB.tag().length(), 2));
    return comparison != 0 ? comparison : shortTagA.compareTo(shortTagB);
  }
} public class NGram extends ArrayList<CoreLabel> implements Cloneable {

  private static PorterStemmer ps = new PorterStemmer();
  private static final long serialVersionUID = 3797853353962652098l;
  private static final CoreLabelComparator coreLabelComparator = new CoreLabelComparator();
  private static WordNetDatabase wn_database;
  private String normalizedForm;

  public enum SequenceType {
    TAG, STEM, LEMMA, WIKI_FROM, ORIGINAL, NORMALIZED
  }

  public NGram() {
    this(4);
  }

  public NGram(int n) {
    super(n);
  }

  public NGram(String toAnnotate) {
    Annotation annotation = new Annotation(toAnnotate);
    KpeReader.sentenceAnalyzer.annotate(annotation);
    for (CoreLabel cl : annotation.get(TokensAnnotation.class)) {
      add(cl);
    }
    setNormalizedForm();
  }

  public NGram(String[] originalForms, String[] lemmas) {
    for (int i = 0; i < originalForms.length; ++i) {
      CoreLabel cl = new CoreLabel();
      cl.set(TextAnnotation.class, originalForms[i]);
      cl.set(OriginalTextAnnotation.class, originalForms[i]);
      cl.set(LemmaAnnotation.class, lemmas[i]);
      cl.set(PartOfSpeechAnnotation.class, "dummy");
      add(cl);
    }
    setNormalizedForm();
  }

  public NGram(CoreLabel ew) {
    this();
    add(ew);
    setNormalizedForm();
  }

  public NGram(Collection<CoreLabel> words) {
    super(words);
    setNormalizedForm();
  }

  public boolean add(CoreLabel element) {
    boolean returnValue = super.add(element);
    setNormalizedForm();
    return returnValue;
  }

  public void add(int index, CoreLabel element) {
    super.add(index, element);
    setNormalizedForm();
  }

  public boolean equals(Object o) {
    if (o instanceof CoreLabel) {
      return equals(new NGram((CoreLabel) o));
      // return new NGram((CoreLabel) o).normalizedForm.equals(this.normalizedForm);
    } else if (o instanceof NGram) {
      // return ((NGram) o).normalizedForm.equals(this.normalizedForm);
      NGram n = (NGram) o;
      if (n.size() != size())
        return false;
      for (int i = 0; i < size(); ++i) {
        CoreLabel clThis = get(i);
        CoreLabel other = n.get(i);
        int comparison = coreLabelComparator.compareForNGramEquality(clThis, other);
        if (comparison != 0)
          return false;
      }
      return true;
    } else
      return false;
  }

  public int compareTo(NGram ngram) {
    return this.normalizedForm.compareTo(ngram.normalizedForm);
  }

  public int hashCode() {
    return normalizedForm.hashCode();
  }

  public Object clone() {
    NGram clone = new NGram();
    for (CoreLabel ew : this)
      clone.add(new CoreLabel(ew));
    clone.normalizedForm = normalizedForm;
    return clone;
  }

  public static boolean initWordNet(String wordNetDir) {
    if (wordNetDir != null && !wordNetDir.matches("(?i)[\\p{Punct}\\s]*false[\\p{Punct}\\s]*")) {
      if (new File(wordNetDir).isDirectory()) {
        System.setProperty("wordnet.database.dir", wordNetDir);
        wn_database = WordNetDatabase.getFileInstance();
        return true;
      } else {
        System.err.println("The WordNet dictionary directory provided does not exist.");
        System.err.println("Either disable the usage of WordNet or set its correct location in the config file.");
        System.exit(1);
      }
    }
    wn_database = null;
    return false;
  }

  private static String getTransformedForm(CoreLabel cl) {
    String pos = cl.getString(PartOfSpeechAnnotation.class).toLowerCase();
    Set<SynsetType> synsetTypes = new HashSet<SynsetType>(Arrays.asList(SynsetType.ALL_TYPES));
    if (pos.startsWith("nn")) {
      synsetTypes = new HashSet<SynsetType>(Arrays.asList(new SynsetType[] { SynsetType.NOUN }));
    } else if (pos.startsWith("vb")) {
      synsetTypes = new HashSet<SynsetType>(Arrays.asList(new SynsetType[] { SynsetType.VERB }));
    } else if (pos.startsWith("jj")) {
      synsetTypes = new HashSet<SynsetType>(Arrays.asList(new SynsetType[] { SynsetType.ADJECTIVE,
          SynsetType.ADJECTIVE_SATELLITE }));
    }

    Map<String, Integer> aggrFreqs = new HashMap<String, Integer>();
    String lemma = cl.get(LemmaAnnotation.class);
    Synset[] synsets = wn_database.getSynsets(lemma);
    // System.out.println(lemma + "\t" + pos);
    for (Synset synset : synsets) {
      SynsetType actualType = synset.getType();
      if (!synsetTypes.contains(actualType)) {
        continue;
      }
      WordSense[] wss = actualType == SynsetType.VERB ? synset.getDerivationallyRelatedForms(lemma)
          : new WordSense[] { new WordSense(lemma, synset) };
      for (WordSense w : wss) {
        Synset s = w.getSynset();
        if (pos.startsWith("vb") && s.getType() != SynsetType.NOUN) {
          continue;
        }

        String[] forms = s.getWordForms();
        for (String form : forms) {
          Integer prevdVal = aggrFreqs.get(form);
          aggrFreqs.put(form, (prevdVal == null ? 0 : prevdVal) + s.getTagCount(form));
        }
      }
    }
    String argMax = lemma.replaceAll("([a-z])\\1{2,}", "$1$1");
    int freqMax = 0;
    for (Entry<String, Integer> transformationEntry : aggrFreqs.entrySet()) {
      if (transformationEntry.getValue() > freqMax && transformationEntry.getKey().split(" ").length == 1) {
        freqMax = transformationEntry.getValue();
        argMax = transformationEntry.getKey();
      }
    }
    // just in case someone would be curious what kind of rewriting might happen there
    // if (!lemma.toLowerCase().equals(argMax.toLowerCase())) {
    // System.err.println(lemma + "-->" + argMax);
    // }
    return ps.stemString(argMax);
  }

  /**
   * Sets and returns the normalized representation of a CoreLabel object at the same time.
   * 
   * @param cl
   *          - the CoreLabel to determine the normalized form of
   * @return - the normalized representation of the CoreLabel parameter (or null if the word is known to be a
   *         stopword)
   */

  public static String getNormalizedCoreLabel(CoreLabel cl) {
    boolean isStopWord = cl.has(StopWordAnnotation.class) && cl.get(StopWordAnnotation.class);
    String normalization = ps.stem(cl.lemma().toLowerCase());

    if (isStopWord) {
      cl.set(NormalizerAnnotation.class, cl.lemma().toLowerCase());
      return null;
    } else if (wn_database != null) {
      normalization = getTransformedForm(cl);
    } else {
      normalization = ps.stemString(cl.lemma());
    }
    cl.set(NormalizerAnnotation.class, normalization);
    return normalization;
  }

  private void setNormalizedForm() {
    List<String> normalizedTokens = new LinkedList<String>();
    for (CoreLabel cl : this) {
      String normalized = getNormalizedCoreLabel(cl);
      if (normalized != null) {
        normalizedTokens.add(normalized);
      }
    }
    Collections.sort(normalizedTokens);
    StringBuilder sb = new StringBuilder();
    for (String token : normalizedTokens) {
      sb.append(token + ' ');
    }
    normalizedForm = sb.toString().trim();
  }

  private String[] getSequence(SequenceType type) {
    String[] sb = new String[size()];
    for (int o = 0; o < size(); ++o) {
      CoreLabel ew = get(o);
      switch (type) {
      case TAG:
        sb[o] = ew.tag();
        break;
      case NORMALIZED:
        sb[o] = ew.get(NormalizerAnnotation.class);
        break;
      case STEM:
        sb[o] = ps.stem(ew.word().toLowerCase());
        break;
      case LEMMA:
        sb[o] = ew.get(LemmaAnnotation.class);
        break;
      case WIKI_FROM:
        sb[o] = (o == size() - 1) ? ew.get(LemmaAnnotation.class) : ew.word();
        break;
      case ORIGINAL:
        sb[o] = ew.word();
        break;

      }
    }
    return sb;
  }

  public String getSequenceAsString(SequenceType type) {
    return getSequenceAsString(type, ' ');
  }

  public String getSequenceAsString(SequenceType type, char joiner) {
    return NLPUtils.join(getSequence(type), joiner);
  }

  public String getCanonicalForm() {
    if (normalizedForm == null) {
      setNormalizedForm();
    }
    return normalizedForm;
  }

  public String getStemmedStringFrom() {
    List<String> tokens = new LinkedList<String>();
    for (CoreLabel cl : this) {
      tokens.add(cl.word().toLowerCase());
    }
    Collections.sort(tokens);
    String ordered = NLPUtils.join(tokens.toArray(new String[tokens.size()]));
    return ps.stemString(ordered);
  }

  public String toString() {
    StringBuffer sb = new StringBuffer();
    for (CoreLabel cl : this) {
      sb.append(cl.word() + ' ');
    }
    return sb.toString().trim();
  }
} public class NGramStats {// implements Comparable<NGramStats> {
  private List<Integer> positions;
  private TreeSet<Integer> sectionIds;
  private List<int[]> sentences;
  private int seenAsHeading;
  private int numOfReferences;
  private boolean presentInAbstract;
  private boolean presentInReferences;

  /**
   * 
   * @param position
   * @param positionWithinDoc
   *          stores the section and the order of the sentence the ngram is contained within the section
   * @param hasReference
   * @param heading
   */
  public NGramStats(Integer position, int[] positionWithinDoc, boolean hasReference) {
    positions = new LinkedList<Integer>();
    sectionIds = new TreeSet<Integer>();
    sentences = new LinkedList<int[]>();

    if (position != null)
      positions.add(position);
    if (positionWithinDoc != null) {
      sectionIds.add(positionWithinDoc[0]);
      sentences.add(positionWithinDoc);
    }
    seenAsHeading = positionWithinDoc[1] == 0 ? 1 : 0;
    // the value of presentInAbstract is enough to be set at the very construation, as if even the first presence
    // of an ngram is not in the first paragraph, then later ones will definitely not have the chance for it
    presentInAbstract = positionWithinDoc[0] == 0;
    numOfReferences = hasReference ? 1 : 0;
  }

  public List<Integer> getPositions() {
    return positions;
  }

  public void setPositions(List<Integer> positions) {
    this.positions = positions;
  }

  public void updatePositions(Integer position) {
    this.positions.add(position);
  }

  public TreeSet<Integer> getSectionIds() {
    return sectionIds;
  }

  public void setSectionIds(TreeSet<Integer> sectionIds) {
    this.sectionIds = sectionIds;
  }

  public int numWithReferences() {
    return numOfReferences;
  }

  public void updateContainsReference(boolean containsReference) {
    this.numOfReferences += containsReference ? 1 : 0;
  }

  public List<int[]> getSentencePositions() {
    return sentences;
  }

  public void setSentences(List<int[]> positionWithinDoc) {
    this.sentences = positionWithinDoc;
  }

  public void addSentence(int[] sentencePosition) {
    sentences.add(sentencePosition);
    sectionIds.add(sentencePosition[0]);
    seenAsHeading += sentencePosition[1] == 0 ? 1 : 0;
  }

  public int getSeenAsHeading() {
    return seenAsHeading;
  }

  public void setSeenAsHeading(boolean seenAsHeading) {
    this.seenAsHeading += seenAsHeading ? 1 : 0;
  }

  public boolean isPresentInAbstract() {
    return presentInAbstract;
  }

  public boolean isPresentInReferences() {
    return presentInReferences;
  }

  public void setPresentInReferences(boolean presentInReferences) {
    this.presentInReferences = this.presentInReferences || presentInReferences;
  }

  // public int compareTo(NGramStats other) {
  // if (other == null)
  // return 1;
  // int comparison = new Integer(positions.size()).compareTo(other.getPositions().size());
  // if (comparison != 0)
  // return comparison;
  // comparison = new Integer(sectionIds.size()).compareTo(other.getSectionIds().size());
  // if (comparison != 0)
  // return comparison;
  // return 1;
  // }

} public abstract class Feature implements Serializable {
  public enum Scale {
    BINARY, NOMINAL, NUMERIC
  }

  private static final long serialVersionUID = 1L;
  protected static boolean binarizeNominals;
  protected boolean canBeRepresentedAsSequential;
  protected boolean employBIESmarkup;
  public static Map<String, Integer[]> dict;
  protected int numDoc;
  public static int numberOfDocs;

  /** its value represents how many documents have been keyphrased (including the one in progress) **/
  protected int documentToExamine = -1;
  /**
   * dummy value is the one that is applied in those situations when a phrase was not present in a document at
   * all
   **/
  protected double dummyValue = 0.0d;
  protected Class<?> collectionToStoreDocVals = ArrayList.class;
  protected Scale scale = null;
  protected Map<String, List<Collection<Number>>> featureVals = new HashMap<String, List<Collection<Number>>>();
  protected int maxPhraseLength;

  public void setBinarization(boolean needToBinarizeNominals) {
    binarizeNominals = needToBinarizeNominals;
  }

  public abstract void value(String phrase, int[] length, Entry<NGram, NGramStats> ngramForm, boolean train,
      int docToCheck, List<Map<String, Map<NGram, NGramStats>>> listOfHashs, List<CoreMap> sentences,
      DocumentData... docs);

  public void setFeatureField(KPEFilter kf) {
    dict = kf.getDictionary();
    numDoc = kf.getDocsNumber();
    maxPhraseLength = kf.getMaxPhraseLength();
  }

  public void setEmployBIESmarkup(Boolean newValue) {
    if (canBeRepresentedAsSequential) {
      if (newValue == null) {
        try {
          throw new Exception("Too little BIES parameters were passed.");
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
      employBIESmarkup = newValue;
    }
  }

  public double getDummyVal() {
    return dummyValue;
  }

  public Map<String, List<Collection<Number>>> getVals() {
    return featureVals;
  }

  protected void updateFeatureVals(Number val, int docToCheck) {
    if (this instanceof SentiWordnetFeature || val.intValue() != -1) {
      updateFeatureVals(this.getClass().getName(), val, docToCheck, collectionToStoreDocVals);
    }
  }

  protected void updateFeatureVals(String key, Number val, int docToCheck) {
    updateFeatureVals(key, val, docToCheck, collectionToStoreDocVals);
  }

  @SuppressWarnings("unchecked")
  protected void updateFeatureVals(String key, Number val, int docToCheck, Class<?> collectionClass) {
    List<Collection<Number>> docFeatVals = featureVals.get(key);
    if (docFeatVals == null) {
      docFeatVals = new ArrayList<Collection<Number>>(numberOfDocs);
      for (int i = 0; i < numberOfDocs; ++i) {
        try {
          docFeatVals.add((Collection<Number>) collectionClass.newInstance());
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
      featureVals.put(key, docFeatVals);
    }

    Collection<Number> vals = docFeatVals.get(docToCheck);
    if (vals instanceof Set<?> || vals instanceof LinkedList<?>) {
      vals.add(val == null ? null : val.doubleValue());
    } else if (vals instanceof ArrayList<?>) {
      if (vals.size() > 0) {
        double lastVal = ((Number) ((ArrayList<?>) vals).get(vals.size() - 1)).doubleValue();
        vals.add(val.doubleValue() + lastVal);
      } else {
        vals.add(val);
      }
    } else {
      System.err.println("No such collection was expected.");
    }
    documentToExamine = docToCheck;
  }

  protected double aggregation(List<Collection<Number>> docVals, String phrase, boolean train, List<int[]> length) {
    double[] perDocFeatureVals = new double[docVals.size()];
    for (int doc = 0; doc < docVals.size(); ++doc) {
      Collection<Number> docNums = docVals.get(doc);
      if (docNums instanceof TreeSet<?>) {
        perDocFeatureVals[doc] = ((TreeSet<Number>) docVals.get(doc)).first().doubleValue();
      } else if (docNums instanceof HashSet<?>) {
        perDocFeatureVals[doc] = docNums.contains(1.0d) ? 1.0d : 0.0d;
      } else if (docNums instanceof ArrayList<?>) {
        perDocFeatureVals[doc] = ((ArrayList<Number>) docNums).get(docNums.size() - 1).doubleValue();
      } else if (docNums instanceof LinkedList<?>) {
        perDocFeatureVals[doc] = NLPUtils.mean(docVals.get(doc));
      } else {
        System.err.println("Unexpected collection type " + docNums.getClass() + "\t" + this.getClass());
      }
    }
    return NLPUtils.mean(perDocFeatureVals);
  }

  public Map<String, Double> aggregateVals(boolean train, String token, List<int[]> length, double[] dedicatedFeatures) {
    Map<String, Double> aggregatedVals = new HashMap<String, Double>();
    for (Entry<String, List<Collection<Number>>> entry : featureVals.entrySet()) {
      double featureVal = aggregation(entry.getValue(), token, train, length);
      aggregatedVals.put(entry.getKey(), featureVal);
      if (this instanceof TfIdfFeature)
        dedicatedFeatures[0] = featureVal;
      else if (this instanceof FirstIndexFeature)
        dedicatedFeatures[1] = featureVal;
    }
    // reset it so the next time a set of documents are to be keyphrased this flag can start to count from the
    // beginning
    documentToExamine = -1;
    featureVals = new HashMap<String, List<Collection<Number>>>();
    return aggregatedVals;
  }
} public class FeatureHolder extends ArrayList<Feature> {

  /**
   * Class for storing the various Feature object and the mapping to their list of double values.
   */
  private static final long serialVersionUID = 4577522680430595054L;
  private transient Map<String, Map<Double, Integer>> featureValDistribution;

  public FeatureHolder(int initialCapacity) {
    super(initialCapacity);
    featureValDistribution = new HashMap<String, Map<Double, Integer>>();
  }

  public void updateDataHandler(double[] dedicatedFeats, String phrase, String instanceId, List<int[]> length,
      List<Map<String, Map<NGram, NGramStats>>> listOfHashs, List<Map<Integer, List<CoreMap>>> grammars,
      DataHandler dataHandler, boolean train, DocumentData... docs) {

    Feature.numberOfDocs = listOfHashs.size();
    double[] ngramAlternations = new double[listOfHashs.size()];
    for (int docToCheck = 0; docToCheck < listOfHashs.size(); ++docToCheck) {
      // DocumentType docType = docs[docToCheck].getDocumentType();
      Map<Integer, List<CoreMap>> documentSentences = grammars.get(docToCheck);
      Map<String, Map<NGram, NGramStats>> docVocabulary = listOfHashs.get(docToCheck);
      Map<NGram, NGramStats> docPhrasesStats = docVocabulary.get(phrase);
      // when keyphrasing multiple documents it is possible that one keyphrase aspirant is not present in some
      // of the documents. Add some dummy feature values in these cases.
      if (docPhrasesStats == null) {
        for (Feature f : this) {
          f.updateFeatureVals(f.getDummyVal(), docToCheck);
        }
        continue;
      }
      ngramAlternations[docToCheck] = docPhrasesStats.size();
      int[] docLength = length.get(docToCheck);
      for (Entry<NGram, NGramStats> ngramModificationOfDoc : docPhrasesStats.entrySet()) {
        List<int[]> sentenceLocations = ngramModificationOfDoc.getValue().getSentencePositions();
        List<CoreMap> sentences = new LinkedList<CoreMap>();
        for (int[] sentenceLocation : sentenceLocations) {
          sentences.add(documentSentences.get(sentenceLocation[0]).get(sentenceLocation[1]));
        }
        for (Feature f : this) {
          f.value(phrase, docLength, ngramModificationOfDoc, train, docToCheck, listOfHashs, sentences, docs);
        }
      }
    }

    double averageAlternations = NLPUtils.mean(ngramAlternations);
    dataHandler.setNumericValue(instanceId, "numOfAppearances", averageAlternations);
    if (DocumentSet.adaptationType % 2 == 1) {
      if (!train || instanceId.matches("target_(\\d+_)+.*")) {
        dataHandler.setNumericValue(instanceId, "T_numOfAppearances", averageAlternations);
      } else {
        dataHandler.setNumericValue(instanceId, "S_numOfAppearances", averageAlternations);
      }
    }

    // set the DataHandler Object from feature to feature
    for (Feature f : this) {
      Map<String, Double> values = f.aggregateVals(train, phrase, length, dedicatedFeats);
      for (Entry<String, Double> val : values.entrySet()) {
        if (Double.isNaN(val.getValue()) || Double.isInfinite(val.getValue())) {
          System.err.println(f + "\n" + phrase + "\n" + val);
          System.exit(2);
        }
        if (f.scale == Scale.BINARY) {
          dataHandler.setBinaryValue(instanceId, val.getKey(), val.getValue() >= 0.5);
          if (DocumentSet.adaptationType % 2 == 1) {
            if (!train || instanceId.matches("target_(\\d+_)+.*")) {
              dataHandler.setBinaryValue(instanceId, "T_" + val.getKey(), val.getValue() >= 0.5);
            } else {
              dataHandler.setBinaryValue(instanceId, "S_" + val.getKey(), val.getValue() >= 0.5);
            }
          }
          if (train) {
            Map<Double, Integer> valsForFeature = featureValDistribution.get(val.getKey());
            valsForFeature = valsForFeature == null ? new TreeMap<Double, Integer>() : valsForFeature;
            featureValDistribution.put(val.getKey(), valsForFeature);
            Integer prevVal = valsForFeature.get(val.getValue() >= 0.5 ? 1.0d : 0.0d);
            valsForFeature.put(val.getValue() >= 0.5 ? 1.0d : 0.0d, prevVal == null ? 1 : ++prevVal);
          }
        } else if (f.scale == Scale.NOMINAL) {
          // TODO implement such cases
          System.out.println("Dealing with nominal attributes in class FeatureHolder is not implemented yet.");
        } else {
          dataHandler.setNumericValue(instanceId, val.getKey(), val.getValue());
          if (DocumentSet.adaptationType % 2 == 1) {
            if (!train || instanceId.matches("target_(\\d+_)+.*")) {
              dataHandler.setNumericValue(instanceId, "T_" + val.getKey(), val.getValue());
            } else {
              dataHandler.setNumericValue(instanceId, "S_" + val.getKey(), val.getValue());
            }
          }
          if (train) {
            Map<Double, Integer> valsForFeature = featureValDistribution.get(val.getKey());
            valsForFeature = valsForFeature == null ? new TreeMap<Double, Integer>() : valsForFeature;
            featureValDistribution.put(val.getKey(), valsForFeature);
            Integer prevVal = valsForFeature.get(val.getValue());
            valsForFeature.put(val.getValue(), prevVal == null ? 1 : ++prevVal);
          }
        }
      }
    }
  }

  public Map<String, Map<Double, Integer>> getFeatureValDistribution() {
    return featureValDistribution;
  }

  public Set<String> getRareFeatures(int threshold) {
    Set<String> unwantedFeatureNames = new HashSet<String>();
    try (PrintWriter out = new PrintWriter("feature.stats")) {
      for (Entry<String, Map<Double, Integer>> featVals : featureValDistribution.entrySet()) {
        out.println(featVals.getKey() + "\t" + featVals.getValue());
        int nonZero = 0;
        for (Entry<Double, Integer> featValOfFeature : featVals.getValue().entrySet()) {
          if (featValOfFeature.getKey() > 0.0d) {
            nonZero += featValOfFeature.getValue();
          }
        }
        if (nonZero <= threshold)
          unwantedFeatureNames.add(featVals.getKey());
      }
    } catch (IOException io) {
      io.printStackTrace();
    }
    return unwantedFeatureNames;
  }
} public class DocumentData implements Comparable<DocumentData>, Serializable {

  private static final long serialVersionUID = -8144005167022088407L;

  /** The total number of DocumentData objects initialized */
  private static int totalDocuments;
  /** Identifier of the document */
  private int documentId;
  /** Location of the document */
  private String file;
  /** Stores ordinal number of the document within the file */
  private int lineNumInFile;
  /** Keyphrases of the document */
  private Map<NGram, Integer> etalonKeyphrases;
  /** Acronyms of the document */
  private Map<String, Integer> acronyms;
  /** Mapping between the formatted strings of the document and their actual formatting */
  protected Map<NGram, Set<String>> formattedStrings;
  /** This can be useful to define reader specific behavior */
  private String documentType;

  public DocumentData(String keyph, String fileName, Class<?> docType) {
    documentId = totalDocuments++;
    etalonKeyphrases = transformKeyphrases(keyph);
    file = fileName;
    documentType = docType.getSimpleName().replace("Reader", "");
  }

  public int getDocId() {
    return documentId;
  }

  public void setDocId(int id) {
    documentId = id;
  }

  public Map<NGram, Set<String>> getFormattedStrings() {
    return formattedStrings;
  }

  public Map<NGram, Integer> getKeyphrases() {
    return etalonKeyphrases;
  }

  public void setKeyphrases(String keyph) {
    etalonKeyphrases = transformKeyphrases(keyph);
  }

  public String getFile() {
    return file;
  }

  public void setFile(String file) {
    this.file = file;
  }

  public Map<String, Integer> getAcronyms() {
    return acronyms;
  }

  public void setAcronyms(Map<String, Integer> acr) {
    acronyms = acr;
  }

  public boolean isFormatted(NGram phraseBuffer) {
    return formattedStrings != null && formattedStrings.containsKey(phraseBuffer);
  }

  // public boolean isFormatted(CoreLabel ew) {
  // return formattedStringParts != null && formattedStringParts.contains(ew);
  // }

  /**
   * Gets all the phrases in the given string and puts them into a map with its occurrences.
   */
  public Map<NGram, Integer> transformKeyphrases(String keyphrases) {
    HashMap<NGram, Integer> hash = new HashMap<NGram, Integer>();
    if (keyphrases == null || keyphrases.length() == 0)
      return hash;
    for (String tok : keyphrases.split("(\r?\n)+")) {
      tok = tok.trim();
      if (tok.length() == 0) {
        continue;
      }

      String newTok;
      if (tok.equalsIgnoreCase("c++")) {
        newTok = tok;
      } else if (tok.toLowerCase().startsWith(".net")) {
        newTok = tok;
      } else {
        newTok = tok.replaceAll("^\\p{Punct}|\\p{Punct}$", "");
      }

      if (newTok.length() < tok.length()) {
        System.err.println("Etalon phrase " + tok + " transformed into " + newTok);
      }
      Annotation annotatedContent = new Annotation(newTok);
      KpeReader.sentenceAnalyzer.annotate(annotatedContent);
      NGram id = new NGram(annotatedContent.get(TokensAnnotation.class));
      Integer value = hash.get(id);
      hash.put(id, value == null ? 1 : ++value);
    }
    return hash;
  }

  public int getLineNumInFile() {
    return lineNumInFile;
  }

  public void setLineNumInFile(int lineNum) {
    lineNumInFile = lineNum;
  }

  public String toString() {
    return documentId + "\t" + file;
  }

  public int compareTo(DocumentData dd) {
    int fileComparison = file.compareTo(dd.getFile());
    return fileComparison == 0 ? (lineNumInFile < dd.getLineNumInFile() ? -1 : 1) : fileComparison;
  }

  public boolean equals(Object o) {
    if (!(o instanceof DocumentData)) {
      return false;
    }
    return file.equals(((DocumentData) o).getFile()) && ((DocumentData) o).getLineNumInFile() == lineNumInFile;
  }

  public int hashCode() {
    return (file + "_" + lineNumInFile).hashCode();
  }

  public boolean containsReference(CoreMap sentence) {
    List<CoreLabel> sentenceTokens = sentence.get(TokensAnnotation.class);
    if (documentType.matches("(?i)semeval|scientific")) {
      nobracket: for (int i = 0; i < sentenceTokens.size(); ++i) {
        if (sentenceTokens.get(i).word().equals("-LRB-")) {
          while (++i < sentenceTokens.size()) {
            CoreLabel nextToken = sentenceTokens.get(i);
            if (!nextToken.word().matches("(,|\\d+|[A-Z&a-z]+\\d{2}|-RRB-)")) {
              continue nobracket;
            } else if (nextToken.word().equals("-RRB-")) {
              return true;
            }
          }
        }
      }
    }
    return false;
  }

  public boolean isScientific() {
    return documentType.matches("(?i)semeval|scientific");
  }

  public TreeMap<Integer, List<CoreMap>> getSections(KpeReader reader, boolean serialize) {
    TreeMap<Integer, List<CoreMap>> sectionsWithSentences = new TreeMap<Integer, List<CoreMap>>();
    List<Annotation> sections = tagAndParse(reader, serialize);
    Iterator<Annotation> sectionIter = sections.iterator();
    while (sectionIter.hasNext()) {
      Annotation sectionAnn = sectionIter.next();
      List<CoreMap> sentencesOfSection = sectionAnn.get(SentencesAnnotation.class);
      sectionsWithSentences.put(sectionsWithSentences.size(), sentencesOfSection);
    }
    return sectionsWithSentences;
  }

  /**
   * Checks for the presence of some critical annotations. In the case some of those entered among the
   * parameters is missing, the texts needs to be re-annotated.
   * 
   * @param a
   *          annotation
   * @param r
   *          reader with the desired annotations
   * @return
   */
  private boolean needsReannotation(Annotation a, KpeReader r) {
    List<CoreMap> sentences = a.get(SentencesAnnotation.class);
    List<CoreLabel> tokens = a.get(TokensAnnotation.class);
    if (tokens == null || sentences == null || tokens.size() == 0 || sentences.size() == 0) {
      return true;
    }
    Set<Class<?>> sentenceAnnotations = sentences.get(0).keySet();
    Set<Class<?>> tokenAnnotations = tokens.get(0).keySet();
    if ((r.getIsMweOn() && !tokenAnnotations.contains(MWEAnnotation.class))
        || (r.getIsNeOn() && !tokenAnnotations.contains(NamedEntityTagAnnotation.class))) {
      return true;
    }
    if (r.getIsSyntaxOn() && !sentenceAnnotations.contains(TreeAnnotation.class)) {
      return true;
    }
    return false;
  }

  @SuppressWarnings("unchecked")
  private List<Annotation> tagAndParse(KpeReader reader, boolean serialize) {
    int numberInDoc = getLineNumInFile();
    File f = new File(file);
    String grammarFile = f.getParent() + "/grammar/" + (numberInDoc > 0 ? numberInDoc : "") + f.getName() + ".gr";
    if (new File(grammarFile).exists()) {
      try {
        ObjectInputStream in = new ObjectInputStream(new BufferedInputStream(new FileInputStream(grammarFile)));
        List<Annotation> documentSections = (List<Annotation>) in.readObject();
        in.close();
        if (documentSections.size() == 0 || needsReannotation(documentSections.get(0), reader)) {
          analyzeSections(documentSections, grammarFile, serialize);
        }
        return documentSections;
      } catch (Exception e) {
        System.err.println("Error with the serialized grammar file " + grammarFile + "\n" + e);
      }
    }
    // text = text.replaceAll("(.)\\1{4,}", "$1");
    // List<String> sectionsOfText = determineSections(text);

    List<String> paragraphs = determineSections(reader.getText(file, lineNumInFile));
    List<Annotation> documentSections = new ArrayList<Annotation>(paragraphs.size());
    try {
      System.err.println(file + " is to be analysed...");
      for (String section : paragraphs) {
        // just some ugly hack to get over such expressions as inequalities that would affect the tokenizer to
        // make dull things
        if (isScientific()) {
          int originalLength = section.length();
          section = section.replaceAll("<([\\S&&[^>]]+) +", "< $1 "); // replaceAll("([<>])(\\S+)", "$1 $2");
          if (originalLength - section.length() < 0) {
            System.err.println("Type-1 scientific document heuristic was applied for " + file);
          }
          // get rid of hyphens as well that might get into the text unintentionally
          originalLength = section.length();
          section = section.replaceAll("([a-z0-9])-\\s+([a-z0-9])", "$1$2");
          if (originalLength - section.length() > 0) {
            System.err.println("Type-2 scientific document heuristic was applied for " + file);
          }
        }
        documentSections.add(new Annotation(section));
      }
      analyzeSections(documentSections, grammarFile, serialize);
    } catch (Exception e) {
      System.err.println("Error occured during the annotation of file " + file + " of line " + lineNumInFile);
      e.printStackTrace();
    }
    return documentSections;
  }

  private void analyzeSections(List<Annotation> documentSections, String grammarFile, boolean serialize) {
    for (Annotation ann : documentSections) {
      KpeReader.sentenceAnalyzer.annotate(ann);
    }
    if (serialize && file != null) {
      NLPUtils.serialize(documentSections, grammarFile);
    }
  }

  /**
   * @param text
   * @return the List of sections of the document
   */
  private List<String> determineSections(String text) {
    // In the simplest case the whole text is handled as one section.
    List<String> sections = new ArrayList<String>(1);
    sections.add(text);
    return sections;
  }

} public abstract class KpeReader {

  public static SzTECoreNLP sentenceAnalyzer;

  /**
   * On default (unless overridden by a descendant class) we shall only treat files having this kind of
   * extension
   */
  protected static final String DEFAULT_EXTENSION = ".txt";
  protected String fileType;
  protected Charset m_encoding;
  private boolean isMweOn;
  private boolean isNeOn;
  private boolean isSyntaxOn;

  protected boolean goldAnnotation;

  /**
   * This method returns a {@link}List of DocumentData class representations of documents being present at
   * location dir+"/"+file <br>
   * It returns a list of {@link} DocumentData objects as one document might contain more than just one
   * document (e.g in the form of an XML). <br>
   * Most often however, this List is going to be just of size 1.
   * 
   * @param dir
   * @param file
   * @return
   */
  public abstract List<DocumentData> getContent(String dir, String file);

  public abstract String getText(String file, int numberWithinFile);

  protected abstract boolean mightBeSectionHeader(String line);

  protected abstract void setDetails();

  public KpeReader() {
    m_encoding = Charset.defaultCharset();
    setDetails();
  }

  public KpeReader(String extension) {
    this(Charset.defaultCharset(), extension);
  }

  public KpeReader(Charset encoding, String extension) {
    m_encoding = encoding;
    fileType = extension;
  }

  public boolean getIsMweOn() {
    return isMweOn;
  }

  public boolean getIsNeOn() {
    return isNeOn;
  }

  public boolean getIsSyntaxOn() {
    return isSyntaxOn;
  }

  public void initGrammar(boolean isMweFeatureOn, boolean isNeFeatureOn, boolean isSyntacticFeatureOn, String lang) {
    isMweOn = isMweFeatureOn;
    isNeOn = isNeFeatureOn;
    isSyntaxOn = isSyntacticFeatureOn;
    if (sentenceAnalyzer == null) {

      String annotators = "tokenize, ssplit, pos, lemma, stopword";
      // the following annotators are optional and only available in English at the moment
      annotators += isMweFeatureOn && lang.equals("en") ? ", mwe" : "";
      annotators += isNeFeatureOn && lang.equals("en") ? ", ner" : "";
      annotators += isSyntacticFeatureOn && lang.equals("en") ? ", parse" : "";

      Properties props = new Properties();
      if (this instanceof EpinionReader) {
        annotators = annotators.replace("tokenize,", "tokenize, cleanxml,");
      }
      props.put("annotators", annotators);
      // TODO this is clearly not necessary in all the cases
      // this should be used only when it is desired for new lines to act as sentence boundaries
      props.put("ssplit.boundariesToDiscard", "*NL*");
      props.put("tokenize.options", "invertible,ptb3Escaping=true,tokenizeNLs");
      props.put("pos.maxlen", "100");
      props.put("lang", lang);

      if (isMweFeatureOn) {
        props.put("mwe.file", System.getProperty("user.dir") + "/resources/wikiMWEfreqs.txt");
      }
      if (isNeFeatureOn) {
        props.put("ner.useSUTime", "false");
      }
      if (isSyntacticFeatureOn) {
        props.put("parser.maxlen", "100");
      }
      sentenceAnalyzer = new SzTECoreNLP(props);
    }
  }

  protected void setFileExtension(String ext) {
    fileType = ext;
  }

  public void setUseGoldAnnotation(boolean ga) {
    goldAnnotation = ga;
  }

  public Charset getEncoding() {
    return m_encoding;
  }

  /**
   * @param dir
   *          - directory to search for the documents
   * @param train
   *          - whether the documents will serve as train instances
   */
  public void addDirectoryOfFiles(String dir, boolean train, DocumentSet docSet) {
    for (File f : new File(dir).listFiles()) {
      if (f.getName().endsWith(fileType)) {
        for (DocumentData dd : getContent(dir, f.getAbsolutePath())) {
          if (!train || (dd.getKeyphrases() != null && dd.getKeyphrases().size() > 0)) {
            docSet.add(dd);
          }
        }
      }
    }
  }
} public abstract class ClassificationResult {
	protected DataHandler dataHandler;
	
  public ClassificationResult(){
  }

  public ClassificationResult(DataHandler dataHandler){
		this.dataHandler = dataHandler;
	}
	
  public <T extends Comparable<?>> T getLabel(String instanceId){
		return dataHandler.getLabel(instanceId);
	}
	
  public int getInstanceCount(){
		return dataHandler.getInstanceCount();
	}
	
	public Set<String> getInstanceIds(){
		return dataHandler.getInstanceIds();
	}
	
	public abstract <T extends Comparable<?>> T getPredictedLabel(String instanceId);
	
	public abstract <T extends Comparable<?>> List<T> getPredictions();
	
	public abstract <T extends Comparable<?>> Map<T,Double> getPredictionProbabilities(String instanceId);
	
	public abstract void savePredictions(String target);
	
	public abstract void loadPredictions(String source);
} public abstract class DataHandler implements Cloneable {

  // dataset create/save/load
  /**
   * creates a new empty dataset using the underlying native datatype
   */
  abstract public void createNewDataset(Map<String, Object> parameters);

  // /*
  // * returns the underlying native datatype
  // */
  // public Object getNativeDataset();

  public DataHandler createEmptyDataHandler() {
    try{
      return this.getClass().newInstance();
    }catch (InstantiationException e){
      e.printStackTrace();
    }catch (IllegalAccessException e){
      e.printStackTrace();
    }
    return null;
  }

  /**
   * loads a native dataset from the given source
   * 
   * @param source
   *          A String denotes the source of the native dataset it contains a native dataset implementation dependent
   *          resource string
   */
  abstract public void loadDataset(String source) throws DataMiningException;

  /**
   * saves the current dataset to the given target
   * 
   * @param target
   *          A String denotes the target of the native dataset it contains a native dataset implementation dependent
   *          resource string
   */
  abstract public void saveDataset(String target);

  // data manipulation

  /**
   * sets the class label of the given instance
   */
  abstract public <T extends Comparable<?>> void setLabel(String instanceId, T label);

  /**
   * @return the class label f the given instance
   */
  abstract public <T extends Comparable<?>> T getLabel(String instanceId);

  /**
   * Sets the value of a numeric feature
   * 
   * @param instanceId
   *          instance identifier
   * @param featureName
   *          name of the feature
   * @param value
   */
  abstract public void setNumericValue(String instanceId, String featureName, double value);

  /**
   * Sets the value of a nominal feature if this is a new nominal value it is added to the dataset
   * 
   * @param instanceId
   *          instance identifier
   * @param featureName
   *          name of the feature
   * @param value
   */
  abstract public void setNominalValue(String instanceId, String featureName, String value);

  /**
   * Sets the value of a binary feature
   * 
   * @param instanceId
   *          instance identifier
   * @param featureName
   *          name of the feature
   * @param value
   */
  abstract public void setBinaryValue(String instanceId, String featureName, Boolean value);

  abstract public void setBinaryValue(String instanceId, String featureName, Boolean value, boolean ternal);

  /**
   * Sets the value of a feature, the type of the feature is given by the beginning of the feature name b_ binary
   * feature n_ numeric feature m_ nominal feature t_ ternal feature
   * 
   * @param instanceId
   *          instance identifier
   * @param featureName
   *          name of the feature
   * @param value
   */
  abstract public <T extends Comparable<?>> void setValue(String instanceId, String featureName, T value)
      throws DataMiningException;

  abstract public Double getNumericValue(String instanceId, String featureName) throws DataMiningException;

  abstract public String getNominalValue(String instanceId, String featureName) throws DataMiningException;

  abstract public Boolean getBinaryValue(String instanceId, String featureName) throws DataMiningException;

  abstract public <T extends Comparable<?>> T getValue(String instanceId, String featureName)
      throws DataMiningException;

  // instance manipulation
  abstract public void removeInstance(String instanceId) throws DataMiningException;

  abstract public Set<String> getInstanceIds();

  abstract public int getInstanceCount();

  // feature manipulation
  abstract public void removeFeature(String featureName) throws DataMiningException;

  abstract public Set<String> getFeatureNames();

  abstract public int getFeatureCount();

  // The value set of a nominal feature.
  // It returns null for numerical and binary features.
  abstract public List<String> getFeatureValues(String featureName);

  // It throws exception for numerical and binary features.
  abstract public void setDefaultFeatureValue(String featureName, String value) throws DataMiningException;

  // classifier functions
  abstract public void initClassifier(Map<String, Object> parameters) throws DataMiningException;

  abstract public Model trainClassifier() throws DataMiningException;

  abstract public ClassificationResult classifyDataset(Model model) throws DataMiningException;

  /**
   * creates a subset of the dataset where only the given instances and/or features are present
   * 
   * @param instancesSelected
   * @param featuresSelected
   * @return
   */
  abstract public DataHandler createSubset(Collection<String> instancesSelected, Set<String> featuresSelected)
      throws DataMiningException;

  abstract public void addDataHandler(DataHandler dh) throws DataMiningException;
} public class DataMiningException extends Exception {
  static final long serialVersionUID = 2451L;
  
  public DataMiningException(String msg){
    super(msg);
  }

  public DataMiningException(String msg, Exception cause){
    super(msg);
    initCause(cause);
  }
  
  public DataMiningException(Exception cause){
    super();
    initCause(cause);
  }
} public abstract class Model implements Serializable {
  /**
   * 
   */
  private static final long serialVersionUID = -1653570677836049203L;

  public void printModel(OutputStream out) {
    printModel(new PrintWriter(out));
  }

  abstract public void printModel(PrintWriter out);
} public class MalletDataHandler extends DataHandler implements Serializable {

  private static final long serialVersionUID = 7555593484247132956L;
  public InstanceList data;
  public Map<String, Integer> instanceIds;
  protected LabelAlphabet labelAlphabet;
  protected FeatureSequence featureAlphabet;
  // protected FeatureSequence featureSequence;
  protected Map<String, List<String>> nominalValues;
  protected ClassifierTrainer<?> trainer = null;

  static {
    ((MalletLogger) MalletProgressMessageLogger.getLogger(MaxEntOptimizableByLabelLikelihood.class.getName())).getRootLogger().setLevel(Level.WARNING);
  }

  private Instance getInstance(String instanceId) {
    if (!instanceIds.containsKey(instanceId)) {
      Instance inst = new Instance(new AugmentableFeatureVector(featureAlphabet.getAlphabet(), 0, false), labelAlphabet.lookupLabel("false"), instanceId,
          instanceId);
      data.add(inst);
      inst.unLock();
      instanceIds.put(instanceId, data.size() - 1);
    }
    return data.get(instanceIds.get(instanceId));
  }

  public FeatureSequence getFeatureSequence() {
    return featureAlphabet;
  }

  public Alphabet getAlphabet(String type) {
    if (type.equalsIgnoreCase("feature")) {
      return featureAlphabet.getAlphabet();
    } else if (type.equals("label")) {
      return labelAlphabet;
    } else
      return null;
  }

  public AugmentableFeatureVector getInstanceData(String instanceId) {
    return (AugmentableFeatureVector) getInstance(instanceId).getData();
  }

  public double getDoubleValue(String instanceId, String featureName) throws DataMiningException {
    if (!featureAlphabet.getAlphabet().contains(featureName))
      throw new DataMiningException("getter for unexisting feature: " + featureName);
    return getInstanceData(instanceId).value(featureAlphabet.getAlphabet().lookupIndex(featureName));
  }

  protected void setDoubleValue(String instanceId, String featureName, double value) {
    AugmentableFeatureVector fv = getInstanceData(instanceId);
    int index = featureAlphabet.getAlphabet().lookupIndex(featureName);
    if (index < 0) {
      return; // it can occur when featureAlphabet.getStopGrowth()==true and the featureset does not contain the feature
    }
    int location = fv.location(index);
    if (location < 0) {
      fv.add(index, value);
      featureAlphabet.add(index);
    } else {
      fv.setValueAtLocation(location, value);
    }
  }

  public void createNewDataset(Map<String, Object> parameters) {
    Object params = parameters != null ? parameters.get("useFeatureSet") : null;
    if (params != null) {
      if (params instanceof MalletDataHandler) {
        MalletDataHandler dh = (MalletDataHandler) params;
        featureAlphabet = dh.featureAlphabet;
        labelAlphabet = dh.labelAlphabet;
      } else if (params instanceof List<?>) {
        for (Object o : (List<?>) params) {
          if (o instanceof LabelAlphabet) {
            labelAlphabet = (LabelAlphabet) o;
          } else if (o instanceof Alphabet) {
            featureAlphabet = new FeatureSequence((Alphabet) o);
          }
        }
        if (featureAlphabet == null || labelAlphabet == null) {
          System.err.println("Uninitialized alphabet");
          System.exit(1);
        }
        featureAlphabet.getAlphabet().stopGrowth();
        labelAlphabet.stopGrowth();
      } else {
        System.err.println("Unsupported useFeatureSet parameter (neither List, nor a MalletDataHandler.)");
        System.exit(1);
      }
    } else {
      featureAlphabet = new FeatureSequence(new Alphabet());
      labelAlphabet = new LabelAlphabet();
      featureAlphabet.getAlphabet().startGrowth();
      labelAlphabet.startGrowth();
    }

    if (labelAlphabet.size() == 0) {
      @SuppressWarnings("unchecked")
      Set<String> classLabels = parameters == null ? null : (Set<String>) parameters.get("classLabels");
      Set<String> defaultFeatures = new HashSet<String>(Arrays.asList(new String[] { new Boolean(true).toString(), new Boolean(false).toString() }));
      if (classLabels == null) {
        classLabels = defaultFeatures;
        System.err.println("A binary classifier is being built now (which might take some time).");
      }
      for (String label : classLabels) {
        labelAlphabet.lookupIndex(label);
      }
      labelAlphabet.stopGrowth();
    }

    data = new InstanceList(featureAlphabet.getAlphabet(), labelAlphabet);
    instanceIds = new HashMap<String, Integer>();
    nominalValues = new HashMap<String, List<String>>();
  }

  public DataHandler createSubset(Collection<String> instancesSelected, Set<String> featuresSelected) throws DataMiningException {
    MalletDataHandler dh = new MalletDataHandler();
    Map<String, Object> param = new HashMap<String, Object>();
    param.put("useFeatureSet", this);
    dh.createNewDataset(param);
    dh.featureAlphabet = new FeatureSequence(new Alphabet());
    dh.featureAlphabet.getAlphabet().startGrowth();
    dh.data = new InstanceList(dh.featureAlphabet.getAlphabet(), dh.labelAlphabet);
    for (String inst : instancesSelected) {
      AugmentableFeatureVector fv = this.getInstanceData(inst);
      for (int i = 0; i < fv.numLocations(); ++i) {
        String featurename = (String) fv.getAlphabet().lookupObject(fv.getIndices()[i]);
        if (featuresSelected.contains(featurename)) {
          dh.setDoubleValue(inst, featurename, fv.getValues()[i]);
        }
      }
      dh.setLabel(inst, this.getLabel(inst));
    }
    return dh;
  }

  /**
   * This feature creates a subset of the InstanceList that the features coded are not pruned (just the instances).
   * 
   * @param instancesSelected
   * @return
   * @throws DataMiningException
   */
  public DataHandler createSubset(Collection<String> instancesSelected) throws DataMiningException {
    return createSubset(instancesSelected, getFeatureNames());
  }

  public void addDataHandler(DataHandler dh) throws DataMiningException {
    if (!(dh instanceof MalletDataHandler)) {
      throw new DataMiningException("MalletDataHandler can add just MalletDataHandlers");
    }
    for (String inst : dh.getInstanceIds()) {
      AugmentableFeatureVector fv = ((MalletDataHandler) dh).getInstanceData(inst);
      for (int i = 0; i < fv.numLocations(); ++i) {
        this.setDoubleValue(inst, (String) fv.getAlphabet().lookupObject(fv.getIndices()[i]), fv.getValues()[i]);
      }
      this.setLabel(inst, dh.getLabel(inst));
    }
  }

  public Boolean getBinaryValue(String instanceId, String featureName) throws DataMiningException {
    return getDoubleValue(instanceId, featureName) > 0.0;
  }

  public int getFeatureCount() {
    return data.getAlphabet().size();
  }

  public Set<String> getFeatureNames() {
    Set<String> featurenames = new HashSet<String>();
    for (Object o : featureAlphabet.getAlphabet().toArray()) {
      featurenames.add((String) o);
    }
    return featurenames;
  }

  public List<String> getFeatureValues(String featureName) {
    return nominalValues.containsKey(featureName) ? nominalValues.get(featureName) : null;
  }

  public int getInstanceCount() {
    return data.size();
  }

  public Set<String> getInstanceIds() {
    return instanceIds.keySet();
  }

  @SuppressWarnings("unchecked")
  public <T extends Comparable<?>> T getLabel(String instanceId) {
    return (T) ((Label) getInstance(instanceId).getTarget()).getEntry();
  }

  public String getNominalValue(String instanceId, String featureName) throws DataMiningException {
    if (!nominalValues.containsKey(featureName))
      throw new DataMiningException(featureName + " is not a nominal feature");
    return nominalValues.get(featureName).get((int) getDoubleValue(instanceId, featureName));
  }

  public Double getNumericValue(String instanceId, String featureName) throws DataMiningException {
    return getDoubleValue(instanceId, featureName);
  }

  @SuppressWarnings("unchecked")
  public <T extends Comparable<?>> T getValue(String instanceId, String featureName) throws DataMiningException {
    return (T) (Double) getDoubleValue(instanceId, featureName);
  }

  public void initClassifier(Map<String, Object> parameters) throws DataMiningException {
    String classifierName = "MaxEntL1";
    // String classifierName = "C45";
    Double gaussianPrior = null;
    if (parameters != null && parameters.containsKey("classifier")) {
      classifierName = (String) parameters.get("classifier");
    }
    if (classifierName.equals("MaxEntL1") && parameters != null && parameters.containsKey("classifier")) {
      gaussianPrior = (Double) parameters.get("gaussianPrior");
    }
    try {
      trainer = (ClassifierTrainer<?>) Class.forName("cc.mallet.classify." + classifierName + "Trainer").newInstance();
      if (classifierName.equals("MaxEntL1")) {
        if (gaussianPrior != null) {
          ((MaxEntTrainer) trainer).setGaussianPriorVariance(gaussianPrior);
        }
      }
      // ((MaxEntTrainer)trainer).setGaussianPriorVariance(1.0);
      /*
       * ((C45Trainer)trainer).setMinNumInsts(3); ((C45Trainer)trainer).setDepthLimited(true); ((C45Trainer)trainer).setMaxDepth(2);
       * ((C45Trainer)trainer).setDoPruning(true);
       */
    } catch (Exception e) {
      throw new DataMiningException("unknown classifier: " + classifierName, e);
    }
  }

  public Model trainClassifier() throws DataMiningException {
    if (trainer == null) {
      initClassifier(null);
    }
    return new MalletClassifier(trainer.train(data));
  }

  public ClassificationResult classifyDataset(Model model) throws DataMiningException {
    if (!(model instanceof MalletClassifier)) {
      throw new DataMiningException("MalletDataHandler can be used only by MALLET classifiers");
    }
    return new MalletClassificationResult(((MalletClassifier) model).getClassifier().classify(data), this);
  }

  public void removeFeature(String featureName) throws DataMiningException {
    this.removeFeature(new String[] { featureName });
  }

  public void removeFeature(String... featureNames) throws DataMiningException {
    removeFeature(new HashSet<String>(Arrays.asList(featureNames)));
  }

  public void removeFeature(Set<String> featureNames) throws DataMiningException {
    Alphabet alphabet = featureAlphabet.getAlphabet();
    FeatureSelection fs = new FeatureSelection(alphabet);
    double[] counts = new double[alphabet.size()];
    for (int feat = 0; feat < counts.length; ++feat) {
      Object featureName = featureAlphabet.getAlphabet().lookupObject(feat);
      counts[feat] = featureNames.contains(featureName) ? 0 : 1;
    }
    Alphabet reducedAlphabet = new Alphabet();
    featureAlphabet.prune(counts, reducedAlphabet, 1);
    Iterator<?> it = reducedAlphabet.iterator();
    while (it.hasNext()) {
      fs.add(it.next());
    }
    data.setFeatureSelection(fs);
  }

  // public Map<String, Map<String, Integer>> removeFeature(int threshold) throws DataMiningException {
  // Map<String, Map<String, Integer>> stats = new TreeMap<String, Map<String, Integer>>();
  // stats.put("classLabel", new HashMap<String, Integer>());
  // double[] counts = new double[featureAlphabet.getAlphabet().size()];
  // for (Entry<String, Integer> instance : instanceIds.entrySet()){
  // Instance inst = data.get(instance.getValue());
  // AugmentableFeatureVector fv = (AugmentableFeatureVector) inst.getData();
  // for (int feat = 0; feat < counts.length; ++feat){
  // String featureName = featureAlphabet.getAlphabet().lookupObject(feat).toString();
  // Map<String, Integer> vals = stats.get(featureName);
  // vals = vals == null ? new HashMap<String, Integer>() : vals;
  // stats.put(featureName, vals);
  // String val;
  // try{
  // // in case the feature was nominal one
  // val = getNominalValue(instance.getKey(), featureName);
  // counts[feat] = threshold;
  // }catch (Exception e){
  // double value = fv.value(feat);
  // if (value > 0.0d)
  // counts[feat]++;
  // val = Double.toString(value);
  // }
  // Integer i = vals.get(val);
  // if (i == null){
  // vals.put(val, 1);
  // }else{
  // vals.put(val, ++i);
  // }
  // }
  // Map<String, Integer> val = stats.get("classLabel");
  // String label = ((Label) inst.getTarget()).getEntry().toString();
  // Integer v = val.get(label);
  // val.put(label, (v == null ? 1 : ++v));
  // stats.put("classLabel", val);
  // }
  // FeatureSelection fs = new FeatureSelection(featureAlphabet.getAlphabet());
  // Alphabet reducedAlphabet = new Alphabet();
  // featureAlphabet.prune(counts, reducedAlphabet, threshold);
  // Iterator<?> it = reducedAlphabet.iterator();
  // while (it.hasNext()){
  // fs.add(it.next());
  // }
  // data.setFeatureSelection(fs);
  // return stats;
  // }

  public void removeInstance(String instanceId) {
    Integer number = instanceIds.remove(instanceId);
    if (number == null) {
      return;
    }
    data.remove((int) number);
    for (Entry<String, Integer> indexPair : instanceIds.entrySet()) {
      if (indexPair.getValue() > number) {
        indexPair.setValue(indexPair.getValue() - 1);
      }
    }
    // throw new DataMiningException("removeFeature is not implemented yet in MalletDataHandler");
  }

  public void removeInstances(Collection<String> instanceIdsToRemove) {
    Iterator<Instance> instanceIt = data.iterator();
    while (instanceIt.hasNext()) {
      Instance next = instanceIt.next();
      if (instanceIdsToRemove.contains(next.getName())) {
        data.remove(next);
        instanceIt = data.iterator();
        Integer prevVal = instanceIds.remove(next.getName());
        if (prevVal != null) {
          for (Entry<String, Integer> indexPair : instanceIds.entrySet()) {
            if (indexPair.getValue() > prevVal) {
              indexPair.setValue(indexPair.getValue() - 1);
            }
          }
        }
      }
    }
  }

  public void loadDataset(String source) throws DataMiningException {
    try (BufferedReader file = new BufferedReader(new FileReader(source))) {
      createNewDataset(null);
      String line;
      while ((line = file.readLine()) != null) {
        String[] tokens = line.split("\\s");
        if (!tokens[0].contains("@")) {
          throw new DataMiningException("Corrput input format. First token should contain @");
        }
        String id = tokens[0].split("@")[0];
        setLabel(id, tokens[0].split("@")[1]);
        for (int i = 1; i < tokens.length; ++i) {
          setDoubleValue(id, tokens[i].split(":")[0], Double.parseDouble(tokens[i].split(":")[1]));
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  public void saveDataset(String target) {
    if (!target.contains("|") || target.split("\\|")[1].equals("mallet")) {
      saveDatasetMallet(target.split("\\|")[0]);
    } else if (target.split("\\|")[1].equals("weka")) {
      saveDatasetWeka(target.split("\\|")[0]);
    } else {
      System.err.println("unknow output format " + target.split("\\|")[1]);
    }
  }

  public void saveDatasetMallet(String target) {
    try {
      PrintWriter out = new PrintWriter(target);
      for (String id : instanceIds.keySet()) {
        out.print(id + "@" + getLabel(id));
        AugmentableFeatureVector fv = getInstanceData(id);
        for (int i = 0; i < fv.numLocations(); ++i) {
          out.print("\t" + featureAlphabet.getAlphabet().lookupObject(fv.getIndices()[i]) + ":" + fv.getValues()[i]);
        }
        out.println();
      }
      out.close();
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    }
  }

  public void saveDatasetWeka(String target) {
    try {
      if (!target.endsWith(".arff")) {
        target += ".arff";
      }
      PrintWriter out = new PrintWriter(target);
      out.println("@relation MalletData");
      for (Object f : data.getAlphabet().toArray()) {
        String name = f.toString().replaceAll("'", "");
        out.println("@attribute '" + name + "' numeric");
      }
      out.println("@attribute classlabel {0,1}");
      out.println("@data");
      for (String id : instanceIds.keySet()) {
        out.print("{");
        AugmentableFeatureVector fv = getInstanceData(id);
        for (int i = 0; i < fv.numLocations(); ++i) {
          if (i > 0) {
            out.print(",");
          }
          out.print((fv.getIndices()[i]) + " " + fv.getValues()[i]);
        }
        if ((boolean) getLabel(id)) {
          out.print("," + data.getAlphabet().size() + " 1");
        }
        out.println("}");
      }
      out.close();
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    }
  }

  public void setBinaryValue(String instanceId, String featureName, Boolean value) {
    setDoubleValue(instanceId, featureName, value ? 1.0 : 0.0);
  }

  public void setBinaryValue(String instanceId, String featureName, Boolean value, boolean ternal) {
    System.err.println("ternal isn't implemented in MalletDataHandler");
  }

  public void setDefaultFeatureValue(String featureName, String value) throws DataMiningException {
    if (!nominalValues.containsKey(featureName)) {
      throw new DataMiningException("setDefaultFeatureValue is called for a feature which is not nominal");
    }
    if (nominalValues.get(featureName).contains(value)) {
      nominalValues.get(featureName).remove(value);
    }
    nominalValues.get(featureName).add(0, value);
  }

  public <T extends Comparable<?>> void setLabel(String instanceId, T label) {
    getInstance(instanceId).setTarget(labelAlphabet.lookupLabel(label.toString()));
  }

  public void setNominalValue(String instanceId, String featureName, String value) {
    if (!nominalValues.containsKey(featureName)) {
      nominalValues.put(featureName, new LinkedList<String>());
      nominalValues.get(featureName).add("MISSINGVALUE");
    }
    int pos = nominalValues.get(featureName).indexOf(value);
    if (pos < 0) {
      nominalValues.get(featureName).add(value);
      pos = nominalValues.get(featureName).size() - 1;
    }
    setDoubleValue(instanceId, featureName, (double) pos);
  }

  public void setNumericValue(String instanceId, String featureName, double value) {
    setDoubleValue(instanceId, featureName, value);
  }

  protected MalletDataHandler clone() {
    MalletDataHandler dh = new MalletDataHandler();
    dh.createNewDataset(null);
    try {
      dh.addDataHandler(this);
    } catch (DataMiningException e) {
      e.printStackTrace();
    }
    return dh;
  }

  public <T extends Comparable<?>> void setValue(String instanceId, String featureName, T value) throws DataMiningException {
    if (featureName.startsWith("b_")) {
      setBinaryValue(instanceId, featureName, (Boolean) value);
    } else if (featureName.startsWith("t_")) {
      setBinaryValue(instanceId, featureName, (Boolean) value, true);
    } else if (featureName.startsWith("m_")) {
      setNominalValue(instanceId, featureName, (String) value);
    } else if (featureName.startsWith("m_")) {
      setNumericValue(instanceId, featureName, (Double) value);
    } else {
      throw new DataMiningException("unknown featuretype " + featureName);
    }
  }
} public class ClassificationInstance {

  private String id;
  private double tfIdf;
  private double firstOcc;
  private double prob;
  private int rank;
  private Object label;
  private Map<NGram, Integer> orthographicForms;

  /**
   * 
   * @param stringFormat
   */
  public ClassificationInstance(String stringFormat) {
    String[] parts = stringFormat.split("\t");
    id = parts[0];
    tfIdf = Double.parseDouble(parts[1]);
    firstOcc = Double.parseDouble(parts[2]);
    prob = Double.parseDouble(parts[3]);
    rank = Integer.parseInt(parts[4]);
    label = parts[5];
    String[] forms = parts[6].split(", ");
    String[] lemmas = parts[7].split(",");
    orthographicForms = new HashMap<NGram, Integer>();
    for (int f = 0; f < forms.length; ++f) {
      String[] formAndOccurrence = forms[f].replaceAll("^\\{|\\}$", "").split("=");
      int occurrence = Integer.parseInt(formAndOccurrence[1]);
      NGram ng = new NGram(formAndOccurrence[0].split(" "), lemmas[f].split(" "));
      orthographicForms.put(ng, occurrence);
    }
  }

  public ClassificationInstance(String malletId, double[] dedicatedFeatures, Map<NGram, NGramStats> ngramForms) {
    id = malletId.replaceAll(" ", "_");
    tfIdf = dedicatedFeatures[0];
    firstOcc = dedicatedFeatures[1];
    label = new Boolean(dedicatedFeatures[2] == 1.0d);
    orthographicForms = new HashMap<NGram, Integer>();
    for (Entry<NGram, NGramStats> form : ngramForms.entrySet()) {
      orthographicForms.put(form.getKey(), form.getValue().getPositions().size());
    }
    rank = Integer.MAX_VALUE;
  }

  public int hashCode() {
    return id.hashCode();
  }

  public boolean equals(Object o) {
    return o instanceof ClassificationInstance ? ((ClassificationInstance) o).getId().equals(id) : o.equals(id);
  }

  public double getTfIdf() {
    return tfIdf;
  }

  public double getFirstOccurr() {
    return firstOcc;
  }

  public int getRanking() {
    return rank;
  }

  public void setRanking(int ranking) {
    rank = ranking;
  }

  public double getProbability() {
    return prob;
  }

  public void setProbability(double probability) {
    this.prob = probability;
  }

  public String getId() {
    return id;
  }

  public Object getClassLabel() {
    return label;
  }

  public Map<NGram, Integer> getOrthographicForms() {
    return orthographicForms;
  }

  public String toString() {
    Map<String, Integer> lemmasToFreqs = new HashMap<String, Integer>();
    for (Entry<NGram, Integer> ng : orthographicForms.entrySet()) {
      String lemmaForm = ng.getKey().getSequenceAsString(SequenceType.LEMMA);
      Integer actualFreq = ng.getValue();
      Integer prevVal = lemmasToFreqs.get(lemmaForm);
      lemmasToFreqs.put(lemmaForm, (prevVal == null ? 0 : prevVal) + actualFreq);
    }
    return id + "\t" + tfIdf + "\t" + firstOcc + "\t" + prob + "\t" + rank + "\t" + label + "\t" + orthographicForms + "\t" + lemmasToFreqs;
  }

  /**
   * Tries to determine the most likely form of a normalized candidate as acting a keyphrase.
   * 
   * @param inst
   * @return
   */
  public String getProbableForm() {
    StringBuffer toWriteOut = new StringBuffer();
    int max = Integer.MIN_VALUE;
    for (Entry<NGram, Integer> ngram : getOrthographicForms().entrySet()) {
      boolean hasStopword = false;
      StringBuffer temp = new StringBuffer();
      NGram ng = ngram.getKey();
      for (int i = 0; i < ng.size(); ++i) {
        CoreLabel cl = ng.get(i);
        temp.append((i > 0 ? " " : "") + cl.word().toLowerCase());
        hasStopword = hasStopword || cl.get(StopWordAnnotation.class);
      }

      if ((!hasStopword && ngram.getValue() > max)) {
        max = ngram.getValue();
        toWriteOut = temp;
      } else if (toWriteOut.length() == 0) {
        toWriteOut = temp;
      }
    }
    return toWriteOut.toString();
  }
} public class NLPUtils {

  private static final String DEFAULT_CHARSET = "UTF-8";
  private static Charset charset = Charset.forName(DEFAULT_CHARSET);
  public static double log2 = Math.log(2);
  public static double SMALL = 1e-6;

  /**
   * Computes the mean for an array of Numbers.
   * 
   * @param vector
   *          the array
   * @return the mean
   */
  public static/* @pure@ */double mean(Number[] vector) {

    double sum = 0;

    if (vector.length == 0) {
      return 0;
    }
    for (int i = 0; i < vector.length; i++) {
      sum += vector[i].doubleValue();
    }
    return sum / (double) vector.length;
  }

  public static/* @pure@ */double mean(Collection<Number> vector) {
    return mean(vector.toArray(new Number[vector.size()]));
  }

  public static/* @pure@ */double mean(double[] vector) {

    double sum = 0;

    if (vector.length == 0) {
      return 0;
    }
    for (int i = 0; i < vector.length; i++) {
      sum += vector[i];
    }
    return sum / (double) vector.length;
  }

  public static/* @pure@ */double[] meanAndVariance(Collection<Number> vector) {
    return meanAndVariance(vector.toArray(new Number[vector.size()]), true);
  }

  public static/* @pure@ */double[] meanAndVariance(Collection<Number> vector, boolean biased) {
    return meanAndVariance(vector.toArray(new Number[vector.size()]), biased);
  }

  public static/* @pure@ */double[] meanAndVariance(Number[] vector, boolean biased) {

    double sum = 0.0d, ssum = 0.0d;

    if (vector.length == 0) {
      return new double[] { 0.0d, 0.0d };
    }
    for (int i = 0; i < vector.length; i++) {
      sum += vector[i].doubleValue();
      ssum += vector[i].doubleValue() * vector[i].doubleValue();
    }
    double mean = sum / (double) vector.length;
    double variance = (ssum / (double) vector.length) - (mean * mean);
    if (!biased) {
      variance *= vector.length / (double) (vector.length - 1);
    }
    return new double[] { mean, variance };
  }

  /**
   * Returns the kth-smallest value in the array.
   * 
   * @param array
   *          the array of integers
   * @param k
   *          the value of k
   * @return the kth-smallest value
   */
  public static double kthSmallestValue(int[] array, int k) {

    int[] index = new int[array.length];

    for (int i = 0; i < index.length; i++) {
      index[i] = i;
    }

    return array[index[select(array, index, 0, array.length - 1, k)]];
  }

  /**
   * Returns the kth-smallest value in the array
   * 
   * @param array
   *          the array of double
   * @param k
   *          the value of k
   * @return the kth-smallest value
   */
  public static double kthSmallestValue(double[] array, int k) {

    int[] index = new int[array.length];

    for (int i = 0; i < index.length; i++) {
      index[i] = i;
    }

    return array[index[select(array, index, 0, array.length - 1, k)]];
  }

  /**
   * Returns index of maximum element in a given array of doubles. First maximum is returned.
   * 
   * @param doubles
   *          the array of doubles
   * @return the index of the maximum element
   */
  public static/* @pure@ */int maxIndex(double[] doubles) {

    double maximum = 0;
    int maxIndex = 0;

    for (int i = 0; i < doubles.length; i++) {
      if ((i == 0) || (doubles[i] > maximum)) {
        maxIndex = i;
        maximum = doubles[i];
      }
    }

    return maxIndex;
  }

  /**
   * Returns index of maximum element in a given array of integers. First maximum is returned.
   * 
   * @param ints
   *          the array of integers
   * @return the index of the maximum element
   */
  public static/* @pure@ */int maxIndex(int[] ints) {

    int maximum = 0;
    int maxIndex = 0;

    for (int i = 0; i < ints.length; i++) {
      if ((i == 0) || (ints[i] > maximum)) {
        maxIndex = i;
        maximum = ints[i];
      }
    }

    return maxIndex;
  }

  /**
   * Returns index of minimum element in a given array of integers. First minimum is returned.
   * 
   * @param ints
   *          the array of integers
   * @return the index of the minimum element
   */
  public static/* @pure@ */int minIndex(int[] ints) {

    int minimum = 0;
    int minIndex = 0;

    for (int i = 0; i < ints.length; i++) {
      if ((i == 0) || (ints[i] < minimum)) {
        minIndex = i;
        minimum = ints[i];
      }
    }

    return minIndex;
  }

  /**
   * Returns index of minimum element in a given array of doubles. First minimum is returned.
   * 
   * @param doubles
   *          the array of doubles
   * @return the index of the minimum element
   */
  public static/* @pure@ */int minIndex(double[] doubles) {

    double minimum = 0;
    int minIndex = 0;

    for (int i = 0; i < doubles.length; i++) {
      if ((i == 0) || (doubles[i] < minimum)) {
        minIndex = i;
        minimum = doubles[i];
      }
    }

    return minIndex;
  }

  private static int select(/* @non_null@ */double[] array, /* @non_null@ */int[] index, int left, int right, int k) {
    if (left == right) {
      return left;
    } else {
      int middle = partition(array, index, left, right);
      if ((middle - left + 1) >= k) {
        return select(array, index, left, middle, k);
      } else {
        return select(array, index, middle + 1, right, k - (middle - left + 1));
      }
    }
  }

  private static int select(/* @non_null@ */int[] array, /* @non_null@ */int[] index, int left, int right, int k) {
    if (left == right) {
      return left;
    } else {
      int middle = partition(array, index, left, right);
      if ((middle - left + 1) >= k) {
        return select(array, index, left, middle, k);
      } else {
        return select(array, index, middle + 1, right, k - (middle - left + 1));
      }
    }
  }

  public <T extends Comparable<? super T>> void sort(T[] arr) {

    // do quicksort
    Arrays.sort(arr);

    Collection<T> list = new ArrayList<T>();
    int i;
    for (i = 0; i < arr.length - 1; i++) {
      if (arr[i].compareTo(arr[i + 1]) != 0) { // if not duplicate, add to the list
        list.add(arr[i]);
      }
    }
    list.add(arr[i]); // add last element
    // btw how do You know that last is not duplicate

  }

  private static <T extends Comparable<? super T>> int partition(T[] array, int[] index, int l, int r) {
    T pivot = array[index[(l + r) / 2]];
    int help;
    while (l < r) {
      while (array[index[l]].compareTo(pivot) < 0 && l < r) {
        l++;
      }
      while (array[index[r]].compareTo(pivot) > 0 && l < r) {
        r--;
      }
      if (l < r) {
        help = index[l];
        index[l] = index[r];
        index[r] = help;
        l++;
        r--;
      }
    }
    if (l == r && array[index[r]].compareTo(pivot) > 0) {
      r--;
    }
    return r;
  }

  private static int partition(double[] array, int[] index, int l, int r) {
    double pivot = array[index[(l + r) / 2]];
    int help;
    while (l < r) {
      while ((array[index[l]] < pivot) && (l < r)) {
        l++;
      }
      while ((array[index[r]] > pivot) && (l < r)) {
        r--;
      }
      if (l < r) {
        help = index[l];
        index[l] = index[r];
        index[r] = help;
        l++;
        r--;
      }
    }
    if ((l == r) && (array[index[r]] > pivot)) {
      r--;
    }
    return r;
  }

  private static int partition(int[] array, int[] index, int l, int r) {
    double pivot = array[index[(l + r) / 2]];
    int help;
    while (l < r) {
      while ((array[index[l]] < pivot) && (l < r)) {
        l++;
      }
      while ((array[index[r]] > pivot) && (l < r)) {
        r--;
      }
      if (l < r) {
        help = index[l];
        index[l] = index[r];
        index[r] = help;
        l++;
        r--;
      }
    }
    if ((l == r) && (array[index[r]] > pivot)) {
      r--;
    }
    return r;
  }

  public static/* @pure@ */double xlogx(double c) {
    if (c == 0) {
      return 0.0;
    }
    return c * NLPUtils.log2((double) c);
  }

  public static/* @pure@ */double log2(double a) {
    return Math.log(a) / log2;
  }

  public static <T extends Comparable<? super T>> int[] stableSort(T[] array) {
    int[] index = new int[array.length];
    int[] newIndex = new int[array.length];
    int[] helpIndex;
    int numEqual;

    array = (T[]) array.clone();
    for (int i = 0; i < index.length; i++) {
      index[i] = i;
    }
    quickSort(array, index, 0, array.length - 1);
    // Make sort stable
    int i = 0;
    while (i < index.length) {
      numEqual = 1;
      for (int j = i + 1; ((j < index.length) && array[index[i]].equals(array[index[j]])); j++)
        numEqual++;
      if (numEqual > 1) {
        helpIndex = new int[numEqual];
        for (int j = 0; j < numEqual; j++)
          helpIndex[j] = i + j;
        quickSort(index, helpIndex, 0, numEqual - 1);
        for (int j = 0; j < numEqual; j++)
          newIndex[i + j] = index[helpIndex[j]];
        i += numEqual;
      } else {
        newIndex[i] = index[i];
        i++;
      }
    }
    return newIndex;
  }

  public static/* @pure@ */int[] stableSort(double[] array) {
    int[] index = new int[array.length];
    int[] newIndex = new int[array.length];
    int[] helpIndex;
    int numEqual;

    array = (double[]) array.clone();
    for (int i = 0; i < index.length; i++) {
      index[i] = i;
      if (Double.isNaN(array[i])) {
        array[i] = Double.MAX_VALUE;
      }
    }
    quickSort(array, index, 0, array.length - 1);
    // Make sort stable
    int i = 0;
    while (i < index.length) {
      numEqual = 1;
      for (int j = i + 1; ((j < index.length) && eq(array[index[i]], array[index[j]])); j++)
        numEqual++;
      if (numEqual > 1) {
        helpIndex = new int[numEqual];
        for (int j = 0; j < numEqual; j++)
          helpIndex[j] = i + j;
        quickSort(index, helpIndex, 0, numEqual - 1);
        for (int j = 0; j < numEqual; j++)
          newIndex[i + j] = index[helpIndex[j]];
        i += numEqual;
      } else {
        newIndex[i] = index[i];
        i++;
      }
    }
    return newIndex;
  }

  private static <T extends Comparable<? super T>> void quickSort(T[] array, int[] index, int left, int right) {
    if (left < right) {
      int middle = partition(array, index, left, right);
      quickSort(array, index, left, middle);
      quickSort(array, index, middle + 1, right);
    }
  }

  private static void quickSort(/* @non_null@ */double[] array, /* @non_null@ */int[] index, int left, int right) {
    if (left < right) {
      int middle = partition(array, index, left, right);
      quickSort(array, index, left, middle);
      quickSort(array, index, middle + 1, right);
    }
  }

  private static void quickSort(/* @non_null@ */int[] array, /* @non_null@ */int[] index, int left, int right) {
    if (left < right) {
      int middle = partition(array, index, left, right);
      quickSort(array, index, left, middle);
      quickSort(array, index, middle + 1, right);
    }
  }

  public static/* @pure@ */boolean grOrEq(double a, double b) {
    return (b - a < SMALL);
  }

  public static/* @pure@ */boolean eq(double a, double b) {
    return (a - b < SMALL) && (b - a < SMALL);
  }

  public static/* @pure@ */double variance(double[] vector) {
    double sum = 0, sumSquared = 0;

    if (vector.length <= 1) {
      return 0;
    }
    for (int i = 0; i < vector.length; i++) {
      sum += vector[i];
      sumSquared += (vector[i] * vector[i]);
    }
    double result = (sumSquared - (sum * sum / (double) vector.length)) / (double) (vector.length - 1);

    // We don't like negative variance
    if (result < 0) {
      return 0;
    } else {
      return result;
    }
  }

  public static List<List<String>> readAsList(String file, String delimiter, Charset characterSet) {
    charset = characterSet;
    return readAsList(file, delimiter);
  }

  public static List<List<String>> readAsList(String file, String delimiter) {
    List<List<String>> list = new LinkedList<List<String>>();
    String line;
    try {
      BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
      while ((line = br.readLine()) != null)
        list.add(Arrays.asList(line.split(delimiter)));
      br.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
    return list;
  }

  public static void readDocToCollection(String file, Collection<String> collection, Charset chSet) {
    charset = chSet;
    readDocToCollection(file, collection);
  }

  public static void readDocToCollection(String file, Collection<String> collection) {
    try {
      readDocToCollection(new FileInputStream(new File(file)), collection, charset);
    } catch (FileNotFoundException e) {
      System.err.println("Error during reading input stream " + file);
    }
  }

  public static void readDocToCollection(File file, Collection<String> collection, Charset chSet) {
    charset = chSet;
    readDocToCollection(file, collection);
  }

  public static void readDocToCollection(File file, Collection<String> collection) {
    try {
      readDocToCollection(new FileInputStream(file), collection, charset);
    } catch (FileNotFoundException e) {
      System.err.println("Error during reading input stream " + file);
      e.printStackTrace();
    }
  }

  public static void readDocToCollection(InputStream is, Collection<String> collection) {
    String line;
    try {
      BufferedReader br = new BufferedReader(new InputStreamReader(is, charset));
      while ((line = br.readLine()) != null)
        collection.add(line);
      br.close();
    } catch (NullPointerException ne) {
      System.err.println("Collection to add elements into not initialized...");
    } catch (IOException e) {
      System.err.println("Error during reading input stream " + is);
    }
  }

  public static void readDocToCollection(InputStream is, Collection<String> collection, Charset chSet) {
    charset = chSet;
    readDocToCollection(is, collection);
  }

  public static Map<String, Integer> readDocToMap(String file, Charset chSet) {
    charset = chSet;
    return readDocToMap(file, chSet);
  }

  public static Map<String, Integer> readDocToMap(String file) {
    Map<String, Integer> map = new HashMap<String, Integer>();
    String line;
    try {
      BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
      while ((line = br.readLine()) != null) {
        String[] split = line.split("\t");
        if (split.length == 2)
          map.put(split[0], Integer.parseInt(split[1]));
      }
      br.close();
    } catch (Exception e) {
      System.err.println("Error during reading file " + file);
    }
    return map;
  }

  public static void serialize(Object obj, String file) {
    try {
      int lastDash = Math.max(file.lastIndexOf('\\'), file.lastIndexOf('/'));
      if (lastDash != -1)
        new File(file.substring(0, lastDash)).mkdirs();
      ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
      out.writeObject(obj);
      out.flush();
      out.close();
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  /**
   * Joins an array of strings to a single string.
   */
  public static String join(String[] str) {
    return join(str, ' ');
  }

  public static String join(String[] str, char joiner) {
    if (str.length == 0)
      return "";
    StringBuffer result = new StringBuffer(str[0]);
    for (int i = 1; i < str.length; i++)
      result.append(joiner + str[i]);
    return result.toString();
  }

  public static String join(AbstractSequentialList<String> str) {
    return join(str, ' ');
  }

  public static String join(AbstractSequentialList<String> str, char joiner) {
    return join(str.toArray(new String[str.size()]), joiner);
  }
} public class StopWordAnnotator implements Annotator {

  private Timing timer;
  private boolean verbose;
  public static Stopword stopWord;

  public StopWordAnnotator() {
    this(false);
  }

  public StopWordAnnotator(boolean verbose) {
    stopWord = new Stopword();
    timer = new Timing();
    this.verbose = verbose;
  }

  @Override
  public void annotate(Annotation annotation) {
    if (verbose) {
      timer.start();
      System.err.print("Adding normalized token annotation...");
    }

    if (annotation.has(CoreAnnotations.SentencesAnnotation.class)) {
      List<CoreMap> sentences = annotation.get(SentencesAnnotation.class);
      for (CoreMap sentence : sentences) {
        List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
        for (int i = 0; i < tokens.size(); i++) {
          CoreLabel token = tokens.get(i);
          token.set(StopWordAnnotation.class, stopWord.isStopword(token));
        }
      }
    } else {
      throw new RuntimeException("unable to find words/tokens in: " + annotation);
    }

    if (verbose)
      timer.stop("done.");
  }

  public static class StopWordAnnotation implements CoreAnnotation<Boolean> {
    public Class<Boolean> getType() {
      return Boolean.class;
    }
  }

  @Override
  public Set<Requirement> requires() {
    return Collections.singleton(TOKENIZE_REQUIREMENT);
  }

  @Override
  public Set<Requirement> requirementsSatisfied() {
    return Collections.singleton(TOKENIZE_REQUIREMENT);
  }
} public static class StopWordAnnotation implements CoreAnnotation<Boolean> {
  public Class<Boolean> getType() {
    return Boolean.class;
  }
}
import hu.u_szeged.kpe.candidates.CoreLabelComparator;
import hu.u_szeged.kpe.candidates.NGram;
import hu.u_szeged.kpe.candidates.NGramStats;
import hu.u_szeged.kpe.features.Feature;
import hu.u_szeged.kpe.features.FeatureHolder;
import hu.u_szeged.kpe.readers.DocumentData;
import hu.u_szeged.kpe.readers.KpeReader;
import hu.u_szeged.ml.ClassificationResult;
import hu.u_szeged.ml.DataHandler;
import hu.u_szeged.ml.DataMiningException;
import hu.u_szeged.ml.Model;
import hu.u_szeged.ml.mallet.MalletDataHandler;
import hu.u_szeged.utils.ClassificationInstance;
import hu.u_szeged.utils.NLPUtils;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
import cc.mallet.types.Alphabet;
import edu.stanford.nlp.ling.CoreAnnotations.LemmaAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.pipeline.StopWordAnnotator;
import edu.stanford.nlp.pipeline.StopWordAnnotator.StopWordAnnotation;
import edu.stanford.nlp.util.CoreMap;
package hu.u_szeged.kpe.main;




public class KPEFilter implements Serializable {

  private static final long serialVersionUID = -4424060992087580944L;
  /** The maximum length of phrases */
  private int maxPhraseLength = 5;
  /** The minimum length of phrases */
  private int minPhraseLength = 1;
  /** The minimum number of occurrences of a phrase */
  private int minNumOccur = 1;
  /**
   * The dictionary containing the global corpus statistics of the keyphrase candidates
   */
  private Map<String, Integer[]> m_Dictionary;
  /** The number of documents in the global frequencies corpus */
  private int numDocs = 0;
  /** The number of sections in the global frequencies corpus */
  private int numSections = 0;
  /** The name of the classifier to apply */
  private String classifierName;
  /** The list of alphabets of the classifier (i.e. labels and features) */
  private List<Alphabet> m_Alphabets;
  /** The actual classifier used to compute probabilities */
  private Model learnedModel;
  /** The list of features */
  private FeatureHolder features;
  /**
   * Collection of interesting words collected for feature value calculation during the processing of
   * documents
   */
  private Map<CoreLabel, Integer> indicatorWords;
  /** Some sort of external knowledge storing Collection */
  public static Map<CoreLabel, List<double[]>> wordList;
  // /** Container that contains possible synonyms */
  // public static Map<CoreLabel, Set<String>> synsets;
  /**
   * Regexp matching only those POS codes that are welcomed in the current setting
   */
  private Pattern posRegexp;
  /**
   * Set of ngrams regarded as too commons thus as such ones that are not supposed to be built in the model
   */
  private Set<String> commonNGrams = new HashSet<String>();
  /** Indicates if stopwords-based pruning of keyphrases is _not_ performed */
  private boolean noStopWordPruning;
  /** Indicates if POS ending-based pruning of keyphrases is _not_ performed */
  private boolean noPosEndingPruning;

  public KPEFilter(boolean noSWpruning, boolean noPOSpruning) {
    noStopWordPruning = noSWpruning;
    noPosEndingPruning = noPOSpruning;
  }

  public FeatureHolder getFeatures() {
    return features;
  }

  public Map<CoreLabel, List<double[]>> getWordList() {
    return wordList;
  }

  public String getClassifierName() {
    return classifierName;
  }

  /**
   * @param limitThreshold
   * @return those ExtendedWords that were present more than limitThreshold times in the training corpus
   */
  public Set<CoreLabel> determineIndicatingVerbs(int limitThreshold) {
    Set<CoreLabel> indicators = new TreeSet<CoreLabel>(new CoreLabelComparator());