// Path: src/main/java/org/dice/helper/DefaultHashTable.java
// public class DefaultHashTable<K,V> extends Hashtable<K,V> {
// 
//     private final Supplier<V> _supplier;
//     public DefaultHashTable(V defaultVal)
//     {
//         // need to make this final
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(V defaultVal, Map<K,V> map){
//         super(map);
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             @Override
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier)
//     {
//         _supplier = supplier;
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier, Map<K,V> map)
//     {
//         super(map);
//         _supplier = supplier;
//     }
// 
//     @Override
//     public V get(Object key)
//     {
//         if(!this.containsKey(key))
//         {
//             super.put((K)key, this._supplier.get());
//         }
//         return super.get(key);
//     }
// }
// 
// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.dice.helper.DefaultHashTable;
import org.dice.solrenhancements.TermExtractionHelper;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
package org.dice.solrenhancements.machinelearning;



/**
 * Created by simon.hughes on 2/1/17.
 */
public class FeatureExtractor {

    private final IndexReader ir;
    private final String[] fields;
    private Map<String, Map<String, Integer>> featureMap = null;
    private Analyzer analyzer = null;
    private final int minDf;

    public static class LABELS {
        public static final String POSITIVE = "POSITIVE";
        public static final String NEGATIVE = "NEGATIVE";
    }

    public FeatureExtractor(IndexReader ir, String[] fields, Analyzer analyzer, int minDf){
        this.ir = ir;
        this.fields = fields;
        this.analyzer = analyzer;
        this.minDf = minDf;
        this.featureMap = createFeatureMap();
    }

    public FeatureExtractor extractFeaturesFromDocuments(Set<Integer> docIds, String featureLabel) throws IOException {
        this.extractFeaturesFromDocuments(docIds, featureLabel, this.featureMap);
        return this;
    }

    public Map<String, Map<String, Integer>> getFeatureMap(){
        return featureMap;
    }

    public void resetFeatures(){
        this.featureMap = createFeatureMap();
    }

    private final static Map<String, Map<String, Integer>> createFeatureMap(){
// Path: src/main/java/org/dice/helper/DefaultHashTable.java
// public class DefaultHashTable<K,V> extends Hashtable<K,V> {
// 
//     private final Supplier<V> _supplier;
//     public DefaultHashTable(V defaultVal)
//     {
//         // need to make this final
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(V defaultVal, Map<K,V> map){
//         super(map);
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             @Override
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier)
//     {
//         _supplier = supplier;
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier, Map<K,V> map)
//     {
//         super(map);
//         _supplier = supplier;
//     }
// 
//     @Override
//     public V get(Object key)
//     {
//         if(!this.containsKey(key))
//         {
//             super.put((K)key, this._supplier.get());
//         }
//         return super.get(key);
//     }
// }
// 
// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


// Path: src/main/java/org/dice/solrenhancements/machinelearning/FeatureExtractor.java
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.dice.helper.DefaultHashTable;
import org.dice.solrenhancements.TermExtractionHelper;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;

package org.dice.solrenhancements.machinelearning;



/**
 * Created by simon.hughes on 2/1/17.
 */
public class FeatureExtractor {

    private final IndexReader ir;
    private final String[] fields;
    private Map<String, Map<String, Integer>> featureMap = null;
    private Analyzer analyzer = null;
    private final int minDf;

    public static class LABELS {
        public static final String POSITIVE = "POSITIVE";
        public static final String NEGATIVE = "NEGATIVE";
    }

    public FeatureExtractor(IndexReader ir, String[] fields, Analyzer analyzer, int minDf){
        this.ir = ir;
        this.fields = fields;
        this.analyzer = analyzer;
        this.minDf = minDf;
        this.featureMap = createFeatureMap();
    }

    public FeatureExtractor extractFeaturesFromDocuments(Set<Integer> docIds, String featureLabel) throws IOException {
        this.extractFeaturesFromDocuments(docIds, featureLabel, this.featureMap);
        return this;
    }

    public Map<String, Map<String, Integer>> getFeatureMap(){
        return featureMap;
    }

    public void resetFeatures(){
        this.featureMap = createFeatureMap();
    }

    private final static Map<String, Map<String, Integer>> createFeatureMap(){
        return new DefaultHashTable<String, Map<String, Integer>>(new Supplier<Map<String, Integer>>() {