// Path: src/main/java/org/dice/helper/DefaultHashTable.java
// public class DefaultHashTable<K,V> extends Hashtable<K,V> {
// 
//     private final Supplier<V> _supplier;
//     public DefaultHashTable(V defaultVal)
//     {
//         // need to make this final
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(V defaultVal, Map<K,V> map){
//         super(map);
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             @Override
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier)
//     {
//         _supplier = supplier;
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier, Map<K,V> map)
//     {
//         super(map);
//         _supplier = supplier;
//     }
// 
//     @Override
//     public V get(Object key)
//     {
//         if(!this.containsKey(key))
//         {
//             super.put((K)key, this._supplier.get());
//         }
//         return super.get(key);
//     }
// }
// 
// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.dice.helper.DefaultHashTable;
import org.dice.solrenhancements.TermExtractionHelper;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;

    private void extractFeaturesFromDocuments(Set<Integer> docIds, String featureLabel, Map<String, Map<String,Integer>> featureMap) throws IOException {
        for(Integer id: docIds){
            extractFeaturesFromDocument(id, ir, featureLabel, featureMap);
        }
    }

    private void extractFeaturesFromDocument(int docNum, IndexReader ir, String featureLabel, Map<String, Map<String,Integer>> featureMap) throws IOException {

        if(fields == null || fields.length == 0){
            return;
        }

        final Fields vectors = ir.getTermVectors(docNum);
        final Document document = ir.document(docNum);

        for (String fieldName : fields) {

            Terms vector = null;
            if (vectors != null) {
                vector = vectors.terms(fieldName);
            }

            // field does not store term vector info
            // even if term vectors enabled, need to extract payload from regular field reader
            if (vector == null) {
                IndexableField docFields[] = document.getFields(fieldName);
                for (IndexableField field : docFields) {
                    final String stringValue = field.stringValue();
                    if (stringValue != null) {
// Path: src/main/java/org/dice/helper/DefaultHashTable.java
// public class DefaultHashTable<K,V> extends Hashtable<K,V> {
// 
//     private final Supplier<V> _supplier;
//     public DefaultHashTable(V defaultVal)
//     {
//         // need to make this final
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(V defaultVal, Map<K,V> map){
//         super(map);
//         final V defaultValCopy = defaultVal;
//         _supplier = new Supplier<V>() {
//             @Override
//             public V get() {
//                 return defaultValCopy;
//             }
//         };
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier)
//     {
//         _supplier = supplier;
//     }
// 
//     public DefaultHashTable(Supplier<V> supplier, Map<K,V> map)
//     {
//         super(map);
//         _supplier = supplier;
//     }
// 
//     @Override
//     public V get(Object key)
//     {
//         if(!this.containsKey(key))
//         {
//             super.put((K)key, this._supplier.get());
//         }
//         return super.get(key);
//     }
// }
// 
// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


// Path: src/main/java/org/dice/solrenhancements/machinelearning/FeatureExtractor.java
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.dice.helper.DefaultHashTable;
import org.dice.solrenhancements.TermExtractionHelper;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;


    private void extractFeaturesFromDocuments(Set<Integer> docIds, String featureLabel, Map<String, Map<String,Integer>> featureMap) throws IOException {
        for(Integer id: docIds){
            extractFeaturesFromDocument(id, ir, featureLabel, featureMap);
        }
    }

    private void extractFeaturesFromDocument(int docNum, IndexReader ir, String featureLabel, Map<String, Map<String,Integer>> featureMap) throws IOException {

        if(fields == null || fields.length == 0){
            return;
        }

        final Fields vectors = ir.getTermVectors(docNum);
        final Document document = ir.document(docNum);

        for (String fieldName : fields) {

            Terms vector = null;
            if (vectors != null) {
                vector = vectors.terms(fieldName);
            }

            // field does not store term vector info
            // even if term vectors enabled, need to extract payload from regular field reader
            if (vector == null) {
                IndexableField docFields[] = document.getFields(fieldName);
                for (IndexableField field : docFields) {
                    final String stringValue = field.stringValue();
                    if (stringValue != null) {
                        List<String> lstTerms = TermExtractionHelper.getTermsFromString(analyzer, fieldName, stringValue);