// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.queries.function.docvalues.StrDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.dice.solrenhancements.TermExtractionHelper;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
        StringBuilder sb = new StringBuilder();

        try {
            if(!tryExtractTermsFromTermVector(docNum, indexedField, ir, sb)) {
                extractTermsFromField(docNum, indexedField, ir, sb);
            }
            String values = sb.toString().trim();
            if(values.length() == 0){
                return "";
            }
            return values.substring(0, sb.length()-DELIM.length());

        }
        catch (Exception ex)
        {
            throw new RuntimeException("caught exception in function " + description()+" : doc=" + docNum, ex);
        }
    }

    private void extractTermsFromField(int docNum, String indexedField, IndexReader ir, StringBuilder sb) throws IOException {

        Set<String> fields = new HashSet<String>();
        fields.add(indexedField);
        Document d = ir.document(docNum, fields);

        IndexableField field = d.getField(indexedField);
        if (field == null) {
            return;
        }

// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


// Path: src/main/java/org/dice/solrenhancements/functionqueries/FieldIndexedValueSource.java
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.queries.function.docvalues.StrDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.dice.solrenhancements.TermExtractionHelper;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

        StringBuilder sb = new StringBuilder();

        try {
            if(!tryExtractTermsFromTermVector(docNum, indexedField, ir, sb)) {
                extractTermsFromField(docNum, indexedField, ir, sb);
            }
            String values = sb.toString().trim();
            if(values.length() == 0){
                return "";
            }
            return values.substring(0, sb.length()-DELIM.length());

        }
        catch (Exception ex)
        {
            throw new RuntimeException("caught exception in function " + description()+" : doc=" + docNum, ex);
        }
    }

    private void extractTermsFromField(int docNum, String indexedField, IndexReader ir, StringBuilder sb) throws IOException {

        Set<String> fields = new HashSet<String>();
        fields.add(indexedField);
        Document d = ir.document(docNum, fields);

        IndexableField field = d.getField(indexedField);
        if (field == null) {
            return;
        }

        List<String> terms = TermExtractionHelper.getTermsFromField(analyzer, field);