// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
import org.apache.solr.common.SolrException;
import org.apache.solr.search.FunctionQParser;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
import org.dice.solrenhancements.TermExtractionHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

        return new TermIntersectsValueSource(paramInfo.field, paramInfo.analyzer, paramInfo.terms, paramInfo.similarity);
    }

    private static ParamInfo parseTerm(FunctionQParser fp) throws SyntaxError {
        ParamInfo paramInfo = new ParamInfo();

        paramInfo.field = fp.parseArg();
        String textVal = fp.parseArg();
        if(textVal == null || textVal.trim().length() == 0){
            return paramInfo;
        }

        if(fp.hasMoreArguments()){
            String similarity = fp.parseArg().toLowerCase().trim();
            if( !similarity.equals(SimilarityType.DOC_LEN) &&
                !similarity.equals(SimilarityType.PARAM_LEN) &&
                !similarity.equals(SimilarityType.DICE) &&
                !similarity.equals(SimilarityType.JACCARD)){

                log.error(String.format("Invalid similarity class: %s. Defaulting to %s", similarity, SimilarityType.DOC_LEN));
                similarity = SimilarityType.DOC_LEN;
            }
            paramInfo.similarity = similarity;
        }

        // need to do analysis on the term
        Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer();
        paramInfo.analyzer = analyzer;
        try {
// Path: src/main/java/org/dice/solrenhancements/TermExtractionHelper.java
// public class TermExtractionHelper {
// 
//     public static List<String> getTermsFromField(Analyzer analyzer, IndexableField field) throws IOException {
//         TokenStream ts = null;
//         try {
//             ArrayList<String> terms = new ArrayList<String>();
// 
//             ts = field.tokenStream(analyzer, ts);
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 terms.add(termAtt.toString());
//             }
//             ts.end();
//             return terms;
//         }
//         finally {
//             if(ts != null){
//                 IOUtils.closeWhileHandlingException(ts);
//             }
//         }
//     }
// 
//     public static List<String> getTermsFromString(Analyzer analyzer, String fieldName, String fieldValue)
//             throws IOException {
// 
//         final StringReader reader = new StringReader(fieldValue);
//         ArrayList<String> terms = new ArrayList<String>();
//         TokenStream ts = analyzer.tokenStream(fieldName, reader);
//         try {
//             // for every token
//             CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
//             ts.reset();
//             while (ts.incrementToken()) {
//                 String term = termAtt.toString();
//                 terms.add(term);
//             }
//             ts.end();
//         } finally {
//             IOUtils.closeWhileHandlingException(ts);
//         }
//         return terms;
//     }
// 
//     public static List<String> getTermsFromTermVectorField(Terms vector) throws IOException {
//         ArrayList<String> terms = new ArrayList<String>();
// 
//         final TermsEnum termsEnum = vector.iterator();
//         CharsRefBuilder spare = new CharsRefBuilder();
//         BytesRef text;
// 
//         while((text = termsEnum.next()) != null) {
//             spare.copyUTF8Bytes(text);
//             final String term = spare.toString();
//             terms.add(term);
//         }
//         return terms;
//     }
// }


// Path: src/main/java/org/dice/solrenhancements/functionqueries/TermIntersectsValueSourceParser.java
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
import org.apache.solr.common.SolrException;
import org.apache.solr.search.FunctionQParser;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
import org.dice.solrenhancements.TermExtractionHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;


        return new TermIntersectsValueSource(paramInfo.field, paramInfo.analyzer, paramInfo.terms, paramInfo.similarity);
    }

    private static ParamInfo parseTerm(FunctionQParser fp) throws SyntaxError {
        ParamInfo paramInfo = new ParamInfo();

        paramInfo.field = fp.parseArg();
        String textVal = fp.parseArg();
        if(textVal == null || textVal.trim().length() == 0){
            return paramInfo;
        }

        if(fp.hasMoreArguments()){
            String similarity = fp.parseArg().toLowerCase().trim();
            if( !similarity.equals(SimilarityType.DOC_LEN) &&
                !similarity.equals(SimilarityType.PARAM_LEN) &&
                !similarity.equals(SimilarityType.DICE) &&
                !similarity.equals(SimilarityType.JACCARD)){

                log.error(String.format("Invalid similarity class: %s. Defaulting to %s", similarity, SimilarityType.DOC_LEN));
                similarity = SimilarityType.DOC_LEN;
            }
            paramInfo.similarity = similarity;
        }

        // need to do analysis on the term
        Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer();
        paramInfo.analyzer = analyzer;
        try {
            List<String> terms = TermExtractionHelper.getTermsFromString(analyzer, paramInfo.field, textVal);