public class DeploymentConfig {

    public static String AolIndexPath;
    public static String OdpIndexPath;
    public static String UserSearchLogPath;
    public static String ReferenceModelPath;
    public static String AolDocFreqRecord;
    public static String OdpHierarchyRecord;
    public static String BackgroundKnowledge;
    public static String TransitionMatrix;
    public static String TransitionProbability;
    public static String OutDirectory;
} public class RunTimeConfig {

    public static int NumberOfCoverQuery;
    public static int NumberOfThreads;
    public static int IntervalInHours;
    public static boolean ClientSideRanking;
    public static int TotalDocInWeb;
    public static boolean removeStopWordsInCQ;
    public static boolean doStemmingInCQ;
} public class OkapiBM25 extends SimilarityBase {

    double k1 = 1.2; // range 1.2 - 2
    double k2 = 750; // range 0 - 1000
    double b = 0.75; // range 0.75 - 1.2

    /**
     * Returns a score for a single term in the document.
     *
     * @param stats Provides access to corpus-level statistics
     * @param termFreq
     * @param docLength
     * @return 
     */
    @Override
    protected float score(BasicStats stats, float termFreq, float docLength) {
        float relevance;
        float part1 = (float) (Math.log((stats.getNumberOfDocuments() - stats.getDocFreq() + 0.5) / (stats.getDocFreq() + 0.5)) / Math.log(Math.E));
        float part2 = (float) (((k1 + 1) * termFreq) / ((k1 * (1 - b + b * (docLength / stats.getAvgFieldLength()))) + termFreq));
        float part3 = (float) (((k2 + 1) * 1) / (k2 + 1));
        relevance = part1 * part2 * part3;
        return relevance;
    }

    @Override
    public String toString() {
        return "Okapi BM25";
    }

} public class Searcher {

    private IndexSearcher indexSearcher;
    private final SpecialAnalyzer analyzer;

    /* User profile which is constructed and maintained in the server side */
    private Profile userProfile;
    /* Flag to turn on or off personalization */
    private boolean activatePersonalization = false;

    /**
     * Sets up the Lucene index Searcher with the specified index.
     *
     * @param indexPath The path to the desired Lucene index.
     */
    public Searcher(String indexPath) {
        analyzer = new SpecialAnalyzer(true, true);
        try {
            IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
            indexSearcher = new IndexSearcher(reader);
        } catch (IOException exception) {
            Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, exception);
        }
    }

    /**
     * Activate or deactivate personalization.
     *
     * @param flag
     */
    public void activatePersonalization(boolean flag) {
        activatePersonalization = flag;
    }

    /**
     * Initialize the user profile maintained by the server side.
     *
     * @param userId
     */
    public void initializeUserProfile(String userId) {
        userProfile = new Profile(userId);
    }

    /**
     * Update user profile based on the clicked document content.
     *
     * @param query
     * @param doc
     * @throws java.io.IOException
     */
    public void clickDocument(UserQuery query, ResultDoc doc) throws IOException {
        Intent intent = (Intent) userProfile.getNodeMap().get(query.getQuery_intent().getName());
        if (intent == null) {
            intent = new Intent(query.getQuery_intent().getName());
            userProfile.addIntent(intent);
        }
        intent.updateUsingClickedDoc(doc.getContent());
    }

    /**
     * Return user profile maintained by the server side.
     *
     * @return user profile
     */
    public Profile getUserProfile() {
        return userProfile;
    }

    /**
     * Sets ranking function for index searching.
     *
     * @param sim
     */
    public void setSimilarity(Similarity sim) {
        indexSearcher.setSimilarity(sim);
    }

    private void updateSessionInformation(UserQuery userQuery) {
        UserQuery lastSubmittedQuery = userProfile.getLastSubmittedQuery();
        boolean isSame = false;
        if (lastSubmittedQuery != null) {
            isSame = Helper.checkSameSession(lastSubmittedQuery, userQuery);
        }

        // current query and previous query (if any) are from different session
        if (!isSame) {
            if (lastSubmittedQuery != null) {
                // set end time of previous session
                userProfile.getLastSession().setEnd_time(lastSubmittedQuery.getQuery_time());
            }
            // start of a new user session
            Session session = new Session(userProfile.getSessions().size());
            session.setStart_time(userQuery.getQuery_time());
            userProfile.addSession(session);
        }
    }

    /**
     * The main search function. Searches the abstract field and returns a the
     * default number of results.
     *
     * @param userQuery
     * @return
     */
    public SearchResult search(UserQuery userQuery) {
        updateSessionInformation(userQuery);
        SearchQuery searchQuery = new SearchQuery(userQuery.getQuery_text(), Constants.DEFAULT_FIELD);
        BooleanQuery combinedQuery = new BooleanQuery();
        for (String field : searchQuery.fields()) {
            QueryParser parser = new QueryParser(Version.LUCENE_46, field, analyzer);
            try {
                Query textQuery = parser.parse(QueryParser.escape(searchQuery.queryText()));
                combinedQuery.add(textQuery, BooleanClause.Occur.MUST);
            } catch (ParseException exception) {
                Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, exception);
            }
        }
        return runSearch(combinedQuery, searchQuery, userQuery);
    }

    /**
     * The main search function. Searches the abstract field and returns a the
     * default number of results.
     *
     * @param query_text
     * @return
     */
    public SearchResult search(String query_text) {
        SearchQuery searchQuery = new SearchQuery(query_text, Constants.DEFAULT_FIELD);
        BooleanQuery combinedQuery = new BooleanQuery();
        for (String field : searchQuery.fields()) {
            QueryParser parser = new QueryParser(Version.LUCENE_46, field, analyzer);
            try {
                Query textQuery = parser.parse(QueryParser.escape(searchQuery.queryText()));
                combinedQuery.add(textQuery, BooleanClause.Occur.MUST);
            } catch (ParseException exception) {
                Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, exception);
            }
        }
        return runSearch(combinedQuery, searchQuery);
    }

    /**
     * Searches for a document content in the index.
     *
     * @param queryText the document title, a URL
     * @param field
     * @param retField
     * @return clicked document content
     */
    public ArrayList<String> search(String queryText, String field, String retField) {
        return runSearch(new SearchQuery(queryText, field), retField);
    }

    /**
     * Performs the actual Lucene search.
     *
     * @param luceneQuery
     * @param numResults
     * @return the SearchResult
     */
    private SearchResult runSearch(Query luceneQuery, SearchQuery searchQuery, UserQuery userQuery) {
        try {
            TopDocs docs = indexSearcher.search(luceneQuery, searchQuery.fromDoc() + searchQuery.numResults());
            ResultDoc[] hits;
            String field = searchQuery.fields().get(0);
            if (activatePersonalization) {
                /* Personalize result using entire user profile. */
                HashMap<String, Integer> tempMap = userProfile.getCompleteHistory();
                hits = Personalization.personalizeResults(indexSearcher, field, tempMap, docs.scoreDocs);
            } else {
                hits = Converter.convertToResultDoc(docs.scoreDocs, indexSearcher, field);
            }

            /* Updating the server side user profile with the query */
            userQuery.setQuery_intent(userProfile.addIntent(userQuery.getQuery_intent()));
            boolean success = userProfile.addQuery(userQuery);
            if (!success) {
                System.err.println("Failed to update server-side user profile and now exiting...");
                System.exit(1);
            }
            userProfile.getLastSession().addUser_queries(userQuery);

            SearchResult searchResult = new SearchResult(searchQuery, docs.totalHits);
            for (ResultDoc hit : hits) {
                searchResult.addResult(hit);
            }
            searchResult.trimResults(searchQuery.fromDoc());
            return searchResult;
        } catch (IOException exception) {
            Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, exception);
        }
        return new SearchResult(searchQuery);
    }

    /**
     * Performs the actual Lucene search.
     *
     * @param luceneQuery
     * @param numResults
     * @return the SearchResult
     */
    private SearchResult runSearch(Query luceneQuery, SearchQuery searchQuery) {
        try {
            TopDocs docs = indexSearcher.search(luceneQuery, searchQuery.fromDoc() + searchQuery.numResults());
            ScoreDoc[] hits = docs.scoreDocs;
            String field = searchQuery.fields().get(0);

            SearchResult searchResult = new SearchResult(searchQuery, docs.totalHits);
            for (ScoreDoc hit : hits) {
                Document doc = indexSearcher.doc(hit.doc);
                ResultDoc rdoc = new ResultDoc(hit.doc);
                String contents = doc.getField(field).stringValue();
                rdoc.setContent(contents);
                rdoc.setUrl(doc.getField("url").stringValue());
                rdoc.setTopic(doc.getField("topic").stringValue());
                searchResult.addResult(rdoc);
            }

            searchResult.trimResults(searchQuery.fromDoc());
            return searchResult;

        } catch (IOException ex) {
            Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, ex);
        }
        return new SearchResult(searchQuery);
    }

    /**
     * Searches and returns relevant document's content.
     *
     * @param searchQuery a clicked URL
     * @param indexableField content of this field needs to be returned
     * @return document's content
     */
    private ArrayList<String> runSearch(SearchQuery searchQuery, String indexableField) {
        Query luceneQuery = new TermQuery(new Term(searchQuery.fields().get(0), searchQuery.queryText()));
        ArrayList<String> returnedResult = new ArrayList<>();
        try {
            TotalHitCountCollector collector = new TotalHitCountCollector();
            indexSearcher.search(luceneQuery, collector);
            TopDocs docs = indexSearcher.search(luceneQuery, Math.max(1, collector.getTotalHits()));
            ScoreDoc[] hits = docs.scoreDocs;
            for (ScoreDoc hit : hits) {
                Document doc = indexSearcher.doc(hit.doc);
                String contents = doc.getField(indexableField).stringValue();
                returnedResult.add(contents);
            }
        } catch (IOException exception) {
            Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, exception);
        }

        return returnedResult;
    }

    /**
     * Searches and returns total hit count for given query terms.
     *
     * @param tokens
     * @param field
     * @return hit document's id
     */
    public int search(List<String> tokens, String field) {
        int hit_count = 0;
        BooleanQuery combinedQuery = new BooleanQuery();
        for (String token : tokens) {
            Query luceneQuery = new TermQuery(new Term(field, token));
            combinedQuery.add(luceneQuery, BooleanClause.Occur.MUST);
        }

        try {
            TotalHitCountCollector collector = new TotalHitCountCollector();
            indexSearcher.search(combinedQuery, collector);
            hit_count = collector.getTotalHits();
        } catch (IOException exception) {
            Logger.getLogger(Searcher.class.getName()).log(Level.SEVERE, null, exception);
        }
        return hit_count;
    }

} public class Helper {

    private static final TextTokenizer TOKENIZER = new TextTokenizer(true, true);

    /**
     * Generate a random number within a range.
     *
     * @param min
     * @param max
     * @return
     */
    public static int getRandom(int min, int max) {
        Random random = new Random();
        return random.nextInt(max - min) + min;
    }

    /**
     * Generate random number using poisson distribution.
     *
     * @param lambda average query length
     * @return
     */
    public static int getPoisson(double lambda) {
        int n = 1;
        double prob = 1.0;
        Random r = new Random();

        while (true) {
            prob *= r.nextDouble();
            if (prob < Math.exp(-lambda)) {
                break;
            }
            n += 1;
        }
        return n - 1;
    }

    /**
     *
     *
     * @param previousQuery
     * @param currentQuery
     * @return
     */
    public static boolean checkSameSession(UserQuery previousQuery, UserQuery currentQuery) {
        /**
         * Measuring the time difference between current query and last
         * submitted query in minutes.
         */
        double diffMinutes = 0;
        try {
            //in milliseconds
            long diff = currentQuery.getQuery_time().getTime() - previousQuery.getQuery_time().getTime();
            diffMinutes = diff / (60.0 * 1000);
        } catch (Exception ex) {
            Logger.getLogger(Helper.class.getName()).log(Level.SEVERE, null, ex);
        }
        /**
         * If time difference is less than 60 minutes between current query and
         * previous query, they belong to the same session .
         */
        return diffMinutes < 60;
    }

    public static boolean checkSameTask(UserQuery previousQuery, UserQuery currentQuery) {
        /**
         * Measuring the similarity between current query and last submitted
         * query in terms of similar tokens they have.
         */
        HashSet<String> currentQuTokens = new HashSet<>(TOKENIZER.TokenizeText(currentQuery.getQuery_text()));
        HashSet<String> prevQuTokens = new HashSet<>(TOKENIZER.TokenizeText(previousQuery.getQuery_text()));
        boolean isSimilar = false;
        int count = (currentQuTokens.size() > prevQuTokens.size() ? prevQuTokens.size() : currentQuTokens.size()) / 2;
        HashSet<String> intersection = new HashSet<>(currentQuTokens);
        intersection.retainAll(prevQuTokens);
        if (intersection.size() >= count) {
            isSimilar = true;
        }
        /**
         * If both queries have 50% similarity in terms of having similar
         * tokens, they belong to the same task .
         */
        return isSimilar;
    }

    public static UserQuery isInSessionTask(Session session, UserQuery currentQuery) {
        for (UserQuery query : session.getUser_queries()) {
            if (checkSameTask(query, currentQuery)) {
                return query;
            }
        }
        return null;
    }

    private static HashMap<String, Integer> computeTermFreq(List<String> tokens) {
        HashMap<String, Integer> termFreqMap = new HashMap<>();
        // computing term frequency of all the unique terms found in the document
        for (String tok : tokens) {
            if (termFreqMap.containsKey(tok)) {
                termFreqMap.put(tok, termFreqMap.get(tok) + 1);
            } else {
                termFreqMap.put(tok, 1);
            }
        }
        return termFreqMap;
    }

    private static ArrayList<ResultDoc> reRank(HashMap<String, Float> docScoreMap, ArrayList<ResultDoc> relDocs) {
        /**
         * Client side re-ranking using true user profile.
         */
        Map<String, Float> resultedMap = SortMap.sortMapByValue(docScoreMap, false);

        /**
         * Re-rank the documents by giving weight to the search engine rank and
         * the client side rank.
         */
        int i = 0;
        for (Map.Entry<String, Float> entry : resultedMap.entrySet()) {
            float score;
            // Giving 30% weight to client-side re-ranking and 70% to server-side ranking.
            if (entry.getValue() == 0) {
                score = 0.7f * (1.0f / (Integer.parseInt(entry.getKey()) + 1));
            } else {
                score = 0.3f * (1.0f / (i + 1)) + 0.7f * (1.0f / (Integer.parseInt(entry.getKey()) + 1));
            }
            docScoreMap.put(entry.getKey(), score);
            i++;
        }

        // sort the documents in descending order according to the new score assigned
        Map<String, Float> result = SortMap.sortMapByValue(docScoreMap, false);
        ArrayList<ResultDoc> retValue = new ArrayList<>();
        for (Map.Entry<String, Float> entry : result.entrySet()) {
            retValue.add(relDocs.get(Integer.parseInt(entry.getKey())));
        }

        return retValue;
    }

    /**
     * Method that re-ranks the result in the client side.
     *
     * @param history
     * @param relDocs all the relevant documents returned by the search engine
     * @return re-ranked resulting documents
     */
    public static ArrayList<ResultDoc> reRankResults(HashMap<String, Integer> history, ArrayList<ResultDoc> relDocs) {
        HashMap<String, Float> docScoreMap = new HashMap<>();
        HashMap<String, Integer> uniqueDocTerms;

        for (int i = 0; i < relDocs.size(); i++) {
            List<String> tokens = TOKENIZER.TokenizeText(relDocs.get(i).getContent());
            uniqueDocTerms = computeTermFreq(tokens);

            float docScore = 0;
            // smoothing parameter for linear interpolation
            float lambda = 0.1f;
            for (Map.Entry<String, Integer> entry : history.entrySet()) {
                if (!uniqueDocTerms.containsKey(entry.getKey())) {
                    continue;
                }
                // maximum likelihood calculation
                Double tokenProb = (uniqueDocTerms.get(entry.getKey()) * 1.0) / tokens.size();
                // probability from reference model for smoothing purpose
                Double refProb = StaticData.SmoothingReference.get(entry.getKey());
                if (refProb == null) {
                    refProb = 0.0;
                }
                // smoothing token probability using linear interpolation
                Double smoothedTokenProb = (1 - lambda) * tokenProb + lambda * refProb;
                smoothedTokenProb = smoothedTokenProb / (lambda * refProb);
                docScore = docScore + (entry.getValue() * (float) Math.log(smoothedTokenProb));
            }
            docScoreMap.put(String.valueOf(i), docScore);
        }

        // return re-ranked documents
        return reRank(docScoreMap, relDocs);
    }

    /**
     * Method that generate the id of all users for evaluation.
     *
     * @param folder folder path where all user search log resides
     * @param count
     * @return list of all user id
     */
    public static ArrayList<String> getAllUserId(String folder, int count) {
        ArrayList<String> allUserIds = new ArrayList<>();
        File dir = new File(folder);
        int userCount = 0;
        for (File f : dir.listFiles()) {
            if (f.isFile()) {
                String fileName = f.getName();
                fileName = fileName.substring(0, fileName.lastIndexOf("."));
                allUserIds.add(fileName);
                userCount++;
            }
            if (userCount == count) {
                break;
            }
        }
        return allUserIds;
    }

} public static int getRandom(int min, int max) {
    Random random = new Random();
    return random.nextInt(max - min) + min;
} public interface TreeNode {

    public TreeNode getParent();

    public void setParent(TreeNode node);

    public List<TreeNode> getChildrens();

    public void setChildrens(List<TreeNode> childrens);

    public void addChildren(TreeNode children);

    public void setNodeLevel(int level);

    public int getNodeLevel();
} public class Topic {

    private final int BUCKET_SIZE = 10;
    private final double LAMBDA = 0.1;
    private final double GAMMA = 0.1;

    private HashMap<String, Integer> unigramLM;
    private HashMap<String, Integer> bigramLM;
    private HashMap<String, Integer> trigramLM;
    private HashMap<String, Integer> fourgramLM;
    private final Double[] probArrayUnigram;
    private final Double[] probArrayBigram;
    private final Double[] probArrayTrigram;
    private final Double[] probArrayFourgram;

    private int totalUnigrams;
    private int totalUniqueUnigrams;

    private double maxProbUnigram;
    private double minProbUnigram;
    private double maxProbBigram;
    private double minProbBigram;
    private double maxProbTrigram;
    private double minProbTrigram;
    private double maxProbFourgram;
    private double minProbFourgram;

    private final TextTokenizer tokenizer;

    public Topic() {
        this.unigramLM = new HashMap<>();
        this.bigramLM = new HashMap<>();
        this.trigramLM = new HashMap<>();
        this.fourgramLM = new HashMap<>();
        this.probArrayUnigram = new Double[BUCKET_SIZE];
        this.probArrayBigram = new Double[BUCKET_SIZE];
        this.probArrayTrigram = new Double[BUCKET_SIZE];
        this.probArrayFourgram = new Double[BUCKET_SIZE];
        this.tokenizer = new TextTokenizer(RunTimeConfig.removeStopWordsInCQ, RunTimeConfig.doStemmingInCQ);
    }

    public void reset() {
        this.unigramLM = new HashMap<>();
        this.bigramLM = new HashMap<>();
        this.trigramLM = new HashMap<>();
        this.fourgramLM = new HashMap<>();
        totalUnigrams = 0;
        totalUniqueUnigrams = 0;
    }

    public HashMap<String, Integer> getUnigramLM() {
        return unigramLM;
    }

    public void setUnigramLM(HashMap<String, Integer> unigramLM) {
        this.unigramLM = unigramLM;
    }

    public HashMap<String, Integer> getBigramLM() {
        return bigramLM;
    }

    public void setBigramLM(HashMap<String, Integer> bigramLM) {
        this.bigramLM = bigramLM;
    }

    public HashMap<String, Integer> getTrigramLM() {
        return trigramLM;
    }

    public void setTrigramLM(HashMap<String, Integer> trigramLM) {
        this.trigramLM = trigramLM;
    }

    public HashMap<String, Integer> getFourgramLM() {
        return fourgramLM;
    }

    public void setFourgramLM(HashMap<String, Integer> fourgramLM) {
        this.fourgramLM = fourgramLM;
    }

    public int getTotalUnigrams() {
        return totalUnigrams;
    }

    public void setTotalUnigrams(int totalUnigrams) {
        this.totalUnigrams = totalUnigrams;
    }

    public int getTotalUniqueUnigrams() {
        return totalUniqueUnigrams;
    }

    public void setTotalUniqueUnigrams(int totalUniqueUnigrams) {
        this.totalUniqueUnigrams = totalUniqueUnigrams;
    }

    public double getMaxProbUnigram() {
        return maxProbUnigram;
    }

    public void setMaxProbUnigram(double maxProbUnigram) {
        this.maxProbUnigram = maxProbUnigram;
    }

    public double getMinProbUnigram() {
        return minProbUnigram;
    }

    public void setMinProbUnigram(double minProbUnigram) {
        this.minProbUnigram = minProbUnigram;
    }

    public double getMaxProbBigram() {
        return maxProbBigram;
    }

    public void setMaxProbBigram(double maxProbBigram) {
        this.maxProbBigram = maxProbBigram;
    }

    public double getMinProbBigram() {
        return minProbBigram;
    }

    public void setMinProbBigram(double minProbBigram) {
        this.minProbBigram = minProbBigram;
    }

    public double getMaxProbTrigram() {
        return maxProbTrigram;
    }

    public void setMaxProbTrigram(double maxProbTrigram) {
        this.maxProbTrigram = maxProbTrigram;
    }

    public double getMinProbTrigram() {
        return minProbTrigram;
    }

    public void setMinProbTrigram(double minProbTrigram) {
        this.minProbTrigram = minProbTrigram;
    }

    public double getMaxProbFourgram() {
        return maxProbFourgram;
    }

    public void setMaxProbFourgram(double maxProbFourgram) {
        this.maxProbFourgram = maxProbFourgram;
    }

    public double getMinProbFourgram() {
        return minProbFourgram;
    }

    public void setMinProbFourgram(double minProbFourgram) {
        this.minProbFourgram = minProbFourgram;
    }

    /**
     *
     * @return
     */
    public Double[] getProbArrayUnigram() {
        return probArrayUnigram;
    }

    /**
     *
     * @return
     */
    public Double[] getProbArrayBigram() {
        return probArrayBigram;
    }

    /**
     *
     * @return
     */
    public Double[] getProbArrayTrigram() {
        return probArrayTrigram;
    }

    /**
     *
     * @return
     */
    public Double[] getProbArrayFourgram() {
        return probArrayFourgram;
    }

    /**
     * Checks whether a topic is empty or not.
     *
     * @return
     */
    public boolean isEmpty() {
        return unigramLM.isEmpty();
    }

    /**
     * Computes probability from reference model for smoothing purpose.
     *
     * @param unigram
     * @return
     */
    public double getReferenceProbability(String unigram) {
        if (StaticData.SmoothingReference.containsKey(unigram)) {
            return StaticData.SmoothingReference.get(unigram);
        }
        return 0.00000001;
    }

    /**
     * Computes joint probability or conditional probability of a unigram.
     *
     * @param unigram
     * @return
     */
    public double getProbabilityUnigram(String unigram) {
        double prob;
        /* Computing probability of a unigram using linear interpolation smoothing */
        if (unigramLM.containsKey(unigram)) {
            prob = (1.0 - LAMBDA) * (unigramLM.get(unigram) / totalUnigrams) + LAMBDA * getReferenceProbability(unigram);
        } else {
            prob = LAMBDA * getReferenceProbability(unigram);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a unigram.
     *
     * @param unigram
     * @param queryTopicNode
     * @return
     */
    public double getProbabilityUnigram(String unigram, TopicTreeNode queryTopicNode) {
        double prob;
        TopicTreeNode parent = (TopicTreeNode) (queryTopicNode.getParent());
        /* Computing probability of a unigram using linear interpolation smoothing */
        if (unigramLM.containsKey(unigram)) {
            if (parent != null) {
                prob = (1.0 - LAMBDA - GAMMA) * (unigramLM.get(unigram) / totalUnigrams)
                        + LAMBDA * getReferenceProbability(unigram)
                        + GAMMA * parent.getTopic().getProbabilityUnigram(unigram, parent);
            } else {
                prob = (1.0 - LAMBDA) * (unigramLM.get(unigram) / totalUnigrams)
                        + LAMBDA * getReferenceProbability(unigram);
            }

        } else {
            prob = LAMBDA * getReferenceProbability(unigram);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a bigram.
     *
     * @param bigram
     * @param isJoint
     * @return
     */
    public double getProbabilityBigram(String bigram, boolean isJoint) {
        double prob;
        /* Computing probability of a bigram using linear interpolation smoothing */
        String[] split = bigram.split(" ");
        if (bigramLM.containsKey(bigram)) {
            prob = (1.0 - LAMBDA) * (bigramLM.get(bigram) / unigramLM.get(split[0]));
        } else {
            prob = 0.0;
        }
        prob += LAMBDA * getProbabilityUnigram(split[0]);
        if (isJoint) {
            prob *= getProbabilityUnigram(split[0]);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a bigram.
     *
     * @param bigram
     * @param isJoint
     * @param queryTopicNode
     * @return
     */
    public double getProbabilityBigram(String bigram, boolean isJoint, TopicTreeNode queryTopicNode) {
        double prob;
        /* Computing probability of a bigram using linear interpolation smoothing */
        TopicTreeNode parent = (TopicTreeNode) (queryTopicNode.getParent());
        String[] split = bigram.split(" ");
        if (bigramLM.containsKey(bigram)) {
            if (parent != null) {
                prob = (1.0 - LAMBDA - GAMMA) * (bigramLM.get(bigram) / unigramLM.get(split[0]));
            } else {
                prob = (1.0 - LAMBDA) * (bigramLM.get(bigram) / unigramLM.get(split[0]));
            }
        } else {
            prob = 0.0;
        }
        if (parent != null) {
            prob = prob + LAMBDA * getProbabilityUnigram(split[0], queryTopicNode)
                    + GAMMA * parent.getTopic().getProbabilityBigram(bigram, isJoint, parent);
        } else {
            prob = prob + LAMBDA * getProbabilityUnigram(split[0], queryTopicNode);
        }
        if (isJoint) {
            prob *= getProbabilityUnigram(split[0], queryTopicNode);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a trigram.
     *
     * @param trigram
     * @param isJoint
     * @return
     */
    public double getProbabilityTrigram(String trigram, boolean isJoint) {
        double prob;
        /* Computing probability of a trigram using linear interpolation smoothing */
        String[] split = trigram.split(" ");
        String prevBigram = split[0] + " " + split[1];
        if (trigramLM.containsKey(trigram)) {
            prob = (1.0 - LAMBDA) * (trigramLM.get(trigram) / bigramLM.get(prevBigram));
        } else {
            prob = 0.0;
        }
        String bigram = split[1] + " " + split[2];
        prob += LAMBDA * getProbabilityBigram(bigram, false);
        if (isJoint) {
            prob *= getProbabilityBigram(prevBigram, false);
            prob *= getProbabilityUnigram(split[0]);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a trigram.
     *
     * @param trigram
     * @param isJoint
     * @param queryTopicNode
     * @return
     */
    public double getProbabilityTrigram(String trigram, boolean isJoint, TopicTreeNode queryTopicNode) {
        double prob;
        /* Computing probability of a trigram using linear interpolation smoothing */
        TopicTreeNode parent = (TopicTreeNode) (queryTopicNode.getParent());
        String[] split = trigram.split(" ");
        String prevBigram = split[0] + " " + split[1];
        if (trigramLM.containsKey(trigram)) {
            if (parent != null) {
                prob = (1.0 - LAMBDA - GAMMA) * (trigramLM.get(trigram) / bigramLM.get(prevBigram));
            } else {
                prob = (1.0 - LAMBDA) * (trigramLM.get(trigram) / bigramLM.get(prevBigram));
            }
        } else {
            prob = 0.0;
        }
        String bigram = split[1] + " " + split[2];
        if (parent != null) {
            prob = prob + LAMBDA * getProbabilityBigram(bigram, false, queryTopicNode)
                    + GAMMA * parent.getTopic().getProbabilityTrigram(trigram, isJoint, parent);
        } else {
            prob = prob + LAMBDA * getProbabilityBigram(bigram, false, queryTopicNode);
        }
        if (isJoint) {
            prob *= getProbabilityBigram(prevBigram, false, queryTopicNode);
            prob *= getProbabilityUnigram(split[0], queryTopicNode);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a fourgram.
     *
     * @param fourgram
     * @param isJoint
     * @return if isJoint is true, returns joint probability, otherwise
     * conditional probability
     */
    public double getProbabilityFourgram(String fourgram, boolean isJoint) {
        double prob;
        /* Computing probability of a fourgram using linear interpolation smoothing */
        String[] split = fourgram.split(" ");
        String prevTrigram = split[0] + " " + split[1] + " " + split[2];
        if (fourgramLM.containsKey(fourgram)) {
            prob = (1.0 - LAMBDA) * (fourgramLM.get(fourgram) / trigramLM.get(prevTrigram));
        } else {
            prob = 0.0;
        }
        String trigram = split[1] + " " + split[2] + " " + split[3];
        prob += LAMBDA * getProbabilityTrigram(trigram, false);
        if (isJoint) {
            prob *= getProbabilityBigram(prevTrigram, false);
            String bigram = split[0] + " " + split[1];
            prob *= getProbabilityBigram(bigram, false);
            prob *= getProbabilityUnigram(split[0]);
        }
        return prob;
    }

    /**
     * Computes joint probability or conditional probability of a fourgram.
     *
     * @param fourgram
     * @param isJoint
     * @param queryTopicNode
     * @return if isJoint is true, returns joint probability, otherwise
     * conditional probability
     */
    public double getProbabilityFourgram(String fourgram, boolean isJoint, TopicTreeNode queryTopicNode) {
        double prob;
        /* Computing probability of a fourgram using linear interpolation smoothing */
        TopicTreeNode parent = (TopicTreeNode) (queryTopicNode.getParent());
        String[] split = fourgram.split(" ");
        String prevTrigram = split[0] + " " + split[1] + " " + split[2];
        if (fourgramLM.containsKey(fourgram)) {
            if (parent != null) {
                prob = (1.0 - LAMBDA - GAMMA) * (fourgramLM.get(fourgram) / trigramLM.get(prevTrigram));
            } else {
                prob = (1.0 - LAMBDA) * (fourgramLM.get(fourgram) / trigramLM.get(prevTrigram));
            }
        } else {
            prob = 0.0;
        }
        String trigram = split[1] + " " + split[2] + " " + split[3];
        if (parent != null) {
            prob = prob + LAMBDA * getProbabilityTrigram(trigram, false, queryTopicNode)
                    + GAMMA * parent.getTopic().getProbabilityFourgram(fourgram, isJoint, parent);
        } else {
            prob = prob + LAMBDA * getProbabilityTrigram(trigram, false, queryTopicNode);
        }
        if (isJoint) {
            prob *= getProbabilityBigram(prevTrigram, false, queryTopicNode);
            String bigram = split[0] + " " + split[1];
            prob *= getProbabilityBigram(bigram, false, queryTopicNode);
            prob *= getProbabilityUnigram(split[0], queryTopicNode);
        }
        return prob;
    }

    /**
     * Computes joint probability of a n-gram where n>4. Suppose n=6, then joint
     * probability formula is, P(w6 w5 w4 w3 w2 w1) = P(w6 | w5 w4 w3) * P(w5 |
     * w4 w3 w2) * P(w4 | w3 w2 w1) * P(w3 | w2 w1) * P(w2 | w1) * P(w1)
     *
     * @param ngram
     * @param n
     * @return
     */
    public double getProbabilityNgram(String ngram, int n) {
        double prob = 1.0;
        /* Computing probability of a n-gram using linear interpolation smoothing */
        String[] split = ngram.split(" ");
        for (int i = split.length - 1; i >= 0; i--) {
            if (i >= 3) {
                String fourgram = split[i - 3] + " " + split[i - 2] + " " + split[i - 1] + " " + split[i];
                prob *= getProbabilityFourgram(fourgram, false);
            } else if (i == 2) {
                String trigram = split[i - 2] + " " + split[i - 1] + " " + split[i];
                prob *= getProbabilityBigram(trigram, false);
            } else if (i == 1) {
                String bigram = split[i - 1] + " " + split[i];
                prob *= getProbabilityBigram(bigram, false);
            } else {
                prob *= getProbabilityUnigram(split[0]);
            }
        }
        return prob;
    }

    /**
     * Computes joint probability of a n-gram where n>4. Suppose n=6, then joint
     * probability formula is, P(w6 w5 w4 w3 w2 w1) = P(w6 | w5 w4 w3) * P(w5 |
     * w4 w3 w2) * P(w4 | w3 w2 w1) * P(w3 | w2 w1) * P(w2 | w1) * P(w1)
     *
     * @param ngram
     * @param n
     * @param queryTopicNode
     * @return
     */
    public double getProbabilityNgram(String ngram, int n, TopicTreeNode queryTopicNode) {
        double prob = 1.0;
        /* Computing probability of a n-gram using linear interpolation smoothing */
        String[] split = ngram.split(" ");
        for (int i = split.length - 1; i >= 0; i--) {
            if (i >= 3) {
                String fourgram = split[i - 3] + " " + split[i - 2] + " " + split[i - 1] + " " + split[i];
                prob *= getProbabilityFourgram(fourgram, false, queryTopicNode);
            } else if (i == 2) {
                String trigram = split[i - 2] + " " + split[i - 1] + " " + split[i];
                prob *= getProbabilityBigram(trigram, false, queryTopicNode);
            } else if (i == 1) {
                String bigram = split[i - 1] + " " + split[i];
                prob *= getProbabilityBigram(bigram, false, queryTopicNode);
            } else {
                prob *= getProbabilityUnigram(split[0], queryTopicNode);
            }
        }
        return prob;
    }

    /**
     * Set the Maximum and Minimum probability of the language model.
     *
     * @param param either unigram or bigram or trigram or fourgram
     */
    public void setMaxMinProb(String param) {
        double max = -1.0;
        double min = -1.0;
        HashMap<String, Double> tempMap = new HashMap<>();
        if (param.equals("unigram")) {
            if (unigramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : unigramLM.entrySet()) {
                double prob = getProbabilityUnigram(entry.getKey());
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbUnigram = max;
            minProbUnigram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayUnigram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayUnigram[i - 1] = list.get(index).getValue();
                }
            }
        } else if (param.equals("bigram")) {
            if (bigramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : bigramLM.entrySet()) {
                double prob = getProbabilityBigram(entry.getKey(), true);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbBigram = max;
            minProbBigram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayBigram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayBigram[i - 1] = list.get(index).getValue();
                }
            }
        } else if (param.equals("trigram")) {
            if (trigramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : trigramLM.entrySet()) {
                double prob = getProbabilityTrigram(entry.getKey(), true);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbTrigram = max;
            minProbTrigram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayTrigram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayTrigram[i - 1] = list.get(index).getValue();
                }
            }
        } else if (param.equals("fourgram")) {
            if (fourgramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : fourgramLM.entrySet()) {
                double prob = getProbabilityFourgram(entry.getKey(), true);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbFourgram = max;
            minProbFourgram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayFourgram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayFourgram[i - 1] = list.get(index).getValue();
                }
            }
        } else {
            System.err.println("Unknown Parameter...!");
        }
    }

    /**
     * Set the Maximum and Minimum probability of the language model.
     *
     * @param param either unigram or bigram or trigram or fourgram
     * @param queryTopicNode
     */
    public void setMaxMinProb(String param, TopicTreeNode queryTopicNode) {
        double max = -1.0;
        double min = -1.0;
        HashMap<String, Double> tempMap = new HashMap<>();
        if (param.equals("unigram")) {
            if (unigramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : unigramLM.entrySet()) {
                double prob = getProbabilityUnigram(entry.getKey(), queryTopicNode);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbUnigram = max;
            minProbUnigram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayUnigram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayUnigram[i - 1] = list.get(index).getValue();
                }
            }
        } else if (param.equals("bigram")) {
            if (bigramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : bigramLM.entrySet()) {
                double prob = getProbabilityBigram(entry.getKey(), true, queryTopicNode);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbBigram = max;
            minProbBigram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayBigram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayBigram[i - 1] = list.get(index).getValue();
                }
            }
        } else if (param.equals("trigram")) {
            if (trigramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : trigramLM.entrySet()) {
                double prob = getProbabilityTrigram(entry.getKey(), true, queryTopicNode);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbTrigram = max;
            minProbTrigram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayTrigram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayTrigram[i - 1] = list.get(index).getValue();
                }
            }
        } else if (param.equals("fourgram")) {
            if (fourgramLM.isEmpty()) {
                return;
            }
            for (Map.Entry<String, Integer> entry : fourgramLM.entrySet()) {
                double prob = getProbabilityFourgram(entry.getKey(), true, queryTopicNode);
                tempMap.put(entry.getKey(), prob);
                if (max < prob) {
                    max = prob;
                }
                if (min > prob) {
                    min = prob;
                }
            }
            maxProbFourgram = max;
            minProbFourgram = min;
            List<Map.Entry<String, Double>> list = SortMap.sortMapGetList(tempMap, false);
            int size = list.size() % BUCKET_SIZE == 0 ? list.size() / BUCKET_SIZE : (list.size() / BUCKET_SIZE) + 1;
            for (int i = 1; i <= BUCKET_SIZE; i++) {
                int index = i * size;
                if (index >= list.size()) {
                    index = list.size() - 1;
                    probArrayFourgram[i - 1] = list.get(index).getValue();
                } else {
                    probArrayFourgram[i - 1] = list.get(index).getValue();
                }
            }
        } else {
            System.err.println("Unknown Parameter...!");
        }
    }

    private boolean checkForNull(Double[] param) {
        boolean nonNullElemExist = false;
        for (Double d : param) {
            if (d != null) {
                nonNullElemExist = true;
                break;
            }
        }
        return nonNullElemExist;
    }

    public boolean prepare(Searcher _searcher, String topic_name, int option) {
        ArrayList<String> documents = _searcher.search(topic_name, "topic", "modified_content");
        if (!documents.isEmpty()) {
            for (String content : documents) {
                analyzeDocument(content, option);
            }
            switch (option) {
                case 1:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram");
                    }
                    break;
                case 2:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram");
                    }
                    if (!checkForNull(probArrayBigram)) {
                        setMaxMinProb("bigram");
                    }
                    break;
                case 3:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram");
                    }
                    if (!checkForNull(probArrayBigram)) {
                        setMaxMinProb("bigram");
                    }
                    if (!checkForNull(probArrayTrigram)) {
                        setMaxMinProb("trigram");
                    }
                    break;
                default:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram");
                    }
                    if (!checkForNull(probArrayBigram)) {
                        setMaxMinProb("bigram");
                    }
                    if (!checkForNull(probArrayTrigram)) {
                        setMaxMinProb("trigram");
                    }
                    if (!checkForNull(probArrayFourgram)) {
                        setMaxMinProb("fourgram");
                    }
                    break;
            }
            return true;
        } else {
            return false;
        }
    }

    public boolean prepare(Searcher _searcher, String topic_name, int option, TopicTreeNode queryTopicNode) {
        ArrayList<String> documents = _searcher.search(topic_name, "topic", "modified_content");
        if (!documents.isEmpty()) {
            for (String content : documents) {
                analyzeDocument(content, option);
            }
            switch (option) {
                case 1:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram", queryTopicNode);
                    }
                    break;
                case 2:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram", queryTopicNode);
                    }
                    if (!checkForNull(probArrayBigram)) {
                        setMaxMinProb("bigram", queryTopicNode);
                    }
                    break;
                case 3:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram", queryTopicNode);
                    }
                    if (!checkForNull(probArrayBigram)) {
                        setMaxMinProb("bigram", queryTopicNode);
                    }
                    if (!checkForNull(probArrayTrigram)) {
                        setMaxMinProb("trigram", queryTopicNode);
                    }
                    break;
                default:
                    if (!checkForNull(probArrayUnigram)) {
                        setMaxMinProb("unigram", queryTopicNode);
                    }
                    if (!checkForNull(probArrayBigram)) {
                        setMaxMinProb("bigram", queryTopicNode);
                    }
                    if (!checkForNull(probArrayTrigram)) {
                        setMaxMinProb("trigram", queryTopicNode);
                    }
                    if (!checkForNull(probArrayFourgram)) {
                        setMaxMinProb("fourgram", queryTopicNode);
                    }
                    break;
            }
            return true;
        } else {
            return false;
        }
    }

    private void analyzeDocument(String document, int option) {
        String previousTrigram = ""; // for four-grams
        String previousBigram = ""; // for trigrams
        String previousUnigram = ""; // for bigrams
        List<String> tokens = tokenizer.TokenizeText(document);
        for (String token : tokens) {
            if (!token.isEmpty()) {
                if (unigramLM.containsKey(token)) {
                    unigramLM.put(token, unigramLM.get(token) + 1);
                } else {
                    unigramLM.put(token, 1);
                    totalUniqueUnigrams++;
                }
                totalUnigrams++;
                if (option == 1) {
                    continue;
                }
                // generating bigrams
                if (!previousUnigram.isEmpty()) {
                    String bigram = previousUnigram + " " + token;
                    if (bigramLM.containsKey(bigram)) {
                        bigramLM.put(bigram, bigramLM.get(bigram) + 1);
                    } else {
                        bigramLM.put(bigram, 1);
                    }
                }
                if (option == 2) {
                    previousUnigram = token;
                    continue;
                }
                // generating trigrams
                if (!previousBigram.isEmpty()) {
                    String trigram = previousBigram + " " + token;
                    if (trigramLM.containsKey(trigram)) {
                        trigramLM.put(trigram, trigramLM.get(trigram) + 1);
                    } else {
                        trigramLM.put(trigram, 1);
                    }
                }
                if (option == 3) {
                    if (!previousUnigram.isEmpty()) {
                        previousBigram = previousUnigram + " " + token;
                    }
                    previousUnigram = token;
                    continue;
                }
                // generating four-grams
                if (!previousTrigram.isEmpty()) {
                    String fourgram = previousTrigram + " " + token;
                    if (fourgramLM.containsKey(fourgram)) {
                        fourgramLM.put(fourgram, fourgramLM.get(fourgram) + 1);
                    } else {
                        fourgramLM.put(fourgram, 1);
                    }
                }
                if (!previousBigram.isEmpty()) {
                    previousTrigram = previousBigram + " " + token;
                }
                if (!previousUnigram.isEmpty()) {
                    previousBigram = previousUnigram + " " + token;
                }
                previousUnigram = token;
            }
        }
    }

} public class UserQuery {

    private int query_id;
    private String query_text;
    private Date query_time;
    private int query_length;
    private ArrayList<ResultDoc> relevant_documents;
    private ArrayList<UserQuery> cover_queries;
    private Intent query_intent;

    public UserQuery(int query_id, String query_text) {
        this.query_id = query_id;
        this.query_text = query_text;
        this.relevant_documents = new ArrayList<>();
        this.cover_queries = new ArrayList<>();
    }

    public int getQuery_id() {
        return query_id;
    }

    public void setQuery_id(int query_id) {
        this.query_id = query_id;
    }

    public String getQuery_text() {
        return query_text;
    }

    public void setQuery_text(String query_text) {
        this.query_text = query_text;
    }

    public Date getQuery_time() {
        return query_time;
    }

    public void setQuery_time(Date query_time) {
        this.query_time = query_time;
    }

    public int getQuery_length() {
        return query_length;
    }

    public void setQuery_length(int query_length) {
        this.query_length = query_length;
    }

    public ArrayList<ResultDoc> getRelevant_documents() {
        return relevant_documents;
    }

    public ResultDoc relDoc_Contains(ResultDoc rdoc) {
        for (ResultDoc clicked_doc : relevant_documents) {
            if (clicked_doc.getUrl().equals(rdoc.getUrl())) {
                return clicked_doc;
            }
        }
        return null;
    }

    public void setRelevant_documents(ArrayList<ResultDoc> clicked_documents) {
        this.relevant_documents = clicked_documents;
    }

    public void addRelevant_document(ResultDoc relDoc) {
        this.relevant_documents.add(relDoc);
    }

    public Intent getQuery_intent() {
        return query_intent;
    }

    public void setQuery_intent(Intent query_intent) {
        this.query_intent = query_intent;
    }

    public ArrayList<UserQuery> getCover_queries() {
        return cover_queries;
    }

    public void setCover_queries(ArrayList<UserQuery> cover_queries) {
        this.cover_queries = new ArrayList<>(cover_queries);
    }

    public void addCover_query(UserQuery cover_query) {
        this.cover_queries.add(cover_query);
    }

} public class Intent implements TreeNode {

    private TreeNode parent;
    private List<TreeNode> childrens;
    private int level;
    private final History history;

    private final TextTokenizer _tokenizer;

    private double totalQueryLength;
    private double totalQuery;
    private int totalTokens;

    private final String intentName;

    public Intent(String name) {
        history = new History();
        this.childrens = new ArrayList<>();

        this._tokenizer = new TextTokenizer(true, true);
        this.intentName = name;

        totalTokens = 0;
        totalQuery = 0;
        totalQueryLength = 0;
    }

    public String getName() {
        return this.intentName;
    }

    @Override
    public TreeNode getParent() {
        return this.parent;
    }

    @Override
    public void setParent(TreeNode node) {
        this.parent = node;
    }

    public boolean isParent(TreeNode node) {
        Intent intent = (Intent) node.getParent();
        if (intent == null) {
            return false;
        }
        return this.intentName.equals(((Intent) node.getParent()).getName());
    }

    @Override
    public List<TreeNode> getChildrens() {
        return this.childrens;
    }

    @Override
    public void setChildrens(List<TreeNode> childrens) {
        this.childrens = new ArrayList<>(childrens);
    }

    @Override
    public void addChildren(TreeNode children) {
        if (this.childrens != null) {
            this.childrens.add(children);
        } else {
            throw new NullPointerException("children is not initialized");
        }
    }

    @Override
    public void setNodeLevel(int level) {
        this.level = level;
    }

    @Override
    public int getNodeLevel() {
        return level;
    }

    public int getTotalTokenCount() {
        return totalTokens;
    }

    public History getHistory() {
        return history;
    }

    /**
     * Returns average length of all queries submitted by the user. Default
     * value is 3.
     *
     * @return average query length
     */
    public double getAvgQueryLength() {
        if (totalQuery == 0) {
            return 3;
        }
        return totalQueryLength / totalQuery;
    }

    /**
     * Update user profile by the user submitted query.
     *
     * @param queryText
     */
    public void updateUsingSubmittedQuery(String queryText) {
        List<String> qParts = _tokenizer.TokenizeText(queryText);
        totalQuery++;
        totalQueryLength = totalQueryLength + qParts.size();
        for (String qPart : qParts) {
            if (qPart.isEmpty()) {
                continue;
            }
            totalTokens++;
            history.addElementInQueryTerms(qPart, 1);
        }
    }

    public HashMap<String, Integer> getRecord(String text, boolean fromDoc) {
        List<String> tokens = _tokenizer.TokenizeText(text);
        HashMap<String, Integer> record = new HashMap<>();
        if (fromDoc) {
            /* To update user profile, select the top k tokens using tf-idf weight */
            int k = (int) (tokens.size() * 0.25);
            HashMap<String, Integer> retVal = selectTopKtokens(tokens, k);
            for (Map.Entry<String, Integer> entry : retVal.entrySet()) {
                if (record.containsKey(entry.getKey())) {
                    record.put(entry.getKey(), record.get(entry.getKey()) + entry.getValue());
                } else {
                    record.put(entry.getKey(), entry.getValue());
//                    record.put(entry.getKey(), 1);
                }
            }
        } else {
            for (String token : tokens) {
                if (token.isEmpty()) {
                    continue;
                }
                if (record.containsKey(token)) {
                    record.put(token, record.get(token) + 1);
                } else {
                    record.put(token, 1);
                }
            }
        }
        return record;
    }

    /**
     * Update user profile by the content of the user clicked document.
     *
     * @param content
     */
    public void updateUsingClickedDoc(String content) {
        List<String> tokens = _tokenizer.TokenizeText(content);
        /* To update user profile, select the top k tokens using tf-idf weight */
        int k = (int) (tokens.size() * 0.25);
        HashMap<String, Integer> retVal = selectTopKtokens(tokens, k);
        for (Map.Entry<String, Integer> entry : retVal.entrySet()) {
            // totalTokens += entry.getValue();
            // history.addElementInDocTerms(entry.getKey(), entry.getValue());
            history.addElementInDocTerms(entry.getKey(), 1);
            totalTokens++;
        }
    }

    /**
     * Method that returns the top k tokens from a list of tokens. Tokens are
     * ranked based on their tf-idf value.
     *
     * @param tokenList list of tokens
     * @param k return only the top k elements
     * @return top k tokens with their term frequency
     */
    private HashMap<String, Integer> selectTopKtokens(List<String> tokenList, int k) {
        HashMap<String, Integer> retValue = new HashMap<>();
        HashMap<String, Integer> tempMap = new HashMap<>();
        /* Stores tf-idf weight of all tokens */
        HashMap<String, Float> unsortedMap = new HashMap<>();
        for (String token : tokenList) {
            Integer n = tempMap.get(token);
            n = (n == null) ? 1 : ++n;
            tempMap.put(token, n);
        }
        for (Map.Entry<String, Integer> entry : tempMap.entrySet()) {
            Double idf = StaticData.IDFRecord.get(entry.getKey());
            if (idf == null) {
                idf = 0.0;
            }
            double tfIdfWeight = entry.getValue() * idf;
            unsortedMap.put(entry.getKey(), (float) tfIdfWeight);
        }
        Map<String, Float> temp = SortMap.sortMapByValue(unsortedMap, false, k);
        for (Map.Entry<String, Float> entry : temp.entrySet()) {
            retValue.put(entry.getKey(), tempMap.get(entry.getKey()));
        }
        return retValue;
    }

} public class Profile implements Tree {

    private final String userId;
    private HashMap<String, TreeNode> intents;
    private final ArrayList<UserQuery> submittedQueries;
    private int totalTokenCount;
    private final TreeNode root;

    private final ArrayList<Session> sessions;
    private final ArrayList<Task> in_session_tasks;

    public Profile(String id) {
        this.submittedQueries = new ArrayList<>();
        this.userId = id;
        this.intents = new HashMap<>();
        this.sessions = new ArrayList<>();
        this.in_session_tasks = new ArrayList<>();
        this.root = new Intent("Top");
        this.root.setNodeLevel(0);
        this.root.setParent(null);
    }

    public boolean isEmpty() {
        return submittedQueries.isEmpty();
    }

    public ArrayList<UserQuery> getSubmittedQueries() {
        return submittedQueries;
    }

    public UserQuery getLastSubmittedQuery() {
        if (submittedQueries.isEmpty()) {
            return null;
        }
        return submittedQueries.get(submittedQueries.size() - 1);
    }

    public UserQuery getQuery(int index) {
        if (index < submittedQueries.size()) {
            return submittedQueries.get(index);
        } else {
            return null;
        }
    }

    public boolean addQuery(UserQuery query) {
        this.submittedQueries.add(query);
        Intent query_intent = (Intent) intents.get(query.getQuery_intent().getName());
        if (query_intent == null) {
            System.err.println("Failed to update user profile!");
            return false;
        } else {
            query_intent.updateUsingSubmittedQuery(query.getQuery_text());
            for (ResultDoc doc : query.getRelevant_documents()) {
                if (doc.isClicked()) {
                    query_intent.updateUsingClickedDoc(doc.getContent());
                }
            }
            return true;
        }
    }

    public Intent addIntent(Intent intent) {
        if (!intentExists(intent.getName())) {
            String[] nodes = intent.getName().split("/");
            String temp = nodes[0];
            TreeNode parent = root;
            for (int i = 1; i < nodes.length; i++) {
                if (i == nodes.length - 1) {
                    intent.setParent(parent);
                    intent.setNodeLevel(parent.getNodeLevel() + 1);
                    parent.addChildren(intent);
                    intents.put(intent.getName(), intent);
                } else {
                    temp += "/" + nodes[i];
                    TreeNode node = new Intent(temp);
                    if (!intentExists(temp)) {
                        node.setParent(parent);
                        node.setNodeLevel(parent.getNodeLevel() + 1);
                        parent.addChildren(node);
                        intents.put(temp, node);
                    }
                    parent = node;
                }
            }
            return intent;
        } else {
            return (Intent) intents.get(intent.getName());
        }
    }

    public boolean intentExists(String intent_name) {
        return intents.containsKey(intent_name);
    }

    @Override
    public HashMap<String, TreeNode> getNodeMap() {
        return this.intents;
    }

    @Override
    public void addNode(String nodePath, TreeNode node) {
        this.intents.put(nodePath, node);
    }

    @Override
    public void setNodes(HashMap<String, TreeNode> nodeMap) {
        this.intents = nodeMap;
    }

    public String getUserId() {
        return userId;
    }

    public int getTotalTokenCount() {
        return totalTokenCount;
    }

    public HashMap<String, TreeNode> getIntents() {
        return intents;
    }

    public TreeNode getRoot() {
        return root;
    }

    public ArrayList<Session> getSessions() {
        return sessions;
    }

    public Session getLastSession() {
        if (sessions.isEmpty()) {
            return null;
        }
        return this.sessions.get(sessions.size() - 1);
    }

    public void addSession(Session sess) {
        this.sessions.add(sess);
    }

    public ArrayList<Task> getIn_session_tasks() {
        return in_session_tasks;
    }

    public Task getLastInSessionTask() {
        return this.in_session_tasks.get(in_session_tasks.size() - 1);
    }

    public void addInSessionTask(Task task) {
        this.in_session_tasks.add(task);
    }

    public HashMap<String, Integer> getCompleteHistory() {
        HashMap<String, Integer> completeHistory = new HashMap<>();
        totalTokenCount = 0;
        for (String key : intents.keySet()) {
            TreeNode node = intents.get(key);
            for (Map.Entry<String, Integer> entry : ((Intent) node).getHistory().getSelectedQueryTerms().entrySet()) {
                if (completeHistory.containsKey(entry.getKey())) {
                    completeHistory.put(entry.getKey(), completeHistory.get(entry.getKey()) + entry.getValue());
                } else {
                    completeHistory.put(entry.getKey(), entry.getValue());
                }
                totalTokenCount += entry.getValue();
            }
            for (Map.Entry<String, Integer> entry : ((Intent) node).getHistory().getSelectedDocTerms().entrySet()) {
                if (completeHistory.containsKey(entry.getKey())) {
                    completeHistory.put(entry.getKey(), completeHistory.get(entry.getKey()) + entry.getValue());
                } else {
                    completeHistory.put(entry.getKey(), entry.getValue());
                }
                totalTokenCount += entry.getValue();
            }
        }
        return completeHistory;
    }

    public HashMap<String, Integer> getBranchHistory(Intent intent) {
        HashMap<String, Integer> branchHistory = new HashMap<>();
        Intent target_intent = (Intent) intents.get(intent.getName());
        totalTokenCount = 0;
        while (target_intent != null) {
            for (Map.Entry<String, Integer> entry : target_intent.getHistory().getSelectedQueryTerms().entrySet()) {
                if (branchHistory.containsKey(entry.getKey())) {
                    branchHistory.put(entry.getKey(), branchHistory.get(entry.getKey()) + entry.getValue());
                } else {
                    branchHistory.put(entry.getKey(), entry.getValue());
                }
                totalTokenCount += entry.getValue();
            }
            for (Map.Entry<String, Integer> entry : target_intent.getHistory().getSelectedDocTerms().entrySet()) {
                if (branchHistory.containsKey(entry.getKey())) {
                    branchHistory.put(entry.getKey(), branchHistory.get(entry.getKey()) + entry.getValue());
                } else {
                    branchHistory.put(entry.getKey(), entry.getValue());
                }
                totalTokenCount += entry.getValue();
            }
            target_intent = (Intent) target_intent.getParent();
        }
        return branchHistory;
    }

    public HashMap<String, Integer> getSessionHistory(Intent intent) {
        HashMap<String, Integer> sessionHistory = new HashMap<>();
        Session currentSession = sessions.get(sessions.size() - 1);
        for (UserQuery query : currentSession.getUser_queries()) {
            for (Map.Entry<String, Integer> entry : intent.getRecord(query.getQuery_text(), false).entrySet()) {
                if (sessionHistory.containsKey(entry.getKey())) {
                    sessionHistory.put(entry.getKey(), sessionHistory.get(entry.getKey()) + entry.getValue());
                } else {
                    sessionHistory.put(entry.getKey(), entry.getValue());
                }
            }
            for (ResultDoc doc : query.getRelevant_documents()) {
                if (doc.isClicked()) {
                    for (Map.Entry<String, Integer> entry : intent.getRecord(doc.getContent(), true).entrySet()) {
                        if (sessionHistory.containsKey(entry.getKey())) {
                            sessionHistory.put(entry.getKey(), sessionHistory.get(entry.getKey()) + entry.getValue());
                        } else {
                            sessionHistory.put(entry.getKey(), entry.getValue());
                        }
                    }
                }
            }
        }
        return sessionHistory;
    }

    /**
     * Computes KL-Divergence between two different user profiles.
     *
     * @param comparedTo
     * @return KL-Divergence value
     */
    public float calculateKLDivergence(Profile comparedTo) {
        float klDiv = 0;
        float lambda = 0.1f;
        HashMap<String, Integer> thisProfile = getCompleteHistory();
        HashMap<String, Integer> profileCompared = comparedTo.getCompleteHistory();

        HashSet<String> keySet = new HashSet<>();
        keySet.addAll(thisProfile.keySet());
        keySet.addAll(profileCompared.keySet());

        for (String name : keySet) {
            Integer value = profileCompared.get(name);
            if (value == null) {
                value = 0;
            }
            Double tokenProb = (value * 1.0) / comparedTo.getTotalTokenCount();
            Double refProb = StaticData.SmoothingReference.get(name);
            if (refProb == null) {
                refProb = 0.0;
            }
            Double smoothedTokenProb = (1 - lambda) * tokenProb + lambda * refProb;
            Double p1 = smoothedTokenProb;

            Integer value2 = thisProfile.get(name);
            if (value2 == null) {
                value2 = 0;
            }
            Double tokenProb2 = (value2 * 1.0) / getTotalTokenCount();
            Double refProb2 = refProb;

            Double smoothedTokenProb2 = (1 - lambda) * tokenProb2 + lambda * refProb2;
            Double p2 = smoothedTokenProb2;
            if (p1 == 0) {
                continue;
            }
            if (p2 == 0) {
                continue;
            }
            klDiv = (float) (klDiv + p1 * Math.log(p1 / p2));
        }
        return klDiv;
    }

} public class TextTokenizer {

    private final SpecialAnalyzer analyzer;

    public TextTokenizer(boolean removeStopWords, boolean doStemming) {
        analyzer = new SpecialAnalyzer(removeStopWords, doStemming);
    }

    /**
     * Method that generates list of tokens from the parameter string.
     *
     * @param string
     * @return list of tokens generated
     */
    public List<String> TokenizeText(String string) {
        List<String> result = new ArrayList<>();
        try {
            TokenStream stream = analyzer.tokenStream(null, new StringReader(string));
            stream.reset();
            while (stream.incrementToken()) {
                result.add(stream.getAttribute(CharTermAttribute.class
                ).toString());
            }
            stream.end();
            stream.close();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return result;
    }
}
import edu.virginia.cs.config.DeploymentConfig;
import edu.virginia.cs.config.RunTimeConfig;
import edu.virginia.cs.engine.OkapiBM25;
import edu.virginia.cs.engine.Searcher;
import edu.virginia.cs.extra.Helper;
import static edu.virginia.cs.extra.Helper.getRandom;
import edu.virginia.cs.interfaces.TreeNode;
import edu.virginia.cs.object.Topic;
import edu.virginia.cs.object.UserQuery;
import edu.virginia.cs.user.Intent;
import edu.virginia.cs.user.Profile;
import edu.virginia.cs.utility.TextTokenizer;
import java.util.ArrayList;
import java.util.List;
/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package edu.virginia.cs.model;


/**
 *
 * @author wua4nw
 */
public class GenerateCoverQuery {

    private final TopicTree topicTree;
    private final TextTokenizer tokenizer;
    private final Searcher SEARCHER;
    private final GenerateQuery gQuery;

    public GenerateCoverQuery(TopicTree tree) {
        this.topicTree = tree;
        this.SEARCHER = new Searcher(DeploymentConfig.OdpIndexPath);
        this.SEARCHER.setSimilarity(new OkapiBM25());