@Component
public class DialogManager {
    protected final static Log logger = LogFactory.getLog(DialogManager.class);

    @Autowired OCUtil ocUtil;

    @Autowired SuggestionsCreator suggestionsCreator;

    @Autowired private LearningModelHelper learningModelHelper;
    @Autowired LearningModel learningModel;

    AtomicInteger incrementer = new AtomicInteger();

    /**
     * check whether dialog needs to be modeled or not for any disambiguations (ocs)
     * 
     * @param question
     * @return
     */
    public boolean requiresOCDialog(Question question, boolean forceDialog) {
        boolean result = false;
        List<List<SemanticConcept>> ocs = question.getSemanticConcepts();
        for (List<SemanticConcept> list : ocs) {
            if (list.size() > 1) {
                if (!ocUtil.ocsVerified(list)) {
                    result = true;
                    logger.info("Overlapped OCs found?" + result);
                    return result;
                }
            }
        }
        logger.info("Overlapped OCs found?" + result);
        return result;
    }

    public boolean requiresOCDialog(Question question) {
        return requiresOCDialog(question, false);
    }


    /**
     * checkes whether there are any pocs so that the dialog is generated
     * 
     * @param question
     * @return
     */
    public boolean requiresPOCDialog(Question question) {
        boolean result = false;
        List<POC> pocs = question.getPocs();
        if (pocs.size() > 0) result = true;
        logger.info("Unresolved POCs found?" + result);
        return result;
    }

    /**
     * generate dialog when ocs are overlapped before starting resolvement of pocs
     * 
     * @param question
     */
    public SuggestionPair generateDisambiguationDialog(Question question,
                    boolean forceDialog) {
        SuggestionPair pair = null;
        // which OC here
        long start = System.currentTimeMillis();
        List<SemanticConcept> list = whichOC(question);
        long end = System.currentTimeMillis();
        logger.info("Which OC took:" + (end - start) + "ms.");
        if (list.size() > 1) {
            start = System.currentTimeMillis();
            pair = getOCVotesFromTheLearningModel(list, question, forceDialog);
            end = System.currentTimeMillis();
            logger.info("getOCVotesFromTheLearningModel took:" + (end - start)
                            + "ms.");
        }
        VoteComparator c = new VoteComparator();
        Collections.sort(pair.getVote(), c);
        return pair;
    }

    /**
     * default is that dialog is not forced for oc
     * 
     * @param question
     * @return
     */
    public SuggestionPair generateDisambiguationDialog(Question question) {
        return generateDisambiguationDialog(question, false);
    }

    /**
     * this method generates the sample key from the sample element in the list and then checks whether the learning
     * model contains the pair with that key; if yes, return it, if no, generate the pair with initialized votes and add
     * it to the learning model
     * 
     * @param list
     * @return
     */
    SuggestionPair getOCVotesFromTheLearningModel(List<SemanticConcept> list,
                    Question question, boolean forceDialog) {
        SuggestionKey key = new SuggestionKey();
        SuggestionPair pair = new SuggestionPair();
        SemanticConcept sampleElement = null;
        // the assumption here is that all elements will have the same text and
        // NNeighbours
        if (list != null && list.size() > 0) sampleElement = list.get(0);
        String text;
        if (sampleElement.getOntologyElement().getAnnotation().getText() == null)
            text =
                            "Annotation for an ontology element is null?!!!this shouldnot happen dfkdjf !!";
        else
            text = sampleElement.getOntologyElement().getAnnotation().getText();
        key.setText(text);
        // finding NN for any sample element as these ocs overlap so they
        // are at the same position
        List<SemanticConcept> nearestNeighbours = null;
        long start = System.currentTimeMillis();
        if (sampleElement != null) {
            nearestNeighbours = ocUtil.findNearestNeighbours(question, list);
        }
        long end = System.currentTimeMillis();
        logger.info("Found " + nearestNeighbours.size() + " NNs for "
                        + (end - start) + "ms.");
        key.setNearestNeighbours(nearestNeighbours);
        List<Key> generatedKeys =
                        generateLearningKey(key);
        List<org.freya.model.learning.Vote> learningVotes =
                        findLearningVotes(generatedKeys);
        pair.setKey(key);
        start = System.currentTimeMillis();
        List<Vote> suggestionPairVotes = getInitialVotes(list);
        end = System.currentTimeMillis();
        logger.info("getInitialVotes for " + (end - start) + "ms.");
        // ///////////////////////////////////////////////////////////////////////
        // ovde sada call find suggestions isto kao za poc
        if (forceDialog) {
            start = System.currentTimeMillis();
            suggestionPairVotes.addAll(findAdditionalVotesAsIfThisOCwereAPOC(
                            question, sampleElement, key, list));
            end = System.currentTimeMillis();
            logger.info("findAdditionalVotesAsIfThisOCwereAPOC for " + (end - start)
                            + "ms.");
        }
        // /////////////////////////////////////////////////////////////////////////
        start = System.currentTimeMillis();
        if (learningVotes != null && learningVotes.size() > 0) {
            // take it from the learning model
            suggestionPairVotes =
                            learningModelHelper.getVotesFromLearningVotes(learningVotes, suggestionPairVotes);
            pair.setVote(suggestionPairVotes);
        } else {
            // nothing is found in the learning model
            // use initialized vote and add it to the learning model
            pair.setVote(suggestionPairVotes);
            // save for each element which needs to be disambiguated one
            // learningKey
            for (org.freya.model.learning.Key learningKey : generatedKeys) {
                List<org.freya.model.learning.Vote> lightVotes =
                                learningModelHelper.getLearningVotesFromVotes(suggestionPairVotes);
                learningModel.getVotesMap().put(learningKey.toString(), lightVotes);
            }
        }
        end = System.currentTimeMillis();
        logger.info("gedting votes from the learning model for " + (end - start)
                        + "ms.");
        return pair;
    }

    /**
     * generates additional votes and updates question.getSemanticConcepts so that addiiotnal elemenst are treated as
     * OCs if they are selected
     * 
     * @param question
     * @param sampleElement
     * @param key
     * @return
     */
    List<Vote> findAdditionalVotesAsIfThisOCwereAPOC(Question question,
                    SemanticConcept sampleElement, SuggestionKey key,
                    List<SemanticConcept> existingList) {
        List<Vote> suggestionPairVotes = new ArrayList<Vote>();
        POC poc = new POC();
        Annotation newAnn = new Annotation();
        newAnn
                        .setText(sampleElement.getOntologyElement().getAnnotation()
                                        .getText());
        newAnn.setStartOffset(sampleElement.getOntologyElement().getAnnotation()
                        .getStartOffset());
        newAnn.setEndOffset(sampleElement.getOntologyElement().getAnnotation()
                        .getEndOffset());
        newAnn.setSyntaxTree(sampleElement.getOntologyElement().getAnnotation()
                        .getSyntaxTree());
        poc.setAnnotation(newAnn);
        List<Vote> pocVotes = new ArrayList<Vote>();
        boolean addNone = false;
        int max = 100000;
        if (key.getNearestNeighbours().size() > 0) {
            long start = System.currentTimeMillis();
            pocVotes.addAll(suggestionsCreator.generateVotes(key, poc, addNone,
                            existingList));
            long end = System.currentTimeMillis();
            logger.info("Generate Votes: " + (end - start) + "ms.");
        } else {
            long start = System.currentTimeMillis();
            pocVotes.addAll(suggestionsCreator.generateGenericVotes(key, poc,
                            addNone, max, existingList));
            long end = System.currentTimeMillis();
            logger.info("generateGenericVotes for " + (end - start) + "ms.");
        }
        logger.debug("Set max property size to " + max);
        long start = System.currentTimeMillis();
        List<SemanticConcept> candidates = new ArrayList<SemanticConcept>();
        for (Vote vote : pocVotes) {
            if (!existingList.contains(vote.getCandidate())
                            || !(vote.getCandidate().getOntologyElement() instanceof NoneElement)) {
                // transfer the score
                vote.getCandidate().setScore(vote.getVote());
                // logger
                // .info("Setting vote:"
                // + vote.getVote()
                // + " for "
                // + vote.getCandidate().getOntologyElement().getData()
                // .toString());
                candidates.add(vote.getCandidate());
            }
        }
        suggestionPairVotes.addAll(getInitialVotes(candidates));
        long end = System.currentTimeMillis();
        logger.debug("Generating candidate elements from pocVotes for "
                        + (end - start) + "ms.");
        logger.info("There were:" + existingList.size()
                        + " elements, and we now add " + candidates.size() + " more.");
        existingList.addAll(candidates);
        logger.info("After adding total suggestions for oc dialog: "
                        + existingList.size());
        return suggestionPairVotes;
    }



    /**
     * finds candidate based on the identifier which is either URI or DatatypePropertyValueIdentifier
     * 
     * @param vote
     * @param list
     * @return
     */
    SemanticConcept findCandidate(org.freya.model.learning.Vote vote,
                    List<SemanticConcept> list) {
        SemanticConcept sc = null;
        for (SemanticConcept sConcept : list) {
            if (vote.getIdentifier().equals(sConcept.getOntologyElement().getData())) {
                // bingo
                return sConcept;
            }
        }
        return sc;
    }

    /**
     * generate initial votes for the semantic elements by setting scores to some initial value
     * 
     * @param elements
     * @return
     */
    List<Vote> getInitialVotes(List<SemanticConcept> elements) {
        List<Vote> votes = new ArrayList<Vote>();
        // 1.0 because we want it before any poc suggestion
        double initialVoteScore = 1.0;
        for (SemanticConcept element : elements) {
            Vote vote = new Vote();
            long id = incrementer.incrementAndGet();
            vote.setId(id);
            SemanticConcept clonedElement = (SemanticConcept) element.clone();
            vote.setCandidate(clonedElement);
            if (element.getScore() != null)
                initialVoteScore = element.getScore().doubleValue();
            vote.setVote(initialVoteScore);
            votes.add(vote);
        }
        return votes;
    }

    /**
     * generate dialog for mapping poc to oc
     * 
     * @param question
     * @return
     */
    public SuggestionPair generateMappingPocToOcDialog(Question question) {
        SuggestionPair pair = new SuggestionPair();
        SuggestionKey key = new SuggestionKey();
        logger.info("Generating dialog with POCs (" + question.getPocs().size()
                        + " POCs)");
        // which poc? start with the one which is closest to the
        // existing semantic concept
        POC poc = whichPoc(question);
        logger.info("Decided to use " + poc.getAnnotation().getText()
                        + " for the dialog");
        key.setText(poc.getAnnotation().getText());
        List<SemanticConcept> nearestNeighbours =
                        ocUtil.findNearestNeighbours(question, poc);
        logger.info("Found " + nearestNeighbours.size() + " neighbours.");
        key.setNearestNeighbours(nearestNeighbours);
        pair.setCurrentDialogSubject(poc);
        List<org.freya.model.learning.Key> generatedKeys =
                        generateLearningKey(key);
        List<org.freya.model.learning.Vote> learningVotes =
                        findLearningVotes(generatedKeys);
        List<Vote> votes = null;
        if (nearestNeighbours.size() > 0)
            votes = suggestionsCreator.generateVotes(key, poc);
        else
            votes = suggestionsCreator.generateGenericVotes(key, poc);
        pair.setKey(key);
        if (learningVotes != null) {
            List<Vote> suggestionPairVotes =
                            learningModelHelper.getVotesFromLearningVotes(learningVotes, votes);
            pair.setVote(suggestionPairVotes);
        } else {
            // generate it and update the learning model
            pair.setVote(votes);
            for (org.freya.model.learning.Key learningKey : generatedKeys) {
                List<org.freya.model.learning.Vote> lightVotes =
                                learningModelHelper.getLearningVotesFromVotes(votes);
                learningModel.getVotesMap().put(learningKey.toString(), lightVotes);
            }
        }
        VoteComparator c = new VoteComparator();
        Collections.sort(pair.getVote(), c);
        return pair;
    }

    /**
     * this method identifies which poc should be used first for the dialog: it iterates through all ocs and then finds
     * the poc which is closest to its nearestNeighbours
     * 
     * @param question
     * @return
     */
    public POC whichPoc(Question question) {
        POC poc = ocUtil.whichPOC(question);
        return poc;
    }

    /**
     * @param question
     */
    public List<SemanticConcept> whichOC(Question question) {
        return ocUtil.whichOC(question);
    }

    /**
     * finds the semanticConcept which is equal to dialogSubject and makes the map with key=index of the column in the
     * table and the value=list with only dialog Subject which will not be removed
     * 
     * @param table
     * @param dialogSubject
     * @return
     */
    public HashMap<Integer, List<SemanticConcept>> findSemanticConceptsToRemove(
                    List<List<SemanticConcept>> table, List<SemanticConcept> candidates) {
        HashMap<Integer, List<SemanticConcept>> toRemove =
                        new HashMap<Integer, List<SemanticConcept>>();
        for (SemanticConcept candidate : candidates) {
            for (List<SemanticConcept> column : table) {
                for (SemanticConcept cell : column) {
                    if ((candidate.getOntologyElement() instanceof NoneElement && candidate
                                    .getOntologyElement().equals(cell.getOntologyElement()))
                                    || (!(candidate.getOntologyElement() instanceof NoneElement) && candidate
                                                    .getOntologyElement().getData().toString().equals(
                                                                    cell.getOntologyElement().getData().toString()))) {
                        // mark only the one that will not be removed
                        List<SemanticConcept> list = toRemove.get(new Integer(table.indexOf(column)));
                        if (list == null)
                            list = new ArrayList<SemanticConcept>();
                        // list.addAll(column);
                        // list.remove(cell);
                        list.add(cell);
                        logger.info("Marked " + (column.size() - 1)
                                        + " elements for removal. Thread:"
                                        + Thread.currentThread().getName());
                        toRemove.put(new Integer(table.indexOf(column)), list);
                    }
                }
            }
        }
        return toRemove;
    }


    /**
     * this method generates learning key based on which it is possible to search the learning model
     * 
     * @param key
     * @return
     */
    List<org.freya.model.learning.Key> generateLearningKey(SuggestionKey key) {
        List<org.freya.model.learning.Key> lk =
                        new ArrayList<org.freya.model.learning.Key>();
        for (SemanticConcept sc : key.getNearestNeighbours()) {
            org.freya.model.learning.Key learningKey =
                            new org.freya.model.learning.Key();
            learningKey.setText(key.getText());
            Object genericElement = learningModelHelper.findGenericElement(sc.getOntologyElement());
            learningKey.setOntologyElementIdentifier(genericElement);
            lk.add(learningKey);
        }
        if (key.getNearestNeighbours() == null
                        || key.getNearestNeighbours().size() == 0) {
            org.freya.model.learning.Key learningKey =
                            new org.freya.model.learning.Key();
            learningKey.setText(key.getText());
            learningKey
                            .setOntologyElementIdentifier(FreyaConstants.NONE_NEIGHBOURING_ONTOLOGY_ELEMENT);
            lk.add(learningKey);
        }
        return lk;
    }

    /**
     * searches the learning model to find existing key and votes if they exist
     * 
     * @return
     */
    List<org.freya.model.learning.Vote> findLearningVotes(
                    List<org.freya.model.learning.Key> generatedKeys) {
        List<org.freya.model.learning.Vote> learningVotes =
                        new ArrayList<org.freya.model.learning.Vote>();
        for (org.freya.model.learning.Key k : generatedKeys) {
            List<org.freya.model.learning.Vote> theVotes =
                            ((Map<String, List<org.freya.model.learning.Vote>>) learningModel
                                            .getVotesMap()).get(k.toString());
            Integer num = null;
            if (theVotes != null) num = theVotes.size();
            logger.debug("Found " + num + " votes for " + k.toString());
            if (theVotes != null) learningVotes.addAll(theVotes);
        }
        return learningVotes;
    }
}
 @Component
public class JsonCreator {
    private static final Log logger = LogFactory.getLog(JsonCreator.class);
    @Autowired GraphUtils graphUtils;
    @Autowired LuceneAnnotator luceneAnnotator;


    public JsonCreator() {}

    /**
     * generates json string from list of json suggestions
     * 
     * @param question
     * @return
     */
    public ObjectNode fromSuggestionsToJson(SuggestionPair pair) {
        List<JsonSuggestion> jsonSuggestions = generateJsonSuggestions(pair);
        ObjectNode topNode = JsonNodeFactory.instance.objectNode();
        ArrayNode clarificationNodes = JsonNodeFactory.instance.arrayNode();
        for (JsonSuggestion suggestion : jsonSuggestions) {
            ObjectNode node = JsonNodeFactory.instance.objectNode();
            node.put("stringToClarify", suggestion.getText());
            // here beautify the key and add label not uri
            ArrayNode optionsNode = JsonNodeFactory.instance.arrayNode();
            List<JsonVote> list = new ArrayList<JsonVote>(suggestion.getVotes());
            for (JsonVote jsonVote : list) {
                ObjectNode userVoteNode = JsonNodeFactory.instance.objectNode();
                userVoteNode.put("id", jsonVote.getId());
                userVoteNode.put("option", jsonVote.getCandidate());
                userVoteNode.put("vote", jsonVote.getVote());
                String function = "-";
                if (jsonVote.getFunction() != null) {
                    function = jsonVote.getFunction();
                }
                userVoteNode.put("function", function);
                optionsNode.add(userVoteNode);
            }
            node.put("clarificationOptions", optionsNode);
            clarificationNodes.add(node);
        }
        topNode.put("clarifications", clarificationNodes);
        return topNode;// topNode.toString();
    }

    @Value("${org.freya.use.labels}") String useLabels;
    @Value("${org.freya.max.suggestions}") String maxSuggestions;

    /**
     * generates json suggestions from suggestions
     * 
     * @param suggestions
     * @return
     */
    public List<JsonSuggestion> generateJsonSuggestions(SuggestionPair pair) {

        List<JsonSuggestion> list = new ArrayList<JsonSuggestion>();
        JsonSuggestion jsonSuggestion = new JsonSuggestion();
        jsonSuggestion.setText(pair.getKey().getText());
        jsonSuggestion.setVotes(new ArrayList<JsonVote>());
        // here beautify the key and add label not uri
        List<Vote> votes = pair.getVote();
        for (Vote vote : votes) {
            // to ensure that java script does not go crazy with 200 000 suggestions
            if (jsonSuggestion.getVotes().size() > new Long(maxSuggestions)) {
                // ovde ne bi trebalo samo ovakoo nego i da se nadje none i da se doda
                break;
            }


            SemanticConcept sc = vote.getCandidate();
            OntologyElement ontologyElement = null;
            if (sc != null) {
                ontologyElement = sc.getOntologyElement();
            } else {
                String v = null;
                if (vote != null) v = vote.toString();
                logger.error("Candidate is null!! How this happened? Vote:" + v);
            }
            String keyLabel = null;// getLabel(uri);
            // change keyLabel based on what is oc as when its instance we want
            // dialog with classes
            keyLabel = findKeyLabel(ontologyElement, new Boolean(useLabels));
            String function = sc.getFunction();
            JsonVote jsonVote =
                            new JsonVote(keyLabel, new Double(vote.getVote()).toString(),
                                            new Long(vote.getId()).toString(), function);
            jsonSuggestion.getVotes().add(jsonVote);
            logger.info("keylabel (shouldn't be null):" + keyLabel);
        }
        list.add(jsonSuggestion);
        // logger.info("jsonSuggestions:" + jsonSuggestion.toString());
        return list;
    }

    /**
     * when there are no interpretations found create relevant json (empty nodes)
     * 
     * @return
     */
    String getNoInterpretationsFoundNode() {
        ObjectNode topNode = JsonNodeFactory.instance.objectNode();
        // ArrayNode interpretations = JsonNodeFactory.instance.arrayNode();
        ArrayNode clarifications = JsonNodeFactory.instance.arrayNode();
        /* this is faking just so that js does not complain */
        // topNode.put("interpretations", interpretations);
        topNode.put("clarifications", clarifications);
        return topNode.toString();
    }

    /**
     * returns suggestion string based on the type of ontology element
     * 
     * @param ontologyElement
     * @param useLabels
     * @return
     */
    String findKeyLabel(OntologyElement ontologyElement, boolean useLabels) {
        String keyLabel = "";
        String uri = (ontologyElement.getData()).toString();
        Set<String> instanceUriLabels = new HashSet<String>();
        if (!useLabels) {
            if (ontologyElement instanceof InstanceElement) {
                List<String> classUris =
                                ((InstanceElement) ontologyElement).getClassURIList();
                StringBuffer classUrisString = new StringBuffer("");
                for (String cUri : classUris) {
                    classUrisString.append(cUri).append("|");
                }
                keyLabel = uri + " (" + classUrisString + ")";
            } else if (ontologyElement instanceof InstanceListElement) {
                List<SerializableURI> instanceUris =
                                ((InstanceListElement) ontologyElement).getData();
                String instanceLabels = "";
                for (SerializableURI instUri : instanceUris) {
                    instanceLabels = instanceLabels + "," + instUri.toString();
                }
                String classUri =
                                ((InstanceListElement) ontologyElement).getClassURI().toString();
                // keyLabel = getLabel(classUri);
                keyLabel = instanceLabels + " (" + classUri + ")";
            } else {
                keyLabel = uri;
            }
        } else if (useLabels) {
            if (ontologyElement instanceof InstanceElement) {
                List<String> classUris =
                                ((InstanceElement) ontologyElement).getClassURIList();
                if (classUris == null) classUris = new ArrayList<String>();
                logger.info("class  LIST for" + uri + " is:\n" + classUris.toString());
                String instanceLabel = getLabels(uri);
                String classesString = "";
                for (String classUri : classUris) {
                    classesString = classesString + getLabels(classUri) + " |OR| ";
                }
                keyLabel = instanceLabel + " (" + getLabels(classesString) + ")";;
            } else if (ontologyElement instanceof InstanceListElement) {
                List<SerializableURI> instanceUris =
                                ((InstanceListElement) ontologyElement).getData();
                String instanceLabels = "";
                for (SerializableURI instUri : instanceUris) {
                    instanceUriLabels.add(getLabels(instUri.toString()));
                }
                List<String> classUris =
                                ((InstanceListElement) ontologyElement).getClassURIList();
                for (String s : instanceUriLabels) {
                    if (s != null && !"".equals(s.trim()))
                        instanceLabels = instanceLabels + s + ",";
                }
                instanceLabels =
                                instanceLabels.substring(0, instanceLabels.lastIndexOf(","));
                // keyLabel = instanceLabels + " (" + getLabels(classUri) + ")";
                String classesString = "";
                for (String classUri : classUris) {
                    classesString = classesString + getLabels(classUri) + " |OR| ";
                }
                keyLabel = instanceLabels + " (" + classesString + ")";
            } else if (ontologyElement instanceof ClassElement
                            || ontologyElement instanceof PropertyElement) {
                keyLabel = uri;
            } else if (ontologyElement instanceof DatatypePropertyValueElement) {
                DatatypePropertyValueElement el =
                                ((DatatypePropertyValueElement) ontologyElement);
                keyLabel =
                                uri;
                // + " ( "
                // + ((DatatypePropertyValueIdentifier)el.getData())
                // .getInstanceURI()
                // + " "
                // + ((DatatypePropertyValueIdentifier)el.getData())
                // .getPropertyUri() + ")";
            } else {
                keyLabel = getLabels(uri);
            }
        }
        return keyLabel;
    }



    /**
     * @param graph
     * @return
     */
    @SuppressWarnings("finally")
    public FreyaResponse createJsonNodeFromResultGraph(List<ResultGraph> aGraph,
                    List<List<String>> table, FreyaResponse response) {
        // ObjectNode topNode = JsonNodeFactory.instance.objectNode();
        //FreyaResponse topNode = new FreyaResponse();
        if (aGraph == null || (aGraph != null && aGraph.size() == 0)) {// result
            // is
            // empty
            // and
            // call
            // some refinement
            return getRefinementNode();
        }
        // ArrayNode nodes = JsonNodeFactory.instance.arrayNode();
        ArrayList<Object> nodes = new ArrayList();
        try {
            // get all labels for the nodes for which we know URIs at the moment
            HashMap nodeNames = getNodeNames(aGraph);
            // then, find out the names for edges
            HashMap propertyNames = getPropertyNames(aGraph, nodeNames);
            // iterate through all nodes in order to find the main subject: this
            // might be stupid for the moment, but cannot figure out the
            // alternative way to find the name of the property which is a main
            // subject
            String propertyHeader = null;
            Integer id = null;
            ResultGraph propertyNode = null;
            List<ResultGraph> bGraph = new ArrayList<ResultGraph>();
            bGraph.addAll(aGraph);
            for (ResultGraph aNode : aGraph) {
                boolean isMSubject = false;
                if (aNode.isMainSubject() != null)
                    isMSubject = aNode.isMainSubject().booleanValue();
                if (isMSubject) {
                    for (ResultGraph bNode : bGraph) {
                        List<Integer> adjs = bNode.getAdjacencies();
                        if (adjs != null && adjs.size() > 0) {
                            if (adjs.contains(aNode.getId())) {
                                id = bNode.getId();
                                logger.debug("main subject is: " + aNode.getURI());
                                propertyNode = aNode;
                                // logger.info("found id for the property node is:"
                                // + id.toString());
                                break;
                            }
                        }
                    }
                }
            }
            // now find the graph with the id above: this will be the node which
            // represents the property which needs to be shown in the answer
            // for (ResultGraph cNode : aGraph) {
            // Integer nodeId = cNode.getId();
            // if (id != null && nodeId.intValue() == id.intValue()) {
            // propertyNode = cNode;
            // logger
            // .info("found node which is a property; that value of this property is a main subject:"
            // + propertyNode.getURI());
            // break;
            // }
            //
            // }
            if (propertyNode != null) {
                propertyHeader = getLabel(propertyNode.getURI());
                // logger
                // .info("now finding the label of the above property...the label is:"
                // + propertyHeader);
            }
            // System.out.print("nodeNames:\n" + nodeNames.toString());
            // update the graph
            List<ResultGraph> newGraph =
                            graphUtils.removePropertiesAsNodesAndOptimizeGraph(aGraph);
            logger.debug("optimized graph (NO properties):" + newGraph.toString());
            logger.debug("optimized graph (NO properties) consistent:"
                            + graphUtils.isGraphConsistent(newGraph));
            for (ResultGraph gNode : newGraph) {
                ObjectNode node = JsonNodeFactory.instance.objectNode();
                String label = getLabel(gNode.getURI());
                node.put("id", gNode.getId());
                String nodeName = label;
                if (CATConstants.FEATURE_TYPE_CLASS.equals(gNode.getType())) {
                    if (gNode.getData().size() > 1)
                        nodeName = nodeName + " (" + gNode.getData().size() + ")";
                }
                String propertyNodeName = label;
                node.put("name", nodeName);
                ObjectNode dataNode = JsonNodeFactory.instance.objectNode();
                ArrayNode answer = JsonNodeFactory.instance.arrayNode();
                ArrayNode blankAnswer = JsonNodeFactory.instance.arrayNode();
                // getData returns list of String (URIs)
                List<String> listOfUris = gNode.getData();
                for (String ch : listOfUris) {
                    answer.add(getLabel(ch));
                }
                dataNode.put("answer", answer);
                Font font = new Font("Arial", Font.PLAIN, 11);
                FontMetrics metrics = new FontMetrics(font) {};
                // Rectangle2D bounds = metrics.getStringBounds(nodeName, null);
                // int widthInPixels = (int)bounds.getWidth();
                // dataNode.put("$width", widthInPixels);
                // dataNode.put("$dim", widthInPixels);
                String type = gNode.getType();
                if (type != null && CATConstants.TYPE_PROPERTY.equals(type)) {
                    dataNode.put("$color", "#9F9595");
                    dataNode.put("$type", "custom-line");
                    // dataNode.put("$type", "square");
                    // just for now comment it out
                    // dataNode.put("$type", "none");
                    node.put("name", propertyNodeName);
                    dataNode.put("answer", blankAnswer);
                    // bounds = metrics.getStringBounds(propertyNodeName, null);
                    // widthInPixels = (int)bounds.getWidth();
                    // dataNode.put("$width", widthInPixels);
                    // dataNode.put("$dim", widthInPixels);
                    // dataNode.put("$color","#FFFFFF");
                } else if (type != null && CATConstants.FEATURE_TYPE_CLASS.equals(type)) {
                    // dataNode.put("$type", "circle");
                    dataNode.put("$color", "#BF7777");
                    // dataNode.put("$dim", widthInPixels);
                } else if (type != null && CATConstants.TYPE_INSTANCE.equals(type)) {
                    // dataNode.put("$type", "star");
                    // dataNode.put("$dim", widthInPixels);
                } else {
                    // dataNode.put("$type", "triangle");// datatype
                    // propertyvalues
                    // if (propertyHeader!=null)
                    // node.put("name", propertyHeader);
                    dataNode.put("$color", "#E0FF5F");
                    // dataNode.put("$dim", widthInPixels);
                }
                boolean isMSubject = false;
                if (gNode.isMainSubject() != null)
                    isMSubject = gNode.isMainSubject().booleanValue();
                dataNode.put("mainSubject", isMSubject);
                node.put("data", dataNode);
                // if (isMSubject && propertyHeader != null) {
                if (propertyHeader != null
                                && CATConstants.TYPE_DATATYPE_PROPERTY_VALUE.equals(gNode
                                                .getType())) {
                    dataNode.put("header", propertyHeader);
                    logger.debug("setting property header: " + propertyHeader);
                    dataNode.put("mainSubject", true);
                }
                ArrayNode newAdjacencies = JsonNodeFactory.instance.arrayNode();
                List<Integer> adjacencies = gNode.getAdjacencies();
                for (Integer adjacency : adjacencies) {
                    // System.out.println("now calling for adjacency:"
                    // + adjacency.toString());
                    ObjectNode adjacencyNode = JsonNodeFactory.instance.objectNode();
                    adjacencyNode.put("nodeTo", adjacency);
                    ObjectNode relationDataNode = JsonNodeFactory.instance.objectNode();
                    ArrayNode fromAndToId = JsonNodeFactory.instance.arrayNode();
                    fromAndToId.add(gNode.getId());
                    fromAndToId.add(adjacency);
                    String labelId =
                                    new Integer(gNode.getId()).toString() + "-"
                                                    + adjacency.toString();
                    // System.out.print("labelId " + labelId);
                    relationDataNode.put("$direction", fromAndToId);
                    String labelText = (String) propertyNames.get(labelId);
                    // System.out.print("\nlabelText " + labelText);
                    if (labelText != null) {
                        relationDataNode.put("labeltext", labelText);
                        if (labelId != null) relationDataNode.put("labelid", labelId);
                        relationDataNode.put("$type", "custom-line");
                    } else {
                        relationDataNode.put("$type", "arrow");
                    }
                    adjacencyNode.put("data", relationDataNode);
                    newAdjacencies.add(adjacencyNode);
                }
                node.put("adjacencies", newAdjacencies);
                nodes.add(node);
            }// for each node in a graph
        } catch (Throwable e) {
            e.printStackTrace();
        } finally {
            List<List<HashMap>> tableResult = generateTable(table);
            response.setTable(tableResult);
            response.setGraph(nodes);
            // topNode.put(FreyaConstants.TABLE, tableResult);
            // topNode.put(FreyaConstants.GRAPH, nodes);
            // String rootnodestring = topNode.toString();
            // return rootnodestring;
            return response;
        }
    }

    /**
     * @param table
     * @return
     */
    List<List<HashMap>> generateTable(List<List<String>> table) {
        // ArrayNode tableResult = JsonNodeFactory.instance.arrayNode();
        List<List<HashMap>> tableResult = new ArrayList<List<HashMap>>();
        for (List<String> row : table) {
            // ArrayNode rowNode = JsonNodeFactory.instance.arrayNode();

            List<HashMap> rowNode = new ArrayList<HashMap>();
            for (String cell : row) {
                // ObjectNode cellNode = JsonNodeFactory.instance.objectNode();
                HashMap map = new HashMap();
                map.put("uri", cell);
                map.put("label", getLabel(cell));
                // cellNode.put("uri", cell);
                // cellNode.put("label", getLabel(cell));
                // rowNode.add(cellNode);
                rowNode.add(map);

            }
            tableResult.add(rowNode);
        }
        return tableResult;
    }

    /**
     * create empty node or add refinements
     * 
     * @return
     */
    public FreyaResponse getRefinementNode() {
        // method
        FreyaResponse response = new FreyaResponse();
        ObjectNode rootNode = JsonNodeFactory.instance.objectNode();
        // ArrayNode rootNodes = JsonNodeFactory.instance.arrayNode();
        List<Object> refinementNodes = new ArrayList<Object>();
        rootNode.put("id", 1);
        String message = "No results found.";// , how
        // about
        // refining your
        // query?";
        rootNode.put("name", message);
        String someRefinementSuggestion = "no refinement suggestions";// "this is refinement suggestion";
        ObjectNode dataNode = JsonNodeFactory.instance.objectNode();
        ArrayNode someRefinementSuggestionArray =
                        JsonNodeFactory.instance.arrayNode();
        someRefinementSuggestionArray.add(someRefinementSuggestion);
        dataNode.put("answer", someRefinementSuggestionArray);
        // dataNode.put("$type", "circle");
        // dataNode.put("$dim", "15");
        // Font font = new Font("Arial", Font.PLAIN, 10);
        /*
         * FontMetrics metrics = new FontMetrics(font) { }; Rectangle2D bounds = metrics.getStringBounds(message, null);
         * int widthInPixels = (int)bounds.getWidth(); dataNode.put("$width", widthInPixels);
         */
        rootNode.put("data", dataNode);
        ArrayNode adjacencies = JsonNodeFactory.instance.arrayNode();
        rootNode.put("adjacencies", adjacencies);
        refinementNodes.add(rootNode);
        
        response.setRefinementNodes(refinementNodes);
        return response;
    }

    /**
     * @param nodeId
     * @param aGraph
     * @return
     */
    private Integer findPrevious(Integer nodeId, List<ResultGraph> aGraph) {
        Integer result = null;
        for (ResultGraph node : aGraph) {
            Integer currentNodeId = node.getId();
            List<Integer> adjs = node.getAdjacencies();
            if (adjs.contains(nodeId)) {
                result = currentNodeId;
                break;
            }
        }
        return result;
    }

    /**
     * returns property names but updated so that when they are removed from the graph label id makes sense so, key is
     * 'node1-node2' and value is the name of the property
     * 
     * @param aGraph
     * @param nodeNames
     * @return
     */
    public HashMap getPropertyNames(List<ResultGraph> aGraph, HashMap nodeNames) {
        HashMap propertyNames = new HashMap();// key='node1-node2'
        for (ResultGraph node : aGraph) {
            Integer nodeId = node.getId();
            if (CATConstants.TYPE_PROPERTY.equals(node.getType())) {
                List<Integer> adjs = node.getAdjacencies();
                for (Integer adj : adjs) {
                    int IdOfPreviousElement = findPrevious(nodeId, aGraph);
                    String labelIds = IdOfPreviousElement + "-" + adj;
                    propertyNames.put(labelIds, nodeNames.get(nodeId));
                }
                // System.out.print("propertyNames:\n" +
                // propertyNames.toString());
            }
            // for node id - adj id pair: store the property name
        }
        return propertyNames;
    }

    /**
     * @param aGraph
     * @param mapOfLabels
     * @return
     */
    public HashMap getNodeNames(List<ResultGraph> aGraph) {
        if (aGraph == null) return null;
        HashMap nodeNames = new HashMap();
        for (ResultGraph node : aGraph) {
            Integer nodeId = node.getId();
            nodeNames.put(nodeId, getLabel(node.getURI()));
        }
        return nodeNames;
    }

    /**
     * This method gets label randomly from the mapOfLabels; this method exist in queryUtils??????????
     * 
     * @param uri
     * @param mapOfLabels
     * @return
     */
    public String getLabel(String uriString) {
        if (uriString == null) return "null";
        SerializableURI uri = null;
        try {
            uri = new SerializableURI(uriString, false);
            String label = null;
            // logger
            // .info("to do: proveri da li je csv of label uris set and if yes use it!!!!!!!!");
            List<String> labels =
                            luceneAnnotator.findLabels(uriString);
            if (labels == null || labels.size() == 0) {
                String propertyURI = "http://purl.org/dc/elements/1.1/title";
                labels =
                                luceneAnnotator.findLiteral(uriString,
                                                propertyURI);
            }
            for (String l : labels) {
                label = l;
                if (l == null)
                    continue;
                else if (l.contains(" ")) return l;
            }
            if (label != null)
                return StringUtil.beautifyString(label);
            else if (label == null)
                return StringUtil.beautifyString(uri.getResourceName());
            else
                return null;
        } catch (Exception ie) {
            return StringUtil.beautifyString(uriString);
        }
    }

    /**
     * returns all existing labels
     * 
     * @param uriString
     * @return
     */
    public String getLabels(String uriString) {
        if (uriString == null) return "null";
        SerializableURI uri = null;
        try {
            uri = new SerializableURI(uriString, false);
            logger
                            .info("to do: proveri da li je csv of label uris set and if yes use it!!!!!!!!");
            List<String> labels =
                            luceneAnnotator.findLabels(uriString);
            if (labels == null || labels.size() == 0) {
                String propertyURI = "http://purl.org/dc/elements/1.1/title";
                labels =
                                luceneAnnotator.findLiteral(uriString,
                                                propertyURI);
            }
            if (labels == null || labels.size() == 0)
                return StringUtil.beautifyString(uri.getResourceName());
            else {
                StringBuffer labelsString = new StringBuffer("");
                for (String thisLabel : labels) {
                    labelsString.append(thisLabel).append(" OR ");
                }
                int endIndex = labelsString.toString().lastIndexOf(" OR ");
                logger.info("Before cutting:" + labelsString.toString());
                return labelsString.toString().substring(0, endIndex);
            }
        } catch (Exception ie) {
            return StringUtil.beautifyString(uriString);
        }
    }
}
 @Component
public class LearningModelHelper {
    protected final Log logger = LogFactory.getLog(getClass());
    
    @Autowired LearningModel learningModel;

    @Autowired LuceneAnnotator luceneAnnotator;


    /**
     * @param pair
     */
    public void updateTheLearningModel(SuggestionPair pair) {
        List<Vote> votes = pair.getVote();
        for (SemanticConcept sc : pair.getKey().getNearestNeighbours()) {
            Key learningKey = new Key();
            learningKey.setText(pair.getKey().getText());
            Object genericElement =
                            findGenericElement(sc.getOntologyElement());
            learningKey.setOntologyElementIdentifier(genericElement);
            List<org.freya.model.learning.Vote> lightVotes =
                            getLearningVotesFromVotes(votes);
            // logger.debug("Updating the learning model (in memory). ");
            learningModel.getVotesMap().put(
                            learningKey.toString(), lightVotes);
        }
        if (pair.getKey().getNearestNeighbours().size() == 0) {
            // still need to generate key and votes
            Key key = new Key();
            key.setText(pair.getKey().getText());
            key
                            .setOntologyElementIdentifier(FreyaConstants.NONE_NEIGHBOURING_ONTOLOGY_ELEMENT);
            List<org.freya.model.learning.Vote> lightVotes =
                            getLearningVotesFromVotes(votes);
            learningModel.getVotesMap().put(key.toString(),
                            lightVotes);
        }
    }

    /**
     * @param element
     * @return
     */
    public Object findGenericElement(OntologyElement element) {
        Object genericElement = null;
        // save class if it is instance
        String classURI = null;
        String uri = null;
        if (element instanceof InstanceElement) {
            classURI = ((InstanceElement) element).getClassURI().toString();
            uri = ((InstanceElement) element).getData().toString();
        } else if (element instanceof InstanceListElement) {
            classURI = ((InstanceListElement) element).getClassURI().toString();
            uri = ((InstanceListElement) element).getData().toString();
        } else if (element instanceof DatatypePropertyValueElement) {
            DatatypePropertyValueIdentifier dtpv =
                            (DatatypePropertyValueIdentifier) element.getData();
            List<SerializableURI> uris = dtpv.getInstanceURIs();
            String sampleUri = uris.get(0).toString();
            classURI = luceneAnnotator.findOneDirectType(sampleUri);
            uri = dtpv.getInstanceURIs().get(0).toString();
        }
        if (element instanceof InstanceElement
                        || element instanceof InstanceListElement
                        || element instanceof DatatypePropertyValueElement) {
            if (classURI != null)
                genericElement = classURI;
            else
                genericElement = uri;
            // else genericElement=FreyaConstants.INSTANCE_WITHOUT_DIRECT_CLASS;
            // TODO find the top classes
        } else
            genericElement = element.getData().toString();
        return genericElement;
    }

    /**
     * @param votes
     * @return
     */
    public List<org.freya.model.learning.Vote> getLearningVotesFromVotes(
                    List<Vote> votes) {
        List<org.freya.model.learning.Vote> lightVotes =
                        new ArrayList<org.freya.model.learning.Vote>();
        for (Vote voteAgain : votes) {
            org.freya.model.learning.Vote newVote =
                            new org.freya.model.learning.Vote();
            newVote.setFunction(voteAgain.getCandidate().getFunction());
            newVote.setId(voteAgain.getId());
            newVote.setIdentifier(voteAgain.getCandidate().getOntologyElement()
                            .getData());
            newVote.setScore(voteAgain.getVote());
            lightVotes.add(newVote);
        }
        return lightVotes;
    }

    /**
     * updates the heavy votes with the scores from the learning ones
     * 
     * @param allVotes
     * @param list
     * @return
     */
    public List<Vote> getVotesFromLearningVotes(
                    List<org.freya.model.learning.Vote> learningVotes,
                    List<Vote> oldVotes) {
        for (org.freya.model.learning.Vote v : learningVotes) {
            // find vote in oldVotes and update the score
            for (Vote oldVote : oldVotes) {
                if (v.getIdentifier().equals(
                                oldVote.getCandidate().getOntologyElement().getData())) {
                    if ((v.getFunction() != null && v.getFunction().equals(
                                    oldVote.getCandidate().getFunction()))
                                    || (v.getFunction() == null || "null".equals(v.getFunction()))) {
                        oldVote.setVote(v.getScore());
                        // logger.debug("Updating the vote's score from:" +
                        // oldVote.getVote()
                        // + " to:" + v.getScore());
                    }
                }
            }
        }
        return oldVotes;
    }

    /**
     * generates 'heavy' votes from the learning onesdddd
     * 
     * @param learningVotes
     * @param sConcepts
     * @return
     */
    List<Vote> getOCVotesFromLearningVotes(
                    List<org.freya.model.learning.Vote> learningVotes,
                    List<SemanticConcept> sConcepts) {
        Iterator iterator = sConcepts.iterator();
        List<Vote> votes = new ArrayList<Vote>();
        for (org.freya.model.learning.Vote v : learningVotes) {
            // find vote in oldVotes and update the score
            Vote vote = new Vote();
            if (iterator.hasNext())
                vote.setCandidate((SemanticConcept) iterator.next());
            vote.setId(v.getId());
            vote.setVote(v.getScore());
            votes.add(vote);
        }
        return votes;
    }
} @Component
public class Consolidator {
    @Autowired OCUtil ocUtil;


    Log logger = LogFactory.getLog(Consolidator.class);

    /**
     * find the answer type
     * 
     * @param question
     * @return
     */
    public Question consolidateTheAnswerType(Question question) {
        POC focus = question.getFocus();
        POC headOfTheFocus = null;
        if (focus != null) headOfTheFocus = focus.getHead();
        List<OntologyElement> foc = new ArrayList<OntologyElement>();
        try {
            List<List<SemanticConcept>> sConceptsTable = question.getSemanticConcepts();
            SemanticConceptListComparator scc = new SemanticConceptListComparator();
            Collections.sort(sConceptsTable, scc);
            for (List<SemanticConcept> columnConcept : sConceptsTable) {
                if (columnConcept.get(0).getOntologyElement() instanceof NoneElement) {
                    continue;
                } else {
                    for (SemanticConcept concept : columnConcept) {
                        foc.add(concept.getOntologyElement());
                    }
                    break;
                }
            }
            if (focus != null)
                logger.info("Head of the focus:" + headOfTheFocus.getAnnotation().toString());
            OntologyElement focSample = null;
            if (foc != null && foc.size() > 0) {
                focSample = foc.get(0);
                logger.info("FOC:" + focSample.getAnnotation().toString());
            } else
                logger.info("No FOC found!!!!!!!!!!!!!!!!!!!!");
            List<OntologyElement> answerType = new ArrayList<OntologyElement>();
            // perfect match: any oc (not foc!) and head of the focus overlap
            for (List<SemanticConcept> columnConcept : sConceptsTable) {
                if (!(columnConcept.get(0).getOntologyElement() instanceof NoneElement)
                                && focus != null
                                && headOfTheFocus.getAnnotation().equals(
                                                columnConcept.get(0).getOntologyElement()
                                                                .getAnnotation())) {
                    for (SemanticConcept concept : columnConcept) {
                        answerType.add(concept.getOntologyElement());
                        concept.getOntologyElement().setMainSubject(true);
                    }
                    question.setAnswerType(answerType);
                    logger.info("Perfect match: OC and head of the focus!");
                    break;
                }
            }
            if (answerType.size() == 0) {
                // if(headOfTheFocus != null
                // && headOfTheFocus.getAnnotation().equals(
                // focSample.getAnnotation())) {
                // answerType = foc;
                // for(OntologyElement focElement : foc) {
                // focElement.setMainSubject(true);
                // }
                // question.setAnswerType(answerType);
                // } else
                if (focus != null
                                && focus.getAnnotation() != null
                                && focSample != null
                                && focSample.getAnnotation() != null
                                && focus.getHead() != null
                                && focSample.getAnnotation().getStartOffset() >= focus.getHead()
                                                .getAnnotation().getEndOffset()) {
                    answerType = foc;
                    for (OntologyElement focElement : foc) {
                        focElement.setMainSubject(true);
                    }
                    question.setAnswerType(answerType);
                    logger.info("FOC is after the focus...needs dialog to identify the answer type...");
                } else if (focus != null
                                && focus.getAnnotation() != null
                                && focSample != null
                                && focus.getHead() != null
                                && focus.getHead().getAnnotation() != null
                                && focSample.getAnnotation().getEndOffset() <= focus.getHead()
                                                .getAnnotation().getStartOffset()) {
                    // if the foc is modifier of the existing element (is of type
                    // property and hasGovernor!=null then the answer type is not
                    // changed!!!)
                    if (foc instanceof PropertyElement
                                    && ((PropertyElement) foc).getGovernor() != null) {
                        // do not change the answer type as this is property.....!!!
                        logger.info("FOC is before the head of the focus HOWEVER it is a property with governor!=null and therefore it cannot become the answer type!");
                    } else {
                        answerType = foc;
                        for (OntologyElement focElement : foc) {
                            focElement.setMainSubject(true);
                        }
                        question.setAnswerType(answerType);
                        logger.info("FOC is before the head of the focus (and it is not property with governor!=null) and thefefore it becomes the answer type!");
                    }
                }
            }

            if (question.getAnswerType() == null) question.setAnswerType(foc);

        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        // question = removeOntologyConceptsBasedOnMainSubjectPriority(question);
        return question;
    }

    /**
     * This method checks if mainsubject's priority is max, if yes, remove ontology concepts which are overlapped
     * 
     * @param query
     * @param mainSubject
     * @return
     */
    public Question removeOntologyConceptsBasedOnMainSubjectPriority(Question question) {
        POC focus = question.getFocus();
        String focusString = null;
        if (focus != null && focus.getAnnotation().getSyntaxTree() != null)
            focusString = focus.getAnnotation().getSyntaxTree().toString();
        logger.info("Question focus:" + focusString);
        List<List<SemanticConcept>> sConceptsTable = null;
        try {
            sConceptsTable = question.getSemanticConcepts();
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        // this map holds key: column and value the row of the element in the
        // selected column
        Map<Integer, Integer> toRemove = new HashMap<Integer, Integer>();
        for (List<SemanticConcept> sConceptList : sConceptsTable) {
            for (SemanticConcept sConcept : sConceptList) {
                if (focus != null
                                && focus.getMainSubject() != null
                                && focus.getMainSubject().getPriority() != null
                                && focus.getMainSubject().getPriority().intValue() == FreyaConstants.MAIN_SUBJECT_PRIORITY_MAX) {
                    logger.debug("Main subject has max priority, removing ontology element (probably question of type how long, where, when, who)...");
                    // remove ontology concept which is overlapped with the main
                    // Subject
                    OntologyElement el = sConcept.getOntologyElement();
                    if (overlaps(focus, el)) {
                        logger.debug("bingo bingo: main subject overlaps with oc");
                        int col = sConceptsTable.indexOf(sConceptList);
                        int row = sConceptList.indexOf(sConcept);
                        toRemove.put(new Integer(col), new Integer(row));
                        // logger.info("col:" + col + " row: " + row);
                    }
                } else {
                    // logger.info("focus was null...");
                }
            }
        }
        Integer col = null;
        Iterator it = toRemove.keySet().iterator();
        if (it.hasNext()) col = (Integer) it.next();
        Integer row = null;
        if (col != null) row = toRemove.get(col);
        if (row != null) {
            // here we need to remove all raws within the col because it
            // overalps with the main subject we delete all!
            sConceptsTable.remove(col.intValue());
            logger.debug("After removing OCs due to priritized main subject:"
                            + sConceptsTable.toString());
        } else {
            logger.debug("No OCs removed after consolidation of prioritised main subject...");
        }
        question.setSemanticConcepts(sConceptsTable);
        return question;
    }

    /**
     * checks whether the main subject overlaps with the ontology concept
     * 
     * @param question
     * @param mainSubject
     * @param el
     * @return
     */
    public boolean overlaps(POC focus, OntologyElement el) {
        if (focus == null || el == null) return false;
        if (focus != null && focus.getAnnotation().getText() == null) return false;
        Long mainSubjectStartOffset = focus.getAnnotation().getStartOffset();
        Long mainSubjectEndOffset = focus.getAnnotation().getEndOffset();
        Long startOffset = el.getAnnotation().getStartOffset();
        Long endOffset = el.getAnnotation().getEndOffset();
        if (mainSubjectStartOffset == null || mainSubjectEndOffset == null || startOffset == null
                        || endOffset == null) {
            // logger.info("an offset is null...which one?");
            return false;
        }
        if (endOffset.longValue() == mainSubjectEndOffset.longValue()
                        && startOffset.longValue() == mainSubjectStartOffset.longValue()) {
            // logger
            // .info("endOffset.longValue() < mainSubjectStartOffset.longValue()");
            return true;
        } else {
            //
            // logger
            // .info("startOffset.longValue() > mainSubjectEndOffset.longValue()");
            return false;
        }
        //
        // logger
        // .info("main subject overlaps with oc***************************************************bingo!!!!");
    }// overlaps
}
 @Component
public class Mapper {
    private static final Log logger = LogFactory.getLog(Mapper.class);

    @Autowired
    private SolrAnnotator solrAnnotator;

    //@Autowired
    //private LuceneAnnotator luceneAnnotator;

    @Autowired
    private RenameAnnotations renameAnnotations;

    @Autowired
    private StanfordExecutor stanfordExecutor;

    @Autowired
    private OCCreator ocCreator;

    @Autowired
    private Consolidator consolidator;

    @Autowired
    private OCUtil ocUtil;

    @Autowired
    private TreeUtils treeUtils;

    /**
     * @param text
     * @param forceDialog
     * @param forceDialogThreshold
     * @param preferLonger
     * @return
     */
    public Question processQuestion(String text, boolean forceDialog,
                    Long forceDialogThreshold, boolean preferLonger) {
        return processQuestion(text, forceDialog, forceDialogThreshold, preferLonger,
                        forceDialog);
    }

    /**
     * this is the initial query processing
     * 
     * @param text
     * @param forceDialog
     * @param forceDialogThreshold
     * @param preferLonger
     * @return
     */
    public Question processQuestion(String text, boolean forceDialog,
                    Long forceDialogThreshold, boolean preferLonger, boolean addNoneToOCDialog) {
        Question question = new Question();
        // get POCs start
        long startTime = System.currentTimeMillis();
        question = stanfordExecutor.findPocs(text);
        question = stanfordExecutor.getStanfordParser().findQuestionType(question);
        long endTime = System.currentTimeMillis();
        logger.info("Found " + question.getPocs().size() + " POCs:" + question.getPocs().toString()
                        + " for " + (endTime - startTime) + " ms.");
        // get POCs end
        // get OCs start
        startTime = System.currentTimeMillis();
        List<Annotation> lookupsList =
                        fromTokensToLuceneLookupAnnotations(question.getTokens(),
                                        question.getSyntaxTree(), preferLonger);


        // filter instance class or class instance
        long start = System.currentTimeMillis();
        Set<Annotation> toRemove = ocUtil.filterInstanceClass(lookupsList);
        logger.info("To remove due to automatic disambiguation:" + toRemove.size());
        /////////////////////////////////////////////////////////////////////////
        ///////////move this somewhere else start ///////////////////////////////
        Set<POC> pocsToRemove = new HashSet<POC>();
        if (toRemove.size() > 0) {
            lookupsList.removeAll(toRemove);
            // also remove POCs with the same startofset, endoffset and string
            List<POC> pocs = question.getPocs();
            for (POC poc : pocs) {
                for (Annotation ocAnnotation : toRemove) {
                    if (poc.getAnnotation().equals(ocAnnotation)) {
                        // if (poc.getAnnotation().overlaps(ocAnnotation)){
                        pocsToRemove.add(poc);
                    } else if (poc.getAnnotation().getStartOffset().longValue() == ocAnnotation
                                    .getStartOffset().longValue()) {
                        pocsToRemove.add(poc);
                    } else if (poc.getAnnotation().getEndOffset().longValue() == ocAnnotation
                                    .getEndOffset().longValue()) {
                        pocsToRemove.add(poc);
                    }
                }
            }

        }
        /////////////////////////////////////////////////////////
        ////////////move this somewhere else end ////////////////

        question.getPocs().removeAll(pocsToRemove);

        long end = System.currentTimeMillis();
        long runtime = end - start;
        logger.info("filterInstanceClass for " + runtime + " ms. There are " + lookupsList.size()
                        + " annotations.");

        // Set<Annotation> lookups = null;
        endTime = System.currentTimeMillis();
        if (lookupsList != null)
            logger.info("Generated " + lookupsList.size() + " lucene lookup annotatioons for "
                            + (endTime - startTime) + " ms.  " + " These are:\n"
                            + lookupsList.toString());
        // get OCs END
        // consolidate POC and OC into Semantic Concepts START
        startTime = System.currentTimeMillis();
        List<List<SemanticConcept>> semanticConcepts =
                        ocCreator.getSemanticConcepts(new HashSet(lookupsList), question);
        endTime = System.currentTimeMillis();
        logger.info("Generated " + semanticConcepts.size()
                        + " column with semantic concepts from Lookup annotatioons for "
                        + (endTime - startTime) + " ms.");
        // consolidate POC and OC into Semantic Concepts END
        // adding now None concept to all
        if (addNoneToOCDialog) {
            semanticConcepts = ocUtil.addNoneToAll(semanticConcepts);
        }
        //logger.info("Finshed adding none.");
        question.setSemanticConcepts(semanticConcepts);
        // remove now wrb, make separate pocs for jjs etc...
        startTime = System.currentTimeMillis();
        question = stanfordExecutor.getStanfordParser().findFocus(question);
        question = stanfordExecutor.getStanfordParser().cleanPocsLucene(question);
        endTime = System.currentTimeMillis();
        logger.info("Cleaned POCs (removed WRBs etc) for " + (endTime - startTime) + " ms.");
        // remove WRBs in how many, which state etc
        question = stanfordExecutor.removePocsToIgnore(question);
        logger.info("Removed POCs to ignore.");
        // check priority of the main subject and remove ocs if they overlap
        // with priritized main subject such as how long;
        startTime = System.currentTimeMillis();
        question = consolidator.removeOntologyConceptsBasedOnMainSubjectPriority(question);
        endTime = System.currentTimeMillis();
        logger.debug("removeOntologyConceptsBasedOnMainSubjectPriority for "
                        + (endTime - startTime) + " ms.");
        // remove POCs if OCs exist over the same span
        startTime = System.currentTimeMillis();
        question = resolvePocsWhichOverlapWithOcs(question);
        logger.info("Current pocs:" + question.getPocs());
        endTime = System.currentTimeMillis();
        logger.info("Resolved POC which overlap with OCs for " + (endTime - startTime) + " ms.");
        return question;
    }

 
    /**
     * checks whether poc spans over oc
     * 
     * @param poc
     * @param oc
     * @return
     */
    private boolean pocSpansOverOC(Annotation poc, Annotation oc) {
        boolean pocSpansOverOC = false;
        if ((oc.getEndOffset().longValue() <= poc.getEndOffset().longValue())
                        && oc.getStartOffset().longValue() >= poc.getStartOffset().longValue()) {
            pocSpansOverOC = true;
        }
        return pocSpansOverOC;
    }

    /**
     * oc spans over poc
     * 
     * @param poc
     * @param oc
     * @return
     */
    private boolean ocSpansOverPoc(Annotation poc, Annotation oc) {
        boolean ocSpansOverPOC = false;
        if ((poc.getEndOffset().longValue() <= oc.getEndOffset().longValue())
                        && poc.getStartOffset().longValue() >= oc.getStartOffset().longValue()) {
            ocSpansOverPOC = true;
        }
        return ocSpansOverPOC;
    }

    /**
     * this method checks whether the poc overlaps over any ocs and returns the list of list of ocs over which the poc
     * overlaps
     * 
     * @param poc
     * @param table
     * @return
     */
    List<List<SemanticConcept>> pocAndOcOverlap(POC poc, List<List<SemanticConcept>> table) {
        List<List<SemanticConcept>> newTable = new ArrayList<List<SemanticConcept>>();
        long realStartOfThePoc = poc.getAnnotation().getStartOffset().longValue();
        long realEndOfThePoc = poc.getAnnotation().getEndOffset().longValue();
        boolean pocMatched = false;
        for (int i = 0; i < table.size(); i++) {
            List<SemanticConcept> column = table.get(i);
            List<SemanticConcept> newColumn = new ArrayList<SemanticConcept>();
            for (int j = 0; j < column.size(); j++) {
                SemanticConcept sConcept = column.get(j);
                // here get the first one, it does not really matter as start offset and
                // endoffset should be equal
                long realStartOfTheOC =
                                sConcept.getOntologyElement().getAnnotation().getStartOffset()
                                                .longValue();
                long realEndOfTheOC =
                                sConcept.getOntologyElement().getAnnotation().getEndOffset()
                                                .longValue();
                // logger.debug("Started checking OC:"
                // + sConcept.getOntologyElement().getAnnotation());
                if (realStartOfThePoc == realStartOfTheOC) {
                    // logger.debug("The start offset of OC matches with that of POC.");
                    if (!newColumn.contains(sConcept)) newColumn.add(sConcept);
                    if (realEndOfThePoc == realEndOfTheOC) {
                        // logger.debug(" The end is also matched...(OC within the POC)...");
                        pocMatched = true;
                        break;
                    } else {
                        // take all remaining elements and iterate through them
                        // in order to check whether the end of one of them
                        // matches with that of poc
                        // logger
                        // .debug("The end was NOT matched; iterating through the remaining elements in the table...");
                        List<List<SemanticConcept>> remainingTable =
                                        new ArrayList<List<SemanticConcept>>();
                        for (int k = i + 1; k < table.size(); k++) {
                            remainingTable.add(table.get(k));
                        }
                        boolean matched = false;
                        for (List<SemanticConcept> col : remainingTable) {
                            List<SemanticConcept> nextColumn = new ArrayList<SemanticConcept>();
                            for (SemanticConcept sc : col) {
                                if (!nextColumn.contains(sc)) nextColumn.add(sc);
                                long endOfTheOC =
                                                sc.getOntologyElement().getAnnotation()
                                                                .getEndOffset().longValue();
                                if (realEndOfThePoc == endOfTheOC) {
                                    // logger.debug("Found OC within the POC.");
                                    matched = true;
                                    pocMatched = true;
                                    break;
                                }
                            }
                            if (nextColumn.size() > 0) newTable.add(nextColumn);
                            // logger
                            // .debug("Adding next column to the list of verified OC elements:"
                            // + nextColumn.toString());
                            if (matched) break;
                        }
                        break;
                    }
                } else {
                    continue;
                }
            }
            if (newColumn.size() > 0) newTable.add(newColumn);
            // logger.debug("Adding this column with verified ocs:"
            // + newColumn.toString());
            if (pocMatched) break;
        }
        return newTable;
    }

    /**
     * lucene checks whether poc overlaps with oc if yes, sConcept.verified=true and poc is removed from the list this
     * method also adds tree to the ontology concepts!!! TODO: find trees for ontology concepts!! it also assigns a main
     * subject from the poc to the semantic concept
     * 
     * @param ocs
     * @return
     */
    public Question resolvePocsWhichOverlapWithOcs(Question question) {
        logger.debug("Checking whether any POC is overlaped with OC so that we can remove it...");
        List<List<SemanticConcept>> table = question.getSemanticConcepts();
        // check here which pocs overlap with ocs
        Set<POC> pocsToRemove = new HashSet<POC>();
        Set<POC> pocsToAdd = new HashSet<POC>();
        List<POC> pocs = question.getPocs();
        // logger.debug(pocs.size() + " pocs...starting verification...");
        for (POC poc : pocs) {
            // logger.debug("Checking poc:" + poc.toString());
            // logger
            // .debug("Above poc needs to be matched against the following ocs in the table:"
            // + table.toString());
            for (int i = 0; i < table.size(); i++) {
                List<SemanticConcept> sList = table.get(i);
                for (int j = 0; j < sList.size(); j++) {
                    SemanticConcept sConcept = sList.get(j);
                    boolean pocSpansOverOC =
                                    pocSpansOverOC(poc.getAnnotation(), sConcept
                                                    .getOntologyElement().getAnnotation());
                    // logger.debug("pocSpansOverOC:" + pocSpansOverOC);
                    boolean ocSpansOverPOC =
                                    ocSpansOverPoc(poc.getAnnotation(), sConcept
                                                    .getOntologyElement().getAnnotation());
                    // logger.debug("ocSpansOverPOC:" + ocSpansOverPOC);
                    List<List<SemanticConcept>> sConceptsToBeVerifiedTable =
                                    pocAndOcOverlap(poc, table);
                    // logger.debug("sConceptsToBeVerifiedTable:"
                    // + sConceptsToBeVerifiedTable.toString());
                    if ((sConceptsToBeVerifiedTable != null && sConceptsToBeVerifiedTable.size() > 0)
                                    || ocSpansOverPOC) {
                        for (List<SemanticConcept> sConceptsToBeVerified : sConceptsToBeVerifiedTable) {
                            for (SemanticConcept vConcept : sConceptsToBeVerified) {
                                // logger.debug("Verified OC "
                                // + vConcept.getOntologyElement().getData().toString());
                                // vConcept.setVerified(true);
                                // TODO:check if this is ok!!!
                                vConcept.getOntologyElement().getAnnotation()
                                                .setSyntaxTree(poc.getAnnotation().getSyntaxTree());
                                if (poc.getMainSubject() != null) {
                                    vConcept.getOntologyElement().setMainSubject(true);
                                }
                            }
                        }
                        logger.debug("POC marked for removal: "
                                        + poc.getAnnotation().getSyntaxTree());
                        pocsToRemove.add(poc);
                    } else if (pocSpansOverOC) {
                        // change the start and end annotation sets of the
                        // poc
                        // and verify the contained oc
                        // delete old poc
                        if (j > 0) {
                            // dont generate new poc...
                        } else {
                            // add new one
                            POC newPOC = new POC();
                            // TODO: make this more generic
                            // at the moment, we assume that the oc to be
                            // verified
                            // is always on the right hand side
                            newPOC.getAnnotation().setStartOffset(
                                            poc.getAnnotation().getStartOffset());
                            Long newEnd =
                                            sConcept.getOntologyElement().getAnnotation()
                                                            .getStartOffset() - 1;
                            newPOC.getAnnotation().setEndOffset(newEnd);
                            int ocSize =
                                            sConcept.getOntologyElement().getAnnotation().getText()
                                                            .length();
                            logger.debug("ocSize:" + ocSize);
                            int endPoc = poc.getAnnotation().getText().length() - ocSize;
                            logger.debug("endPoc:" + endPoc);
                            String newText = poc.getAnnotation().getText().substring(0, endPoc - 1);
                            newPOC.getAnnotation().setText(newText);
                            // TODO the tree should be
                            // checked!!!!!!!!!!!!!!!!!!!!!!!!!
                            List<Tree> ontologyConceptTrees =
                                            sConcept.getOntologyElement().getAnnotation()
                                                            .getSyntaxTree();
                            List<Tree> syntaxTree = poc.getAnnotation().getSyntaxTree();
                            List<Tree> newTree = new ArrayList<Tree>();
                            for (Tree tr : syntaxTree) {
                                if (ontologyConceptTrees != null
                                                && ontologyConceptTrees.contains(tr)) {
                                    // do nothing
                                } else {
                                    newTree.add(tr);
                                }
                            }
                            newPOC.getAnnotation().setSyntaxTree(newTree);
                            pocsToAdd.add(newPOC);
                            pocsToRemove.add(poc);
                        }
                        // sConcept.setVerified(true);
                        sConcept.getOntologyElement().getAnnotation()
                                        .setSyntaxTree(poc.getAnnotation().getSyntaxTree());
                    }
                }
            }
            // logger.debug("Finished checking POC: " +
            // poc.getAnnotation().getText());
        }
        logger.info("Marked " + pocsToRemove.size()
                        + " POCs to remove (i.e.they overlap with OCs).");
        question.getPocs().removeAll(pocsToRemove);
        question.getPocs().addAll(pocsToAdd);
        return question;
    }

    /**
     * check whether word is in the specified list this should be moved somewhere to the configuration file
     * 
     * @param preTerminal
     * @return
     */
    boolean isInTheIgnoreList(Tree preTerminal) {
        String word = preTerminal.getChildrenAsList().get(0).label().value().toLowerCase();
        logger.debug("Checking:" + word);
        String[] ignoreArray =
        {"do", "does", "did", "are", "were", "was", "is", "has", "have", "give",
                        "show", "list"};
        List<String> ignoreList = Arrays.asList(ignoreArray);
        if (ignoreList.contains(word)) {
            logger.debug(word + " found in the ignore list.");
            return true;
        } else {
            logger.debug(word + " NOT in the ignore list.");
            return false;
        }
    }

    /**
     * @deprecated
     * @param root
     * @param start
     * @param word
     * @return
     */
    Tree locateTheTreeWithThisWord(Tree root, int start, String word) {
        boolean locatedWord = false;
        Tree thisTree = null;
        while (!locatedWord) {
            thisTree = Trees.getTerminal(root, start);
            String treeString = thisTree.label().value();
            logger.info("Tree string:" + treeString + " searching for word:" + word);
            if (treeString.equals(word)) {
                thisTree = Trees.getPreTerminal(root, start);
                logger.info("Found:" + thisTree.toString());
                locatedWord = true;
            } else {
                start++;
            }
        }
        return thisTree;
    }

    /**
     * for phrases such as give me and show me
     * 
     * @param root
     * @param start
     * @param end
     * @return
     */
    boolean checkWhetherThisIsATwoWordToIgnore(Tree root, int start, int end) {
        boolean foundWordToIgnore = false;
        List<Tree> list = new ArrayList();
        for (int i = start; i < end; i++) {
            Tree tree = Trees.getTerminal(root, i);
            list.add(tree);
        }
        String niceString =
                        treeUtils
                                        .getNiceString(list).toLowerCase();
        String[] ignoreTwoWordList = {"give me", "show me"};
        List ignoreList = Arrays.asList(ignoreTwoWordList);
        if (ignoreList.contains(niceString)) foundWordToIgnore = true;
        return foundWordToIgnore;
    }

    /**
     * check whether word should be skipped by lucene annotator
     * 
     * @param root
     * @param start
     * @return
     */
    boolean checkWhetherItShouldBeIgnored(Tree root, int start, Tree thisTree) {
        boolean foundWordToIgnore = false;
        logger.debug("Checking " + thisTree.toString());
        if (FreyaConstants.DT_TAG_TREEBANK.equals(thisTree.label().value()))
            foundWordToIgnore = true;
        else if (thisTree.label().value().startsWith("VB") && isInTheIgnoreList(thisTree))
            foundWordToIgnore = true;
        else if (thisTree.label().value().startsWith("PRP"))
            foundWordToIgnore = true;
        // check for IN used to be here but now is in the phrase part because
        else if (thisTree.getChildrenAsList().get(0).label().value().equals("a"))
            foundWordToIgnore = true;// this is to correct SP when it identifies that
        // 'a' is a foreign word?!
        return foundWordToIgnore;
    }

    /**
     * checks whether word starts with WH or IN and then does not do lookup for these
     * 
     * @param root
     * @param start
     * @return
     */
    boolean checkWhetherThisIsAWHPhrase(Tree root, int start, Tree thisTree) {
        boolean foundWordToIgnore = false;
        // Tree thisTree = locateTheTreeWithThisWord(root,start,word);
        // thisTree.parent() is null; we must supply the root tree in order to find
        // parent
        if (thisTree.parent(root).label().value().startsWith("WH")
                        && !thisTree.label().value().startsWith(FreyaConstants.NN_TAG_TREEBANK)
                        && !thisTree.label().value().startsWith(FreyaConstants.JJ_TAG_TREEBANK))
            foundWordToIgnore = true;
        // also if it starts with IN than this phrase should be skipped
        if (thisTree.label().value().startsWith("IN")) foundWordToIgnore = true;
        return foundWordToIgnore;
    }

    /**
     * this method takes question with tokens and
     *
     * @param tokens
     * @param root
     * @param preferLonger
     * @return a list of Annotations.
     */
    public List<Annotation> fromTokensToLuceneLookupAnnotations(List<HasWord> tokens, Tree root, boolean preferLonger) {
        List<Annotation> lookups = new ArrayList<Annotation>();
        logger.debug("There are this many tokens in the query:" + tokens.size());

        for (int start = 0; start < tokens.size(); start++) {
            Tree thisTree = Trees.getPreTerminal(root, start);
            for (int end = start + 1; end <= tokens.size(); end++) {
                boolean foundWordToIgnore = checkWhetherItShouldBeIgnored(root, start, thisTree);
                boolean foundPhraseToIgnore = checkWhetherThisIsAWHPhrase(root, start, thisTree);
                boolean foundTwoWordsToIgnore = checkWhetherThisIsATwoWordToIgnore(root, start, end);
                if (start == end - 2 && foundTwoWordsToIgnore) {
                    logger.debug("NOT searching solr for this (found in ignore list):" + start + " end:" + end);
                    break;
                } else if (start == end - 1 && foundWordToIgnore) {
                    logger.debug("NOT searching solr for this (found in ignore list):" + thisTree.toString());
                    // do nothing, skip this DT
                } else if (foundPhraseToIgnore) {
                    // do nothing skip this is for words starting with some special words
                    // such as WH phrase eg how many
                    logger.debug("NOT searching solr for this (found in ignore list):" + thisTree.toString());
                } else {
                    // String to be looked up
                    String span = "";
                    Annotation ann = new Annotation();
                    Long startNode = null;
                    Long endNode = null;
                    List<Tree> trees = new ArrayList<Tree>();
                    for (int k = start; k < end; k++) {
                        span = span + " " + tokens.get(k).word();
                        Tree tree = Trees.getPreTerminal(root, k);
                        trees.add(tree);
                    }
                    startNode = new Long(start);
                    endNode = new Long(end);
                    ann.setStartOffset(startNode);
                    ann.setEndOffset(endNode);
                    ann.setText(span.trim());
                    ann.setSyntaxTree(trees);

                    // check here whether this is common noun plural (NNS);
                    // if yes, and if start=end-1 then this should have special treatment:
                    // i.e. sent it to stemmer immediately; if nothing found lowercase;
                    // then exact...
                    boolean specialTreatment = false;
                    if (start == end - 1 && requiresSpecialTreatment(trees)) {
                        specialTreatment = true;
                    }

                    // this is to support all live albums so that we can search for live
                    // albums not all live albums and also use stemming for phrases
                    if (phraseRequiresSpecialTreatment(trees, root)) {
                        specialTreatment = true;
                        // remove DT from the phrase if any
                        // if(trees.get(0).label().value().startsWith("DT")) {
                        // ann.getSyntaxTree().remove(0);
                        // logger.info("After Removing DT:"+trees.toString());
                        // }
                        // remove everything but not JJ or NN
                        // How many live albums did The Rolling Stones release?
                        // Are there any live albums by the Beatles?
                        List<Tree> newTrees = new ArrayList<Tree>();
                        for (Tree t : trees) {
                            if (t.label().value().contains("JJ") && !t.parent(root).label().value().startsWith("WH")) {
                                newTrees.add(t);
                            }
                            if (t.label().value().contains("NN")) {
                                newTrees.add(t);
                            }
                        }
                        ann.setSyntaxTree(newTrees);
                    }

                    //List<Annotation> tmplookups = luceneAnnotator.searchIndex(ann, specialTreatment);
                    List<Annotation> tmplookups;
//                    if (specialTreatment) {
//                        tmplookups = solrAnnotator.searchStemmedContentFirst(ann);
//                    } else {
                        tmplookups = solrAnnotator.searchExactContentFirst(ann);
//                    }
                    lookups.addAll(tmplookups);
                    if (tmplookups.size() > 0) {
                        logger.info("Found " + tmplookups.size() + " entries in the solr index for " + ann.getText());
                    }
                }
            }
        }
        logger.debug("before filtering:" + lookups.size());
        if (preferLonger) {
            lookups = ocUtil.disambiguateBasedOnLength(lookups);
        }
        logger.debug("after filtering:" + lookups.size());
        // rename here
        int before = lookups.size();
        List<Annotation> toReturn = renameAnnotations.rename(lookups);
        int after = lookups.size();
        if (before != after)
            logger.debug("After renaming the number changed from " + before + " to " + after);
        return toReturn;
    }

    /**
     * @param trees
     * @return
     */
    boolean requiresSpecialTreatment(List<Tree> trees) {
        boolean yesorno = false;
        Tree aTree = trees.get(0);
        if (aTree.label().value().equals("NNS")) yesorno = true;
        // some mass nouns can be NN
        if (aTree.label().value().equals("NN")) yesorno = true;
        return yesorno;
    }

    /**
     * if the parent is NP or WHNP
     * 
     * @param trees
     * @return
     */
    boolean phraseRequiresSpecialTreatment(List<Tree> trees, Tree root) {
        boolean yesorno = false;
        Tree aTree = trees.get(0).parent(root);
        logger.info("Parent is:" + aTree.toString());
        if (aTree.label().value().contains("NP"))
            yesorno = true;
        // not always there are prepreterminals see:
        // How many live albums did The Rolling Stones release?
        else if (aTree.parent(root).label().value().contains("NP")) yesorno = true;
        return yesorno;
    }

    /**
     * finds annotations with annotation.getText()
     * 
     * @param annotation
     * @return
     */
    public List<Annotation> findLiteral(Annotation annotation) {
        List<Annotation> anns = solrAnnotator.searchExactContentFirst(annotation);
        logger.info("For " + annotation.getText() + " found:" + anns.toString());
        List toReturn = renameAnnotations.rename(anns);
        logger.info("After renaming these are " + toReturn.toString());
        return toReturn;
    }
}
 public interface OntologyElement extends Serializable {

	public String getFunction();

	public void setFunction(String function);

	public boolean isAlreadyAdded();

	public void setAlreadyAdded(boolean alreadyAdded);

	public Annotation getAnnotation();
	
	public void setAnnotation(Annotation annotation);

	public String getVariable();

	public void setVariable(String key);

	public boolean isMainSubject();

	public void setMainSubject(boolean value);

	public List<String> getResults();

	public void setResults(List<String> results);

	public Object getData();

	public void setData(Object data);

	public void setScore(Score score);

	public Score getScore();

	public boolean isAnswer();

	public void setAnswer(boolean isAnswer);

	public boolean equals(Object o);

}
 public class POC {

	Annotation annotation = new Annotation();

	public Annotation getAnnotation() {
		return annotation;
	}

	public void setAnnotation(Annotation annotation) {
		this.annotation = annotation;
	}

	POC head;

	MainSubject mainSubject;
	List<String> modifiers;

	public POC getHead() {
		return head;
	}

	public void setHead(POC head) {
		this.head = head;
	}

	public List<String> getModifiers() {
		return modifiers;
	}

	public void setModifiers(List<String> modifiers) {
		this.modifiers = modifiers;
	}

	public MainSubject getMainSubject() {
		return mainSubject;
	}

	public void setMainSubject(MainSubject mainSubject) {
		this.mainSubject = mainSubject;
	}

	public String toString() {
		StringBuffer buffer = new StringBuffer("");
		buffer.append("POC:" + getAnnotation().toString());
		return buffer.toString();
	}
}
 @XmlRootElement
public class Question {

    /**
     * question type such as boolean, number, classic (all the others);
     */

    public String sentenceText;
    public Type type;

    public Type getType() {
        return type;
    }

    public void setType(Type type) {
        this.type = type;
    }

    List<HasWord> tokens;

    Tree syntaxTree;

    /* this is focus */
    POC focus;

    List<OntologyElement> answerType;

    List<POC> pocs = new ArrayList<POC>();


    public List<HasWord> getTokens() {
        return tokens;
    }

    public void setTokens(List<HasWord> tokens) {
        this.tokens = tokens;
    }


    List<List<SemanticConcept>> semanticConcepts =
                    new ArrayList<List<SemanticConcept>>();

    public List<OntologyElement> getAnswerType() {
        return answerType;
    }

    public void setAnswerType(List<OntologyElement> answerType) {
        this.answerType = answerType;
    }

    public POC getFocus() {
        return focus;
    }

    public void setFocus(POC focus) {
        this.focus = focus;
    }

    public Tree getSyntaxTree() {
        return syntaxTree;
    }

    public void setSyntaxTree(Tree syntaxTree) {
        this.syntaxTree = syntaxTree;
    }

    public List<List<SemanticConcept>> getSemanticConcepts() {
        return semanticConcepts;
    }

    public void setSemanticConcepts(List<List<SemanticConcept>> semanticConcepts) {
        this.semanticConcepts = semanticConcepts;
    }

    public List<POC> getPocs() {
        return pocs;
    }

    public void setPocs(List<POC> pocs) {
        this.pocs = pocs;
    }

    public String toString() {
        StringBuffer b = new StringBuffer("");
        b.append("sem concepts:");
        if (this.semanticConcepts != null)
            b.append(this.semanticConcepts.toString());
        b.append("pocs:");
        if (this.pocs != null) b.append(pocs.toString());
        b.append("Tree:");
        if (this.syntaxTree != null) b.append(this.syntaxTree.toString());
        return b.toString();
    }
}
 public class ResultGraph {

	List data = new ArrayList();
	String type;
	Integer id;
	private Boolean mainSubject;

	public Boolean isMainSubject() {
		return mainSubject;
	}

	public void setMainSubject(boolean mainSubject) {
		this.mainSubject = mainSubject;
	}

	public Integer getId() {
		return id;
	}

	public void setId(Integer id) {
		this.id = id;
	}

	public List getAdjacencies() {
		return adjacencies;
	}

	public void setAdjacencies(List adjacencies) {
		this.adjacencies = adjacencies;
	}

	List adjacencies = new ArrayList();

	public String getType() {
		return type;
	}

	public void setType(String type) {
		this.type = type;
	}

	public List getData() {
		return data;
	}

	public void setData(List data) {
		this.data = data;
	}

	String URI;

	public String getURI() {
		return URI;
	}

	public void setURI(String uri) {
		URI = uri;
	}

	/**
	 * @see java.lang.Object#toString()
	 */
	public String toString() {
		StringBuffer result = new StringBuffer("URI:").append(this.URI).append(
				" ID:").append(this.id).append(
				" ADJACENCIES:" + this.getAdjacencies()).append("TYPE:")
				.append(this.getType()).append("\n");
		return result.toString();
	}
}
 @XmlRootElement
public class SemanticConcept implements Serializable {

    OntologyElement ontologyElement;
    String function;
    Boolean verified = false;
    Double score;

    public Double getScore() {
        return score;
    }

    public void setScore(Double score) {
        this.score = score;
    }

    public Boolean isVerified() {
        return verified;
    }

    public void setVerified(Boolean verified) {
        this.verified = verified;
    }

    public OntologyElement getOntologyElement() {
        return ontologyElement;
    }

    public void setOntologyElement(OntologyElement ontologyElement) {
        this.ontologyElement = ontologyElement;
    }

    public String getFunction() {
        return function;
    }

    public void setFunction(String function) {
        this.function = function;
    }

    public String toString() {
        StringBuffer b = new StringBuffer("");
        b.append("Ontology Elements:\n");
        if (ontologyElement != null) b.append(ontologyElement.toString());
        b.append(" Verified:").append(verified).append("\n");
        b.append(" Function:").append(function).append("\n");
        if (ontologyElement != null)
            b.append("Is ontology element already added:"
                            + ontologyElement.isAlreadyAdded() + "\n");
        return b.toString();
    }

    public Object clone() {
        Object deepCopy = null;
        ByteArrayOutputStream byteArrOs = new ByteArrayOutputStream();
        ObjectOutputStream objOs;
        try {
            objOs = new ObjectOutputStream(byteArrOs);
            objOs.writeObject(this);
            ByteArrayInputStream byteArrIs =
                            new ByteArrayInputStream(byteArrOs.toByteArray());
            ObjectInputStream objIs = new ObjectInputStream(byteArrIs);
            deepCopy = objIs.readObject();
            return deepCopy;
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return deepCopy;
    }

    public boolean equals(Object theOtherConcept) {
        boolean isEqual = false;
        String function1 = this.getFunction();
        String function2 = ((SemanticConcept) theOtherConcept).getFunction();
        OntologyElement element1 = this.getOntologyElement();
        OntologyElement element2 =
                        ((SemanticConcept) theOtherConcept).getOntologyElement();
        boolean functionEquals = false;
        if (function1 != null && function2 != null)
            functionEquals = function1.equals(function2);
        else if (function1 == null && function2 == null) functionEquals = true;
        boolean verifiedEquals = false;
        if (this.isVerified().booleanValue() == ((SemanticConcept) theOtherConcept)
                        .isVerified().booleanValue()) verifiedEquals = true;
        if (functionEquals && element1.equals(element2))
            isEqual = true;
        // System.out
        // .print("************************************************************calling equals method..result is:"
        // + isEqual);
        return isEqual;
    }
}
 public class SuggestionPair {

	SuggestionKey key;

	public SuggestionKey getKey() {
		return key;
	}

	public void setKey(SuggestionKey key) {
		this.key = key;
	}

	public List<Vote> getVote() {
		return vote;
	}

	public void setVote(List<Vote> vote) {
		this.vote = vote;
	}

	List<Vote> vote;

	Object currentDialogSubject;
	
	public Object getCurrentDialogSubject() {
		return currentDialogSubject;
	}

	public void setCurrentDialogSubject(Object currentDialogSubject) {
		this.currentDialogSubject = currentDialogSubject;
	}
	
	public String toString() {
		StringBuffer s = new StringBuffer("");
		s.append("Key:");
		if (key != null)
			s.append(key.toString()).append("\n");
		s.append("Votes:");
		if (s != null)
			s.append(vote.toString());
		return s.toString();
	}
}
 public class Vote {
  double vote;

  SemanticConcept candidate;

  long id;

  public long getId() {
    return id;
  }

  public void setId(long id) {
    this.id = id;
  }

  public double getVote() {
    return vote;
  }

  public void setVote(double vote) {
    this.vote = vote;
  }

  public SemanticConcept getCandidate() {
    return candidate;
  }

  public void setCandidate(SemanticConcept candidate) {
    this.candidate = candidate;
  }

  public String toString() {
    StringBuffer s = new StringBuffer("");
    s.append("Vote id:").append(id).append(" Score:").append(vote);
    if(candidate != null)
      s.append("\nCandidate:" + candidate.toString()).append("\n");
    return s.toString();
  }
}
 @XmlRootElement
public class FreyaResponse {
    
    String repositoryId;
    String repositoryUrl;
    String sparqlQuery;
    String preciseSparql;
    Object textResponse;
    ArrayList clarifications;
    List<List<HashMap>> table;
    List<Object> refinementNodes;
    List<List<SemanticConcept>> semanticConcepts;
    ArrayList<Object> graph;
    List<Annotation> annotations = new ArrayList<Annotation>();

    public String getPreciseSparql() {
        return preciseSparql;
    }

    public void setPreciseSparql(String preciseSparql) {
        this.preciseSparql = preciseSparql;
    }
    public List<Object> getRefinementNodes() {
        return refinementNodes;
    }

    public void setRefinementNodes(List<Object> refinementNodes) {
        this.refinementNodes = refinementNodes;
    }

    public ArrayList getClarifications() {
        return clarifications;
    }

    public void setClarifications(ArrayList clarifications) {
        this.clarifications = clarifications;
    }

    public List<List<HashMap>> getTable() {
        return table;
    }

    public void setTable(List<List<HashMap>> table) {
        this.table = table;
    }

    public ArrayList<Object> getGraph() {
        return graph;
    }

    public void setGraph(ArrayList<Object> graph) {
        this.graph = graph;
    }

    public List<Annotation> getAnnotations() {
        return annotations;
    }

    public void setAnnotations(List<Annotation> annotations) {
        this.annotations = annotations;
    }

    public List<List<SemanticConcept>> getSemanticConcepts() {
        return semanticConcepts;
    }

    public void setSemanticConcepts(List<List<SemanticConcept>> semanticConcepts) {
        this.semanticConcepts = semanticConcepts;
    }

    public Object getTextResponse() {
        return textResponse;
    }

    public void setTextResponse(Object textResponse) {
        this.textResponse = textResponse;
    }

    public String getRepositoryId() {
        return repositoryId;
    }

    public void setRepositoryId(String repositoryId) {
        this.repositoryId = repositoryId;
    }

    public String getRepositoryUrl() {
        return repositoryUrl;
    }

    public void setRepositoryUrl(String repositoryUrl) {
        this.repositoryUrl = repositoryUrl;
    }

    public String getSparqlQuery() {
        return sparqlQuery;
    }

    public void setSparqlQuery(String sparqlQuery) {
        this.sparqlQuery = sparqlQuery;
    }
} @Component
public class OCAnalyser {

    protected final Log logger = LogFactory.getLog(getClass());
    /*
     * whether the sparql results should be postprocessed or not if yes then we will show the max(R(i)), where i=1..n, n
     * is the number of returned results (rows)
     */
    boolean filterResultsBasedOnTheFunction = true;

    @Autowired QueryUtil queryUtil;
    @Autowired OCUtil ocUtil;
    @Autowired TripleCreator tripleCreator;
    @Autowired FormalQueryMaker queryMaker;
    @Autowired QueryResultsMaker queryResultsMaker;
    @Qualifier("rdfRepository") @Autowired SesameRepositoryManager repository;

    /**
     * @param elements
     * @return
     */
    private List<TripleElement> getTriples(List<List<OntologyElement>> elements) {
        List<TripleElement> triples = null;
        // run tripleCreator here
        try {
            triples = tripleCreator.getTriples(elements);
        } catch (Exception e) {
            logger.error(e);
        }
        // interpretation = QueryUtil
        // .attachNiceDescriptionsToInterpretations(interpretation);
        // triples = interpretation.getTriples();
        return triples;
    }

    /**
     * @TODO ovde input parameters treba da budu konkretniji
     * @param map
     * @return
     */
    public Map getResultGraph(HashMap map) {

        HashMap toReturn = new HashMap();

        List<List<OntologyElement>> preparedOntologyElements =
                        (List<List<OntologyElement>>) map.get("elements");

        List<List<String>> list2DOfResults = (List<List<String>>) map.get("table");
        String preciseSparql = (String) map.get("preciseSparql");
        // ovde ga promeni

        List<ResultGraph> resultGraph = null;
        try {
            List<TripleElement> triples = getTriples(preparedOntologyElements);
            logger.info("Passing now these elements to query maker:\n" + triples);
            // resultGraph =
            // queryResultsMaker.getQueryUtil().getResultGraph(list2DOfResults,
            // triples, filterResultsBasedOnTheFunction);

            Map newMap;

            newMap =
                            queryUtil.getResultGraph(list2DOfResults,
                                            triples, filterResultsBasedOnTheFunction);
            resultGraph = (List<ResultGraph>) newMap.get(FreyaConstants.RESULTS_GRAPH);
            toReturn.put(FreyaConstants.RESULTS_GRAPH, resultGraph);
            toReturn.put(FreyaConstants.NUMBER_OF_RESULTS,
                            newMap.get(FreyaConstants.NUMBER_OF_RESULTS));

            // logger.debug("Generated graph:" + resultGraph.toString());
            if (resultGraph != null)
                logger.info("Generated graph has " + resultGraph.size() + " nodes.");
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        return toReturn;
    }

    /**
     * if all overlapped elements are of the same type returns true otherwise false
     * 
     * @param table
     * @return
     */
    public boolean elementsConsistent(List<List<SemanticConcept>> table) {
        boolean consistent = true;
        for (List<SemanticConcept> column : table) {
            boolean colConsistent = false;
            if (column.size() == 1)
                colConsistent = true;
            else {
                String previous = null;
                for (SemanticConcept cell : column) {
                    OntologyElement current = cell.getOntologyElement();
                    String currentClass = current.getClass().toString();
                    // logger.info("CurrentClass:"+currentClass);
                    if (previous == null) {
                        previous = currentClass;
                        continue;
                    } else if (previous != null && currentClass.equals(previous)) {
                        previous = currentClass;
                        colConsistent = true;
                    } else {
                        colConsistent = false;
                        return colConsistent;
                    }
                }
            }
        }
        return consistent;
    }

    /**
     * input is a list of columns; output is list of raws generated using backtracking
     * 
     * @param table
     * @return
     */
    List<List<SemanticConcept>> generateAllCombinations(List<List<SemanticConcept>> table) {
        List<List<SemanticConcept>> allCombinations = new ArrayList<List<SemanticConcept>>();
        int numberOfElements = table.size();
        BacktrackEngine bcte = new BacktrackEngine(new int[numberOfElements]);
        int[] newArray = bcte.getSizes();
        int[] clock = new int[numberOfElements];
        for (int i = 0; i < numberOfElements; i++) {
            List<SemanticConcept> object = table.get(i);
            Integer size = object.size();
            newArray[i] = size;
            clock[i] = 0;
        }
        bcte.setSizes(newArray);
        long start = System.currentTimeMillis();
        while (clock != null) {
            List listOfIndexes = new ArrayList();
            for (int n : clock) {
                // logger.debug("" + n + ":");
                listOfIndexes.add(n);
            }
            List<SemanticConcept> oneCombination = new ArrayList<SemanticConcept>();
            for (int l = 0; l < listOfIndexes.size(); l++) {
                for (List<SemanticConcept> column : table) {
                    SemanticConcept oneElement = column.get(l);
                    oneCombination.add(oneElement);
                }
                allCombinations.add(oneCombination);
            }
            clock = bcte.nextIndexes(clock);
        }
        long duration = System.currentTimeMillis() - start;
        logger.debug("Exec time : " + duration);
        return allCombinations;
    }

    /**
     * returns map where one key is table with results the other one is list of elements
     * 
     * @param table
     * @return
     */
    public HashMap getResultMap(List<List<SemanticConcept>> table, Question question) {
        List<List<OntologyElement>> preparedOntologyElements = null;
        HashMap map = new HashMap();
        List<List<String>> list2DOfResults = null;
        SemanticConceptListComparator semanticConceptComparator =
                        new SemanticConceptListComparator();
        Collections.sort(table, semanticConceptComparator);
        StringBuffer elms = new StringBuffer("");
        String sparqlQuery = null;
        String preciseSparql = null;
        // added to fix Bug
        ArrayList<Integer> toBeRemoved = new ArrayList();
        String uriTracker =null;
        Set<String> tempSet = new HashSet();
        for (List<SemanticConcept> column : table) {
            elms.append("Start new column:");
            for (SemanticConcept cell : column) {
                elms.append(cell.getOntologyElement().getData().toString());
                elms.append("(");
                uriTracker=cell.getOntologyElement().getData().toString();
                if (cell.getOntologyElement().getFunction() != null)
                    elms.append(cell.getFunction().toString());
                elms.append(") ");
                elms.append("|");
                // added to remove duplicate uri entry
                if (tempSet.add(uriTracker)) {
                    logger.debug("Added The following URI entry:"+ uriTracker);
                    logger.debug("Function of the corresponding URI is:"+cell.getOntologyElement().getFunction());
                }
                else
                {
                	logger.debug(" Not Added the duplciate URI entry:"+ uriTracker);
                	toBeRemoved.add(table.indexOf(column));
                	logger.debug("Function of the duplicate URI is:"+cell.getOntologyElement().getFunction());
                }
            }
        }
        logger.info("Duplicate Entries To Be removed:"+toBeRemoved);
        for (int i: toBeRemoved)
        {
        	table.remove(i);
        }
        logger.info("Now generating SPARQL from " + table.size() + " columns: " + elms);
        logger.info("Elements are:\n " + table.toString());
        try {
            List<List<OntologyElement>> elementsTable = new ArrayList<List<OntologyElement>>();
            for (List<SemanticConcept> column : table) {
                List<OntologyElement> columnOfOE = new ArrayList<OntologyElement>();
                for (SemanticConcept sc : column) {
                    if (sc.getOntologyElement() instanceof NoneElement) {
                        logger.info("Ignoring NONE element.");
                        continue;
                    }
                    if (!sc.getOntologyElement().isAlreadyAdded()) {
                        // TODO sc should not have function ...only oe..or the
                        // other
                        // way around
                        sc.getOntologyElement().setFunction(sc.getFunction());
                        columnOfOE.add(sc.getOntologyElement());
                        logger.debug("Adding function from sc to oe..." + sc.getFunction());
                    } else {
                        logger.debug("Found element which is already added:"
                                        + sc.getOntologyElement().getData().toString()
                                        + " and with function: " + sc.getFunction());
                        // change function
                        SemanticConcept changedSc = sc;
                        List<SemanticConcept> sElementsNoDuplicates =
                                        new ArrayList<SemanticConcept>();
                        sElementsNoDuplicates.addAll(column);
                        // remove the one flagged 'already added' but update the
                        // function
                        for (SemanticConcept concept : sElementsNoDuplicates) {
                            if (concept.getOntologyElement().getAnnotation()
                                            .equals(changedSc.getOntologyElement().getAnnotation())
                            // function is equal?
                            ) {
                                concept.getOntologyElement().setFunction(changedSc.getFunction());
                                logger.debug("Updating function of already existing ontology element: "
                                                + changedSc.getOntologyElement().getAnnotation()
                                                                .getText()
                                                + " from "
                                                + concept.getFunction()
                                                + " to "
                                                + changedSc.getFunction());
                            }
                        }
                    }
                }
                if (columnOfOE.size() > 0) elementsTable.add(columnOfOE);
                columnOfOE = new ArrayList();
            }
            int col = 0;
            for (List<OntologyElement> column : elementsTable) {
                logger.debug("These are elements in col " + col + ":" + column.toString());
                col++;
            }
            OntologyElementListComparator c = new OntologyElementListComparator();
            Collections.sort(elementsTable, c);
            logger.debug("Calling triple creator...");
            preparedOntologyElements = tripleCreator.prepareOntologyElements(elementsTable);
            // now check all property elements: if they are datatypeProperty
            // elements and not surrounded by datatype property value element or
            // Joker, then shift them to the right or to the left
            // query maker will also add setAnswer(true) for the non-governors
            // of the datatype properties
            QueryElement queryElement = queryMaker.transform(preparedOntologyElements);
            logger.debug("after query maker:\n" + queryElement.toString());
            FormalQueryResult formalQueryResult =
                            queryResultsMaker.transform(queryElement, question);
            logger.debug("Results:\n" + formalQueryResult.toString());
            list2DOfResults = formalQueryResult.getData();
            sparqlQuery = formalQueryResult.getQuery();
            preciseSparql = formalQueryResult.getPreciseSparqlQuery();
            logger.debug("list2DOfResults:\n" + list2DOfResults.toString());
            logger.info("SPARQL result has " + list2DOfResults.size() + " rows.");
        } catch (Exception e) {
            logger.error(e);
        }
        // return list2DOfResults;
        map.put("elements", preparedOntologyElements);
        map.put("table", list2DOfResults);
        map.put("sparql", sparqlQuery);
        map.put(FreyaConstants.PRECISE_SPARQL, preciseSparql);
        // map.put(FreyaConstants.REPOSITORY_URL, ((SparqlQueryExecuter) getQueryResultsMaker()
        // .getQueryUtil().getQueryExecuter()).getSparqlUtils().getRepositoryURL());
        // map.put(FreyaConstants.REPOSITORY_ID, ((SparqlQueryExecuter) getQueryResultsMaker()
        // .getQueryUtil().getQueryExecuter()).getSparqlUtils().getRepositoryId());
        map.put(FreyaConstants.REPOSITORY_URL, repository.getRepositoryURL());
        map.put(FreyaConstants.REPOSITORY_ID, repository.getRepositoryId());
        return map;
    }


}
 @Component
public class OCUtil {
    private static final Log logger = LogFactory.getLog(OCUtil.class);

    AtomicInteger id = new AtomicInteger();
    @Autowired TreeUtils treeUtils;



    /**
     * 
     * @param anns
     * @return
     */
    public Set<Annotation> filterInstanceClass(List<Annotation> anns) {
        HashMap<Long, List<Annotation>> map = new HashMap<Long, List<Annotation>>();

        for (Annotation ann : anns) {
            List<Annotation> tmp = map.get(ann.getStartOffset());
            if (tmp == null) tmp = new ArrayList<Annotation>();
            tmp.add(ann);
            map.put(ann.getStartOffset(), tmp);
        }
        // now we have all start offsets
        logger.info("Now finished puting all anns into the map where key is offsets, these are the keys:"
                        + map.keySet().toString());
        List<Long> startOffsets = new ArrayList<Long>();
        startOffsets.addAll(map.keySet());
        Collections.sort(startOffsets);
        List toRemove = new ArrayList();

        for (int i = 0; i < startOffsets.size() - 1; i++) {

            Long current = startOffsets.get(i);
            Long next = startOffsets.get(i + 1);
            logger.info("Current ofsset" + current.toString() + " next offset:" + next.toString());
            List<Annotation> currentAnnotations = map.get(current);
            List<Annotation> nextAnnotations = map.get(next);
            toRemove.addAll(whichToRemove(currentAnnotations, nextAnnotations));
            toRemove.addAll(whichToRemove(nextAnnotations, currentAnnotations));
        }

        // if (toRemove!=null)
        // anns.removeAll(toRemove);
        logger.info("Marked to remove due to classInstance:" + toRemove.size());
        // return anns;
        return new HashSet(toRemove);

    }

    // }
    /**
     * if first is class and second is instance then it checks the class of the instance and if the two match marks the
     * class to remove
     */
    List<Annotation> whichToRemove(List<Annotation> first, List<Annotation> second) {
        List<Annotation> toRemove = new ArrayList<Annotation>();
        // save in this list members from 'first' only if they are classes
        List<Annotation> firstClassesOnly = new ArrayList<Annotation>();
        // save in this list members from 'second' only if they are instances only
        List<Annotation> secondInstancesOnly = new ArrayList<Annotation>();
        for (Annotation ann1 : first) {
            String type = (String) ann1.getFeatures().get(CATConstants.TYPE_FEATURE);
            if (type.equals(CATConstants.FEATURE_TYPE_CLASS)) {
                firstClassesOnly.add(ann1);
            }
        }
        logger.info("There are " + firstClassesOnly.size() + " classes in the first column.");
        // second column
        for (Annotation ann1 : second) {
            String type = (String) ann1.getFeatures().get(CATConstants.TYPE_FEATURE);
            if (type.equals(CATConstants.TYPE_INSTANCE)) {
                secondInstancesOnly.add(ann1);
            }
        }
        logger.info("There are " + secondInstancesOnly.size() + " instances in the second column.");
        // collect classUris from first

        // now we know for sure that the first column is class and second is instance
        // so we take the second only, find all classUris of instances, and if we do find
        // that instance X is of the existing type of class then we do this:
        // - remove all instances from first
        // - remove all but X from the second column
        Set<String> allClassUrisFirst = new HashSet<String>();
        for (Annotation ann1 : firstClassesOnly) {
            String classUri = (String) ann1.getFeatures().get(CATConstants.FEATURE_URI);
            logger.info("class uri:" + classUri);
            allClassUrisFirst.add(classUri);
        }
        Set<Annotation> secondToKeep = new HashSet<Annotation>();
        for (Annotation ann1 : secondInstancesOnly) {
            logger.info("inst annotation:" + ann1.toString());

            Set<String> classUris = new HashSet((List) (ann1.getFeatures().get(CATConstants.CLASS_URI_LIST)));
            String classUri = (String) ann1.getFeatures().get(CATConstants.CLASS_URI);
            classUris.add(classUri);
            logger.info("classUris:" + classUris);
            // if classUris contain any of the classes in allClassUrisFirst then
            if (classInstanceFound(classUris, allClassUrisFirst))
            {
                secondToKeep.add(ann1);
            }
        }
        if (secondToKeep.size() > 0) {
            toRemove.addAll(first);
            toRemove.addAll(second);
            toRemove.removeAll(secondToKeep);
        }
        logger.info("marked to remove:" + toRemove.size());
        return toRemove;
    }

    /**
     * 
     * @param instanceClassUris
     * @param allClassUris
     * @return
     */
    boolean classInstanceFound(Set<String> instanceClassUris, Set<String> allClassUris) {
        boolean toReturn = false;
        for (String instClassUri : instanceClassUris) {
            logger.info("Checking:" + instClassUri);
            for (String classUri : allClassUris) {
                logger.info(" and:" + classUri);
                if (instClassUri.equals(classUri)) {
                    logger.info("Annotation should be removed?");
                    toReturn = true;
                    break;
                }
            }
        }
        return toReturn;
    }

    /**
     * finding nearest neighbour for poc
     * 
     * @param question
     * @param poc
     * @return
     */
    public List<SemanticConcept> findNearestNeighbours(Question question, POC poc) {
        List<SemanticConcept> nearestNeighbours = new ArrayList<SemanticConcept>();
        Tree root = question.getSyntaxTree();
        List<SemanticConcept> scList = null;
        try {
            scList = getLinearisedSemanticConcepts(question.getSemanticConcepts());
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        int min = 1000;
        logger.info("Finding nearest neighbour for POC:"
                        + poc.getAnnotation().getText());
        logger.info("There are:" + scList.size()
                        + " candidates (semantic concepts)");
        for (SemanticConcept sConcept : scList) {
            if (sConcept.getOntologyElement() instanceof NoneElement) {
                continue;
            }
            int distance =
                            treeUtils.getDistance(poc.getAnnotation().getSyntaxTree(),
                                            sConcept.getOntologyElement().getAnnotation()
                                                            .getSyntaxTree(), root);
            // logger.debug("Distance is:"
            // + distance
            // + " between "
            // + poc.getAnnotation().getSyntaxTree().toString()
            // + " \n and \n"
            // + sConcept.getOntologyElement().getAnnotation()
            // .getSyntaxTree().toString());
            if (min > distance) {
                min = distance;
                nearestNeighbours = new ArrayList<SemanticConcept>();
                nearestNeighbours.add(sConcept);
            } else if (min == distance) {
                nearestNeighbours.add(sConcept);
            }
        }
        return nearestNeighbours;
    }

    /**
     * @param question
     * @param keySemConcept
     * @return
     */
    public List<SemanticConcept> findNearestNeighbours(Question question,
                    List<SemanticConcept> list) {
        SemanticConcept keySemConcept = list.get(0);
        List<Tree> treeOfTheKeySemanticConcept =
                        keySemConcept.getOntologyElement().getAnnotation().getSyntaxTree();
        List<SemanticConcept> nearestNeighbours = new ArrayList<SemanticConcept>();
        Tree root = question.getSyntaxTree();
        List<SemanticConcept> newList = null;
        // this currentList is introduced so that the element is not found to
        // be the nearest neighbour of itself
        List<List<SemanticConcept>> currentList =
                        new ArrayList<List<SemanticConcept>>();
        currentList.addAll(question.getSemanticConcepts());
        currentList.remove(list);
        try {
            newList = getAllSemanticConceptsLinearised(currentList);
            logger.debug("currently:" + newList.size()
                            + " in the semantic elements list");
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        List noneToRemove = new ArrayList();
        // if any of te elements in the neighbourhood is None then ignore them
        for (SemanticConcept theOne : newList) {
            if (theOne.getOntologyElement() instanceof NoneElement) {
                noneToRemove.add(theOne);
            }
        }
        newList.removeAll(noneToRemove);
        logger.info("after removing none elements:" + newList.size()
                        + " in the semantic elements list");
        logger.debug("Removed:" + noneToRemove.size()
                        + " none elements as they are not counted when finding NN.");
        int min = 1000;
        logger.debug("Found " + newList.size() + " neighbours for: "
                        + keySemConcept.getOntologyElement().getData().toString());
        for (SemanticConcept sConcept : newList) {
            int distance =
                            treeUtils.getDistance(treeOfTheKeySemanticConcept, sConcept
                                            .getOntologyElement().getAnnotation().getSyntaxTree(),
                                            root);
            // logger.debug("distance is:" + distance + " between "
            // // + treeOfTheKeySemanticConcept + " \n and \n"
            // + sConcept.getOntologyElement().getAnnotation().getSyntaxTree());
            if (min > distance) {
                min = distance;
                nearestNeighbours = new ArrayList<SemanticConcept>();
                nearestNeighbours.add(sConcept);
                // logger.debug("Adding:"
                // + sConcept.getOntologyElement().getData().toString()
                // + " to the list of NNs");
            } else if (min == distance) {
                nearestNeighbours.add(sConcept);
                // logger.debug("Adding:"
                // // + sConcept.getOntologyElement().getData().toString()
                // + " to the list of NNs");
            }
        }
        return nearestNeighbours;
    }

    /**
     * @param question
     * @param poc
     * @return
     */
    public POC whichPOC(Question question) {
        List<POC> pocs = question.getPocs();
        POC theOne = null;
        Tree root = question.getSyntaxTree();
        List<SemanticConcept> scList = null;
        try {
            scList = getLinearisedSemanticConcepts(question.getSemanticConcepts());
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        logger.debug("This many sConcepts:" + scList.size());
        logger
                        .debug("Trying to decide which poc to use for the dialog; this many pocs:"
                                        + pocs.size());
        int min = 1000;
        for (POC poc : pocs) {
            List<SemanticConcept> nearestNeighbours =
                            findNearestNeighbours(question, poc);
            SemanticConcept sc = null;
            // take one, it does not matter which
            if (nearestNeighbours != null && nearestNeighbours.size() > 0)
                sc = nearestNeighbours.get(0);
            if (sc != null) {
                int distance =
                                treeUtils.getDistance(poc.getAnnotation().getSyntaxTree(), sc
                                                .getOntologyElement().getAnnotation().getSyntaxTree(),
                                                root);
                if (min > distance) {
                    min = distance;
                    theOne = poc;
                } else if (min == distance && theOne != null) {
                    // take the one which is on the right-hand side
                    // this means that theOne and poc are on the same level:
                    // take the one on the right
                    if (poc.getAnnotation().getStartOffset().longValue() > theOne
                                    .getAnnotation().getStartOffset().longValue()) theOne = poc;
                }
            }
        }
        // this means if there are not ocs, use the first one, which should also
        // be mainsubject TODO: this is more a workaround than a solution but I
        // live it like this for the time being
        if (pocs.size() > 0 && theOne == null) {
            theOne = pocs.get(0);
        }
        return theOne;
    }

    public boolean ocsVerified(List<SemanticConcept> list) {
        boolean allVerified = false;
        for (SemanticConcept c : list) {
            if (c.isVerified().booleanValue() == true)
                allVerified = true;
        }
        return allVerified;
    }

    /**
     * finds the OC to be put into dialog: looks which one is closest to focus
     * 
     * @param question
     * @return
     */
    public List<SemanticConcept> whichOC(Question question) {

        if (question.getFocus() == null) {
            return question.getSemanticConcepts().get(0);
        }

        SemanticConceptListComparator c = new SemanticConceptListComparator();
        long start = System.currentTimeMillis();
        if (question.getSemanticConcepts().size() > 1)
            Collections.sort(question.getSemanticConcepts(), c);
        long end = System.currentTimeMillis();
        logger.info("Sorting sem elements took:" + (end - start) + "ms");
        List<SemanticConcept> theOne = null;
        Tree root = question.getSyntaxTree();
        int min = 1000;
        start = System.currentTimeMillis();
        for (List<SemanticConcept> list : question.getSemanticConcepts()) {
            if (!ocsVerified(list)) {
                SemanticConcept sampleElement = list.get(0);
                int distance =
                                treeUtils.getDistance(question.getFocus().getAnnotation()
                                                .getSyntaxTree(), sampleElement.getOntologyElement()
                                                .getAnnotation().getSyntaxTree(), root);
                logger.debug("Distance:" + distance);
                if (min > distance && list.size() > 1) {
                    min = distance;
                    theOne = list;
                }
            }
        }
        end = System.currentTimeMillis();
        logger.info("Calculating distance for which oc took:" + (end - start) + "ms");
        // this means if there are no ocs, use the first one, which should also
        // be mainsubject TODO: this is more a workaround than a solution but I
        // live it like this for the time being
        logger.debug("Focus:" + question.getFocus());
        logger.debug("Sem elements to use for the OC dialog:" + theOne);
        return theOne;

    }

    public List<SemanticConcept> getLinearisedSemanticConcepts(
                    List<List<SemanticConcept>> table) throws Exception {
        List<SemanticConcept> linearisedList = new ArrayList<SemanticConcept>();
        for (List<SemanticConcept> row : table) {
            if (row.size() > 1) {
                logger
                                .debug("List of OCs is ambiguious and needs to be disambiguated using dialog");
                linearisedList.add(row.get(0));
            } else if (row.size() > 0) linearisedList.add(row.get(0));
        }
        return linearisedList;
    }

    /**
     * utility method to transform list of lists into one list
     * 
     * @param table
     * @return
     * @throws Exception
     */
    public List<SemanticConcept> getAllSemanticConceptsLinearised(
                    List<List<SemanticConcept>> table) throws Exception {
        List<SemanticConcept> linearisedList = new ArrayList<SemanticConcept>();
        for (List<SemanticConcept> row : table) {
            // logger
            // .debug("List of OCs is ambiguious and needs to be disambiguated using dialog!");
            linearisedList.addAll(row);
        }
        return linearisedList;
    }

    /**
     * if two semantic concepts are in the same column and refer to the same ontology concept (same instance URI or
     * class URI or propertyURI) then remove it
     * 
     * @param Question
     * @return Question
     */
    public Question removeDuplicateOcs(Question question) {
        List<List<SemanticConcept>> table = question.getSemanticConcepts();
        // logger.debug("input table with semantic concepts:" + table.toString());
        // key: column index within table; value: list of indexes in the column to
        // be removed
        Map<Integer, List<Integer>> mapToRemove =
                        new HashMap<Integer, List<Integer>>();
        for (List<SemanticConcept> sList : table) {
            List<String> noDuplicates = new ArrayList<String>();
            List<DatatypePropertyValueIdentifier> noDuplicatesDTVI =
                            new ArrayList<DatatypePropertyValueIdentifier>();
            for (SemanticConcept sConcept : sList) {
                OntologyElement oe = sConcept.getOntologyElement();
                // uri for classes, instances or properties
                String uri = null;
                DatatypePropertyValueIdentifier dpvIdentifier = null;
                if (oe instanceof DatatypePropertyValueElement) {
                    dpvIdentifier =
                                    (DatatypePropertyValueIdentifier) ((DatatypePropertyValueElement) oe)
                                                    .getData();
                } else {
                    Object data = oe.getData();
                    if (data != null) uri = data.toString();
                }
                logger.debug("dpvIdentifier:" + dpvIdentifier);
                logger.debug("URI:" + uri);
                // different treatment for classes,properties and instances
                // versus datatypepropertyvalues
                if (uri != null && !noDuplicates.contains(uri)) {
                    noDuplicates.add(uri);
                } else if (dpvIdentifier != null
                                && !noDuplicatesDTVI.contains(dpvIdentifier)) {
                    noDuplicatesDTVI.add(dpvIdentifier);
                } else {
                    List<Integer> list =
                                    mapToRemove.get(new Integer(table.indexOf(sList)));
                    if (list == null) list = new ArrayList<Integer>();
                    list.add(sList.indexOf(sConcept));
                    mapToRemove.put(new Integer(table.indexOf(sList)), list);
                    logger.debug("List with elements to be removed:" + list.toString());
                }
            }
        }
        if (mapToRemove.keySet().size() > 0)
            logger.info("Removing " + mapToRemove.keySet().size()
                            + " OCs as they are duplicated.");
        logger.debug("map with semantic concepts to be removed:"
                        + mapToRemove.toString());
        logger
                        .debug("before removing:"
                                        + question.getSemanticConcepts().toString());
        question = removeMarkedOcsByIndex(question, mapToRemove);
        logger.debug("after removing:" + question.getSemanticConcepts().toString());
        return question;
    }

    /**
     * removes marked ocs from the table: the map contains key=index of the column; and value=list of elements to be
     * removed
     * 
     * @param question
     * @param toRemove
     * @return
     */
    public Question removeMarkedOcsByIndex(Question question,
                    Map<Integer, List<Integer>> toRemove) {
        List<Integer> keys = new ArrayList<Integer>(toRemove.keySet());
        Integer index = null;
        for (int i = 0; i < keys.size(); i++) {
            index = keys.get(i);
            if (index != null) {
                List<SemanticConcept> column =
                                question.getSemanticConcepts().get(index.intValue());
                logger.debug("the columns has " + column.size() + " elements.");
                List<Integer> elementsToRemove = toRemove.get(index);
                logger.debug("now removing " + elementsToRemove.size()
                                + " element(s) from the column.");
                for (Integer rowIndex : elementsToRemove) {
                    column.remove(rowIndex.intValue());
                }
                logger.debug("after removing this many left: " + column.size()
                                + " elements.");
            }
        }
        return question;
    }

    /**
     * removes marked ocs from the table: the map contains key=index of the column; and value=list of elements to be
     * removed
     * 
     * @param question
     * @param toRemove
     * @return
     */
    public Question removeMarkedOcs(Question question,
                    Map<Integer, List<SemanticConcept>> toRemove) {
        List<Integer> keys = new ArrayList<Integer>(toRemove.keySet());
        for (Integer index : keys) {
            if (index != null) {
                List<SemanticConcept> column =
                                question.getSemanticConcepts().get(index.intValue());
                List<SemanticConcept> onlySelectedElements = toRemove.get(index);
                logger.info("About to remove elements from column:" + index + " there are:"
                                + (column.size()));
                column.clear();
                // verified means the user selected it
                for (SemanticConcept c : onlySelectedElements) {
                    c.setVerified(true);
                }

                column.addAll(onlySelectedElements);

            }
        }
        return question;
    }

    /**
     * @param question
     * @return
     */
    public List<List<SemanticConcept>> addNoneToAll(
                    List<List<SemanticConcept>> elements) {
        for (List<SemanticConcept> column : elements) {
            for (SemanticConcept sampleElement : column) {
                if (sampleElement.getOntologyElement().getAnnotation() != null) {
                    SemanticConcept scEl = new SemanticConcept();
                    NoneElement noneElement = new NoneElement();
                    Annotation newAnn =
                                    (Annotation) sampleElement
                                                    .getOntologyElement().getAnnotation().clone();
                    noneElement.setAnnotation(newAnn);
                    noneElement.setId(new Integer(id.getAndIncrement()).toString());
                    logger.debug("Setting annotation for none element:"
                                    + noneElement.toString());
                    scEl.setOntologyElement(noneElement);
                    column.add(scEl);
                    break;
                }
            }
        }
        return elements;
    }

    /**
     * checkes whether there are any overlapped annotations, if there are leaves only the longest (of type Lookup! only)
     * 
     * @param allAnnotations
     * @return
     */
    public List<Annotation> disambiguateBasedOnLength(
                    List<Annotation> allAnnotations) {
        AnnotationOffsetComparator c = new AnnotationOffsetComparator();
        Collections.sort(allAnnotations, c);
        ArrayList<Annotation> aCopyOfLookupAnnotations =
                        null;
        ArrayList<Annotation> checkedAnnotations =
                        new ArrayList<Annotation>();
        ArrayList<Annotation> toRemove =
                        new ArrayList<Annotation>();
        // startofsset, endoffset
        Map<Long, Long> removed = new HashMap();
        for (Annotation ann : allAnnotations) {
            if (!checkedAnnotations.contains(ann)) {
                checkedAnnotations.add(ann);
                aCopyOfLookupAnnotations =
                                new ArrayList<Annotation>();
                aCopyOfLookupAnnotations.addAll(allAnnotations);
                aCopyOfLookupAnnotations.remove(ann);
                for (Annotation newAnn : aCopyOfLookupAnnotations) {
                    if (ann.overlaps(newAnn)) {
                        // logger.info(ann.toString()
                        // + "\n***************overlaps with***********\n"
                        // + newAnn.toString());
                        // which one is longer
                        // add it to toRemove
                        long diffAnn =
                                        ann.getEndOffset().longValue()
                                                        - ann.getStartOffset().longValue();
                        long diffNewAnn =
                                        newAnn.getEndOffset().longValue()
                                                        - newAnn.getStartOffset().longValue();
                        if (diffAnn > diffNewAnn) {
                            if (!toRemove.contains(newAnn)) {
                                toRemove.add(newAnn);
                                removed.put(newAnn.getStartOffset(), newAnn.getEndOffset());
                                if (!checkedAnnotations.contains(newAnn))
                                    checkedAnnotations.add(newAnn);
                                // logger.info("\nadding to remove:\n" + newAnn.toString());
                            }
                        } else if (diffAnn < diffNewAnn) {
                            if (!toRemove.contains(ann)) {
                                toRemove.add(ann);
                                removed.put(ann.getStartOffset(), ann.getEndOffset());
                                if (!checkedAnnotations.contains(ann))
                                    checkedAnnotations.add(ann);
                                // logger.info("\nadding to remove:\n" + ann.toString());
                            }
                        }
                    } else {
                        // they do not overlap, continue to the next one
                        // logger.debug(ann.toString()
                        // + "***************DO NOT overlap with***********"
                        // + newAnn.toString());
                    }
                }// newAnn
            }
        }
        logger
                        .debug("********************************disambiguation******************\n");
        // logger.info("before:" + allAnnotations.toString());
        if (toRemove.size() > 0)
            logger.info("Removing:" + toRemove.size() + " columns (short annotations).");
        allAnnotations.removeAll(toRemove);
        // logger.info("after:" + allAnnotations.toString());
        toRemove = new ArrayList();
        for (Annotation remove : allAnnotations) {
            Set<Long> startKeys = removed.keySet();
            // logger.info("startOfsets:"+startKeys);
            if (startKeys.contains(remove.getStartOffset())) {
                Long endOffset = removed.get(remove.getStartOffset());
                // logger.info("endOffset:"+endOffset);
                if (endOffset.longValue() == remove.getEndOffset().longValue()) {
                    toRemove.add(remove);
                }
            }
        }
        // logger.info("and again before:" + allAnnotations.toString());
        if (toRemove.size() > 0)
            logger.info("Removing again:" + toRemove.size() + " annotations.");
        allAnnotations.removeAll(toRemove);
        // logger.info("after:" + allAnnotations.toString());
        return allAnnotations;
    }

    public static void main(String[] args) {
        OCUtil u = new OCUtil();
        List<Annotation> set =
                        new ArrayList<Annotation>();
        AnnotationOffsetComparator c = new AnnotationOffsetComparator();
        Annotation ann =
                        new Annotation();
        ann.setText("Michael");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(1));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Michael");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(1));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Michael Jackson");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(2));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Michael Jackson");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(2));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Jackson");
        ann.setStartOffset(new Long(1));
        ann.setEndOffset(new Long(2));
        set.add(ann);
        logger.info("Before:" + set.size());
        List sortedList = new ArrayList(set);
        Collections.sort(sortedList, c);
        set = u.disambiguateBasedOnLength(sortedList);
        logger.info("After:" + set.size());
    }
}
 @Component
public class StanfordSentenceSplitter {
   
    StanfordCoreNLP pipeline;

    @PostConstruct
    void init() {
        Properties props = new Properties();
        // props.put("annotators", "tokenize, ssplit, pos, lemma, ner, parse, dcoref");
        props.put("annotators", "tokenize, ssplit");
        pipeline = new StanfordCoreNLP(props);
    }

    public List<String> split(String input) {
        List<String> allSentences = new ArrayList<String>();
        Annotation document = new Annotation(input);
        pipeline.annotate(document);
        // a CoreMap is essentially a Map that uses class objects as keys and has values with custom types
        List<CoreMap> sentences = document.get(SentencesAnnotation.class);
        for (CoreMap sentence : sentences) {
            allSentences.add(sentence.toString());
        }
        return allSentences;
    }
} @Component("rdfRepository")
public class SesameRepositoryManager implements RepositoryManager {
	static Logger log = LoggerFactory.getLogger(SesameRepositoryManager.class);

	RepositoryConnection connection = null;

	private org.openrdf.repository.manager.RepositoryManager repositoryManager;
	private static final String PREFIX_FILE_URI = "file://";
	Repository repository;

	protected RepositoryConfig repConfig;

	@Value("${org.freya.rdf.repository.url}")
	String repositoryURL;

	@Value("${org.freya.rdf.repository.id}")
	String repositoryId;

	@Value("${org.freya.rdf.sesame.repository.username}")
	String username;

	@Value("${org.freya.rdf.sesame.repository.password}")
	String password;

	@Value("${org.freya.rdf.sesame.repository.ontology.load.pattern}")
	String ontologyLoadPattern;

	// @Value("${org.freya.rdf.owlim.config}")
	// String config;

	@Value("${org.freya.rdf.sesame.repository.localmanagerstoragedir}")
	String localRepositoryManagerDir;

	public String getRepositoryURL() {
		return repositoryURL;
	}

	public void setRepositoryURL(String repositoryURL) {
		this.repositoryURL = repositoryURL;
	}

	public void setRepositoryId(String repositoryId) {
		this.repositoryId = repositoryId;
	}

	public String getRepositoryId() {
		return repositoryId;
	}

	public TupleQueryResult executeQuery(final String query) {
		return executeQuery(query, true);
	}

	public TupleQueryResult executeQuery(final String query, boolean includeInferred) {
		try {
			final TupleQuery tuple = connection.prepareTupleQuery(QueryLanguage.SPARQL, query);
			tuple.setIncludeInferred(includeInferred);
			return tuple.evaluate();

		} catch (final Exception e) {
			log.error("error executing query:{}", query, e);
			throw new RuntimeException(e);
		}
	}

	public void loadResources(org.springframework.core.io.Resource[] resources) {
		int numberOfFilesToLoad = resources.length;
		int currentFileNumber = 1;
		for (org.springframework.core.io.Resource resource : resources) {
			String uri;
			try {
				uri = resource.getURI().toString();
			} catch (IOException e) {
				log.error("couldn't fetch uri from resource:{}", resource, e);
				throw new RuntimeException(e);
			}
			String ext = uri.substring(uri.lastIndexOf('.') + 1);
			RDFFormat format = null;
			boolean skip = false;
			if ("owl".equalsIgnoreCase(ext) || "rdf".equalsIgnoreCase(ext) || "rdfs".equalsIgnoreCase(ext)) {
				format = RDFFormat.RDFXML;
			} else if ("nt".equalsIgnoreCase(ext)) {
				format = RDFFormat.NTRIPLES;
			} else if ("n3".equalsIgnoreCase(ext)) {
				format = RDFFormat.N3;
			} else if ("ttl".equalsIgnoreCase(ext)) {
				format = RDFFormat.TURTLE;
			} else {
				log.info("Skipping uri:" + uri);
				skip = true;
			}
			if (!skip) {
				assert format != null : ext;
				// log.info("Started loading {} ({}/{})", uri,
				// currentFileNumber, numberOfFilesToLoad);
				try {
					connection.add(resource.getInputStream(), "http://example.org#", format);
				} catch (Exception /*
									 * RDFParseException | RepositoryException |
									 * IOException
									 */ e) {
					log.error("There was a problem with this uri, uri not loaded:{}", uri, e);
					throw new RuntimeException(e);
				}
				log.info("{} loaded.", uri);
			}
			currentFileNumber++;
		}
		try {
			connection.commit();
		} catch (RepositoryException e) {
			log.error("couldn't commit repo changes", e);
			throw new RuntimeException(e);
		}
	}

	public void loadData() throws IOException, QueryEvaluationException {
		ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
		org.springframework.core.io.Resource[] resources = resolver.getResources(ontologyLoadPattern);
		loadResources(resources);
	}

	/**
	 * load ontologies from URLs
	 */
	void initLocalRepository() {
		// InputStream configFileInputStream;
		try {
			// ResourcePatternResolver resolver = new
			// PathMatchingResourcePatternResolver();
			// org.springframework.core.io.Resource[] resources = resolver
			// .getResources(config);
			// configFileInputStream = resources[0].getInputStream();

			// repositoryManager = new LocalRepositoryManager(new File(
			// localRepositoryManagerDir));
			//
			// repositoryManager.initialize();

			repository = new SailRepository(new ForwardChainingRDFSInferencer(new MemoryStore()));
			repository.initialize();

			log.debug("Local repository initialised, ontologies not yet added");
		} catch (Exception e) {
			log.error("error loading local repository", e);
			throw new RuntimeException(e);
		}

		// final Graph graph = new GraphImpl();
		// RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
		// RDFHandler handler = new RDFHandler() {
		// public void endRDF() throws RDFHandlerException {
		// }
		//
		// public void handleComment(String arg0) throws RDFHandlerException {
		// }
		//
		// public void handleNamespace(String arg0, String arg1)
		// throws RDFHandlerException {
		// }
		//
		// public void handleStatement(Statement arg0)
		// throws RDFHandlerException {
		// graph.add(arg0);
		// }
		//
		// public void startRDF() throws RDFHandlerException {
		// }
		// };
		// parser.setRDFHandler(handler);

		// try {
		// parser.parse(configFileInputStream, "http://example.org#");
		// Iterator<Statement> iter = graph.match(null, RDF.TYPE, new URIImpl(
		// "http://www.openrdf.org/config/repository#Repository"));
		// org.openrdf.model.Resource repNode = null;
		// if (iter.hasNext()) {
		// Statement st = iter.next();
		// repNode = st.getSubject();
		// }
		// repConfig = RepositoryConfig.create(graph, repNode);
		// repositoryManager.addRepositoryConfig(repConfig);

		// } catch (Exception e) {
		// log.error("error parsing file:{}", config, e);
		// throw new RuntimeException(e);
		// }
	}

	@PostConstruct
	public void afterPropertiesSet() throws RepositoryConfigException, RepositoryException {
		log.info("afterPropertiesSet() - initialising");

		boolean loadLocal = false;
		if (repositoryURL != null && !repositoryURL.isEmpty()) {

			if (repositoryId == null || "".equals(repositoryId))
				repository = new SPARQLRepository(repositoryURL);
			else {
				if (username != null && password != null) {
					repository = new HTTPRepository(repositoryURL, repositoryId);
					// ((HTTPRepository)
					// repository).setUsernameAndPassword(username,
					// password);
				} else {
					throw new RuntimeException("Not valid username & password for the http repository");
				}
			}
		} else {
			loadLocal = true;
			initLocalRepository();
			if (repositoryManager == null) {
				throw new RuntimeException("couldn't create local repository manager");
			}
			// repository = repositoryManager.getRepository(repositoryId);
		}

		try {
			repository.initialize();
			connection = repository.getConnection();
			if (connection == null) {
				throw new RuntimeException("Connection NOT established!");
			} else {
				log.debug("Connection established");
			}

			if (loadLocal)
				loadData();
		} catch (final RepositoryException e) {
			// log.error("Not able to connect to the Repository -
			// repositoryId:{}, repositoryURL:{}",
			// repositoryId, repositoryURL, e);
			throw new RuntimeException(e);
		} catch (final QueryEvaluationException e) {
			log.error("Problem evaluating SPARQL query.");
			throw new RuntimeException(e);
		} catch (IOException e) {
			log.error("Problem reading ontology files from the file system.");
			throw new RuntimeException(e);
		}
		log.info("afterPropertiesSet() - Successfully initialised Sesame Repository using impl: {}",
				repository.getClass().getName());
	}

	/**
	 * This method is called to reload {@link #preloadInputStream} to
	 * repository.
	 * 
	 * @param sourceUri
	 *            A string of source file URI.
	 * @throws RepositoryException
	 */
	public void loadByInputStream(String sourceUri, InputStream preloadInputStream) throws RepositoryException {
		if (this.repositoryURL == null) {
			throw new NullPointerException("Please set repositoryURL where you want to upload.");
		}
		if (this.repositoryId == null) {
			throw new NullPointerException("Please set a repositoryId.");
		}

		if (this.repository.getConnection() == null) {
			throw new NullPointerException("Please initialize SparqlUtils");
		}

		RDFFormat format = null;
		boolean skip = false;
		String ext = sourceUri.substring(sourceUri.lastIndexOf('.') + 1);
		if ("owl".equalsIgnoreCase(ext) || "rdf".equalsIgnoreCase(ext) || "rdfs".equalsIgnoreCase(ext)) {
			format = RDFFormat.RDFXML;
		} else if ("nt".equalsIgnoreCase(ext)) {
			format = RDFFormat.NTRIPLES;
		} else if ("n3".equalsIgnoreCase(ext)) {
			format = RDFFormat.N3;
		} else if ("ttl".equalsIgnoreCase(ext)) {
			format = RDFFormat.TURTLE;
		} else {
			log.info("Skipping uri:" + sourceUri);
			skip = true;
		}

		if (skip) {
			log.warn("Skipped unknown source: " + sourceUri);
			return;
		}

		try {
			log.info("Loading:" + sourceUri);
			Resource resource = getFileResourceFromUri(sourceUri);
			repository.getConnection().clear(resource);
			repository.getConnection().add(preloadInputStream, "http://example.org#", format, resource);
			repository.getConnection().commit();
			// this.conn.clear(resource);
			// this.conn.add(preloadInputStream, "http://example.org#", format,
			// resource);
			// this.conn.commit();
		} catch (IOException e) {
			log.error(e.getMessage());
		} catch (RDFParseException e) {
			log.error(e.getMessage());
		} catch (RepositoryException e) {
			log.error(e.getMessage());
		} finally {
			try {
				preloadInputStream.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	private Resource getFileResourceFromUri(String sourceUri) {
		if (sourceUri == null || "".equals(sourceUri)) {
			throw new NullPointerException("Resource URI cannot be empty!");
		}

		int beginIndex = sourceUri.lastIndexOf("/");
		if (beginIndex != -1) {
			sourceUri = PREFIX_FILE_URI + sourceUri.substring(beginIndex + 1);
		}

		return new URIImpl(sourceUri);
	}

	@PreDestroy
	public void destroy() throws Exception {
		if (repository != null)
			repository.shutDown();
		if (repositoryManager != null)
			repositoryManager.shutDown();
		if (connection != null)
			connection.close();
	}
} public class CATConstants {

  public static String TYPE_INSTANCE = "instance";

  public static String FEATURE_INSTANCE_URI = "instanceURI";

  /* used to set feature names for Lookups inside OntoRootGaz */
  public static String FEATURE_URI = "URI";
  public static String FEATURE_TYPE_CLASS = "class";
  
  public static String FEATURE_PROPERTY_URI = "propertyURI";

  public static String FEATURE_PROPERTY_VALUE = "propertyValue";

  public static String FEATURE_HEURISTIC_LEVEL = "heuristic_level";

  public static String FEATURE_HEURISTIC_VALUE = "heuristic_value";



  public static String CLASS_URI = "classURI";

  public static String CLASS_URI_LIST = "classURIList";

  public static String TYPE_PROPERTY = "property";

  public static String TYPE_DATATYPE_PROPERTY_VALUE = "datatypePropertyValue";

  public static String ANNOTATION_TYPE_ONTORES = "OntoRes";

  public static String ANNOTATION_TYPE_ONTORESCHUNK = "OntoResChunk";

  /** used to set feature names for Lookups inside OntoRootGaz */
  public static String TYPE_FEATURE = "type";

  /**
   * Name of the common logger
   */

  public static String LOGGER_OUPUT_LEVEL = "2000";

  /** separator used during formatting of results */
  public static String NEW_LINE =System.getProperty("line.separator");// "\n";

  public static String REGEX_CAMEL_CASE = "[a-z][A-Z]";

}
 public class AnnotationScoreComparator implements Comparator<Annotation> {
    public int compare(Annotation a1, Annotation a2) {
        // compare annotation scores
        return a2.getScore().compareTo(a1.getScore());
    }
} public class FreyaConstants {
    
    public static final String SENTENCE_DELIMITER = "\n ";
    
    public static String QUESTION = "processedQuestion";
    public static String SEMANTIC_CONCEPTS = "semanticConcepts";

    public static String VARIABLE_E = "E";
    public static String VARIABLE_L = "L";
    public static String VARIABLE_P = "P";
    public static String VARIABLE_T = "T";

    public static String POC_TYPE = "POC";
    public static String OC_TYPE = "OC";

    public static String OWL_CLASS = "http://www.w3.org/2002/07/owl#Class";

    public static String SOLR_FIELD_ID = "id";
    public static String SOLR_FIELD_CLASS = "class_s";
    public static String SOLR_FIELD_INSTANCE = "inst_s";
    public static String SOLR_FIELD_PROPERTY = "pred_s";
    public static String SOLR_FIELD_STEMMED_CONTENT = "stem_en";
    /** see also: setting in solr schema.xml <copyField source="prec_s" dest="prec_lowercase"/> */
    public static String SOLR_FIELD_EXACT_CONTENT = "prec_s";
    public static String SOLR_FIELD_LOWERCASE_CONTENT = "prec_lowercase";


    public static String CLASS_FEATURE_LKB = "class";

    public static String INST_FEATURE_LKB = "inst";

    public static String PROPERTY_FEATURE_LKB = "pred";

    public static String FIELD_EXACT_CONTENT = "prec";

    public static String FIELD_EXACT_LOWERCASED_CONTENT = "low";

    public static String FIELD_STEMMED_CONTENT = "stem";

    public static String SCORE = "score";

    public static String MAX_FUNCTION = "max";

    public static String MIN_FUNCTION = "min";

    public static String SUM_FUNCTION = "sum";
    
    public static String AVG_FUNCTION = "avg";

    public static String NONE_NEIGHBOURING_ONTOLOGY_ELEMENT = "NONENEIGHBOURINGONTOLOGYELEMENT";

    public static String INSTANCE_WITHOUT_DIRECT_CLASS = "INSTANCEWITHOUTDIRECTCLASS";

    public static String NUMBER_OF_RESULTS = "numberOfResults";
    public static String TRIPLE_ELEMENTS = "tripleElements";
    public static String RESULTS_GRAPH = "resultsGraph";

    public static String ANSWER = "answer";



    public static String ELEMENTS = "elements";

    public static String TABLE = "table";
    public static String GRAPH = "graph";
    
    public static String SPARQL = "sparql";

    public static String PRECISE_SPARQL = "preciseSparql";

    public static String REPOSITORY_URL = "repositoryUrl";

    public static String REPOSITORY_ID = "repositoryId";
    // ///////////////////////////////////////
    // query specific constants ////////////
    // //////////////////////////////////////
    public static String NP_TAG_TREEBANK = "NP";
    public static String NN_TAG_TREEBANK = "NN";
    public static String NX_TAG_TREEBANK = "NX";
    public static String PRP_TAG_TREEBANK = "PRP";
    public static String EX_TAG_TREEBANK = "EX";
    public static String WHADVP_TAG_TREEBANK = "WHADVP";
    public static String WHADJP_TAG_TREEBANK = "WHADJP";
    public static String WHNP_TAG_TREEBANK = "WHNP";
    public static String WRB_TAG_TREEBANK = "WRB";
    public static String WP_TAG_TREEBANK = "WP";
    public static String RB_TAG_TREEBANK = "RB";
    /** JJ adjective, JJR comparative, JJS superlative */
    public static String JJ_TAG_TREEBANK = "JJ";
    /** VBN verb, past participle */
    public static String VBN_TAG_TREEBANK = "VBN";
    /** RBS adverb, superlative */
    public static String RBS_TAG_TREEBANK = "RBS";

    public static String ADJP_TAG_TREEBANK = "ADJP";
    public static String DT_TAG_TREEBANK = "DT";

    public static boolean DEBUG_MODE = false;

    public static String ANNOTATION_FEATURE_TREE = "Tree";

    public static String STANFORD_TREE = "StanfordTree";

    public static String RESULT_TYPE_GRAPH = "Graph";

    public static String RESULT_TYPE_TREE = "Tree";

    public static String RESULT_TYPE_STRING = "string";

    public static String ANNOTATION_TYPE_ONTORES = "OntoRes";

    public static String ANNOTATION_TYPE_ONTORESCHUNK = "OntoResChunk";

    /* used when grouping elements to indicate if it is a conjunction element */
    public static String CONJUNCTION = "and";

    /* used when grouping elements to indicate if it is a disjunction element */
    public static String DISJUNCTION = "or";

    /**
     * this constant is used to indicate wheather the keyword is from the gazetteer with this name
     */
    public static String LIST_COMMANDS = "listCommands";

    /**
     * Name of the common logger
     */
    public static String LOGGER_NAME = "CLOnE-QL-logger";

    public static String LOGGER_OUPUT_LEVEL = "2000";

    /* separator used during formatting of results */
    public static String TRIPLES_SEPARATOR = " --> ";

    /*
     * flag indicating the output of the result, in this case it is refering to the resource names
     */
    public static String SHOWING_TRIPLES_WITH_RESOURCE_NAMES = "showingTriplesWithResourceNames";

    /*
     * flag indicating the output of the result, in this case it is refering to the labels
     */
    public static String SHOWING_TRIPLES_WITH_LABELS = "showingTriplesWithLabels";

    public static String REGEX_GROUPS_SEPARATED_BY_AND = "i\\d+-and-i\\d+(-and-i\\d+)*";

    public static String REGEX_GROUPS_SEPARATED_BY_OR = "i\\d+-or-i\\d+(-or-i\\d+)*";

    public static String GROUP_PREFIX = "gs:";

    public static String GROUP_SUFFIX = ":ge";

    public static String REGEX_PPP = "[i,c,d,g]\\d+-([k,o,p]\\d+-)*[i,c,d,g]\\d+";

    public static String REGEX_FIND_JOKER = "[i,c,d,g]\\d+-[i,c,d,g]\\d+";

    public static String REGEX_FIND_CLASS_JOKER = "[p]\\d+-[p]\\d+";

    public static String POTENTIAL_PROPERTY_POSITION = "-r";

    public static String EXACT_PROPERTY = "-ep";

    public static String POTENTIAL_RELATED_ELEMENTS = ":";

    public static String MAX_SIMILARITY_VALUE = "1";

    /* three types of weights - to show the importance of every type of score */
    public static double SIMILARITY_SCORE_WEIGHT = 3.0;
    public static double SPECIFICITY_SCORE_WEIGHT = 1.0;
    public static double DISTANCE_SCORE_WEIGHT = 1.0;

    public static String SELECT = "select distinct";
    
    public static String SELECT_SUM ="select (SUM(xsd:decimal(";
    
    public static String SELECT_AVG ="select (AVG(xsd:decimal(";

    public static String WHERE = "where";

    public static String FROM = "from";

    public static String INVERSE_PROPERTY = "[inverseProperty]";

    public static String JOKER = "joker";

    public static String LITERAL_VALUE_CONNECTOR = "is";

    // these are used as annotation type names in order to transwer the results
    // from the gate pipeline elsewhere
    public static String QUERY = "query";
    public static String INTERPRETATIONS_LIST = "interpretationsList";
    public static String SELECTED_INTERPRETATION = "selectedInterpretation";
    public static String INTERPRETATION_RESULTS = "interpretationResults";
    public static String QUERY_INTERPRETATIONS = "queryInterpretations";
    public static String MAP_OF_LABELS = "mapOfLabels";

    public static double LEVENSHTEIN_THRESHOLD = 0.1;
    public static double MONGE_THRESHOLD = 0.5;
    public static double MAX_MONGE_THRESHOLD = 1.0;

    public static double REINFORCEMENT_REWARD = 1.0;
    public static long REINFORCEMENT_NULL_STATE = 0;
    public static double REINFORCEMENT_NEGATIVE_REWARD = -1.0;

    public static String CLARIFICATION_OPTIONS_NONE = "none";

    public static String CLARIFIED_INTERPRETATION_DETAILS = "clarifiedInterpretationDetails";

    public static String CIPIN_SEPARATOR = ",";

    /* priority over ontology annotations */
    public static int MAIN_SUBJECT_PRIORITY_MAX = 100;

    public static int MAIN_SUBJECT_PRIORITY_MIN = 0;


}
 public class NumberUtils {
	  public static double roundTwoDecimals(double d) {
	    DecimalFormatSymbols decimalSymbols = new DecimalFormatSymbols(Locale.getDefault());
	    decimalSymbols.setDecimalSeparator('.');
	    decimalSymbols.setGroupingSeparator(' ');
	    DecimalFormat twoDForm = new DecimalFormat("#.##");
	    twoDForm.setGroupingUsed(false);
	    twoDForm.setDecimalFormatSymbols(decimalSymbols);
	    return Double.valueOf(twoDForm.format(d));
	  }
} public class OntologyElementsUtil {
    /**
     * @param list
     * @return
     */
    public static String beautifyListOfOntologyElements(List<OntologyElement> list) {
        StringBuffer buff = new StringBuffer();
        for (OntologyElement el : list) {
            buff.append(el.getClass().getSimpleName()).append(":").append(el.getData());

        }
        if (list != null && list.size() > 0) {
            buff.append("(").append(list.get(0).getAnnotation().getText()).append(",")
                            .append(list.get(0).getAnnotation().getStartOffset()).append(",")
                            .append(list.get(0).getAnnotation().getEndOffset()).append(")");
        }
        return buff.toString();
    }

    /**
     * utility method takes ontology elements from semantic concepts and generates a list
     * 
     * @param candidates
     * @return
     */
    public static List<OntologyElement> fromSemanticConceptsToOntologyElements(
                    List<SemanticConcept> candidates) {
        List list = new ArrayList();
        for (SemanticConcept c : candidates) {
            list.add(c.getOntologyElement());
        }
        return list;
    }
} public class ProfilerUtil {
 public static String profileString(String sessionId, String query, long runTime, String[] voteIds){
    StringBuilder sb = new StringBuilder();
    sb.append("SESSION ID: ");
    sb.append(sessionId);
    sb.append("; QUERY: ");
    sb.append(query);
    if (voteIds!=null){
    sb.append("; VOTE ID: ");
    for (String id:voteIds){
    sb.append(id).append(" ");
    }
    }
    sb.append("; TIME: ");
    sb.append(runTime);
    return sb.toString();
  }
 
 public static String profileSelection(long runTime, String sessionId, String query, List<SemanticConcept> candidates, Integer[] ranks){
   StringBuilder sb = new StringBuilder();
   sb.append("SESSION ID: ");
   sb.append(sessionId);
   sb.append("; QUERY: ");
   sb.append(query);
   sb.append("; SELECTION: ");
   List list=OntologyElementsUtil.fromSemanticConceptsToOntologyElements(candidates);
  
   StringBuffer ranksString=new StringBuffer("");
   for (Integer rank:ranks){
     ranksString.append(rank.toString()).append(" ");
   }
   sb.append(OntologyElementsUtil.beautifyListOfOntologyElements(list));
   sb.append("; RANK: ");
   sb.append(ranksString.toString());
   sb.append("; TIME: ");
   sb.append(runTime);
   return sb.toString();
 }
 
 
 
}
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpSession;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.freya.dialogue.DialogManager;
import org.freya.format.JsonCreator;
import org.freya.learning.LearningModelHelper;
import org.freya.mapping.Consolidator;
import org.freya.mapping.Mapper;
import org.freya.model.OntologyElement;
import org.freya.model.POC;
import org.freya.model.Question;
import org.freya.model.ResultGraph;
import org.freya.model.SemanticConcept;
import org.freya.model.SuggestionPair;
import org.freya.model.Vote;
import org.freya.model.service.FreyaResponse;
import org.freya.oc.OCAnalyser;
import org.freya.oc.OCUtil;
import org.freya.parser.stanford.StanfordSentenceSplitter;
import org.freya.rdf.SesameRepositoryManager;
import org.freya.rdf.query.CATConstants;
import org.freya.util.AnnotationScoreComparator;
import org.freya.util.FreyaConstants;
import org.freya.util.NumberUtils;
import org.freya.util.OntologyElementsUtil;
import org.freya.util.ProfilerUtil;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.config.RepositoryConfigException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
package org.freya.service;




@Component
public class FreyaServiceHelper {
	protected final Log logger = LogFactory.getLog(getClass());

	@Autowired
	private Mapper mapper;

	@Autowired
	Consolidator consolidator;

	@Autowired
	private DialogManager dialogManager;

	@Autowired
	private OCAnalyser ocAnalyser;

	@Autowired
	private OCUtil ocUtil;