@Component
public class RenameAnnotations {
    private static final Log logger = LogFactory.getLog(RenameAnnotations.class);
    @Autowired LuceneAnnotator luceneAnnotator;
    /*
     * this list contains those properties which are used to identify the entity; for example label property or
     * 'resourceHasName' type properties; the idea is to make this configurable as various ontologies use different
     * naming conventions and we need distinction btw. labels and 'those-serving-as-labels' properties as opposed to
     * others such as date-of-birth for example, which is not identifying the entity but the specific feature of it
     */
    @Value("${org.freya.annotate.rename.annotations.properties.list}") String propertiesListString;

    List<String> propertiesList;

    /*
     * list of properties for which the values will be ignored...such as abbreviation in the case of mooney...or
     * comment..etc
     */
    @Value("${org.freya.annotate.rename.annotations.ignore.properties.list}") String ignorePropertiesListString;
    /**
     * "The list of property URIs which are used to identify an entity. For example label property or 'resourceHasName' properties. Properties which are not in this list will generate 'datatypePropertyValue' annotation feature type"
     * )
     * 
     * @param propertiesList
     */
    List<String> ignorePropertiesList;

    @PostConstruct
    void init() {
        propertiesList = new ArrayList<String>();
        ignorePropertiesList = new ArrayList<String>();
        if (propertiesListString != null && ignorePropertiesListString != null) {
            String[] list = propertiesListString.split(",");
            for (String s : list) {
                if (s != null && !"".equals(s))
                    propertiesList.add(s);
            }

            list = ignorePropertiesListString.split(",");
            for (String s : list) {
                if (s != null && !"".equals(s))
                    ignorePropertiesList.add(s);
            }
        }
    }

    /**
     * @param annotations
     * @param lookup
     * @return
     */
    boolean anyOfAlreadyAddedAnnotationsOverlapWithLookup(
                    List<Annotation> annotations, Annotation lookup) {
        boolean result = false;
        for (Annotation ann : annotations) {
            if (lookup.overlaps(ann)) {
                result = true;
                break;
            }
        }
        return result;
    }

    /**
	 * 
	 */
    private static final long serialVersionUID = -2085562223086784633L;


    /**
     * 
     * @param lookupAnnotations
     * @return
     */
    public List<Annotation> rename(List<Annotation> lookupAnnotations) {

        // map of uris with the value list of annotations
        Map<String, List<Annotation>> uris =
                        new HashMap<String, List<Annotation>>();
        Map<DatatypePropertyValueIdentifier, List<Annotation>> dtpvi =
                        new HashMap<DatatypePropertyValueIdentifier, List<Annotation>>();
        // now read Lookup annotations
        List<Annotation> toRemove = new ArrayList<Annotation>();
        List<Annotation> newLookupAnnotations = new ArrayList<Annotation>();
        for (Annotation lookup : lookupAnnotations) {
            // now read feature class
            String classFeature =
                            (String) lookup.getFeatures()
                                            .get(FreyaConstants.CLASS_FEATURE_LKB);
            String instFeature =
                            (String) lookup.getFeatures().get(FreyaConstants.INST_FEATURE_LKB);
            String predFeature =
                            (String) lookup.getFeatures().get(
                                            FreyaConstants.PROPERTY_FEATURE_LKB);
            String annText = (String) lookup.getFeatures().get(
                            FreyaConstants.RESULT_TYPE_STRING);
            Float score = (Float) lookup.getFeatures().get(
                            FreyaConstants.SCORE);
            // //////////////////////////////
            // //////////////hack: this is due to the bug in the jiant sparql
            // / where duplicates are retrieved for each class/property/inst ///////
            // //////////////////////////////
            if ("http://www.w3.org/2000/01/rdf-schema#Resource".equals(classFeature))
                // remove lookup
                toRemove.add(lookup);
            if (instFeature == null && classFeature == null) {
                toRemove.add(lookup);
            }
            // //////////////////////////////
            // //////////////instances///////
            // //////////////////////////////
            else if (OWL.CLASS.toString().equals(classFeature)
                            || RDFS.CLASS.toString().equals(classFeature)) {
                if (uris.keySet().contains(instFeature)
                                && anyOfAlreadyAddedAnnotationsOverlapWithLookup(uris
                                                .get(instFeature), lookup)) {
                    // and the annotations overlap
                    // && ((uris.get(instFeature)).contains(lookup))) {
                    // delete the annotation
                    toRemove.add(lookup);
                } else {
                    List<Annotation> annotations = uris.get(instFeature);
                    if (annotations == null) annotations = new ArrayList<Annotation>();
                    annotations.add(lookup);
                    uris.put(instFeature, annotations);
                    // generate class type
                    HashMap features = new HashMap();
                    features.put(CATConstants.TYPE_FEATURE, CATConstants.FEATURE_TYPE_CLASS);
                    features.put(CATConstants.FEATURE_URI, instFeature);
                    features.put(FreyaConstants.SCORE, score);
                    features.put(FreyaConstants.RESULT_TYPE_STRING, annText);
                    lookup.setFeatures(features);
                }
                // features.put(
                // CATConstants.FEATURE_PROPERTY_URI, propUri);
            } else if (OWL.OBJECTPROPERTY.toString().equals(classFeature)
                            || RDF.PROPERTY.toString().equals(classFeature)
                            || OWL.DATATYPEPROPERTY.toString().equals(classFeature)) {
                if (uris.keySet().contains(instFeature)
                                && anyOfAlreadyAddedAnnotationsOverlapWithLookup(uris
                                                .get(instFeature), lookup)) {
                    // delete the annotation
                    toRemove.add(lookup);
                } else {
                    List<Annotation> annotations = uris.get(instFeature);
                    if (annotations == null) annotations = new ArrayList<Annotation>();
                    annotations.add(lookup);
                    uris.put(instFeature, annotations);
                    // generate property
                    HashMap features = new HashMap();
                    features.put(CATConstants.TYPE_FEATURE, CATConstants.TYPE_PROPERTY);
                    features.put(CATConstants.FEATURE_URI, instFeature);
                    features.put(FreyaConstants.SCORE, score);
                    features.put(FreyaConstants.RESULT_TYPE_STRING, annText);
                    lookup.setFeatures(features);
                }
            } else if (ignorePropertiesList.contains(predFeature)) {
                // do nothing, ignore the property value...
                logger
                                .debug("Removing annotation, as ignoring the value for property: "
                                                + predFeature);
                toRemove.add(lookup);
            } else {
                // first check whether there is feature 'pred' if yes:
                // check whether it is in the propertiesList, if yes:
                // generate instance
                // if not: generate datatypeproperty value
                // if pred==null generate instance

                if (predFeature != null && propertiesList != null
                                && !propertiesList.contains(predFeature)) {
                    DatatypePropertyValueIdentifier datatypePropertyValueIdentifier =
                                    new DatatypePropertyValueIdentifier();
                    List<SerializableURI> instanceUriList = new ArrayList();
                    try {
                        instanceUriList.add(new SerializableURI(
                                        instFeature, false));
                    } catch (Exception e) {
                        logger.info(e.getMessage());
                    }
                    datatypePropertyValueIdentifier.setInstanceURIs(instanceUriList);
                    datatypePropertyValueIdentifier.setPropertyUri(predFeature);
                    datatypePropertyValueIdentifier.setPropertyValue(lookup.getText());
                    // if (dtpvi.contains(datatypePropertyValueIdentifier)) {
                    if (dtpvi.keySet().contains(datatypePropertyValueIdentifier)
                                    && anyOfAlreadyAddedAnnotationsOverlapWithLookup(dtpvi
                                                    .get(datatypePropertyValueIdentifier), lookup)) {
                        // delete the annotation
                        toRemove.add(lookup);
                    } else {
                        List<Annotation> annotations =
                                        dtpvi.get(datatypePropertyValueIdentifier);
                        if (annotations == null) annotations = new ArrayList<Annotation>();
                        annotations.add(lookup);
                        dtpvi.put(datatypePropertyValueIdentifier, annotations);
                        // generate datatype property value
                        HashMap features = new HashMap();
                        features.put(CATConstants.TYPE_FEATURE,
                                        CATConstants.TYPE_DATATYPE_PROPERTY_VALUE);
                        features.put(CATConstants.FEATURE_PROPERTY_URI, predFeature);
                        features.put(CATConstants.FEATURE_INSTANCE_URI, instFeature);
                        features.put(CATConstants.FEATURE_PROPERTY_VALUE, lookup.getText());
                        features.put(FreyaConstants.SCORE, score);
                        features.put(FreyaConstants.RESULT_TYPE_STRING, annText);
                        lookup.setFeatures(features);
                    }
                } else {
                    if (uris.keySet().contains(instFeature)
                                    && anyOfAlreadyAddedAnnotationsOverlapWithLookup(uris
                                                    .get(instFeature), lookup)) {
                        // delete the annotation
                        logger.info("Removing:" + lookup.toString());
                        toRemove.add(lookup);
                    } else {
                        List<Annotation> annotations = uris.get(instFeature);
                        if (annotations == null) annotations = new ArrayList<Annotation>();
                        annotations.add(lookup);
                        uris.put(instFeature, annotations);
                        // instances
                        HashMap features = new HashMap();
                        features.put(CATConstants.TYPE_FEATURE, CATConstants.TYPE_INSTANCE);
                        features.put(CATConstants.FEATURE_URI, instFeature);
                        features.put(CATConstants.CLASS_URI, classFeature);

                        features.put(FreyaConstants.SCORE, score);
                        features.put(FreyaConstants.RESULT_TYPE_STRING, annText);

                        // List<String> classUris=(List<String> )features.get(CATConstants.CLASS_URI_LIST);

                        List<String> dTypes = luceneAnnotator.findDirectTypes(instFeature);
                        // logger.info("Found "+dTypes.size()+" direct types for "+instFeature);
                        // List<String> dTypesFiltered =filterDirectTypes(dTypes);

                        features.put(CATConstants.CLASS_URI_LIST, dTypes);
                        lookup.setFeatures(features);
                    }
                }// generating type instance
            }
            // if (features != null) lookup.setFeatures(features);
        }
        // ////////////////////////////////////////////////////////////
        // ////////////////////test to see whether it will be faster than jape
        // /////////////////////////////////////////////////////////////////////
        // this is now done through jape
        // filterBasedOnPriority();
        for (Annotation annToRemove : toRemove) {
            int index = lookupAnnotations.indexOf(annToRemove);
            lookupAnnotations.remove(index);
        }

        // lookupAnnotations.removeAll(toRemove);

        // lookupAnnotations=filterInstanceClass(lookupAnnotations);
        return lookupAnnotations;
    }


    /**
     * this method is checking whether there are any annotations with feature priority=high this means that other
     * annotations which are overlapped should be removed such as in mississippi river where mississippi refers to state
     * and river but because of the following river the annotation referring to mississippi river will have feature
     * priority=high meaning that the annotation referring to state should be removed this method does exactly the same
     * as the FilterInstanceClass.jape
     */
    Set<Annotation> filterBasedOnPriority(Set<Annotation> allAnnotations) {
        List toRemove = new ArrayList();
        logger
                        .debug("===========================================BEGIN filterBasedOnPriority ================================");
        logger.debug("This many annotations:" + allAnnotations.size());
        if (allAnnotations.size() > 5000) {
            // skip this filtering
        } else {
            int i = 0;
            java.util.List<Annotation> allAnnotationsCopy =
                            new java.util.ArrayList<Annotation>();
            allAnnotationsCopy.addAll(allAnnotations);
            // System.out.println("=======allannotations=====\n"+allAnnotationsCopy.toString());
            for (Annotation lookup : allAnnotations) {
                allAnnotationsCopy.remove(lookup);
                logger.debug("=======checking lookup=====  numer\n" + i + ":"
                                + lookup.toString());
                for (Annotation ann : allAnnotationsCopy) {
                    logger.debug("=======checking annotation=====\n" + ann.toString());
                    // remove annotations which are 'within' the annotations
                    // with
                    // feature priority=high
                    // so that mississippi river always refers to either river
                    // or
                    // low point with name
                    // mississippi river and never to the state
                    if ((ann.getStartOffset().longValue() > lookup.getStartOffset()
                                    .longValue() && ann.getEndOffset().longValue() <= lookup
                                    .getEndOffset().longValue())
                                    || (ann.getStartOffset().longValue() >= lookup
                                                    .getStartOffset().longValue() && ann.getEndOffset()
                                                    .longValue() < lookup.getEndOffset().longValue())) {
                        String priority = (String) lookup.getFeatures().get("priority");
                        if (priority != null && priority.equals("high")) {
                            logger.debug("REMOVING:" + ann.toString()
                                            + "because lookup is prioritized:" + lookup.toString());
                            if (allAnnotations.contains(ann)) toRemove.add(ann);
                        }
                    }
                    // now what if we need to actually delete the lookup
                    // annotation?
                    if ((ann.getStartOffset().longValue() < lookup.getStartOffset()
                                    .longValue() && ann.getEndOffset().longValue() >= lookup
                                    .getEndOffset().longValue())
                                    || (ann.getStartOffset().longValue() <= lookup
                                                    .getStartOffset().longValue() && ann.getEndOffset()
                                                    .longValue() > lookup.getEndOffset().longValue())) {
                        String priority = (String) ann.getFeatures().get("priority");
                        if (priority != null && priority.equals("high")) {
                            logger.debug("REMOVING lookup:" + lookup.toString()
                                            + "because ann is prioritized:" + ann.toString());
                            if (allAnnotations.contains(lookup)) toRemove.add(lookup);
                        }
                    }
                }
                logger
                                .debug("=====================END filtering based on priority ================================");
            }
        }
        allAnnotations.removeAll(toRemove);
        return allAnnotations;
    }
}
 @Component
public class SolrAnnotator {

    private static final Logger log = LoggerFactory.getLogger(SolrAnnotator.class);

    @Autowired
    private TripleIndexer indexer;

    public TripleIndexer getIndexer() {
		return indexer;
	}

	public void setIndexer(TripleIndexer indexer) {
		this.indexer = indexer;
	}

	public SolrDocumentList query(String k, String v) {
        SolrQuery query = new SolrQuery();
        query.setParam("fl", "*,score");
        query.setQuery(k + ":\"" + v + "\"");
        return query(query);
    }

    public SolrDocumentList query(SolrParams params) {
        SolrDocumentList documentList = null;
        try {
            QueryResponse response = indexer.getSolrServer().query(params);
            documentList = response.getResults();
        } catch (SolrServerException e) {
            log.error("Error while communicating with SolrServer", e);
            throw new RuntimeException(e);
        }
        return documentList;
    }

    public List<String> findDirectTypes(String instanceUri) {
        Set<String> dTypes = new HashSet<String>();
        SolrDocumentList docs = query(FreyaConstants.SOLR_FIELD_INSTANCE, instanceUri);
        for (SolrDocument doc : docs) {
            dTypes.add(doc.get(FreyaConstants.SOLR_FIELD_CLASS).toString());
        }

        log.debug("there are " + dTypes.size() + " unique direct types");

        return new ArrayList<String>(dTypes);
    }

    public List<String> findLabels(String instanceUri) {
        Set<String> labels = new HashSet<String>();
        SolrQuery params = new SolrQuery();

        params.setQuery(FreyaConstants.SOLR_FIELD_INSTANCE + ":\"" + instanceUri + "\" AND " +
                FreyaConstants.SOLR_FIELD_PROPERTY + ":\"http://www.w3.org/2000/01/rdf-schema#label\"");
        SolrDocumentList docs = query(params);
        for (SolrDocument doc: docs) {
            labels.add(doc.get(FreyaConstants.SOLR_FIELD_EXACT_CONTENT).toString());
        }

        log.debug("there are " + labels.size() + " unique labels");

        return new ArrayList<String>(labels);
    }

    public List<String> findLiteral(String instanceUri, String propertyURI) {
        Set<String> labels = new HashSet<String>();
        SolrQuery params = new SolrQuery(
                FreyaConstants.SOLR_FIELD_INSTANCE + ":\"" + instanceUri + "\" AND " +
                        FreyaConstants.SOLR_FIELD_PROPERTY + ":\"" + propertyURI + "\""
        );
        SolrDocumentList docs = query(params);
        for (SolrDocument doc: docs) {
            labels.add(doc.get(FreyaConstants.SOLR_FIELD_EXACT_CONTENT).toString());
        }

        log.debug("there are " + labels.size() + " unique literals");

        return new ArrayList<String>(labels);
    }

    public List<Annotation> searchExactContentFirst(Annotation annotation) {
        return searchIndexByThreeSteps(annotation, FreyaConstants.SOLR_FIELD_EXACT_CONTENT,
                FreyaConstants.SOLR_FIELD_LOWERCASE_CONTENT, FreyaConstants.SOLR_FIELD_STEMMED_CONTENT);
    }

    public List<Annotation> searchStemmedContentFirst(Annotation annotation) {
        return searchIndexByThreeSteps(annotation, FreyaConstants.SOLR_FIELD_STEMMED_CONTENT,
                FreyaConstants.SOLR_FIELD_LOWERCASE_CONTENT, FreyaConstants.SOLR_FIELD_EXACT_CONTENT);
    }

    public List<Annotation> searchIndexByThreeSteps(Annotation annotation, String first, String second, String third) {
        SolrDocumentList docs = query(first, annotation.getText());
        if (docs.size() == 0) {
            docs = query(second, annotation.getText());
        }
        if (docs.size() == 0) {
            docs = query(third, annotation.getText());
        }

        return populateAnnotationList(docs, annotation);
    }

    private List<Annotation> populateAnnotationList(SolrDocumentList docs, Annotation annotation) {
        List<Annotation> annotations = new ArrayList<Annotation>();
        for (SolrDocument doc : docs) {
            Annotation ann = new Annotation();
            HashMap<String, Object> features = new HashMap<String, Object>();
            features.put(FreyaConstants.CLASS_FEATURE_LKB, doc.get(FreyaConstants.SOLR_FIELD_CLASS));
            features.put(FreyaConstants.INST_FEATURE_LKB, doc.get(FreyaConstants.SOLR_FIELD_INSTANCE));
            features.put(FreyaConstants.PROPERTY_FEATURE_LKB, doc.get(FreyaConstants.SOLR_FIELD_PROPERTY));
            features.put("string", doc.get(FreyaConstants.SOLR_FIELD_EXACT_CONTENT));
            features.put(FreyaConstants.SCORE, doc.get(FreyaConstants.SCORE));
            ann.setFeatures(features);
            ann.setEndOffset(annotation.getEndOffset());
            ann.setStartOffset(annotation.getStartOffset());
            ann.setSyntaxTree(annotation.getSyntaxTree());
            ann.setText(annotation.getText());
            annotations.add(ann);
        }
        //sort annotations here
        System.out.println("Before sorting:"+annotations.toString());
        Comparator c = new AnnotationSolrScoreComparator();
        Collections.sort(annotations, c);
        System.out.println("After sorting:"+annotations.toString());
        return annotations;
    }

} @Component
public class StanfordExecutor {
    private static final Log logger = LogFactory.getLog(StanfordExecutor.class);
    @Autowired private SimpleStanfordParser stanfordParser;
    @Autowired TreeUtils treeUtils;

    public SimpleStanfordParser getStanfordParser() {
        return stanfordParser;
    }

    public void setStanfordParser(SimpleStanfordParser stanfordParser) {
        this.stanfordParser = stanfordParser;
    }

    /**
     * finding POCs independently from GATE
     * 
     * @param question
     * @return
     * @throws Exception
     */
    public Question findPocs(String text) {
        Question question = null;
        try {
            question = stanfordParser.parseQuestion(text);
        } catch (ExecutionException e) {
            logger.error(e);
        }
        return question;
    }

    /**
     * this method is removing pocs such as 'how' 'which' 'me' etc. given that they have certain tags attached there
     * might be better (more robust way) to handle this, but for now...
     * 
     * @param question
     * @return
     */
    public Question removePocsToIgnore(Question question) {
        logger
                        .info("Removing POCs at the beginning  of the sentence (who,give,which,what..)...");
        List<POC> pocsToRemove = new ArrayList<POC>();
        List<POC> pocs = question.getPocs();
        for (int i = 0; i < pocs.size(); i++) {
            POC poc = pocs.get(i);
            for (int k = 0; k < poc.getAnnotation().getSyntaxTree().size(); k++) {
                Tree tree = poc.getAnnotation().getSyntaxTree().get(k);
                logger.debug("Checking poc:" + poc + "\n  is prepreterminal:"
                                + tree.isPrePreTerminal() + " and is preterminal:"
                                + tree.isPreTerminal());
                if (tree.isPreTerminal()) {
                    String annText = poc.getAnnotation().getText().toLowerCase();
                    logger.debug("Poc text:" + annText);
                    if (annText.startsWith("how")
                                    || annText.startsWith("many")
                                    || annText.startsWith("what")
                                    || annText.startsWith("which")
                                    || annText.startsWith("list")
                                    || annText.startsWith("show")
                                    || annText.startsWith("give")
                                    || (tree != null && tree.label().value().startsWith(
                                                    FreyaConstants.PRP_TAG_TREEBANK))) {
                        logger.debug("annText text:" + annText);
                        logger.debug("tree.label().value() text:" + tree.label().value());
                        if (poc.getAnnotation().getSyntaxTree().size() > 1) {
                            // upate the tree of this poc, do not remove it
                            List<Tree> newTree = new ArrayList<Tree>();
                            for (int j = 1; i < poc.getAnnotation().getSyntaxTree().size(); i++) {
                                newTree.add(poc.getAnnotation().getSyntaxTree().get(j));
                                // newTree.remove(tree);
                                long startNode =
                                                Trees.leftEdge(poc.getAnnotation().getSyntaxTree().get(
                                                                k + 1), question.getSyntaxTree());
                                poc.getAnnotation().setStartOffset(startNode);
                                poc.getAnnotation().setSyntaxTree(newTree);
                                String newText =
                                                treeUtils.getNiceString(
                                                                newTree);
                                poc.getAnnotation().setText(newText);
                                logger.debug("The new poc is now:" + poc.toString());
                            }
                        } else {
                            logger.info("POC marked for removal: "
                                            + poc.getAnnotation().getSyntaxTree());
                            pocsToRemove.add(poc);
                        }
                    }
                } else if (tree.isPrePreTerminal()) {
                    // this does not happen often but if it does then split it before
                    // running this method again
                    List<Tree> children = tree.getChildrenAsList();
                    poc.getAnnotation().setSyntaxTree(children);
                    removePocsToIgnore(question);
                }
            }
        }
        logger.info("Marked " + pocsToRemove.size() + " POCs to remove.");
        question.getPocs().removeAll(pocsToRemove);
        return question;
    }

    /**
     * this method is removing pocs such as 'how' 'which' 'me' etc. given that they have certain tags attached there
     * might be better (more robust way) to handle this, but for now...
     * 
     * @param question
     * @return
     */
    public Question removePocsToIgnoreNew(Question question) {
        logger.info("Removing POCs which are 'perhaps' useless...");
        List<POC> pocsToRemove = new ArrayList<POC>();
        List<POC> pocsToAdd = new ArrayList<POC>();
        List<POC> pocs = question.getPocs();
        for (int i = 0; i < pocs.size(); i++) {
            POC poc = pocs.get(i);
            Tree tree = poc.getAnnotation().getSyntaxTree().get(0);
            String annText = poc.getAnnotation().getText().toLowerCase();
            if (annText.startsWith("how")
                            || annText.startsWith("many")
                            || annText.startsWith("what")
                            || annText.startsWith("which")
                            || annText.startsWith("list")
                            || annText.startsWith("show")
                            || (tree != null && tree.label().value().startsWith(
                                            FreyaConstants.PRP_TAG_TREEBANK))) {
                logger.info("POC marked for removal: "
                                + poc.getAnnotation().getSyntaxTree());
                pocsToRemove.add(poc);
            }
        }
        logger.info("Marked " + pocsToRemove.size() + " POCs to remove.");
        question.getPocs().removeAll(pocsToRemove);
        return question;
    }

    /**
     * Closes the GATE application (and by extension all the PRs it contains) and the corpus using
     * {@link Factory#deleteResource(gate.Resource) deleteResource}.
     */
    public void shutdown() {
        logger.debug("SHUTDOWN stanfordExecutor called");
        stanfordParser = null;
    }
} public class Annotation implements Serializable {

    String text;
    HashMap features;
    Long startOffset;
    Long endOffset;

    public HashMap getFeatures() {
        return features;
    }

    public void setFeatures(HashMap features) {
        this.features = features;
    }

    List<Tree> syntaxTree;

    public List<Tree> getSyntaxTree() {
        return syntaxTree;
    }

    public void setSyntaxTree(List<Tree> syntaxTree) {
        this.syntaxTree = syntaxTree;
    }

    public String getText() {
        return text;
    }

    public void setText(String text) {
        this.text = text;
    }

    public Long getStartOffset() {
        return startOffset;
    }

    public void setStartOffset(Long startOffset) {
        this.startOffset = startOffset;
    }

    public Long getEndOffset() {
        return endOffset;
    }

    public void setEndOffset(Long endOffSet) {
        this.endOffset = endOffSet;
    }

    public String toString() {
        StringBuffer buffer = new StringBuffer("");
        buffer.append(" Annotation: StartOffset:" + startOffset);
        buffer.append(" End offset:" + endOffset);
        buffer.append(" String:" + text);
        buffer.append(" Syntax tree:" + syntaxTree);
        if (features != null)
            buffer.append(" Features:" + features.toString());
        return buffer.toString();
    }

    public boolean equals(Object anotherObject) {
        if (this == anotherObject)
            return true;
        if (!(anotherObject instanceof Annotation))
            return false;
        if (this.startOffset == null && this.endOffset == null
                        && ((Annotation) anotherObject).startOffset != null
                        && ((Annotation) anotherObject).endOffset != null)
            return true;
        if ((anotherObject != null)
                        && (this.startOffset.longValue() == ((Annotation) anotherObject)
                                        .getStartOffset().longValue())
                        && (this.endOffset.longValue() == ((Annotation) anotherObject)
                                        .getEndOffset().longValue())
                        && (this.features.equals(((Annotation) anotherObject).features)
                        ))

            return true;
        else
            return false;
    }

    /**
     * 
     * @param aAnnot
     * @return
     */
    public boolean overlaps(Annotation aAnnot) {
        if (aAnnot == null) return false;
        if (aAnnot.getStartOffset() == null ||
                        aAnnot.getEndOffset() == null) return false;

        if (aAnnot.getEndOffset().longValue() <= this.getStartOffset().longValue())
            return false;

        if (aAnnot.getStartOffset().longValue() >= this.getEndOffset().longValue())
            return false;

        return true;
    }// overlaps


    public Object clone() {
        Object deepCopy = null;
        ByteArrayOutputStream byteArrOs = new ByteArrayOutputStream();
        ObjectOutputStream objOs;
        try {
            objOs = new ObjectOutputStream(byteArrOs);
            objOs.writeObject(this);
            ByteArrayInputStream byteArrIs =
                            new ByteArrayInputStream(byteArrOs.toByteArray());
            ObjectInputStream objIs = new ObjectInputStream(byteArrIs);
            deepCopy = objIs.readObject();
            return deepCopy;
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return deepCopy;
    }

}
 public class POC {

	Annotation annotation = new Annotation();

	public Annotation getAnnotation() {
		return annotation;
	}

	public void setAnnotation(Annotation annotation) {
		this.annotation = annotation;
	}

	POC head;

	MainSubject mainSubject;
	List<String> modifiers;

	public POC getHead() {
		return head;
	}

	public void setHead(POC head) {
		this.head = head;
	}

	public List<String> getModifiers() {
		return modifiers;
	}

	public void setModifiers(List<String> modifiers) {
		this.modifiers = modifiers;
	}

	public MainSubject getMainSubject() {
		return mainSubject;
	}

	public void setMainSubject(MainSubject mainSubject) {
		this.mainSubject = mainSubject;
	}

	public String toString() {
		StringBuffer buffer = new StringBuffer("");
		buffer.append("POC:" + getAnnotation().toString());
		return buffer.toString();
	}
}
 @XmlRootElement
public class Question {

    /**
     * question type such as boolean, number, classic (all the others);
     */

    public String sentenceText;
    public Type type;

    public Type getType() {
        return type;
    }

    public void setType(Type type) {
        this.type = type;
    }

    List<HasWord> tokens;

    Tree syntaxTree;

    /* this is focus */
    POC focus;

    List<OntologyElement> answerType;

    List<POC> pocs = new ArrayList<POC>();


    public List<HasWord> getTokens() {
        return tokens;
    }

    public void setTokens(List<HasWord> tokens) {
        this.tokens = tokens;
    }


    List<List<SemanticConcept>> semanticConcepts =
                    new ArrayList<List<SemanticConcept>>();

    public List<OntologyElement> getAnswerType() {
        return answerType;
    }

    public void setAnswerType(List<OntologyElement> answerType) {
        this.answerType = answerType;
    }

    public POC getFocus() {
        return focus;
    }

    public void setFocus(POC focus) {
        this.focus = focus;
    }

    public Tree getSyntaxTree() {
        return syntaxTree;
    }

    public void setSyntaxTree(Tree syntaxTree) {
        this.syntaxTree = syntaxTree;
    }

    public List<List<SemanticConcept>> getSemanticConcepts() {
        return semanticConcepts;
    }

    public void setSemanticConcepts(List<List<SemanticConcept>> semanticConcepts) {
        this.semanticConcepts = semanticConcepts;
    }

    public List<POC> getPocs() {
        return pocs;
    }

    public void setPocs(List<POC> pocs) {
        this.pocs = pocs;
    }

    public String toString() {
        StringBuffer b = new StringBuffer("");
        b.append("sem concepts:");
        if (this.semanticConcepts != null)
            b.append(this.semanticConcepts.toString());
        b.append("pocs:");
        if (this.pocs != null) b.append(pocs.toString());
        b.append("Tree:");
        if (this.syntaxTree != null) b.append(this.syntaxTree.toString());
        return b.toString();
    }
}
 @XmlRootElement
public class SemanticConcept implements Serializable {

    OntologyElement ontologyElement;
    String function;
    Boolean verified = false;
    Double score;

    public Double getScore() {
        return score;
    }

    public void setScore(Double score) {
        this.score = score;
    }

    public Boolean isVerified() {
        return verified;
    }

    public void setVerified(Boolean verified) {
        this.verified = verified;
    }

    public OntologyElement getOntologyElement() {
        return ontologyElement;
    }

    public void setOntologyElement(OntologyElement ontologyElement) {
        this.ontologyElement = ontologyElement;
    }

    public String getFunction() {
        return function;
    }

    public void setFunction(String function) {
        this.function = function;
    }

    public String toString() {
        StringBuffer b = new StringBuffer("");
        b.append("Ontology Elements:\n");
        if (ontologyElement != null) b.append(ontologyElement.toString());
        b.append(" Verified:").append(verified).append("\n");
        b.append(" Function:").append(function).append("\n");
        if (ontologyElement != null)
            b.append("Is ontology element already added:"
                            + ontologyElement.isAlreadyAdded() + "\n");
        return b.toString();
    }

    public Object clone() {
        Object deepCopy = null;
        ByteArrayOutputStream byteArrOs = new ByteArrayOutputStream();
        ObjectOutputStream objOs;
        try {
            objOs = new ObjectOutputStream(byteArrOs);
            objOs.writeObject(this);
            ByteArrayInputStream byteArrIs =
                            new ByteArrayInputStream(byteArrOs.toByteArray());
            ObjectInputStream objIs = new ObjectInputStream(byteArrIs);
            deepCopy = objIs.readObject();
            return deepCopy;
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return deepCopy;
    }

    public boolean equals(Object theOtherConcept) {
        boolean isEqual = false;
        String function1 = this.getFunction();
        String function2 = ((SemanticConcept) theOtherConcept).getFunction();
        OntologyElement element1 = this.getOntologyElement();
        OntologyElement element2 =
                        ((SemanticConcept) theOtherConcept).getOntologyElement();
        boolean functionEquals = false;
        if (function1 != null && function2 != null)
            functionEquals = function1.equals(function2);
        else if (function1 == null && function2 == null) functionEquals = true;
        boolean verifiedEquals = false;
        if (this.isVerified().booleanValue() == ((SemanticConcept) theOtherConcept)
                        .isVerified().booleanValue()) verifiedEquals = true;
        if (functionEquals && element1.equals(element2))
            isEqual = true;
        // System.out
        // .print("************************************************************calling equals method..result is:"
        // + isEqual);
        return isEqual;
    }
}
 @Component
public class OCCreator {
    private static final Log logger = LogFactory.getLog(OCCreator.class);

    private AtomicInteger idCounter = new AtomicInteger();
    @Autowired private JsonCreator jsonCreator;
    @Autowired LuceneAnnotator luceneAnnotator;


    /**
     * takes a list of overlapped elements
     * 
     * @param lookups
     * @return
     */
    private List<List<OntologyElement>> getOntologyElementsLucene(
                    Set<Annotation> lookups) {
        logger.info("Received:" + lookups.size()
                        + " annotations, now generating Ontology Elements.");
        List<List<OntologyElement>> ontologyElements =
                        new ArrayList<List<OntologyElement>>();
        AnnotationOffsetComparator oc = new AnnotationOffsetComparator();
        List<Annotation> lookupsList =
                        new ArrayList<Annotation>();
        if (lookups != null) lookupsList.addAll(lookups);
        Collections.sort(lookupsList, oc);
        List<List<Annotation>> overlappedGroups =
                        getOverlappedGroupsOfLuceneAnnotations(lookupsList);
        logger.debug("from above annotations generated:" + overlappedGroups.size()
                        + " columns with annoations, now generating ontology elements...");
        ontologyElements =
                        getOverlappedGroupsOfOntologyElementsLucene(overlappedGroups);
        int col = 0;
        List<List<OntologyElement>> toRemove =
                        new ArrayList<List<OntologyElement>>();
        List<List<OntologyElement>> toAdd = new ArrayList<List<OntologyElement>>();
        for (List<OntologyElement> column : ontologyElements) {
            // logger.debug("column:" + col + " size:" + column.size());
            HashMap<String, List<OntologyElement>> map =
                            new HashMap<String, List<OntologyElement>>();
            for (OntologyElement el : column) {
                List<OntologyElement> list = map.get(el.getAnnotation().getText());
                if (list == null) list = new ArrayList<OntologyElement>();
                list.add(el);
                map.put(el.getAnnotation().getText(), list);
            }
            if (map.keySet().size() > 1) {
                logger
                                .debug("Now making separate column for overlapped OCs which have different text...");
                // this means that overlapped annotations have different text so
                // separate this one column into text.size
                toRemove.add(column);
                List keys = new ArrayList(map.keySet());
                for (int i = 0; i < map.keySet().size(); i++) {
                    String key = (String) keys.get(i);
                    toAdd.add(map.get(key));
                }
            }
            col++;
        }
        logger.info("Generated " + ontologyElements.size() + " columns.");
        ontologyElements.removeAll(toRemove);
        logger
                        .info("Removed those that will be added as split in a moment, this many left: "
                                        + ontologyElements.size() + " columns.");
        ontologyElements.addAll(toAdd);
        logger.info("Added the above split column, there are  "
                        + ontologyElements.size()
                        + " columns (because overlapped OCs have different text).");
        return ontologyElements;
    }

    /**
     * lucene generates semanticElements from ontologyElements found by Lucene
     * 
     * @param oElements
     * @return
     */
    public List<List<SemanticConcept>> getSemanticConcepts(
                    Set<Annotation> lookups, Question question) {
        List<List<OntologyElement>> oElements = getOntologyElementsLucene(lookups);
        logger.info("from " + lookups.size() + " generated ontogy elements:"
                        + oElements.size() + " columns");
        int colN = 0;
        List<List<SemanticConcept>> semanticElements =
                        new ArrayList<List<SemanticConcept>>();
        for (List<OntologyElement> ontologyElements : oElements) {
            // logger.info("col " + colN + ontologyElements.toString());
            List<SemanticConcept> scColumn = new ArrayList<SemanticConcept>();
            // so this is now a list of lists with grouped ontology elements, this is
            // not only overlapped elements but this grouping is within one column
            List<OntologyElement> listOfGroupedElements =
                            groupOntologyElements(ontologyElements);
            logger.info("After grouping elements these shrink into:"
                            + listOfGroupedElements.size());
            // + " elements and these are:" + listOfGroupedElements.toString());
            // listOfGroupedElements=OCUtil.filterInstanceClass(listOfGroupedElements);
            // sada ovde uradi filter class instance: ako su jedno do drugog izbrisi class i takodje sve ostale instance
            // daj im neki los score

            for (OntologyElement ontologyElement : listOfGroupedElements) {
                // here we want to check whether any of ontologyElements needs to go
                // together into SemanticConcept
                // for example if they share classURI and are InstanceElements than they
                // are grouped together
                SemanticConcept sc = new SemanticConcept();
                sc.setOntologyElement(ontologyElement);
                scColumn.add(sc);
            }
            semanticElements.add(scColumn);
            colN++;
        }
        SemanticConceptListComparator c = new SemanticConceptListComparator();
        Collections.sort(semanticElements, c);
        return semanticElements;
    }

    /**
     * grouping by type: group all elements with different instanceURI and same classURI so that they will be
     * represented by one SemanticConcept and InstanceListElement input is a list of overlapped ontology elements: those
     * that cover the same span
     * 
     * @return
     */
    List<OntologyElement> groupOntologyElements(
                    List<OntologyElement> ontologyElements) {
        List<OntologyElement> finalList = new ArrayList<OntologyElement>();
        Map<String, List<OntologyElement>> map =
                        new HashMap<String, List<OntologyElement>>();
        logger.info("There are " + ontologyElements.size() + " elements.");
        long counter = 0;
        for (OntologyElement el : ontologyElements) {
            if (el instanceof InstanceElement) {
                String classURI = ((InstanceElement) el).getClassURI().toString();
                Collections.sort(((InstanceElement) el).getClassURIList());// sortiraj
                                                                           // kako bi
                                                                           // iste
                                                                           // instance
                                                                           // bile na
                                                                           // istom
                String classURIList =
                                ((InstanceElement) el).getClassURIList().toString();
                String text = ((InstanceElement) el).getAnnotation().getText();
                // logger.debug("Text is:" + text);
                // String key = "I" + classURI + text;
                String key = "I" + classURIList + text;
                List<OntologyElement> lista = map.get(key);
                if (lista == null) lista = new ArrayList<OntologyElement>();
                lista.add(el);
                map.put(key, lista);
            } else if (el instanceof DatatypePropertyValueElement) {
                DatatypePropertyValueIdentifier dpve =
                                (DatatypePropertyValueIdentifier) ((DatatypePropertyValueElement) el)
                                                .getData();
                String propertyURI = dpve.getPropertyUri();
                String propertyValue = dpve.getPropertyValue();
                String text =
                                ((DatatypePropertyValueElement) el).getAnnotation().getText();
                // logger.debug("Text is:" + text);
                String key = "D" + propertyURI + propertyValue + text;
                List<OntologyElement> lista = map.get(key);
                if (lista == null) lista = new ArrayList<OntologyElement>();
                lista.add(el);
                map.put(key, lista);
            } else {
                // for any other element just generate a list and return it with one
                // element
                finalList.add(el);
                // ?
            }
            // logger.debug("Now finished no:"+counter);
            counter++;
        }
        logger.debug("Finished puting elements in the map, now rearanging t"
                        + "hem i.e. making InstanceList or making a set of URIs for DTPV:");
        for (String classURIAndText : map.keySet()) {
            List<OntologyElement> thoseGrouped = map.get(classURIAndText);
            logger.info("Key:" + classURIAndText + ": " + thoseGrouped.size()
                            + " elements.");
            // if(thoseGrouped != null && thoseGrouped.size() == 1) {
            // // do not group as this is only one instance element then just add it
            // finalList.add(thoseGrouped.get(0));
            // continue;
            // }
            // is it instance or dtpv
            OntologyElement sample = thoseGrouped.get(0);
            if (sample instanceof InstanceElement) {
                InstanceListElement instanceListElement = new InstanceListElement();
                // this means we now need to generate a new OntologyElement for now
                // the only supported is InstanceListElement which contains list of
                // instanceURIs and classURI
                List<SerializableURI> uris = new ArrayList<SerializableURI>();
                for (OntologyElement el : thoseGrouped) {
                    uris.add((SerializableURI) el.getData());
                }
                instanceListElement.setData(uris);
                instanceListElement.setAlreadyAdded(sample.isAlreadyAdded());
                instanceListElement.setVariable(sample.getVariable());
                instanceListElement.setAnnotation(sample.getAnnotation());
                instanceListElement.setAnswer(sample.isAnswer());
                instanceListElement.setFunction(sample.getFunction());
                instanceListElement.setMainSubject(sample.isMainSubject());
                Set<String> classUriList = new HashSet<String>();
                for (OntologyElement el : thoseGrouped) {
                    List<String> tempList = ((InstanceElement) el).getClassURIList();
                    String classUri = ((InstanceElement) el).getClassURI().toString();
                    if (tempList == null || tempList.size() == 0) {
                        classUriList.add(classUri);
                    } else {
                        classUriList.addAll(tempList);
                    }
                }
                instanceListElement.setClassURIList(new ArrayList(classUriList));
                instanceListElement.setClassURI(((InstanceElement) thoseGrouped.get(0))
                                .getClassURI());
                finalList.add(instanceListElement);
            } else if (sample instanceof DatatypePropertyValueElement) {
                Set<SerializableURI> instanceUriList = new HashSet<SerializableURI>();
                DatatypePropertyValueIdentifier dpve =
                                (DatatypePropertyValueIdentifier) ((DatatypePropertyValueElement) sample)
                                                .getData();
                for (OntologyElement el : thoseGrouped) {
                    DatatypePropertyValueIdentifier tmp =
                                    (DatatypePropertyValueIdentifier) ((DatatypePropertyValueElement) el)
                                                    .getData();
                    // at this stage each dpv has only one element in instancelist that's
                    // why we can do get(0)
                    instanceUriList.add(tmp.getInstanceURIs().get(0));
                }
                dpve.setInstanceURIs(new ArrayList(instanceUriList));
                // dodaj klasu? nee...
                finalList.add(sample);
            }
        }
        logger
                        .debug("Finished rearranging the elements and grouping those with same classURI/predURi+value");
        OntologyElementComparator comparator = new OntologyElementComparator();
        Collections.sort(finalList, comparator);
        logger.debug("Finished sorting elements.");
        return finalList;
    }

    /**
     * translates annotations into OntologyElements
     * 
     * @return
     */
    public List<List<OntologyElement>> getOverlappedGroupsOfOntologyElementsLucene(
                    List<List<Annotation>> overlappedGroups) {
        List<List<OntologyElement>> tableOfOntologyElements =
                        new ArrayList<List<OntologyElement>>();
        // each column represents overlapped annotations
        int col = 0;
        for (List<Annotation> column : overlappedGroups) {
            List<OntologyElement> newColumn = getOntologyElementsLucene(column);
            tableOfOntologyElements.add(newColumn);
            logger
                            .info("Generating " + newColumn.size() + " elements for col:"
                                            + col);
            // + " these are: " + newColumn.toString());
            col++;
        }
        return tableOfOntologyElements;
    }

    /**
     * translates annotations into ontology elements of various types: class, instance, class, property and
     * datatypepropertyValue this method does whatever old OntoResTransformer used to do
     * 
     * @param row
     * @return
     */
    List<OntologyElement> getOntologyElementsLucene(
                    List<Annotation> column) {
        List<OntologyElement> newRow = new ArrayList<OntologyElement>();
        for (Annotation cell : column) {
            OntologyElement element = null;
            String type = (String) cell.getFeatures().get(CATConstants.TYPE_FEATURE);
            Float solrScore = (Float) cell.getFeatures().get(FreyaConstants.SCORE);
            
            if (CATConstants.FEATURE_TYPE_CLASS.equals(type)) {
                element = new ClassElement();
                SerializableURI uri;
                try {
                    uri = new SerializableURI((String) cell.getFeatures().get(
                                    CATConstants.FEATURE_URI), false);

                    // transform element to be ClassElement
                    element.setData(uri);
                    Double specificityScore = 0.0;
                    if (luceneAnnotator.getSpecificityScores() != null) {
                        specificityScore =
                                        (Double) luceneAnnotator.getSpecificityScores()
                                                        .get(uri);
                    }


                    Score score = new Score();
                    if (specificityScore!=null)
                    score.setSpecificityScore(specificityScore);

                    if (cell != null && cell.getFeatures() != null
                                    && cell.getFeatures().get("score") != null)
                        score.setSimilarityScore((Float) cell.getFeatures().get("score"));
                    element.setScore(score);
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            } else if (CATConstants.TYPE_INSTANCE.equals(type)) {
                element = new InstanceElement();
                String classUri =
                                (String) cell.getFeatures().get(CATConstants.CLASS_URI);
                String uriString =
                                (String) cell.getFeatures().get(CATConstants.FEATURE_URI);
                SerializableURI uri;
                try {
                    uri = new SerializableURI(uriString, false);

                    element.setData(uri);
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                List<String> classURIList =
                                (List<String>) cell.getFeatures().get(
                                                CATConstants.CLASS_URI_LIST);
                if (classURIList == null || classURIList.size() == 0)
                    classURIList.add(classUri);
                ((InstanceElement) element).setClassURIList(classURIList);
                // logger.info("Setting class uri list of size "+classURIList.size());
                if (classUri != null) {
                    // }
                    // Map specificityScoreSet = jsonCreator.getOntology2Map()
                    // .getSpecificityScores();
                    //
                    // Double specificityScore = (Double) jsonCreator
                    // .getOntology2Map().getSpecificityScores().get(classUri);
                    SerializableURI classUriUri;
                    try {
                        classUriUri = new SerializableURI(classUri, false);

                        ((InstanceElement) element).setClassURI(classUriUri);
                    } catch (Exception e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                    }
                } else {
                    logger.info("Class URI for the instance:" + uriString
                                    + " is NULLLLLLLLLLLLLLLLLLLLLlll; this should not happen");
                }
                Double specificityScore = 0.0;
                Score score = new Score();
                score.setSpecificityScore(specificityScore * 1.2);
                if (cell != null && cell.getFeatures() != null
                                && cell.getFeatures().get("score") != null)
                    score.setSimilarityScore((Float) cell.getFeatures().get("score"));
                element.setScore(score);// this times 1.2 is
                // because we want to assign higher score to instances in comparison to
                // classes
            } else if (CATConstants.TYPE_PROPERTY.equals(type)) {
                String propertyUri = (String) cell.getFeatures().get("URI");
                element = new PropertyElement();
                SerializableURI uri;
                try {
                    uri = new SerializableURI(propertyUri, false);

                    element.setData(uri);
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                boolean isDatatypeProperty =
                                luceneAnnotator
                                                .isItDatatypeProperty(propertyUri);
                // if(datatypePropertiesList.contains(propertyUri))
                ((PropertyElement) element).setDatatypeProperty(isDatatypeProperty);
                Set<String> rangeSet =
                                luceneAnnotator
                                                .findPropertyRange(propertyUri);
                String range = null;
                Iterator<String> it = null;
                if (rangeSet != null) it = rangeSet.iterator();
                if (it != null && it.hasNext()) {
                    range = it.next();
                    ((PropertyElement) element).setRange(range);
                }
                Set<String> domainSet =
                                luceneAnnotator
                                                .findPropertyDomain(propertyUri);
                String domain = null;
                Iterator<String> itDomain = null;
                if (domainSet != null) itDomain = domainSet.iterator();
                if (itDomain != null && itDomain.hasNext()) {
                    domain = itDomain.next();
                    ((PropertyElement) element).setDomain(domain);
                }
            } else if (CATConstants.TYPE_DATATYPE_PROPERTY_VALUE.equals(type)) {
                element = new DatatypePropertyValueElement();
                // String propertyValue = (String) cell.getFeatures().get(
                // "propertyValue");
                SerializableURI instanceURI;
                try {
                    instanceURI = new SerializableURI((String) cell.getFeatures().get(
                                    CATConstants.FEATURE_INSTANCE_URI), false);

                    String propertyUri =
                                    (String) cell.getFeatures().get(
                                                    CATConstants.FEATURE_PROPERTY_URI);
                    DatatypePropertyValueIdentifier dtvIdentifier =
                                    new DatatypePropertyValueIdentifier();
                    dtvIdentifier.setPropertyUri(propertyUri);
                    dtvIdentifier.setPropertyValue(cell.getText());
                    List<SerializableURI> instanceUriList = new ArrayList();
                    instanceUriList.add(instanceURI);

                    dtvIdentifier.setInstanceURIs(instanceUriList);
                    element.setData(dtvIdentifier);
                } catch (Exception e) {
                  logger.info(e.getMessage());
                }
            }
            Score newScore = new Score();
            newScore.setSolrScore(solrScore);
            element.setScore(newScore);
            element.setAnnotation(cell);
            if (element != null) newRow.add(element);
        }
        return newRow;
    }

    /**
     * returns a list of lists of annotations in case they overlap; if there is no overlapped annotations then this will
     * be a list of lists with 1 element only
     * 
     * @param lookups
     * @return
     */
    public List<List<Annotation>> getOverlappedGroupsOfLuceneAnnotations(
                    List<Annotation> lookups) {
        List<Annotation> processedAnnotations =
                        new ArrayList<Annotation>();
        List<List<Annotation>> overlappedGroups =
                        new ArrayList<List<Annotation>>();
        // generate a list of lists of annotations where these second list is
        // overlapped annotations
        for (Annotation annotation : lookups) {
            // System.out
            // .println("****************************processing started...for...\n"
            // + annotation.toString());
            if (processedAnnotations.contains(annotation)) {
                // System.out.println("****************************skipping\n");
                continue;
            }
            processedAnnotations.add(annotation);
            List<Annotation> aCopyOfAllAnnotations =
                            new ArrayList<Annotation>();
            aCopyOfAllAnnotations.addAll(lookups);
            // System.out
            // .println("****************************aCopyOfAllAnnotations.....\n"
            // + aCopyOfAllAnnotations.toString());
            aCopyOfAllAnnotations.remove(annotation);
            List<Annotation> overlapped =
                            new ArrayList<Annotation>();
            overlapped.add(annotation);
            // System.out
            // .println("***************again....*************aCopyOfAllAnnotations.....\n"
            // + aCopyOfAllAnnotations.toString());
            for (Annotation anotherAnnotation : aCopyOfAllAnnotations) {
                // System.out
                // .println("****************************checking now.....\n"
                // + anotherAnnotation);
                if (annotation.overlaps(anotherAnnotation)) {
                    // System.out
                    // .println("****************************overlap found");
                    overlapped.add(anotherAnnotation);
                    processedAnnotations.add(anotherAnnotation);
                }
            }
            overlappedGroups.add(overlapped);
            // System.out.println("****************************overlaped:"
            // + overlapped.toString());
        }
        return overlappedGroups;
    }
}
 @Component
public class OCUtil {
    private static final Log logger = LogFactory.getLog(OCUtil.class);

    AtomicInteger id = new AtomicInteger();
    @Autowired TreeUtils treeUtils;



    /**
     * 
     * @param anns
     * @return
     */
    public Set<Annotation> filterInstanceClass(List<Annotation> anns) {
        HashMap<Long, List<Annotation>> map = new HashMap<Long, List<Annotation>>();

        for (Annotation ann : anns) {
            List<Annotation> tmp = map.get(ann.getStartOffset());
            if (tmp == null) tmp = new ArrayList<Annotation>();
            tmp.add(ann);
            map.put(ann.getStartOffset(), tmp);
        }
        // now we have all start offsets
        logger.info("Now finished puting all anns into the map where key is offsets, these are the keys:"
                        + map.keySet().toString());
        List<Long> startOffsets = new ArrayList<Long>();
        startOffsets.addAll(map.keySet());
        Collections.sort(startOffsets);
        List toRemove = new ArrayList();

        for (int i = 0; i < startOffsets.size() - 1; i++) {

            Long current = startOffsets.get(i);
            Long next = startOffsets.get(i + 1);
            logger.info("Current ofsset" + current.toString() + " next offset:" + next.toString());
            List<Annotation> currentAnnotations = map.get(current);
            List<Annotation> nextAnnotations = map.get(next);
            toRemove.addAll(whichToRemove(currentAnnotations, nextAnnotations));
            toRemove.addAll(whichToRemove(nextAnnotations, currentAnnotations));
        }

        // if (toRemove!=null)
        // anns.removeAll(toRemove);
        logger.info("Marked to remove due to classInstance:" + toRemove.size());
        // return anns;
        return new HashSet(toRemove);

    }

    // }
    /**
     * if first is class and second is instance then it checks the class of the instance and if the two match marks the
     * class to remove
     */
    List<Annotation> whichToRemove(List<Annotation> first, List<Annotation> second) {
        List<Annotation> toRemove = new ArrayList<Annotation>();
        // save in this list members from 'first' only if they are classes
        List<Annotation> firstClassesOnly = new ArrayList<Annotation>();
        // save in this list members from 'second' only if they are instances only
        List<Annotation> secondInstancesOnly = new ArrayList<Annotation>();
        for (Annotation ann1 : first) {
            String type = (String) ann1.getFeatures().get(CATConstants.TYPE_FEATURE);
            if (type.equals(CATConstants.FEATURE_TYPE_CLASS)) {
                firstClassesOnly.add(ann1);
            }
        }
        logger.info("There are " + firstClassesOnly.size() + " classes in the first column.");
        // second column
        for (Annotation ann1 : second) {
            String type = (String) ann1.getFeatures().get(CATConstants.TYPE_FEATURE);
            if (type.equals(CATConstants.TYPE_INSTANCE)) {
                secondInstancesOnly.add(ann1);
            }
        }
        logger.info("There are " + secondInstancesOnly.size() + " instances in the second column.");
        // collect classUris from first

        // now we know for sure that the first column is class and second is instance
        // so we take the second only, find all classUris of instances, and if we do find
        // that instance X is of the existing type of class then we do this:
        // - remove all instances from first
        // - remove all but X from the second column
        Set<String> allClassUrisFirst = new HashSet<String>();
        for (Annotation ann1 : firstClassesOnly) {
            String classUri = (String) ann1.getFeatures().get(CATConstants.FEATURE_URI);
            logger.info("class uri:" + classUri);
            allClassUrisFirst.add(classUri);
        }
        Set<Annotation> secondToKeep = new HashSet<Annotation>();
        for (Annotation ann1 : secondInstancesOnly) {
            logger.info("inst annotation:" + ann1.toString());

            Set<String> classUris = new HashSet((List) (ann1.getFeatures().get(CATConstants.CLASS_URI_LIST)));
            String classUri = (String) ann1.getFeatures().get(CATConstants.CLASS_URI);
            classUris.add(classUri);
            logger.info("classUris:" + classUris);
            // if classUris contain any of the classes in allClassUrisFirst then
            if (classInstanceFound(classUris, allClassUrisFirst))
            {
                secondToKeep.add(ann1);
            }
        }
        if (secondToKeep.size() > 0) {
            toRemove.addAll(first);
            toRemove.addAll(second);
            toRemove.removeAll(secondToKeep);
        }
        logger.info("marked to remove:" + toRemove.size());
        return toRemove;
    }

    /**
     * 
     * @param instanceClassUris
     * @param allClassUris
     * @return
     */
    boolean classInstanceFound(Set<String> instanceClassUris, Set<String> allClassUris) {
        boolean toReturn = false;
        for (String instClassUri : instanceClassUris) {
            logger.info("Checking:" + instClassUri);
            for (String classUri : allClassUris) {
                logger.info(" and:" + classUri);
                if (instClassUri.equals(classUri)) {
                    logger.info("Annotation should be removed?");
                    toReturn = true;
                    break;
                }
            }
        }
        return toReturn;
    }

    /**
     * finding nearest neighbour for poc
     * 
     * @param question
     * @param poc
     * @return
     */
    public List<SemanticConcept> findNearestNeighbours(Question question, POC poc) {
        List<SemanticConcept> nearestNeighbours = new ArrayList<SemanticConcept>();
        Tree root = question.getSyntaxTree();
        List<SemanticConcept> scList = null;
        try {
            scList = getLinearisedSemanticConcepts(question.getSemanticConcepts());
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        int min = 1000;
        logger.info("Finding nearest neighbour for POC:"
                        + poc.getAnnotation().getText());
        logger.info("There are:" + scList.size()
                        + " candidates (semantic concepts)");
        for (SemanticConcept sConcept : scList) {
            if (sConcept.getOntologyElement() instanceof NoneElement) {
                continue;
            }
            int distance =
                            treeUtils.getDistance(poc.getAnnotation().getSyntaxTree(),
                                            sConcept.getOntologyElement().getAnnotation()
                                                            .getSyntaxTree(), root);
            // logger.debug("Distance is:"
            // + distance
            // + " between "
            // + poc.getAnnotation().getSyntaxTree().toString()
            // + " \n and \n"
            // + sConcept.getOntologyElement().getAnnotation()
            // .getSyntaxTree().toString());
            if (min > distance) {
                min = distance;
                nearestNeighbours = new ArrayList<SemanticConcept>();
                nearestNeighbours.add(sConcept);
            } else if (min == distance) {
                nearestNeighbours.add(sConcept);
            }
        }
        return nearestNeighbours;
    }

    /**
     * @param question
     * @param keySemConcept
     * @return
     */
    public List<SemanticConcept> findNearestNeighbours(Question question,
                    List<SemanticConcept> list) {
        SemanticConcept keySemConcept = list.get(0);
        List<Tree> treeOfTheKeySemanticConcept =
                        keySemConcept.getOntologyElement().getAnnotation().getSyntaxTree();
        List<SemanticConcept> nearestNeighbours = new ArrayList<SemanticConcept>();
        Tree root = question.getSyntaxTree();
        List<SemanticConcept> newList = null;
        // this currentList is introduced so that the element is not found to
        // be the nearest neighbour of itself
        List<List<SemanticConcept>> currentList =
                        new ArrayList<List<SemanticConcept>>();
        currentList.addAll(question.getSemanticConcepts());
        currentList.remove(list);
        try {
            newList = getAllSemanticConceptsLinearised(currentList);
            logger.debug("currently:" + newList.size()
                            + " in the semantic elements list");
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        List noneToRemove = new ArrayList();
        // if any of te elements in the neighbourhood is None then ignore them
        for (SemanticConcept theOne : newList) {
            if (theOne.getOntologyElement() instanceof NoneElement) {
                noneToRemove.add(theOne);
            }
        }
        newList.removeAll(noneToRemove);
        logger.info("after removing none elements:" + newList.size()
                        + " in the semantic elements list");
        logger.debug("Removed:" + noneToRemove.size()
                        + " none elements as they are not counted when finding NN.");
        int min = 1000;
        logger.debug("Found " + newList.size() + " neighbours for: "
                        + keySemConcept.getOntologyElement().getData().toString());
        for (SemanticConcept sConcept : newList) {
            int distance =
                            treeUtils.getDistance(treeOfTheKeySemanticConcept, sConcept
                                            .getOntologyElement().getAnnotation().getSyntaxTree(),
                                            root);
            // logger.debug("distance is:" + distance + " between "
            // // + treeOfTheKeySemanticConcept + " \n and \n"
            // + sConcept.getOntologyElement().getAnnotation().getSyntaxTree());
            if (min > distance) {
                min = distance;
                nearestNeighbours = new ArrayList<SemanticConcept>();
                nearestNeighbours.add(sConcept);
                // logger.debug("Adding:"
                // + sConcept.getOntologyElement().getData().toString()
                // + " to the list of NNs");
            } else if (min == distance) {
                nearestNeighbours.add(sConcept);
                // logger.debug("Adding:"
                // // + sConcept.getOntologyElement().getData().toString()
                // + " to the list of NNs");
            }
        }
        return nearestNeighbours;
    }

    /**
     * @param question
     * @param poc
     * @return
     */
    public POC whichPOC(Question question) {
        List<POC> pocs = question.getPocs();
        POC theOne = null;
        Tree root = question.getSyntaxTree();
        List<SemanticConcept> scList = null;
        try {
            scList = getLinearisedSemanticConcepts(question.getSemanticConcepts());
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        logger.debug("This many sConcepts:" + scList.size());
        logger
                        .debug("Trying to decide which poc to use for the dialog; this many pocs:"
                                        + pocs.size());
        int min = 1000;
        for (POC poc : pocs) {
            List<SemanticConcept> nearestNeighbours =
                            findNearestNeighbours(question, poc);
            SemanticConcept sc = null;
            // take one, it does not matter which
            if (nearestNeighbours != null && nearestNeighbours.size() > 0)
                sc = nearestNeighbours.get(0);
            if (sc != null) {
                int distance =
                                treeUtils.getDistance(poc.getAnnotation().getSyntaxTree(), sc
                                                .getOntologyElement().getAnnotation().getSyntaxTree(),
                                                root);
                if (min > distance) {
                    min = distance;
                    theOne = poc;
                } else if (min == distance && theOne != null) {
                    // take the one which is on the right-hand side
                    // this means that theOne and poc are on the same level:
                    // take the one on the right
                    if (poc.getAnnotation().getStartOffset().longValue() > theOne
                                    .getAnnotation().getStartOffset().longValue()) theOne = poc;
                }
            }
        }
        // this means if there are not ocs, use the first one, which should also
        // be mainsubject TODO: this is more a workaround than a solution but I
        // live it like this for the time being
        if (pocs.size() > 0 && theOne == null) {
            theOne = pocs.get(0);
        }
        return theOne;
    }

    public boolean ocsVerified(List<SemanticConcept> list) {
        boolean allVerified = false;
        for (SemanticConcept c : list) {
            if (c.isVerified().booleanValue() == true)
                allVerified = true;
        }
        return allVerified;
    }

    /**
     * finds the OC to be put into dialog: looks which one is closest to focus
     * 
     * @param question
     * @return
     */
    public List<SemanticConcept> whichOC(Question question) {

        if (question.getFocus() == null) {
            return question.getSemanticConcepts().get(0);
        }

        SemanticConceptListComparator c = new SemanticConceptListComparator();
        long start = System.currentTimeMillis();
        if (question.getSemanticConcepts().size() > 1)
            Collections.sort(question.getSemanticConcepts(), c);
        long end = System.currentTimeMillis();
        logger.info("Sorting sem elements took:" + (end - start) + "ms");
        List<SemanticConcept> theOne = null;
        Tree root = question.getSyntaxTree();
        int min = 1000;
        start = System.currentTimeMillis();
        for (List<SemanticConcept> list : question.getSemanticConcepts()) {
            if (!ocsVerified(list)) {
                SemanticConcept sampleElement = list.get(0);
                int distance =
                                treeUtils.getDistance(question.getFocus().getAnnotation()
                                                .getSyntaxTree(), sampleElement.getOntologyElement()
                                                .getAnnotation().getSyntaxTree(), root);
                logger.debug("Distance:" + distance);
                if (min > distance && list.size() > 1) {
                    min = distance;
                    theOne = list;
                }
            }
        }
        end = System.currentTimeMillis();
        logger.info("Calculating distance for which oc took:" + (end - start) + "ms");
        // this means if there are no ocs, use the first one, which should also
        // be mainsubject TODO: this is more a workaround than a solution but I
        // live it like this for the time being
        logger.debug("Focus:" + question.getFocus());
        logger.debug("Sem elements to use for the OC dialog:" + theOne);
        return theOne;

    }

    public List<SemanticConcept> getLinearisedSemanticConcepts(
                    List<List<SemanticConcept>> table) throws Exception {
        List<SemanticConcept> linearisedList = new ArrayList<SemanticConcept>();
        for (List<SemanticConcept> row : table) {
            if (row.size() > 1) {
                logger
                                .debug("List of OCs is ambiguious and needs to be disambiguated using dialog");
                linearisedList.add(row.get(0));
            } else if (row.size() > 0) linearisedList.add(row.get(0));
        }
        return linearisedList;
    }

    /**
     * utility method to transform list of lists into one list
     * 
     * @param table
     * @return
     * @throws Exception
     */
    public List<SemanticConcept> getAllSemanticConceptsLinearised(
                    List<List<SemanticConcept>> table) throws Exception {
        List<SemanticConcept> linearisedList = new ArrayList<SemanticConcept>();
        for (List<SemanticConcept> row : table) {
            // logger
            // .debug("List of OCs is ambiguious and needs to be disambiguated using dialog!");
            linearisedList.addAll(row);
        }
        return linearisedList;
    }

    /**
     * if two semantic concepts are in the same column and refer to the same ontology concept (same instance URI or
     * class URI or propertyURI) then remove it
     * 
     * @param Question
     * @return Question
     */
    public Question removeDuplicateOcs(Question question) {
        List<List<SemanticConcept>> table = question.getSemanticConcepts();
        // logger.debug("input table with semantic concepts:" + table.toString());
        // key: column index within table; value: list of indexes in the column to
        // be removed
        Map<Integer, List<Integer>> mapToRemove =
                        new HashMap<Integer, List<Integer>>();
        for (List<SemanticConcept> sList : table) {
            List<String> noDuplicates = new ArrayList<String>();
            List<DatatypePropertyValueIdentifier> noDuplicatesDTVI =
                            new ArrayList<DatatypePropertyValueIdentifier>();
            for (SemanticConcept sConcept : sList) {
                OntologyElement oe = sConcept.getOntologyElement();
                // uri for classes, instances or properties
                String uri = null;
                DatatypePropertyValueIdentifier dpvIdentifier = null;
                if (oe instanceof DatatypePropertyValueElement) {
                    dpvIdentifier =
                                    (DatatypePropertyValueIdentifier) ((DatatypePropertyValueElement) oe)
                                                    .getData();
                } else {
                    Object data = oe.getData();
                    if (data != null) uri = data.toString();
                }
                logger.debug("dpvIdentifier:" + dpvIdentifier);
                logger.debug("URI:" + uri);
                // different treatment for classes,properties and instances
                // versus datatypepropertyvalues
                if (uri != null && !noDuplicates.contains(uri)) {
                    noDuplicates.add(uri);
                } else if (dpvIdentifier != null
                                && !noDuplicatesDTVI.contains(dpvIdentifier)) {
                    noDuplicatesDTVI.add(dpvIdentifier);
                } else {
                    List<Integer> list =
                                    mapToRemove.get(new Integer(table.indexOf(sList)));
                    if (list == null) list = new ArrayList<Integer>();
                    list.add(sList.indexOf(sConcept));
                    mapToRemove.put(new Integer(table.indexOf(sList)), list);
                    logger.debug("List with elements to be removed:" + list.toString());
                }
            }
        }
        if (mapToRemove.keySet().size() > 0)
            logger.info("Removing " + mapToRemove.keySet().size()
                            + " OCs as they are duplicated.");
        logger.debug("map with semantic concepts to be removed:"
                        + mapToRemove.toString());
        logger
                        .debug("before removing:"
                                        + question.getSemanticConcepts().toString());
        question = removeMarkedOcsByIndex(question, mapToRemove);
        logger.debug("after removing:" + question.getSemanticConcepts().toString());
        return question;
    }

    /**
     * removes marked ocs from the table: the map contains key=index of the column; and value=list of elements to be
     * removed
     * 
     * @param question
     * @param toRemove
     * @return
     */
    public Question removeMarkedOcsByIndex(Question question,
                    Map<Integer, List<Integer>> toRemove) {
        List<Integer> keys = new ArrayList<Integer>(toRemove.keySet());
        Integer index = null;
        for (int i = 0; i < keys.size(); i++) {
            index = keys.get(i);
            if (index != null) {
                List<SemanticConcept> column =
                                question.getSemanticConcepts().get(index.intValue());
                logger.debug("the columns has " + column.size() + " elements.");
                List<Integer> elementsToRemove = toRemove.get(index);
                logger.debug("now removing " + elementsToRemove.size()
                                + " element(s) from the column.");
                for (Integer rowIndex : elementsToRemove) {
                    column.remove(rowIndex.intValue());
                }
                logger.debug("after removing this many left: " + column.size()
                                + " elements.");
            }
        }
        return question;
    }

    /**
     * removes marked ocs from the table: the map contains key=index of the column; and value=list of elements to be
     * removed
     * 
     * @param question
     * @param toRemove
     * @return
     */
    public Question removeMarkedOcs(Question question,
                    Map<Integer, List<SemanticConcept>> toRemove) {
        List<Integer> keys = new ArrayList<Integer>(toRemove.keySet());
        for (Integer index : keys) {
            if (index != null) {
                List<SemanticConcept> column =
                                question.getSemanticConcepts().get(index.intValue());
                List<SemanticConcept> onlySelectedElements = toRemove.get(index);
                logger.info("About to remove elements from column:" + index + " there are:"
                                + (column.size()));
                column.clear();
                // verified means the user selected it
                for (SemanticConcept c : onlySelectedElements) {
                    c.setVerified(true);
                }

                column.addAll(onlySelectedElements);

            }
        }
        return question;
    }

    /**
     * @param question
     * @return
     */
    public List<List<SemanticConcept>> addNoneToAll(
                    List<List<SemanticConcept>> elements) {
        for (List<SemanticConcept> column : elements) {
            for (SemanticConcept sampleElement : column) {
                if (sampleElement.getOntologyElement().getAnnotation() != null) {
                    SemanticConcept scEl = new SemanticConcept();
                    NoneElement noneElement = new NoneElement();
                    Annotation newAnn =
                                    (Annotation) sampleElement
                                                    .getOntologyElement().getAnnotation().clone();
                    noneElement.setAnnotation(newAnn);
                    noneElement.setId(new Integer(id.getAndIncrement()).toString());
                    logger.debug("Setting annotation for none element:"
                                    + noneElement.toString());
                    scEl.setOntologyElement(noneElement);
                    column.add(scEl);
                    break;
                }
            }
        }
        return elements;
    }

    /**
     * checkes whether there are any overlapped annotations, if there are leaves only the longest (of type Lookup! only)
     * 
     * @param allAnnotations
     * @return
     */
    public List<Annotation> disambiguateBasedOnLength(
                    List<Annotation> allAnnotations) {
        AnnotationOffsetComparator c = new AnnotationOffsetComparator();
        Collections.sort(allAnnotations, c);
        ArrayList<Annotation> aCopyOfLookupAnnotations =
                        null;
        ArrayList<Annotation> checkedAnnotations =
                        new ArrayList<Annotation>();
        ArrayList<Annotation> toRemove =
                        new ArrayList<Annotation>();
        // startofsset, endoffset
        Map<Long, Long> removed = new HashMap();
        for (Annotation ann : allAnnotations) {
            if (!checkedAnnotations.contains(ann)) {
                checkedAnnotations.add(ann);
                aCopyOfLookupAnnotations =
                                new ArrayList<Annotation>();
                aCopyOfLookupAnnotations.addAll(allAnnotations);
                aCopyOfLookupAnnotations.remove(ann);
                for (Annotation newAnn : aCopyOfLookupAnnotations) {
                    if (ann.overlaps(newAnn)) {
                        // logger.info(ann.toString()
                        // + "\n***************overlaps with***********\n"
                        // + newAnn.toString());
                        // which one is longer
                        // add it to toRemove
                        long diffAnn =
                                        ann.getEndOffset().longValue()
                                                        - ann.getStartOffset().longValue();
                        long diffNewAnn =
                                        newAnn.getEndOffset().longValue()
                                                        - newAnn.getStartOffset().longValue();
                        if (diffAnn > diffNewAnn) {
                            if (!toRemove.contains(newAnn)) {
                                toRemove.add(newAnn);
                                removed.put(newAnn.getStartOffset(), newAnn.getEndOffset());
                                if (!checkedAnnotations.contains(newAnn))
                                    checkedAnnotations.add(newAnn);
                                // logger.info("\nadding to remove:\n" + newAnn.toString());
                            }
                        } else if (diffAnn < diffNewAnn) {
                            if (!toRemove.contains(ann)) {
                                toRemove.add(ann);
                                removed.put(ann.getStartOffset(), ann.getEndOffset());
                                if (!checkedAnnotations.contains(ann))
                                    checkedAnnotations.add(ann);
                                // logger.info("\nadding to remove:\n" + ann.toString());
                            }
                        }
                    } else {
                        // they do not overlap, continue to the next one
                        // logger.debug(ann.toString()
                        // + "***************DO NOT overlap with***********"
                        // + newAnn.toString());
                    }
                }// newAnn
            }
        }
        logger
                        .debug("********************************disambiguation******************\n");
        // logger.info("before:" + allAnnotations.toString());
        if (toRemove.size() > 0)
            logger.info("Removing:" + toRemove.size() + " columns (short annotations).");
        allAnnotations.removeAll(toRemove);
        // logger.info("after:" + allAnnotations.toString());
        toRemove = new ArrayList();
        for (Annotation remove : allAnnotations) {
            Set<Long> startKeys = removed.keySet();
            // logger.info("startOfsets:"+startKeys);
            if (startKeys.contains(remove.getStartOffset())) {
                Long endOffset = removed.get(remove.getStartOffset());
                // logger.info("endOffset:"+endOffset);
                if (endOffset.longValue() == remove.getEndOffset().longValue()) {
                    toRemove.add(remove);
                }
            }
        }
        // logger.info("and again before:" + allAnnotations.toString());
        if (toRemove.size() > 0)
            logger.info("Removing again:" + toRemove.size() + " annotations.");
        allAnnotations.removeAll(toRemove);
        // logger.info("after:" + allAnnotations.toString());
        return allAnnotations;
    }

    public static void main(String[] args) {
        OCUtil u = new OCUtil();
        List<Annotation> set =
                        new ArrayList<Annotation>();
        AnnotationOffsetComparator c = new AnnotationOffsetComparator();
        Annotation ann =
                        new Annotation();
        ann.setText("Michael");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(1));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Michael");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(1));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Michael Jackson");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(2));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Michael Jackson");
        ann.setStartOffset(new Long(0));
        ann.setEndOffset(new Long(2));
        set.add(ann);
        ann = new Annotation();
        ann.setText("Jackson");
        ann.setStartOffset(new Long(1));
        ann.setEndOffset(new Long(2));
        set.add(ann);
        logger.info("Before:" + set.size());
        List sortedList = new ArrayList(set);
        Collections.sort(sortedList, c);
        set = u.disambiguateBasedOnLength(sortedList);
        logger.info("After:" + set.size());
    }
}
 @Component
public class TreeUtils {
    static org.apache.commons.logging.Log logger =
                    org.apache.commons.logging.LogFactory.getLog(TreeUtils.class);

    @Value("${org.freya.parser.preterminal.pocs}") String preterminalPOCs;

    public Tree getAnswerType(Tree tree) throws Exception {
        Tree mainSubjectTree = findTheHeadOfTheNounPhrase(tree);
        return mainSubjectTree;
    }

    /**
     * @param tree
     * @return
     */
    public Tree getMainSubjectTree(Tree tree) {
        List<Tree> pocs = findPocs(tree);
        if (pocs != null && pocs.size() > 0)
            return pocs.get(0);
        else
            return null;
    }

    /**
     * finds the first NP/NN* (main subject/answer type/focus);
     * 
     * @param tree
     * @return
     */
    public Tree findTheFirstNounPhraseOld(Tree tree) {
        Tree value = null;
        // treeLabel is the pos tag: NP, VP, etc.
        String treeLabel = tree.label().value();
        if (treeLabel != null
                        && (treeLabel.startsWith("NP") || treeLabel.startsWith("NN"))
                        && !(treeLabel.startsWith("PRP")))
        // || treeLabel.startsWith("WHNP")
        {
            return tree;
        } else if (treeLabel != null
                        && (treeLabel.startsWith("WHADVP") || treeLabel.startsWith("WHNP") || treeLabel
                                        .startsWith("WHADJP"))// this is to support how long,
        // how
        // large, how big, etc.
        ) {
            List children = tree.getChildrenAsList();
            if (children != null && children.size() <= 2) {// we always expect
                // one or two
                Tree firstChild = (Tree) children.get(0);
                Tree secondChild = null;
                if (children.size() > 1) secondChild = (Tree) children.get(1);
                // this is to support where what etc
                if (secondChild == null
                                && (firstChild.label().value().startsWith("WRB"))) {
                    // || firstChild.label().value().startsWith("WP"))) {
                    return tree;
                } else if (secondChild != null) {
                    if (firstChild.label().value().startsWith("WRB")
                                    && ((secondChild.label().value().startsWith("JJ")) || secondChild
                                                    .label().value().startsWith("ADJP"))
                                    || secondChild.label().value().startsWith("NN")
                                    || secondChild.label().value().startsWith("NP")
                                    || firstChild.label().value().startsWith("NN")
                                    || firstChild.label().value().startsWith("NP"))
                        // return secondChild;// well?
                        return tree;
                }
            } else if (children != null) {
                Iterator it = children.iterator();
                while (it.hasNext()) {
                    Tree something = (Tree) it.next();
                    if (something.label().value().startsWith("NP")
                                    || (something.label().value().startsWith("NN")))
                        return something;
                }
            }
        } else {
            List<Tree> listOfChildren = tree.getChildrenAsList();
            // put them in reverse order
            // Collections.reverse(listOfChildren);
            for (Tree currentTree : listOfChildren) {
                Tree mainSubjectTree = getMainSubjectTree(currentTree);
                if (mainSubjectTree != null) {
                    value = mainSubjectTree;
                    break;
                }
            }
        }
        return value;
    }

    /**
     * fins preterminals
     * 
     * @param tree
     * @return
     */
    public List<Tree> findPrePreTerminals(Tree tree) {
        List<Tree> pocs = new ArrayList<Tree>();
        Tree value = null;
        // treeLabel is the pos tag: NP, VP, etc.
        String treeLabel = tree.label().value();
        // logger.info("label is:"+treeLabel);
        if (tree.isPrePreTerminal()) {
            pocs.add(tree);
            logger.debug(treeLabel
                            + " is prepreterminal so adding it to the list of POCs.");
            return pocs;
        } else {
            List<Tree> listOfChildren = tree.getChildrenAsList();
            // put them in reverse order
            // Collections.reverse(listOfChildren);
            // logger.info("now listing children...");
            for (Tree currentTree : listOfChildren) {
                // logger.info("child..."+currentTree.label().value());
                if (currentTree.isPrePreTerminal())
                    pocs.add(currentTree);
                else
                    pocs.addAll(findPrePreTerminals(currentTree));
                // logger.info("currently total pocs are:"+pocs.size());
            }
        }
        return pocs;
    }


    /**
     * lucene split poc which contains adjective into two pocs: adjective and the rest
     * 
     * @param pocs
     * @param root
     * @param stanfordSentence
     * @return
     */
    public List<POC> generateSeparatePOCForAdjectives(List<POC> pocs, Tree root) {
        // if there is no JJ* generate a new poc and update the current
        List<Tree> adjectives = new ArrayList<Tree>();
        List<POC> toRemove = new ArrayList<POC>();
        logger.debug("before checking jjs...pocs:" + pocs.toString());
        for (POC pocElement : pocs) {
            List<Tree> trees = pocElement.getAnnotation().getSyntaxTree();
            List<Tree> splitTrees = new ArrayList<Tree>();
            for (Tree eachTree : trees) {
                if (eachTree.isPrePreTerminal()) {
                    // split into preterminals
                    splitTrees.addAll(eachTree.getChildrenAsList());
                } else {
                    splitTrees.add(eachTree);
                }
            }
            List<Tree> remainingList = new ArrayList<Tree>();
            for (Tree pocTree : splitTrees) {
                if (isDescriptive(pocTree) && splitTrees.size() > 1) {
                    // so that if JJs are already split we do not generate
                    // duplicates
                    adjectives.add(pocTree);
                } else {
                    if (!remainingList.contains(pocTree)) remainingList.add(pocTree);
                }
            }
            // //////////////////////////////////////update
            if (remainingList != null && remainingList.size() > 0) {
                pocElement = updatePOCAnnotation(remainingList, root, pocElement);
            } else {
                // mark the poc for removal
                toRemove.add(pocElement);
            }
        }
        logger.debug("before splitting jjs...pocs:" + pocs.toString());
        // //////////////////////////////////////////
        // generate poc from the adjective
        // ////////////////////////////////////////
        if (adjectives.size() > 0) {
            for (Tree adjective : adjectives) {
                POC newPoc = new POC();
                List<Tree> adjectiveTree = new ArrayList<Tree>();
                adjectiveTree.add(adjective);
                newPoc = updatePOCAnnotation(adjectiveTree, root, newPoc);
                pocs.add(newPoc);
            }
        }
        logger.debug("after adding jjs...:" + pocs.toString());
        if (toRemove.size() > 0) pocs.removeAll(toRemove);
        return pocs;
    }

    /**
     * checks whether the tree is ajdective, adverb superlative or verb, past participle (as this verb is always part of
     * the NP we assume it has the function of adjective such as 'populated state')
     * 
     * @param tree
     * @return
     */
    boolean isDescriptive(Tree tree) {
        boolean isDescriptive = false;
        if (tree.label().value().startsWith(FreyaConstants.JJ_TAG_TREEBANK)
                        || tree.label().value().startsWith(
                                        FreyaConstants.VBN_TAG_TREEBANK)
                        || tree.label().value().startsWith(
                                        FreyaConstants.RBS_TAG_TREEBANK)
                        || tree.label().value().startsWith("ADJP")) {
            isDescriptive = true;
        }
        return isDescriptive;
    }

    /**
     * checks whether there are any JJ* or ADJP inside poc
     * 
     * @param poc
     * @return
     */
    public boolean pocContainsJJs(POC poc) {
        boolean containsJJs = false;
        for (Tree tree : poc.getAnnotation().getSyntaxTree()) {
            String treeLabel = tree.label().value();
            // logger.info("Checking whether "+treeLabel+ " is JJ");
            if (isDescriptive(tree)) {
                containsJJs = true;
                break;
            } else {
                List<Tree> children = tree.getChildrenAsList();
                for (Tree child : children) {
                    // String childLabel = child.label().value();
                    if (isDescriptive(child)) {
                        containsJJs = true;
                        break;
                    }
                }
            }
        }
        return containsJJs;
    }

    /**
     * if pocs are prepreterminals then split their tree into children
     * 
     * @param pocs
     * @return
     */
    public List<POC> splitPocs(List<POC> pocs) {
        List<POC> toRemove = new ArrayList<POC>();
        for (POC poc : pocs) {
            List<Tree> trees = poc.getAnnotation().getSyntaxTree();
            if (trees != null && trees.size() == 1) {
                // check if they are prepreterminals
                Tree tree = trees.get(0);
                if (tree.isPrePreTerminal()) {
                    // split into preterminals
                    List<Tree> newTree = new ArrayList<Tree>();
                    for (Tree child : tree.getChildrenAsList()) {
                        newTree.add(child);
                    }
                    poc.getAnnotation().setSyntaxTree(newTree);
                }
            }
        }
        return pocs;
    }


    /**
     * update the poc with new syntax tree and also update it's annotation start and end offsets and the text
     * 
     * @param treeList
     * @param root
     * @param stanfordSentence
     * @param poc
     * @return poc updated with the syntax tree; null if the tree is empty
     */
    POC updatePOCAnnotation(List<Tree> treeList, Tree root, POC poc) {
        if (treeList == null || treeList.size() == 0) return null;
        Tree firstTree = treeList.get(0);
        long parserStartOffset = Trees.leftEdge(firstTree, root);
        Tree lastTree = treeList.get(treeList.size() - 1);
        long parserEndOffset = Trees.rightEdge(lastTree, root);
        poc.getAnnotation().setStartOffset(parserStartOffset);
        poc.getAnnotation().setEndOffset(parserEndOffset);
        poc.getAnnotation().setText(getNiceString(treeList));
        poc.getAnnotation().setSyntaxTree(treeList);
        return poc;
    }


    /**
     * independent of stanfordSentence remove DT from pocs: if poc has DT then transform it to the list of its children
     * but remove DT now poc can be a list of trees which are on the same level, not only one tree
     * 
     * @param pocs
     * @return
     */
    public List<POC> removeDTFromPOCs(List<POC> pocs, Tree root) {
        // if there is no DT leave pocs as they are (one Tree in the list)
        for (POC pocElement : pocs) {
            // before executing this method all pocs are stored in list each
            // list having only one element
            Tree pocTree = pocElement.getAnnotation().getSyntaxTree().get(0);
            List<Tree> list = new ArrayList<Tree>();
            List<Tree> children = pocTree.getChildrenAsList();
            for (Tree child : children) {
                if (FreyaConstants.DT_TAG_TREEBANK.equals(child.label().value())) {
                    List<Tree> withoutDT = new ArrayList<Tree>();
                    withoutDT.addAll(children);
                    withoutDT.remove(child);
                    logger.debug("Removing DT from POC.");
                    list = withoutDT;
                    break;
                } else {
                    if (!list.contains(pocTree)) list.add(pocTree);
                }
            }
            pocElement = updatePOCAnnotation(list, root, pocElement);
        }
        return pocs;
    }

    /**
     * finds pocs: CURRENTLY these are NN* and NP*
     * 
     * @param tree
     * @return
     */
    public List<Tree> findPocs(Tree tree) {
        List<Tree> pocs = new ArrayList<Tree>();
        List<Tree> prePreTerminals = findPrePreTerminals(tree);
        for (Tree node : prePreTerminals) {
            String nodeLabel = node.label().value();
            String nodeChildLabel = node.getChildrenAsList().get(0).label().value();
            // NPs and NNs
            if (nodeLabel != null
                            && (nodeLabel.startsWith(FreyaConstants.NP_TAG_TREEBANK)
                                            || nodeLabel.startsWith(FreyaConstants.NN_TAG_TREEBANK)
                                            || nodeLabel.startsWith(FreyaConstants.NX_TAG_TREEBANK) || (nodeLabel
                                                .startsWith(FreyaConstants.ADJP_TAG_TREEBANK)))
                            && !(nodeChildLabel.startsWith(FreyaConstants.PRP_TAG_TREEBANK))
                            && !(nodeChildLabel.startsWith(FreyaConstants.EX_TAG_TREEBANK)))
                pocs.add(node);
            // //////////////////////////////////////////////////////////////////
            // this is to support how long, //////
            // how //////
            // large, how big, etc. //////
            // //////////////////////////////////////////////////////////////////
            else if (nodeLabel != null
                            && (nodeLabel.startsWith(FreyaConstants.WHADVP_TAG_TREEBANK)
                                            || nodeLabel
                                                            .startsWith(FreyaConstants.WHNP_TAG_TREEBANK) || nodeLabel
                                                // added because of high high: high is prepreterminal but
                                                // the whole how high is not
                                                .startsWith(FreyaConstants.WHADJP_TAG_TREEBANK))) {
                List children = node.getChildrenAsList();
                if (children != null && children.size() <= 2) {// we always
                    // expect
                    // one or two
                    Tree firstChild = (Tree) children.get(0);
                    Tree secondChild = null;
                    if (children.size() > 1) secondChild = (Tree) children.get(1);
                    // this is to support where what etc
                    if (secondChild == null
                                    && (firstChild.label().value().startsWith(
                                                    FreyaConstants.WRB_TAG_TREEBANK) || firstChild
                                                    .label().value().startsWith(
                                                                    FreyaConstants.WP_TAG_TREEBANK))) {
                        // || firstChild.label().value().startsWith("WP"))) {
                        List<Tree> leaves = firstChild.getLeaves();
                        for (Tree leaf : leaves) {
                            if (leaf.label().value().toLowerCase().equals("where")
                                            || leaf.label().value().toLowerCase().equals("when")
                                            || leaf.label().value().toLowerCase().equals("who")) {
                                logger
                                                .info("Found WH-");
                                // logger.info("leaf.label().value():" + leaf.label().value());
                                pocs.add(node);
                            }
                        }
                    } else if (secondChild != null) {
                        if (firstChild.label().value().startsWith(
                                        FreyaConstants.WRB_TAG_TREEBANK)
                                        && ((secondChild.label().value()
                                                        .startsWith(FreyaConstants.JJ_TAG_TREEBANK)) || secondChild
                                                        .label().value().startsWith(
                                                                        FreyaConstants.ADJP_TAG_TREEBANK))
                                        || secondChild.label().value().startsWith(
                                                        FreyaConstants.NN_TAG_TREEBANK)
                                        || secondChild.label().value().startsWith(
                                                        FreyaConstants.NP_TAG_TREEBANK)
                                        || firstChild.label().value().startsWith(
                                                        FreyaConstants.NN_TAG_TREEBANK)
                                        || firstChild.label().value().startsWith(
                                                        FreyaConstants.NP_TAG_TREEBANK)
                                        || firstChild.label().value().startsWith(
                                                        FreyaConstants.RB_TAG_TREEBANK))
                            // return secondChild;// well?
                            // this is to remove WRB in 'how long' how many etc
                            pocs.add(node);
                    }
                } else if (children != null) {
                    Iterator it = children.iterator();
                    while (it.hasNext()) {
                        Tree something = (Tree) it.next();
                        if (something.label().value().startsWith("NP")
                                        || (something.label().value().startsWith("NN")))
                            pocs.add(something);
                    }
                }
            }// if prepreterminal is WHNP, WHADJP or WHADVP
        }
        // now also add those that are preterminals and NNs but were missed in
        // findPocs method
        logger.info("Number of pocs as prepreterminals:" + pocs.size());
        pocs.addAll(findPreTerminalsThatAreNNOrINs(tree, pocs));
        logger
                        .info("Number of pocs as preterminals and prepreterminals (final num of pocs):"
                                        + pocs.size());
        return pocs;
    }

    /**
     * find those preterminals that are NNs as they were missed due to the way sp parses tree
     * 
     * @param root
     * @param pocs
     * @return
     */
    List<Tree> findPreTerminalsThatAreNNOrINs(Tree root, List<Tree> pocs) {
        long numberOfTokens = root.getLeaves().size();
        List<Tree> newList = new ArrayList<Tree>();
        Set<Long> skipList = new HashSet();

        for (Tree poc : pocs) {
            logger.debug("Checking poc:" + poc.toString());
            long start = Trees.leftEdge(poc, root);
            long end = Trees.rightEdge(poc, root);
            logger.debug("Start poc:" + start);
            logger.debug("End poc:" + end);
            for (long j = start; j < end; j++) {
                skipList.add(new Long(j));
                logger.debug("Adding element to skip list:" + j);
            }
        }
        Set<Long> listToConsider = new HashSet<Long>();
        for (long i = 0; i < numberOfTokens; i++) {
            boolean iFound = false;
            for (Long toSkip : skipList) {
                if (toSkip.longValue() == i) {
                    iFound = true;
                    break;
                } else {}
            }
            if (!iFound) listToConsider.add(i);
        }
        for (Long element : listToConsider) {
            Tree preTerminal = null;
            preTerminal = Trees.getPreTerminal(root, element.intValue());
            logger.debug("checking now:" + element.intValue()
                            + " and its preterminal:" + preTerminal);
            if (preTerminal != null
                            && startsWithTagsToConsider(preTerminal.label().value())) {
                newList.add(preTerminal);
                logger.info("Found preTerminal:" + preTerminal.toString());
            }
        }
        return newList;
    }

    /**
     * reads pocs to consider form properties file and checks whether it should be considered or not; this applies to
     * preterminal pocs only
     * 
     * @param preterminalLabelValue
     * @return
     */
    boolean startsWithTagsToConsider(String preterminalLabelValue) {
        boolean consider = false;

        // InputStream is = this.getClass().getResourceAsStream("/Service.properties");
        // Properties ps = new Properties();
        // try {
        // ps.load(is);
        // } catch(IOException e1) {
        // // TODO Auto-generated catch block
        // e1.printStackTrace();
        // }
        // String preterminalPOCs = new String(ps.getProperty("preterminalPOCs"));
        String[] tagsToConsider = preterminalPOCs.split(",");
        for (String tag : tagsToConsider) {
            if (preterminalLabelValue.startsWith(tag)) {
                consider = true;
                break;
            }
        }
        return consider;
    }

    /**
     * this method finds the head of the phrase
     * 
     * @param tree
     * @return
     */
    public Tree findTheHeadOfTheNounPhrase(Tree tree) {
        Tree toReturn = null;
        if (tree != null) {
            HeadFinder hf = new ModCollinsHeadFinder();
            Tree head = tree.headTerminal(hf);
            if (head.label().value() != null
                            && !(head.label().value().startsWith("PRP"))) {
                toReturn = head;
            }
        }
        return toReturn;
        // System.out.println("the head is:" + head.toString());
    }

    /**
     * this method finds the head of the phrase
     * 
     * @param tree
     * @return
     */
    public List<TreeGraphNode> findModifiersOfTheNounPhrase(Tree tree, Tree head) {
        if (tree == null || head == null) return null;
        HeadFinder hf = new ModCollinsHeadFinder();
        // Tree head = tree.headTerminal(hf);
        List<TreeGraphNode> toReturn = new ArrayList<TreeGraphNode>();
        TreebankLanguagePack tlp = new PennTreebankLanguagePack();
        GrammaticalStructureFactory gsf = tlp.grammaticalStructureFactory();
        GrammaticalStructure gs = gsf.newGrammaticalStructure(tree);
        //Collection tdl = gs.typedDependenciesCollapsed();
        List<TypedDependency> tdl = gs.typedDependenciesCCprocessed();
        Iterator it = tdl.iterator();
        while (it.hasNext()) {
            TypedDependency td = (TypedDependency) it.next();
            String depName = td.reln().getShortName();
            if ((depName.equals("amod") || depName.equals("dep") || depName
                            .equals("advmod"))
                            && td.gov().value() == head.value()) {
                toReturn.add(td.dep());
            }
        }
        TreePrint tp = new TreePrint("penn,typedDependenciesCollapsed");
        //tp.printTree(head);
        return toReturn;
        // System.out.println("the head is:" + head.toString());
    }

    /**
     * generates a nice string from the tree
     * 
     * @param firstNPorNN
     * @return
     */
    public String getNiceString(List<Tree> trees) {
        String niceString = "";
        for (Tree tree : trees) {
            for (Tree leaf : tree.getLeaves()) {
                if (leaf.isLeaf()) {
                    niceString = niceString + " " + leaf.value();
                } else {
                    List<Tree> leafs = new ArrayList<Tree>();
                    leafs.add(leaf);
                    niceString = niceString + " " + getNiceString(leafs);
                }
            }
        }
        return niceString.trim();
    }

    /**
     * returns the distance from one node to the other.
     * 
     * @param tree1
     * @param tree2
     * @return
     */
    public int getDistance(List<Tree> tree1List, List<Tree> tree2List, Tree root) {
        Tree tree1 = null;
        Tree tree2 = null;
        if (tree1List != null && tree1List.size() > 0) tree1 = tree1List.get(0);
        if (tree2List != null && tree2List.size() > 0) tree2 = tree2List.get(0);
        List nodes = root.pathNodeToNode(tree1, tree2);
        if (tree1 != null && tree2 != null && nodes != null) {
            logger.debug("Distance btw" + tree1.toString() + "and "
                            + tree2.toString() + " is:" + nodes.size());
        } else {
            logger.debug("Distance btw tree1:" + tree1 + "and tree2 " + tree2
                            + " is:" + nodes);
        }
        int nodeSize = 0;
        if (nodes != null) nodeSize = nodes.size();
        return nodeSize;
    }

    /**
     * get closest oc to the given poc
     * 
     * @param poc
     * @param ocs
     * @param root
     * @return
     */
    int getClosestOc(List<Tree> poc, List<List<Tree>> ocs, Tree root) {
        int min = 0;
        for (List<Tree> oc : ocs) {
            int dist = getDistance(poc, oc, root);
            if (dist < min) min = dist;
        }
        return min;
    }
} public class FreyaConstants {
    
    public static final String SENTENCE_DELIMITER = "\n ";
    
    public static String QUESTION = "processedQuestion";
    public static String SEMANTIC_CONCEPTS = "semanticConcepts";

    public static String VARIABLE_E = "E";
    public static String VARIABLE_L = "L";
    public static String VARIABLE_P = "P";
    public static String VARIABLE_T = "T";

    public static String POC_TYPE = "POC";
    public static String OC_TYPE = "OC";

    public static String OWL_CLASS = "http://www.w3.org/2002/07/owl#Class";

    public static String SOLR_FIELD_ID = "id";
    public static String SOLR_FIELD_CLASS = "class_s";
    public static String SOLR_FIELD_INSTANCE = "inst_s";
    public static String SOLR_FIELD_PROPERTY = "pred_s";
    public static String SOLR_FIELD_STEMMED_CONTENT = "stem_en";
    /** see also: setting in solr schema.xml <copyField source="prec_s" dest="prec_lowercase"/> */
    public static String SOLR_FIELD_EXACT_CONTENT = "prec_s";
    public static String SOLR_FIELD_LOWERCASE_CONTENT = "prec_lowercase";


    public static String CLASS_FEATURE_LKB = "class";

    public static String INST_FEATURE_LKB = "inst";

    public static String PROPERTY_FEATURE_LKB = "pred";

    public static String FIELD_EXACT_CONTENT = "prec";

    public static String FIELD_EXACT_LOWERCASED_CONTENT = "low";

    public static String FIELD_STEMMED_CONTENT = "stem";

    public static String SCORE = "score";

    public static String MAX_FUNCTION = "max";

    public static String MIN_FUNCTION = "min";

    public static String SUM_FUNCTION = "sum";
    
    public static String AVG_FUNCTION = "avg";

    public static String NONE_NEIGHBOURING_ONTOLOGY_ELEMENT = "NONENEIGHBOURINGONTOLOGYELEMENT";

    public static String INSTANCE_WITHOUT_DIRECT_CLASS = "INSTANCEWITHOUTDIRECTCLASS";

    public static String NUMBER_OF_RESULTS = "numberOfResults";
    public static String TRIPLE_ELEMENTS = "tripleElements";
    public static String RESULTS_GRAPH = "resultsGraph";

    public static String ANSWER = "answer";



    public static String ELEMENTS = "elements";

    public static String TABLE = "table";
    public static String GRAPH = "graph";
    
    public static String SPARQL = "sparql";

    public static String PRECISE_SPARQL = "preciseSparql";

    public static String REPOSITORY_URL = "repositoryUrl";

    public static String REPOSITORY_ID = "repositoryId";
    // ///////////////////////////////////////
    // query specific constants ////////////
    // //////////////////////////////////////
    public static String NP_TAG_TREEBANK = "NP";
    public static String NN_TAG_TREEBANK = "NN";
    public static String NX_TAG_TREEBANK = "NX";
    public static String PRP_TAG_TREEBANK = "PRP";
    public static String EX_TAG_TREEBANK = "EX";
    public static String WHADVP_TAG_TREEBANK = "WHADVP";
    public static String WHADJP_TAG_TREEBANK = "WHADJP";
    public static String WHNP_TAG_TREEBANK = "WHNP";
    public static String WRB_TAG_TREEBANK = "WRB";
    public static String WP_TAG_TREEBANK = "WP";
    public static String RB_TAG_TREEBANK = "RB";
    /** JJ adjective, JJR comparative, JJS superlative */
    public static String JJ_TAG_TREEBANK = "JJ";
    /** VBN verb, past participle */
    public static String VBN_TAG_TREEBANK = "VBN";
    /** RBS adverb, superlative */
    public static String RBS_TAG_TREEBANK = "RBS";

    public static String ADJP_TAG_TREEBANK = "ADJP";
    public static String DT_TAG_TREEBANK = "DT";

    public static boolean DEBUG_MODE = false;

    public static String ANNOTATION_FEATURE_TREE = "Tree";

    public static String STANFORD_TREE = "StanfordTree";

    public static String RESULT_TYPE_GRAPH = "Graph";

    public static String RESULT_TYPE_TREE = "Tree";

    public static String RESULT_TYPE_STRING = "string";

    public static String ANNOTATION_TYPE_ONTORES = "OntoRes";

    public static String ANNOTATION_TYPE_ONTORESCHUNK = "OntoResChunk";

    /* used when grouping elements to indicate if it is a conjunction element */
    public static String CONJUNCTION = "and";

    /* used when grouping elements to indicate if it is a disjunction element */
    public static String DISJUNCTION = "or";

    /**
     * this constant is used to indicate wheather the keyword is from the gazetteer with this name
     */
    public static String LIST_COMMANDS = "listCommands";

    /**
     * Name of the common logger
     */
    public static String LOGGER_NAME = "CLOnE-QL-logger";

    public static String LOGGER_OUPUT_LEVEL = "2000";

    /* separator used during formatting of results */
    public static String TRIPLES_SEPARATOR = " --> ";

    /*
     * flag indicating the output of the result, in this case it is refering to the resource names
     */
    public static String SHOWING_TRIPLES_WITH_RESOURCE_NAMES = "showingTriplesWithResourceNames";

    /*
     * flag indicating the output of the result, in this case it is refering to the labels
     */
    public static String SHOWING_TRIPLES_WITH_LABELS = "showingTriplesWithLabels";

    public static String REGEX_GROUPS_SEPARATED_BY_AND = "i\\d+-and-i\\d+(-and-i\\d+)*";

    public static String REGEX_GROUPS_SEPARATED_BY_OR = "i\\d+-or-i\\d+(-or-i\\d+)*";

    public static String GROUP_PREFIX = "gs:";

    public static String GROUP_SUFFIX = ":ge";

    public static String REGEX_PPP = "[i,c,d,g]\\d+-([k,o,p]\\d+-)*[i,c,d,g]\\d+";

    public static String REGEX_FIND_JOKER = "[i,c,d,g]\\d+-[i,c,d,g]\\d+";

    public static String REGEX_FIND_CLASS_JOKER = "[p]\\d+-[p]\\d+";

    public static String POTENTIAL_PROPERTY_POSITION = "-r";

    public static String EXACT_PROPERTY = "-ep";

    public static String POTENTIAL_RELATED_ELEMENTS = ":";

    public static String MAX_SIMILARITY_VALUE = "1";

    /* three types of weights - to show the importance of every type of score */
    public static double SIMILARITY_SCORE_WEIGHT = 3.0;
    public static double SPECIFICITY_SCORE_WEIGHT = 1.0;
    public static double DISTANCE_SCORE_WEIGHT = 1.0;

    public static String SELECT = "select distinct";
    
    public static String SELECT_SUM ="select (SUM(xsd:decimal(";
    
    public static String SELECT_AVG ="select (AVG(xsd:decimal(";

    public static String WHERE = "where";

    public static String FROM = "from";

    public static String INVERSE_PROPERTY = "[inverseProperty]";

    public static String JOKER = "joker";

    public static String LITERAL_VALUE_CONNECTOR = "is";

    // these are used as annotation type names in order to transwer the results
    // from the gate pipeline elsewhere
    public static String QUERY = "query";
    public static String INTERPRETATIONS_LIST = "interpretationsList";
    public static String SELECTED_INTERPRETATION = "selectedInterpretation";
    public static String INTERPRETATION_RESULTS = "interpretationResults";
    public static String QUERY_INTERPRETATIONS = "queryInterpretations";
    public static String MAP_OF_LABELS = "mapOfLabels";

    public static double LEVENSHTEIN_THRESHOLD = 0.1;
    public static double MONGE_THRESHOLD = 0.5;
    public static double MAX_MONGE_THRESHOLD = 1.0;

    public static double REINFORCEMENT_REWARD = 1.0;
    public static long REINFORCEMENT_NULL_STATE = 0;
    public static double REINFORCEMENT_NEGATIVE_REWARD = -1.0;

    public static String CLARIFICATION_OPTIONS_NONE = "none";

    public static String CLARIFIED_INTERPRETATION_DETAILS = "clarifiedInterpretationDetails";

    public static String CIPIN_SEPARATOR = ",";

    /* priority over ontology annotations */
    public static int MAIN_SUBJECT_PRIORITY_MAX = 100;

    public static int MAIN_SUBJECT_PRIORITY_MIN = 0;


}

import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.trees.Tree;
import edu.stanford.nlp.trees.Trees;
import java.util.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.freya.annotate.lucene.RenameAnnotations;
import org.freya.annotate.solr.SolrAnnotator;
import org.freya.exe.StanfordExecutor;
import org.freya.model.Annotation;
import org.freya.model.POC;
import org.freya.model.Question;
import org.freya.model.SemanticConcept;
import org.freya.oc.OCCreator;
import org.freya.oc.OCUtil;
import org.freya.parser.stanford.TreeUtils;
import org.freya.util.FreyaConstants;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

package org.freya.mapping;




/**
 * this class communicates with stanford executor and query (gate/solr) executor and mapps results of one to the other (pocs
 * to ocs)
 * 
 * @author danica
 */
@Component
public class Mapper {
    private static final Log logger = LogFactory.getLog(Mapper.class);

    @Autowired
    private SolrAnnotator solrAnnotator;

    //@Autowired
    //private LuceneAnnotator luceneAnnotator;

    @Autowired
    private RenameAnnotations renameAnnotations;

    @Autowired
    private StanfordExecutor stanfordExecutor;

    @Autowired

    private OCCreator ocCreator;
