public interface Annotator extends Stage {
    
    /**
     * The abstract annotation method. All classes that perform some kind of 
     * annotation (splitting, PoS tagging, entity linking...) must inherit from
     * Annotator. They annotate the blackboard given as first parameter or a 
     * component of the blackboard, given as second parameter. 
     * 
     * @param blackboard blackboard to annotate
     * @param component component to annotate.
     */
    public void annotate(Blackboard blackboard, DocumentComponent component);
    
    
    @Override
    default void run(Blackboard b) {
        this.annotate(b, b.getStructure());
    }    
    
}
 public class Blackboard {

    /**
     * The default document identifier.
     */
    private static final String DEFAULT_DOCUMENT_ID = "DocumentRoot";

    /**
     * The full raw text of the document.
     */
    private String rawText;

    /**
     * The root block of the document.
     */
    private DocumentComponent document;

    /**
     * Container for the n-grams of the document. Every n-gram is part of a
     * specific list identifying the type of n-gram. The types of n-grams are
     * the key for searching in the main n-gram list.
     */
    private Map<String, Map<String, Gram>> generalNGramsContainer;

    /**
     * Document-wide annotations. This space can be used to add annotations that
     * belong to the whole document, as for example extracted concepts, or
     * tagset used by a POS-tagger, or the overall sentiment.
     */
    private List<Annotation> annotations;

    /**
     * Instantiates an empty blackboard.
     */
    public Blackboard() {
        createDocument("");
    }

    /**
     * Initializes the blackboard with a new document. This will destroy any
     * information previously held by the blackboard.
     *
     * @param rawText the text of the new document.
     * @param documentId the output-friendly identifier of the document
     */
    public final void createDocument(String rawText, String documentId) {
        this.rawText = rawText;
        this.document = new DocumentComposite(rawText, documentId);
        this.generalNGramsContainer = new HashMap<>();
        this.annotations = new ArrayList<>();
    }

    /**
     * Initializes the blackboard with a new document. This will destroy any
     * information previously held by the blackboard.
     *
     * @param rawText the text of the new document.
     */
    public final void createDocument(String rawText) {
        this.rawText = rawText;
        this.document = new DocumentComposite(rawText, DEFAULT_DOCUMENT_ID);
        this.generalNGramsContainer = new HashMap<>();
        this.annotations = new ArrayList<>();
    }

    /**
     * Gets the root of the document.
     *
     * @return the {@link it.uniud.ailab.dcore.persistence.DocumentComponent}
     * root object.
     */
    @JsonIgnore
    public DocumentComponent getStructure() {
        return document;
    }

    /**
     * Gets the raw text (i.e. unprocessed) of the document.
     *
     * @return the original document string.
     */
    public String getText() {
        return rawText;
    }

    /**
     * Adds a Gram in the Gram Container, merging grams with the same
     * identifier. If the gram is already present, the method updates it adding
     * the new occurrence. Annotations of the new gram are <b>not</b> merged
     * into the old gram. This is because it's good practice to annotate grams
     * only when they've <b>all</b> been added into the blackboard.
     *
     * @param unit the concept unit where the gram appears
     * @param newGram the gram to add
     */
    public void addGram(DocumentComponent unit, Gram newGram) {

        Map<String, Gram> grams = generalNGramsContainer.get(newGram.getType());
        if (grams == null) {
            grams = new HashMap<>();
        }
        Gram gram = grams.get(newGram.getIdentifier());

        // Deep clone the object instead of referencing the found one.
        // this way, we're free to modify it by adding annotations without
        // modifying the old object.
        if (gram == null) {
            Gram cloned = (new Cloner()).deepClone(newGram);
            grams.put(cloned.getIdentifier(), cloned);
            gram = cloned;
        } else {
            // add the surface of the new gram
            gram.addSurfaces(newGram.getSurfaces(), newGram.getTokenLists());
        }

        gram.addAppaerance(unit);
        unit.addGram(gram);
        generalNGramsContainer.put(newGram.getType(), grams);
    }

    public void addGram(Gram newGram) {

        Map<String, Gram> grams = generalNGramsContainer.get(newGram.getType());
        if (grams == null) {
            grams = new HashMap<>();
        }
        Gram gram = grams.get(newGram.getIdentifier());

        // Deep clone the object instead of referencing the found one.
        // this way, we're free to modify it by adding annotations without
        // modifying the old object.
        if (gram == null) {
            Gram cloned = (new Cloner()).deepClone(newGram);
            grams.put(cloned.getIdentifier(), cloned);
        } else {
            // add the surface of the new gram
            gram.addSurfaces(newGram.getSurfaces(), newGram.getTokenLists());
        }

        generalNGramsContainer.put(newGram.getType(), grams);
    }
    
    /**
     * Get the all the different kind of grams found in the document. This
     * grams are divided by type, stored in a Map using their identifier as 
     * key.
     * 
     * @return all the maps found in the document.
     */
    public Map<String,Map<String,Gram>> getGrams() {
        return generalNGramsContainer;
    }

    /**
     * Get all the grams of a given type found in the blackboard.
     * 
     * @param <T> the type of the grams to achieve
     * @param gramType the identifier of the gram type
     * @return a collection with all the grams with match type and identifier,
     * null if there is no match.
     */
    public <T> Collection<T> getGramsByType(String gramType) {        
        return 
                generalNGramsContainer.containsKey(gramType) ?
                (Collection<T>)generalNGramsContainer.get(gramType).values() :
                null;
    }

    /**
     * Retrieves the keyphrases found in the document.
     *
     * @return a collection of
     * {@link it.uniud.ailab.dcore.persistence.Keyphrase}s.
     */
    @Deprecated
    @JsonIgnore
    public List<Gram> getKeyphrases() {

        Map<String, Gram> kps = generalNGramsContainer.get(Keyphrase.KEYPHRASE);
        return kps != null ? new ArrayList(kps.values()) : new ArrayList();
    }

    /**
     * Removes a keyphrase from the document because it's no more relevant, or
     * useful, or for whatever reason an annotator thinks so.
     *
     * @param g the gram to remove.
     */
    @Deprecated
    public void removeKeyphrase(Keyphrase g) {
        removeGram(Keyphrase.KEYPHRASE,g);
    }
    
    /**
     * Removes a gram from the document because it's no more relevant, or
     * useful, or for whatever reason an annotator thinks so.
     *
     * @param type the type of the gram to remove
     * @param g the gram to remove.
     */
    public void removeGram(String type,Gram g) {
        generalNGramsContainer.get(type)
                .remove(g.getIdentifier());

        for (Sentence s : DocumentUtils.getSentences(document)) {
            s.removeGram(g);
        }
    }

    /**
     * Adds an annotation in the blackboard.
     *
     * @param a the annotation to add
     */
    public void addAnnotation(Annotation a) {
        annotations.add(a);
    }

    /**
     * Get all the annotations.
     *
     * @return the annotations stored in the blackboard
     */
    public List<Annotation> getAnnotations() {
        return annotations;
    }

    /**
     * Gets the annotations produced by a specific annotator.
     *
     * @param annotator the annotator identifier
     * @return the annotations produced by the specified annotator
     */
    public List<Annotation> getAnnotations(String annotator) {
        return annotations.stream().filter((a)
                -> (a.getAnnotator().equals(annotator))).
                collect(Collectors.toList());
    }

    /**
     * Remove an annotation from the blackboard.
     * 
     * @param ann the annotation to remove.
     */
    public void removeAnnotation(Annotation ann) {
        annotations.remove(ann);
    }
    
    /**
     * Get the language of the document root.
     * 
     * @return the language of the document root.
     */
    public String getDocumentLanguage() {
        return getStructure().getLanguage().getLanguage();
    }

}
 public class AnnotationException extends RuntimeException {
    
    public AnnotationException(Annotator sender, String message) {
        super("Error while annotating\n\t" + 
                "Annotator " + sender.getClass().getName() + 
                " caused an exception with message:\n\t" + message);
    }
    
    public AnnotationException(Annotator sender,
            Throwable cause) {
        super("Error while annotating\n\t" + 
                "Annotator " + sender.getClass().getName() + 
                " caused an exception. ",cause);
    }
    
    public AnnotationException(Annotator sender, String message,
            Throwable cause) {
        super("Error while annotating\n\t" + 
                "Annotator " + sender.getClass().getName() + 
                " caused an exception with message:\n\t" + message,cause);
    }
    
    
    
} public abstract class DocumentComponent extends Annotable {

    private final String text;
    private Locale language;

    /**
     * Creates a document component.
     *
     * @param text the text of the component
     * @param language the language of the component
     * @param identifier the unique identifier for the component
     */
    public DocumentComponent(String text, Locale language, String identifier) {
        super(identifier);
        this.text = text;
        this.language = language;
    }

    // <editor-fold desc="getters and setters">
    /**
     * Returns the text of the component
     *
     * @return the text of the component.
     */
    public String getText() {
        return text;
    }

    /**
     * Sets the language of the component.
     *
     * @param language the language of the component
     */
    public void setLanguage(Locale language) {
        this.language = language;
        if (hasComponents()) {
            for (DocumentComponent c : getComponents()) {
                c.setLanguage(language);
            }
        }
    }

    /**
     * Returns the language of the component.
     *
     * @return the language of the component.
     * @see <a href="http://tools.ietf.org/html/rfc5646">RFC5646</a>
     * specification to know about language tags used by Java.
     */
    public Locale getLanguage() {
        return this.language;
    }

    /**
     * Check if the component is a leaf ( so it can be a
     * {@link it.uniud.ailab.dcore.persistence.Sentence}), or if it has
     * children, so it's surely a
     * {@link it.uniud.ailab.dcore.persistence.DocumentComposite}.
     *
     * @return true if the component has children; otherwise false.
     */
    public boolean hasComponents() {
        List<DocumentComponent> comps = getComponents();
        return !(comps == null);
    }

    // </editor-fold>
    // <editor-fold desc="abstract methods">
    /**
     * Returns the children of the document component, or null if the current
     * concept unit has no children (a sentence, the leaf of the tree).
     *
     * @return the children of the document component, or null if the current
     * concept unit has no children.
     */
    public abstract List<DocumentComponent> getComponents();
    
    /**
     * Adds a gram to the component.
     *
     * @param g the gram to add
     */
    public abstract void addGram(Gram g);

    /**
     * Returns the gram associated with the component.
     *
     * @return the gram associated with the component.
     */
    public abstract List<Gram> getGrams();
    
    /**
     * Remote a gram from the component.
     * 
     * @param g the gram tor remove
     */
    public abstract void removeGram(Gram g);

    // </editor-fold>

    /**
     * Get the string representation of the component.
     *
     * @return the string that represent the component (which has been set in
     * the constructor).
     */
    @Override
    public String toString() {
        return text;
    }

} public abstract class Gram extends Annotable {
    
    /**
     * The type of n-gram:it can be a concept, a keyphrase, a mention...
     */
    private final String type;
    
    /**
     * The different list of words forming the surface of the gram.
     */
    private List<List<Token>> tokenLists;
    
    /**
     * The different string representation of the gram.
     */
    private final List<String> surfaces;
    
    /**
     * The concept Units in which the gram appears.
     */
    private List<DocumentComponent> appareances;
    
    /**
     * The identifier for a GRAM object.
     */
    public static final String GRAM = "GRAM";
    
    /**
     * Instantiates an n-gram. Usually, the surface should be simply the the 
     * concatenation of the text of the tokens. The signature can be used for 
     * comparison, so be sure to generate different signatures for n-grams
     * that are different in your domain. For example, you can use the sequence 
     * of the stems of the tokens, so that n-grams with the same stemmed form 
     * are considered equal.
     * 
     *
     * @param sequence the tokens that form the gram
     * @param identifier unique identifier of the gram.
     * @param surface the pretty-printed string representation of the gram
     * @param type the type of gram that will be generated.
     */
    public Gram(String identifier, List<Token> sequence, String surface, 
            String type) {
        
        super(identifier);
        
        this.type = type;
        
        tokenLists = new ArrayList<>();
        tokenLists.add(sequence);
        
        surfaces = new ArrayList<>();
        surfaces.add(surface);
    }
    
    /**
     * Adds a surface to the n-gram. Duplicates are permitted.
     * 
     * @param surface the surface to add
     * @param tokens the tokens that form the surface
     */
    public void addSurface(String surface,List<Token> tokens) {
        surfaces.add(surface);
        tokenLists.add(tokens);
    }
    
     /**
     * Adds a group of surfaces to the n-gram. Duplicates are permitted.
     * 
     * @param surfaces the surface to add
     * @param tokenLists the tokens that form the surface
     */
    public void addSurfaces(List<String> surfaces,List<List<Token>> tokenLists) {
        
        if (surfaces.size() != tokenLists.size())
            throw new IllegalArgumentException(
                "Mismatching size of surfaces and token lists.");
        
        this.surfaces.addAll(surfaces);
        
        // note: do not use addAll. The references are lost if you don't copy
        for (List<Token> t : tokenLists) {
            this.tokenLists.add(new ArrayList<Token>(t));
        }
    }
    
    /**
     * Get the type of the Gram that depends on the type of Gram implementation.
     * 
     * @return the type of gram. 
     */
    public String getType(){
        return type;
    }

    /**
     * The tokens that form the most common surface of the gram.
     *
     * @return the tokens of the surface of the gram.
     */
    public List<Token> getTokens() {
        return tokenLists.get(surfaces.indexOf(ListUtils.mostCommon(surfaces)));
    }
    
    /**
     * Returns all the possible lists of tokens that form the gram.
     * 
     * @return all the possible lists of tokens that form the gram.
     */
    @JsonIgnore
    public List<List<Token>> getTokenLists() {
        return tokenLists;
    }

    /**
     * The human-readable form of the gram. This is the most common surface
     * between all the surfaces associated with the gram; if there are more than
     * one, the first one that has been added to the gram is selected.
     *
     * @return the human-readable form of the gram.
     */
    public String getSurface() {
        return ListUtils.mostCommon(surfaces);
    }
    
    /**
     * Returns all the surfaces of the gram. Note: may contain 
     * duplicates.
     * 
     * @return all the surfaces of the gram.
     */
    @JsonIgnore
    public List<String> getSurfaces() {
        return surfaces;
    }
    
    /**
     * Adds an appearance of the gram; in other words, adds the component in
     * which the gram appears to the list of the appearances.
     *
     * @param component the component in which the gram appears
     */
    public void addAppaerance(DocumentComponent component) {
        appareances.add(component);
    }

    /**
     * Gets all the components in which the gram appears.
     *
     * @return all the components in which the gram appears.
     */
    @JsonIgnore
    public List<DocumentComponent> getAppaerances() {
        return appareances;
    }
    
    
    /**
     * The identifier of the gram. Please note that it is possible that two
     * grams with different surface or tokens may have the same identifier, 
     * based on the policy of the class that generated the gram.
     * 
     * For example, "italian" and "Italy" may have the same identifier, because
     * the identifier has been generated using the same stem "ital". Otherwise,
     * the identifier may be the same link on an external ontology: in this 
     * case, both words may have been associated with the entity "Italy".
     * 
     *
     * @return the signature of the gram.
     */
    @Override
    public String getIdentifier() {
        return super.getIdentifier();
    }
    
} public class Keyphrase extends Gram {

    /**
     * The different list of words forming the surface of the gram.
     */
    private List<List<Token>> tokenLists;
    /**
     * The different string representation of the gram.
     */
    private final List<String> surfaces;
    /**
     * The concept Units in which the gram appears.
     */
    private List<DocumentComponent> appareances;
    
    /**
     * The identifier for a Keyphrase object.
     */
    public static final String KEYPHRASE = "Keyphrase"; 

    /**
     * Instantiated an n-gram. Usually, the surface should be simply the the 
     * concatenation of the text of the tokens. The signature can be used for 
     * comparison, so be sure to generate different signatures for n-grams
     * that are different in your domain. For example, you can use the sequence 
     * of the stems of the tokens, so that n-grams with the same stemmed form 
     * are considered equal.
     * 
     *
     * @param sequence the tokens that form the gram
     * @param identifier unique identifier of the gram.
     * @param surface the pretty-printed string representation of the gram
     */
    public Keyphrase(String identifier, List<Token> sequence, String surface) {
        
        super(identifier, sequence, surface,KEYPHRASE);
        
        tokenLists = new ArrayList<>();
        tokenLists.add(sequence);
        
        surfaces = new ArrayList<>();
        surfaces.add(surface);
        appareances = new ArrayList<>();
    }
    
    /**
     * Adds a surface to the n-gram. Duplicates are permitted.
     * 
     * @param surface the surface to add
     * @param tokens the tokens that form the surface
     */
    @Override
    public void addSurface(String surface,List<Token> tokens) {
        surfaces.add(surface);
        tokenLists.add(tokens);
    }
    
     /**
     * Adds a group of surfaces to the n-gram. Duplicates are permitted.
     * 
     * @param surfaces the surface to add
     * @param tokenLists the tokens that form the surface
     */
    @Override
    public void addSurfaces(List<String> surfaces,List<List<Token>> tokenLists) {
        
        if (surfaces.size() != tokenLists.size())
            throw new IllegalArgumentException(
                "Mismatching size of surfaces and token lists.");
        
        this.surfaces.addAll(surfaces);
        
        // note: do not use addAll. The references are lost if you don't copy
        for (List<Token> t : tokenLists) {
            this.tokenLists.add(new ArrayList<Token>(t));
        }
    }

    /**
     * The tokens that form the most common surface of the gram.
     *
     * @return the tokens of the surface of the gram.
     */
    @Override
    public List<Token> getTokens() {
        return tokenLists.get(surfaces.indexOf(ListUtils.mostCommon(surfaces)));
    }
    
    /**
     * Returns all the possible lists of tokens that form the gram.
     * 
     * @return all the possible lists of tokens that form the gram.
     */
    @Override
    @JsonIgnore
    public List<List<Token>> getTokenLists() {
        return tokenLists;
    }

    /**
     * The human-readable form of the gram. This is the most common surface
     * between all the surfaces associated with the gram; if there are more than
     * one, the first one that has been added to the gram is selected.
     *
     * @return the human-readable form of the gram.
     */
    @Override
    public String getSurface() {
        return ListUtils.mostCommon(surfaces);
    }
    
    /**
     * Returns all the surfaces of the gram. Note: may contain 
     * duplicates.
     * 
     * @return all the surfaces of the gram.
     */
    @Override
    @JsonIgnore
    public List<String> getSurfaces() {
        return surfaces;
    }

    /**
     * The identifier of the gram. Please note that it is possible that two
     * grams with different surface or tokens may have the same identifier, 
     * based on the policy of the class that generated the gram.
     * 
     * For example, "italian" and "Italy" may have the same identifier, because
     * the identifier has been generated using the same stem "ital". Otherwise,
     * the identifier may be the same link on an external ontology: in this 
     * case, both words may have been associated with the entity "Italy".
     * 
     *
     * @return the signature of the gram.
     */
    @Override
    public String getIdentifier() {
        return super.getIdentifier();
    }

    // <editor-fold desc="Feature and annotation Management">

    /**
     * Adds a feature to the gram.
     *
     * @param feature the identifier of the feature
     * @param value the value of the feature
     */
    public void putFeature(String feature, double value) {        
        addAnnotation(new FeatureAnnotation(feature,value));
        
    }

    /**
     * Adds a feature to the gram,
     *
     * @param f the feature to add.
     */
    public void putFeature(FeatureAnnotation f) {
        addAnnotation(f);
    }

    /**
     * Check if the gram has been annotated by the annotator specified via input
     * string.
     *
     * @param featureName the name of the feature to search
     * @return true if the gram has the feature; false otherwise
     */
    public boolean hasFeature(String featureName) {
        return this.hasAnnotation(featureName);
    }

    /**
     * Gets the feature generated by the annotator specified via input string.
     *
     * Please note that this method makes no difference between a feature that
     * has been assigned with value 0 and a feature that has not been assigned
     * to the gram, since in both cases the value 0 will be returned.
     *
     * @param featureName the name of the feature to search
     * @return the value of the feature. Returns 0 if the feature is not in the
     * gram.
     */
    public double getFeature(String featureName) {
        // null check; if the feature is not specified, we assume it's 0.
        if (!this.hasAnnotation(featureName)) {
            return 0;
        }
        return ((FeatureAnnotation) getAnnotation(featureName))
                .getScore();
    }

    /**
     * Returns all the features associated with the gram.
     *
     * @return all the features associated with the gram.
     */
    @JsonIgnore
    public FeatureAnnotation[] getFeatures() {
        
        List<FeatureAnnotation> features = new ArrayList<>();
        
        for (Annotation ann : getAnnotations()) {
            if (ann instanceof FeatureAnnotation)
                features.add((FeatureAnnotation)ann);
        }
        
        return features.toArray(new FeatureAnnotation[features.size()]);        
    }

    /**
     * Sets the features of the gram, deleting the previous ones (if any).
     *
     * @param features the new features of the gram.
     */
    public void setFeatures(FeatureAnnotation[] features) {
        for (FeatureAnnotation f : features)
            this.addAnnotation(f);
    }
    // </editor-fold> 

    // <editor-fold desc="Location Management">      
    /**
     * Adds an appearance of the gram; in other words, adds the component in
     * which the gram appears to the list of the appearances.
     *
     * @param component the component in which the gram appears
     */
    @Override
    public void addAppaerance(DocumentComponent component) {
        appareances.add(component);
    }

    /**
     * Gets all the components in which the gram appears.
     *
     * @return all the components in which the gram appears.
     */
    @Override
    public List<DocumentComponent> getAppaerances() {
        return appareances;
    }
    // </editor-fold>
} public class Sentence extends DocumentComponent {
    
    // <editor-fold desc="Private fields">

    /**
     * The {@link it.uniud.ailab.dcore.persistence.Token}s that form the sentence.
     */
    private List<Token> tokenizedSentence;

    /**
     * The {@link it.uniud.ailab.dcore.persistence.Keyphrase}s that have been detected 
     * in the sentence.
     */
    private List<Gram> grams;
    
    // </editor-fold>
    
    /**
     * Creates a sentence with the specified text and language.
     * 
     * @param text the text of the sentence
     * @param language the language of the sentence
     * @param identifier the output friendly identifier of the sentence
     */    
    public Sentence(String text,Locale language,String identifier) {
        super(text,language,identifier);
        tokenizedSentence = new ArrayList<>();
        grams = new ArrayList<>(); 
    }
    
    /**
     * Creates a sentence with the specified text. This requires
     * a call to setLanguage before many of the annotators can actually work.
     * 
     * @param text the text of the sentence
     * @param identifier the output friendly identifier of the sentence
     */
    public Sentence(String text,String identifier) {
        this(text,null,identifier);
    }
    
    /**
     * Sets the tokens of the sentence.
     * 
     * @param tokenzedSentence the tokens of the sentence.
     */
    public void setTokens(List<Token> tokenzedSentence) {
        this.tokenizedSentence = tokenzedSentence;
    }
    
    /**
     * Appends a tokens at the end of the token list of the sentence.
     * 
     * @param t the token to add
     */
    public void addToken(Token t) {
        this.tokenizedSentence.add(t);
    }
    
    /**
     * Returns the tokens of the sentence.
     * 
     * @return the tokens of the sentence.
     */
    public List<Token> getTokens() {
        return tokenizedSentence;
    }      
    
    /**
     * Returns an array containing the POS tags of the tokens of the sentence.
     * 
     * @return an array containing the POS tags of the sentence.
     */
    public String[] getPosTaggedSentence() {
        String[] output = new String[tokenizedSentence.size()];
        for (int i = 0; i < tokenizedSentence.size(); i++) {
            output[i] = tokenizedSentence.get(i).getPoS();
        }
        return output;
    }    
    // </editor-fold>
    
    /**
     * Get the text of the sentence.
     * 
     * @return the text of the sentence.
     */
    @Override
    public String toString() {
        return getText();
    }

    /**
     * A sentence has no sub-components in the document model, so a null value
     * is returned.
     * 
     * @return null, because the sentence has no children.
     */
    @Override
    public List<DocumentComponent> getComponents() {
        return null;
    }

    /**
     * Add a n-gram to the sentence.
     * 
     * @param g the gram to add. 
     */
    @Override
    public void addGram(Gram g) {
        grams.add(g);
    }
    
    /**
     * Get all the grams of a sentence.
     * 
     * @return the grams that have been found in the sentence.
     */
    @Override
    public List<Gram> getGrams() {
        return grams;
    }

    /**
     * Removes a gram from the sentence.
     * 
     * @param gramToRemove the gram to remove
     */
    @Override
    public void removeGram(Gram gramToRemove) {

        for (Iterator<Gram> gramIterator = grams.iterator(); 
                gramIterator.hasNext();)
        {
            Gram gram = gramIterator.next();
            
            if (gram.getIdentifier().equals(gramToRemove.getIdentifier())) {
                gramIterator.remove();
            }
                
        }
    }
}
 public class Token extends Annotable {

    /**
     * The stem of the word.
     */
    private String stem;
    
    /**
     * The part-of-speech tag of the word.
     */
    private String PoS;
    
    /**
     * The lemmatized form of the token, that is obtained removing inflectional 
     * endings only and returning the base or dictionary form of a word.
     */
    private String lemma;

    /**
     * Creates a token.
     * 
     * @param text the text of the token.
     */
    public Token(String text){
        super(text);
    }
    
    // <editor-fold desc="Id, stem and PoS">
    /**
     * Set the stem of the token; if you don't have a stemmer for a certain
     * language, the lemmatized version of the word should work fine as well.
     * 
     * @param stem the stemmed token
     */
    public void setStem(String stem) {
        this.stem = stem;
    }
    
    /**
     * Set the POS tag of the token.
     * 
     * @param PoS the POS tag.
     */
    public void setPoS(String PoS) {
        this.PoS = PoS;
    }   
    
    /**
     * Set the lemmatize form of the token.
     * 
     * @param lemma the lemma for the token. 
     */
    public void setLemma(String lemma){
        this.lemma = lemma;
    }
    
    /**
     * Returns the text of the token.
     * 
     * @return the text of the token.
     */
    public String getText() {
        return super.getIdentifier();
    }

    /**
     * Returns the stem of the token.
     * 
     * @return the stem of the token.
     */
    public String getStem() {
        return stem;
    }
    
    /**
     * Returns the POS tag of the token.
     * 
     * @return the POS tag of the token.
     */
    public String getPoS() {
        return PoS;
    }    
    
    /**
     * Returns the lemmatize form of the token.
     * 
     * @return the lemma for token. 
     */
    public String getLemma(){
        return lemma;
    }
    // </editor-fold>
    
    
    // <editor-fold desc="Annotations">    
    /**
     * Gets all the annotations associated with the token that have been
     * generated by a specific annotator.
     * 
     * @param annotator the identifier of an annotator.
     * @return the annotations generated by the specified annotator.
     */
//    public List<TextAnnotation> getAnnotations(String annotator) {
//        List<TextAnnotation> ret = new ArrayList<>();
//        for (TextAnnotation ann : this.getAnnotations())
//        {
//            if (ann.getAnnotator().equals(annotator))  {
//                ret.add(ann);
//            }
//        }
//        return ret;
//    }
    
    /**
     * Check if the token has been annotated by a given annotator. Please note
     * that to retrieve all the annotations generated by an annotator you should
     * use getAnnotations() instead.
     * 
     * @param annotator the identifier of an annotator.
     * @return the first annotation in the list generated by the given annotator.
     */
//    public TextAnnotation hasAnnotation(String annotator) {
//        TextAnnotation a = null;
//        for (TextAnnotation b : this.getAnnotations())
//        {
//            if (b.getAnnotator().equals(annotator))  {
//                a = b;
//                break;
//            }
//        }
//        return a;
//    }
    // </editor-fold>
    
    /**
     * A full string representation of the token, which returns not only the text,
     * but also the stem and the annotations of the token.
     * 
     * @return 
     */
    @Override
    public String toString() {
        String ret = getText() + " {(POS:" + getPoS() + "), " 
                + "(Lemma: " + getLemma() + "), "
                + "(Stem: " +getStem()+ ")";
        for (Annotation a : getAnnotations()) {
            if (a instanceof TextAnnotation)
                ret = ret + ", (" + a.getAnnotator() + ":" + 
                        ((TextAnnotation) a).getAnnotation() + ")";
        }
        return ret + "}";
    }            

    /**
     * Two tokens are equal if they have the same text, stem and POS tag. 
     * Tokens with different annotation may just refer to same word in different
     * sentences; while the annotations are different, the word is the same.
     * 
     * For example, "Engineering" per se and the word "Engineering" in
     * "Software Engineering" should be treated as equal, even if they may
     * be annotated with different Wikipedia entities.
     * 
     * @param obj the token to compare with
     * @return true if the tokens are equal, false otherwise
     */
    @Override
    public boolean equals(Object obj) {
        if (obj == null) {
            return false;
        }
        if (getClass() != obj.getClass()) {
            return false;
        }
        final Token other = (Token) obj;
        if (!getText().equals(other.getText())) {
            return false;
        }
        if (!stem.equals(other.stem)) {
            return false;
        }
        return PoS.equals(other.PoS);
    }

    @Override
    public String getIdentifier() {
        return getText();
    }

   
} public class DocumentUtils {

    /**
     * Gets all the sentences of a document component.
     *
     * @param component the component of which you want the sentences
     * @return the sentences of the component
     */
    public static List<Sentence> getSentences(DocumentComponent component) {
        List<Sentence> ret = new ArrayList<>();

        for (DocumentComponent c : component.getComponents()) {
            if (c.getComponents() == null) {
                ret.add((Sentence) c);
            } else {
                ret.addAll(getSentences(c));
            }
        }

        return ret;
    }

    /**
     * Get the total number of phrases from which the document id composed. 
     * 
     * @param component
     * @return the total # of simple phrases in the document
     */
    public static double getNumberOfPhrasesInDocument(DocumentComponent component) {
        double numberOfPhrases = 0;
        
        //get the sentences in the document
        List<Sentence> sentences = getSentences(component);
        
        for (Sentence sentence : sentences) {
            
            //get the annotation for phrase count in the single sentence
            FeatureAnnotation annotation = (FeatureAnnotation)sentence
                    .getAnnotation(DefaultAnnotations.PHRASES_COUNT);
            
            //update the total number of phrases with the one of the current 
            //sentence
            numberOfPhrases += annotation.getScore();
        }
        
        assert(numberOfPhrases >= 1):"total number of phrases must be positive";
        
        return numberOfPhrases;
    }
} public class FileSystem {

    /**
     * Get an {@link java.io.InputStreamReader} from the provided path, where
     * the path can point both to the underlying OS' file system or to a file
     * contained inside a JAR file.
     *
     * @param path the path which needs to be converted to an InputStream
     * @return the resulting {@link java.io.InputStreamReader}
     * @throws FileNotFoundException if the file does not exists
     */
    public static InputStreamReader getInputStreamReaderFromPath(String path)
            throws FileNotFoundException {
        InputStreamReader isr;

        // running from command-line and loading inside the JAR
        if (path.contains("!")) {
            isr = new InputStreamReader(
                    (new FileSystem()).getClass().getResourceAsStream(
                            path.substring(
                                    path.lastIndexOf("!") + 1)));
        } else {
            // normal operation
            isr = new FileReader(path);
        }

        return isr;
    }

    /**
     * Get an {@link java.io.InputStream} from the provided path, where the path
     * can point both to the underlying OS' file system or to a file contained
     * inside a JAR file.
     *
     * @param path the path which needs to be converted to an InputStream
     * @return the resulting {@link java.io.InputStream}
     * @throws FileNotFoundException if the file does not exists
     */
    public static InputStream getInputStreamFromPath(String path) throws FileNotFoundException {

        InputStream is;

        // running from command-line and loading inside the JAR
        if (path.contains("!")) {
            is = (new FileSystem()).getClass().getResourceAsStream(
                    path.substring(
                            path.lastIndexOf("!") + 1));
        } else {
            // normal operation
            is = new FileInputStream(path);
        }

        return is;
    }

    /**
     * Get the temporary directory path of the underlying file system. On
     * Windows, it should return the value of the %TMP% (or %TEMP%) environment
     * variable.
     *
     * @return the temporary directory path.
     * @see
     * <a href="https://blogs.msdn.microsoft.com/oldnewthing/20150417-00/?p=44213/">
     * Why are there both TMP and TEMP environment variables, and which one is
     * right?</a>
     */
    public static String getTmpPath() {
        return System.getProperty("java.io.tmpdir");
    }

    /**
     * Get the separator for the underlying OS.
     *
     * @return the path separator for the underlying OS.
     */
    public static String getSeparator() {
        return java.io.File.separator;
    }

    /**
     * Creates an empty directory in the specified path. If the directory
     * already exists, nothing is performed.
     * 
     * @param directoryPath the path where the new directory will be created
     */
    public static void createDirectoryIfNotExists(String directoryPath) {
        File newDir = new File(directoryPath);

        // if the directory does not exist, create it
        if (!newDir.exists()) {
            newDir.mkdir();
        }
    }
    
    /**
     * Get the path where the Distiller puts temporary files. NOTE: you should
     * check if the directory exists before trying to put something inside.
     * 
     * @return the path of the temporary folder of the Distiller.
     */
    public static String getDistillerTmpPath() {
        return getTmpPath().concat(getSeparator()).concat("distiller-tmp");
        
    }
}
import it.uniud.ailab.dcore.persistence.Keyphrase;
import it.uniud.ailab.dcore.persistence.Sentence;
import it.uniud.ailab.dcore.persistence.Token;
import it.uniud.ailab.dcore.utils.DocumentUtils;
import it.uniud.ailab.dcore.utils.FileSystem;
import java.io.InputStreamReader;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import it.uniud.ailab.dcore.annotation.Annotator;
import it.uniud.ailab.dcore.Blackboard;
import it.uniud.ailab.dcore.annotation.AnnotationException;
import it.uniud.ailab.dcore.persistence.DocumentComponent;
import it.uniud.ailab.dcore.persistence.Gram;
/*
 * Copyright (C) 2015 Artificial Intelligence
 * Laboratory @ University of Udine.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
package it.uniud.ailab.dcore.annotation.annotators;


/**
 * Annotates the emotional intensity of grams; currently only the English
 * language is supported.
 *
 * @author Marco Basaldella
 * @see
 * <a href="http://www2.imm.dtu.dk/pubdb/views/publication_details.php?id=6010">The
 * AFINN Dataset page</a>
 * @see <a href="https://www.cs.uic.edu/~liub/FBS/sentiment-analysis.html">Liu
 * Bing's Sentiment Analysis works</a>
 */
public class SyuzhetAnnotator implements Annotator {

    public static final String INTENSITY_AVG = "Intensity_SentenceAverage";
    public static final String INTENSITY_SUM = "Intensity_SentenceSum";
    private static final String INTENSITY_COUNTER = "Intensity_Counter";

    public static final String POLARITY = "Polarity";
    private static final String POLARITY_COUNTER = "Polarity_Counter";

    private enum Dataset {

        BING,
        AFINN;
    }

    private Dataset dataset = Dataset.AFINN;

    private Map<String, Integer> weights = new HashMap<>();

    /**
     * Loads the word valence database created by Finn Årup Nielsen.
     */
    private void loadDefinitions() {

        if (dataset == Dataset.AFINN) {
            loadAfinn();
        } else {
            loadBing();
        }
    }

    /**
     * Loads Finn Årup Nielsen polarity database.
     */
    private void loadAfinn() {

        BufferedReader br = null;

        String line;
        String cvsSplitBy = "\t";

        try {

            InputStreamReader is = FileSystem.getInputStreamReaderFromPath(
                    getClass().
                    getClassLoader()
                    .getResource("afinn/afinn.txt").getFile());

            br = new BufferedReader(is);
            while ((line = br.readLine()) != null) {
                // use comma as separator
                String[] parsedLine = line.split(cvsSplitBy);
                weights.put(parsedLine[0], Integer.parseInt(parsedLine[1]));

            }

        } catch (FileNotFoundException e) {
            throw new AnnotationException(this, "Can't read the AFINN database.", e);
        } catch (IOException e) {
            throw new AnnotationException(this, "Can't read the AFINN database.", e);
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    // don't care
                }
            }
        }
    }

    /**
     * Loads Liu Bing's polarity database.
     */
    private void loadBing() {

        BufferedReader br = null;
        String line;

        try {

            InputStreamReader is = FileSystem.getInputStreamReaderFromPath(
                    getClass().
                    getClassLoader()
                    .getResource("bingliu/positive-words.txt").getFile());

            br = new BufferedReader(is);
            while ((line = br.readLine()) != null) {
                if (!line.startsWith(";")) {
                    weights.put(line, 1);
                }

            }

        } catch (FileNotFoundException e) {
            throw new AnnotationException(this, "Can't read Bing's positive words database.", e);
        } catch (IOException e) {
            throw new AnnotationException(this, "Can't read Bing's positive words database.", e);
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    // don't care
                }
            }
        }

        try {

            InputStreamReader is = FileSystem.getInputStreamReaderFromPath(
                    getClass().
                    getClassLoader()
                    .getResource("bingliu/negative-words.txt").getFile());

            br = new BufferedReader(is);
            while ((line = br.readLine()) != null) {
                if (!line.startsWith(";")) {
                    weights.put(line, -1);
                }

            }

        } catch (FileNotFoundException e) {
            throw new AnnotationException(this,
                    "Can't read Bing's negative words database.", e);
        } catch (IOException e) {
            throw new AnnotationException(this,
                    "Can't read Bing's negative words database.", e);
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    // don't care
                }
            }
        }
    }

    /**
     * Annotates the n-grams with their document intensity. The gram document
     * intensity is calculated as the mean intensity of the phrases where the
     * gram appears, where the intensity of a sentence is the sum of the
     * intensity of all its words.
     *
     * @param component
     */
    @Override
    public void annotate(Blackboard blackboard, DocumentComponent component) {

        // skip non-english components
        if (!component.getLanguage().getLanguage().equals("en")) {
            return;
        }

        loadDefinitions();

        List<Sentence> sentences = DocumentUtils.getSentences(component);

        for (Sentence s : sentences) {

            if (s.getGrams().isEmpty()) {
                continue;
            }

            double intensity = 0;
            int markedWords = 0;
            for (Token t : s.getTokens()) {