pipeline/post_processors.py

from typing import List, Sequence, Tuple, Optional, Dict, Union, Callable
import pandas as pd
import spacy
from spacy.language import Language
from skbio import diversity as dv

SPAN_ATTRS = ["text", "label_", "start", "end"]
CATEGORIES = [
    "ATTRIBUTION",
    "CITATION",
    "COUNTER",
    "DENY",
    "ENDOPHORIC",
    "ENTERTAIN",
    "JUSTIFYING",
    "MONOGLOSS",
    "PROCLAIM",
    "SOURCES",
]


def simple_table(
    doc: Union[spacy.tokens.Doc, Dict[str, str]],
    spans_key: str = "sc",
    attrs: List[str] = SPAN_ATTRS,
):
    columns = attrs + ["Conf. score"]
    data = [
        [str(getattr(span, attr)) for attr in attrs] + [score]  # [f'{score:.5f}']
        for span, score in zip(
            doc.spans[spans_key], doc.spans[spans_key].attrs["scores"]
        )
    ]
    return data, columns


# def span_info_aggregator()


def construction_classifier(doc, span):
    category = None
    spanroot = span.root

    ## Grabbing lexico-grammatical information
    span_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in span]
    span_dep = [t.dep_ for t in span]
    span_token = [t.norm_ for t in span]
    span_tag = [t.tag_ for t in span]

    c = [c for c in spanroot.children]
    c_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in spanroot.children]

    c_norm = [c.norm_ for c in spanroot.children]
    c_dep = [c.dep_ for c in spanroot.children]
    c_pos = [c.pos_ for c in spanroot.children]
    c_tag = [c.tag_ for c in spanroot.children]

    right_dep = [c.dep_ for c in spanroot.rights]

    # conditionals
    subjless = all(
        c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
        for c in spanroot.children
    )
    argmentless = all(
        c.dep_
        not in [
            "nsubj",
            "nsubjpass",
            "csubj",
            "csubjpass",
            "dobj",
            "ccomp",
            "xcomp",
            "dative",
            "attr",
            "oprd",
            "acomp",
        ]
        for c in spanroot.children
    )
    argless_span = all(
        c.dep_
        not in [
            "nsubj",
            "nsubjpass",
            "csubj",
            "csubjpass",
            "dobj",
            "ccomp",
            "xcomp",
            "dative",
            "attr",
            "oprd",
            "acomp",
        ]
        for c in span
    )

    ## nesting classifiers
    if spanroot.dep_ == "conj":
        while spanroot.dep_ == "conj":
            spanroot = spanroot.head
    # if spanroot.dep_ == "poss":
    #     while spanroot.dep_ == 'poss':
    #         spanroot = spanroot.head

    ## Conjunctions
    # Preconjunctions
    if spanroot.dep_ in ["preconj", "cc"]:
        category = "Conjunction"

    ## NOUN PHRASES
    # adverbial phrases
    if spanroot.dep_ in ["amod"]:
        category = "Adjectival modifier"
        # adverbial phrases
    if spanroot.dep_ in ["compound"]:
        category = "Compound noun"

    ## Nominal category
    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj", "dative"]:
        if "acl" in c_dep:
            category = "Noun + Complement (Object)"
        else:
            category = "Object"

    if spanroot.dep_ in ["nsubj", "nsubjpass"]:
        if "acl" in c_dep:
            category = "Noun + Complement (Subject)"
        else:
            category = "Subject"

    ## ADJUNCTS
    # prep phrases
    if spanroot.dep_ in ["prep", "agent"]:
        category = "Prepositional phrase"
    # adverbial phrases
    if spanroot.dep_ in ["advmod", "npadvmod", "nmod", "npmod", "quantmod"]:
        category = "Adverbial phrase"

    ## Predication patterns
    if spanroot.dep_ in ["acomp", "oprd"]:
        if "xcomp" in c_dep:
            category = "Subject predicate to-cl"
        else:
            category = "Adjectival complement"

    if spanroot.dep_ in ["attr"]:
        subjless = all(
            c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
            for c in spanroot.children
        )

        c_head = [c.dep_ for c in spanroot.head.children]
        if "expl" in c_head and "no_det" in span_t_dep_:
            category = "There is/are no NOUN"
        elif "expl" in c_head and spanroot.pos_ in ["NOUN"]:
            category = "There is/are + Noun complement"
        elif "expl" in c_head and spanroot.tag_ in ["NN", "NNS"]:
            category = "There is/are + Noun complement"

        elif spanroot.pos_ in ["NOUN", "PRON"]:
            if "acl" in c_dep:
                category = "Noun + Complement (attr)"
            else:
                category = "Nominal complement"

        elif not subjless and spanroot.pos_ in ["VERB", "AUX"]:
            category = "Main verb 4"

        elif spanroot.tag_ in ["NNP"]:
            category = "Nominal complement"

    ####################################
    ### clausal ####
    ####################################
    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", "acl", "pcomp", "relcl"]:
        _check_to = [
            c.dep_
            for c in spanroot.subtree
            if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"])
            and c.head.dep_ == "xcomp"
        ]
        _check_ing = [
            c.dep_
            for c in spanroot.subtree
            if "Prog" in str(c.morph) and c.dep_ == "xcomp"
        ]
        root_before_ccomp = [
            c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"
        ]

        _check_for_to = [
            "_".join([c.norm_, c.dep_])
            for c in spanroot.subtree
            if c.head.dep_ == "advcl" and (c.dep_ == "mark" or c.dep_ == "aux")
        ]
        entire_cl = (
            spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
        )

        ## Start with broad category, which is then re-evaluated for specific constructions.
        if spanroot.dep_ in ["advcl", "mark", "acl", "pcomp"]:
            ## Adverbial clauses
            ### Finite-adverbial clauses
            ### Non-finite adverbial clauses
            subjless = all(
                c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
                for c in spanroot.children
            )
            entire_cl = (
                spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
            )

            if "mark" in span_dep and spanroot.pos_ in ["VERB", "AUX"]:
                category = "Finite adverbial clause"
            elif "mark" in span_dep and "aux" in span_dep:
                category = "Finite adverbial clause"

            elif (
                "mark" in span_dep
                and spanroot.pos_ in ["VERB", "AUX"]
                and "expl" in c_dep
            ):
                category = "Finite adverbial clause"

            elif "advmod" in span_dep and ("WRB" in span_tag or "WDT" in span_tag):
                if spanroot.pos_ in ["VERB", "AUX"]:
                    category = "Finite adverbial clause"

                elif spanroot.pos_ not in ["VERB", "AUX"] and subjless:
                    category = "Non-finite adv clause 1"

                elif entire_cl:
                    category = "Finite adverbial clause"

            elif (
                str(spanroot.morph)
                in [
                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
                    "Aspect=Perf|Tense=Past|VerbForm=Part",
                ]
                and "aux" not in c_dep
            ):
                # he doing his job
                if argmentless:
                    # e.g., frankly speaking, strictly speaking
                    category = "Adverbial Phrase"
                else:
                    category = "Non-finite adv clause 2"

            elif (
                spanroot.pos_ not in ["VERB", "AUX"] and "mark" in span_dep and subjless
            ):
                category = "Non-finite adv clause 3"

            elif "aux" in c_dep and "TO" in c_tag:
                category = "Adverbial Phrase"

            elif "mark" not in span_dep and spanroot.pos_ in ["VERB", "AUX"]:
                category = "Dependent Verb phrase"

            elif not argmentless:
                category = "Adverbial clause"

            elif spanroot.dep_ == "advcl":
                category = "Adverbial phrase"

        if spanroot.dep_ in ["relcl", "ccomp", "acl"]:
            head = spanroot.head
            if ";" in [t.norm_ for t in head.children]:
                category = "Main verb 3"
            elif "nsubj" not in span_dep:
                category = "Dependent verb 1"
            elif "mark" in span_dep:
                category = "Complement clause"
            elif (
                str(spanroot.morph)
                in [
                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
                    "Aspect=Perf|Tense=Past|VerbForm=Part",
                ]
                and "aux" not in c_dep
            ):
                category = "Non-finite complement clause"
            elif spanroot.dep_ in ["relcl"]:
                category = "Relative clause"
            elif spanroot.dep_ in ["ccomp"]:
                category = "Complement clause"
            elif spanroot.dep_ in ["acl"]:
                category = "Noun Complement clause"
            else:
                # print(_check_for_to)
                category = "this one"

        ## Specific constructions
        # Extraposed that-clause or to-infinitives
        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in [
            "VERB",
            "AUX",
        ]:
            print(c_dep)
            if ("acomp" in c_dep or "oprd" in c_dep) and "ccomp" in c_dep:
                # eg it seems odd (oprd) that X.
                # eg it is certain (acomp) that X.
                category = (
                    "Extraposed that-cl (adj-complement)"  # e.g., it is certain that X.
                )

            elif "xcomp" in c_dep or ("advcl" in c_dep):
                if "for_mark" in _check_for_to:
                    category = (
                        "Extraposed to-cl (explicit subj)"  # eg It is possible to .
                    )
                elif _check_to:
                    category = "Extraposed to-cl 1"  # eg It is possible to .
                elif _check_ing:
                    category = "Extraposed -ing 1"  # eg It is possible to .
            elif (
                ("prep" in right_dep or "npadvmod" in right_dep)
                and "ccomp" in right_dep
                and spanroot.lemma_ == "be"
            ):
                category = "Cleft construction"

            elif "attr" in c_dep:
                category = "Extraposed that-cl (copula)"  # eg It is a wonder that X.

            else:
                category = "Extraposed that-cl (VERB)"

        # if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
        #     category = "Extraposed that-cl (VERB)1" #e.g., it has been shown that X.
        elif (
            "it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_
        ) and "acomp" in c_dep:
            if "xcomp" in c_dep:
                if _check_to:
                    category = "Extraposed to-cl 2"  # eg it is difficult to decide.
                elif _check_ing:
                    category = "Extraposed -ing 2"  # eg it is difficult to decide.

            else:
                category = "Extraposed that-cl (adj-complement) 2"

        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "oprd" in c_dep:
            category = (
                "Extraposed that-cl (adj-complement) 3"  # eg it seems odd that X.
            )

        # something without dummy subject "it"
        elif (
            (("nsubj" in c_dep and spanroot.lemma_ in ["be"]) or "nsubjpass" in c_dep)
            and spanroot.pos_ in ["AUX", "VERB"]
            and "it" not in c_norm
        ):
            # store xcomp, if the head of the xcomp is acomp
            _check_xcomp = [
                c.dep_
                for c in spanroot.subtree
                if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"
            ]
            _check_ccomp = [
                c.dep_
                for c in spanroot.subtree
                if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"
            ]
            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]

            if ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in c_dep:
                if any(root_before_ccomp):
                    category = "Post-predicate that-cl"
                else:
                    category = "Comment clause"

            elif ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in _check_ccomp:
                category = "Post-predicate that-cl 2"

            elif ("attr" in c_dep or "acomp" in c_dep) and "xcomp" in _check_xcomp:
                category = "Post-predicate to-cl"

            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_to:
                category = "Subject predicate to-cl"

            elif "xcomp" in c_dep and "auxpass" in c_dep and _check_to:
                category = "Subject predicate to-cl (passive)"

            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_ing:
                category = "Subject predicate -ing"
            elif "ccomp" in c_dep:
                category = "Subject predicate that-cl"
            elif "acomp" in c_dep:
                category = "Adjectival predicate"

            elif "mark" in c_dep and ("nsubj" in c_dep or "nsubjpass" in c_dep):
                category = "Finite-adverbial clause"
            else:
                category = "Main verb 1"

        ## without dummy subject it, and lexical verbs
        elif (
            ("nsubj" in c_dep or "nsubjpass" in c_dep) in c_dep
            and spanroot.pos_ in ["AUX", "VERB"]
            and "it" not in c_norm
            and spanroot.lemma_ not in ["be"]
        ):
            _check_wh = [
                c.dep_
                for c in spanroot.subtree
                if (
                    c.dep_ in ["attr", "advmod", "dobj", "nsubj"]
                    and c.tag_ in ["WP", "WRB", "WDT", "WP$"]
                )
                and c.head.dep_ == "ccomp"
            ]
            _check_if = [
                c.dep_
                for c in spanroot.subtree
                if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"])
                and c.head.dep_ == "ccomp"
            ]

            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]

            if "ccomp" in c_dep and (_check_wh or _check_if):
                category = "Post-predicate wh-cl"

            elif "ccomp" in c_dep:
                if any(root_before_ccomp):
                    category = "Post-predicate that-cl"
                else:
                    category = "Comment clause"

            elif "xcomp" in c_dep:
                if _check_to:
                    category = "Post-predicate to-cl"
                elif _check_ing:
                    category = "Post-predicate -ing"

        # Existential
        elif "expl" in c_dep and "NOUN" in c_pos and "mark" not in c_dep:
            category = "There is/are NOUN"

        elif (
            "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]
        ):
            category = "Cleft construction"

    if spanroot.dep_ in ["parataxis"]:
        if "_".join(span_dep) in [
            "nsubj_parataxis",
            "aux_parataxis",
            "nsubj_aux_parataxis",
        ]:
            category = "Comment clause"
        else:
            category = "parataxis (for now)"

    ## External comp
    if spanroot.dep_ in ["xcomp"]:
        if spanroot.head.pos_ == "ADJ" and "to_aux" in c_t_dep_:
            category = "Adjective complement to-cl"
        if spanroot.head.pos_ == "VERB" and "to_aux" in c_t_dep_:
            category = "Verb complement to-cl"

    if spanroot.dep_ in ["pcomp"]:
        if (
            str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]
            and "ccomp" in c_dep
        ):
            category = "Participle + that-cl"
        elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
            category = "Participle"

    ## Simple classifier
    # if spanroot.dep_ in ['pcomp']:
    #     if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
    #         category = "Gerund"

    if spanroot.dep_ in ["neg"]:
        category = "Negative particle"
    if spanroot.dep_ in ["aux", "auxpass"]:
        category = "Auxiliary"

    # Modal verbs
    if spanroot.tag_ == "MD":
        category = "Modal auxiliary"

    if spanroot.dep_ in ["dep", "csubj", "csubjpass"]:
        if (
            spanroot.head.dep_ in ["ROOT", "ccomp"]
            and spanroot.head.pos_ in ["AUX", "VERB"]
            and spanroot.pos_ in ["AUX", "VERB"]
        ):
            if spanroot.morph == spanroot.head.morph:
                category = "Main verb 4"
            else:
                category = "Dependent verb 2"
        elif str(spanroot.morph) == "Aspect=Prog|Tense=Pres|VerbForm=Part":
            category = "Gerund"
        elif spanroot.head.dep_ in ["conj", "acl", "relcl"]:
            if spanroot.morph == spanroot.head.morph:
                category = "Main verb 4"
            else:
                category = "Dependent verb 2"
        elif "VerbForm=Fin" in str(spanroot.morph):
            category = "Dependent verb 2"

    # Appositive phrases
    if spanroot.dep_ in ["appos"]:
        if "nummod" in c_dep:
            category = "Apposition"
        elif spanroot.pos_ in ["PROPN"]:
            category = "Appositive Proper Nouns"
        elif spanroot.pos_ in ["NOUN"]:
            category = "Appositive Noun Phrase"
        elif spanroot.pos_ in ["VERB", "AUX"]:
            _check = any(
                c.dep_ in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
                for c in spanroot.children
            )
            if _check:
                category = "Appositive Finite-clause"

    if spanroot.dep_ in ["appos", "dep", "attr"]:
        if not subjless and spanroot.pos_ in ["VERB", "AUX"]:
            category = "Main verb 5"

    if spanroot.dep_ in ["dep", "mark"]:
        if spanroot.tag_ in ["RB", "IN", "CC"]:
            category = "Conjunction"

    # sometimes the extra-clausal links are not accurate
    if spanroot.dep_ in ["aux", "auxpass", "oprd", "appos", "xcomp"]:
        if spanroot.head.dep_ == "ROOT":
            category = "Main verb"
        else:
            category = "dependent verb 5"

    if span.label_ == "CITATION":
        if "NNP" in span_tag or "NNPS" in span_tag:
            if span_dep[0] == "punct" and span_dep[-1] == "punct":
                category = "Parenthetical Citation"
            elif span_tag[0] in ["NNP", "NNPS"]:
                category = "Narrative Citation"
        else:
            category = "Other Citation"

    if category == None:
        category = spanroot.dep_

    return category


def construction_classifier2(doc, span):
    category = None
    spanroot = span.root

    ## Grabbing lexico-grammatical information
    span_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in span]
    span_dep = [t.dep_ for t in span]
    span_token = [t.norm_ for t in span]
    span_tag = [t.tag_ for t in span]

    c = [c for c in spanroot.children]
    c_t_dep_ = ["_".join([t.norm_, t.dep_]) for t in spanroot.children]

    c_norm = [c.norm_ for c in spanroot.children]
    c_dep = [c.dep_ for c in spanroot.children]
    c_pos = [c.pos_ for c in spanroot.children]
    c_tag = [c.tag_ for c in spanroot.children]

    right_dep = [c.dep_ for c in spanroot.rights]

    # conditionals
    subjless = all(
        c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
        for c in spanroot.children
    )
    argmentless = all(
        c.dep_
        not in [
            "nsubj",
            "nsubjpass",
            "csubj",
            "csubjpass",
            "dobj",
            "ccomp",
            "xcomp",
            "dative",
            "attr",
            "oprd",
            "acomp",
        ]
        for c in spanroot.children
    )
    argless_span = all(
        c.dep_
        not in [
            "nsubj",
            "nsubjpass",
            "csubj",
            "csubjpass",
            "dobj",
            "ccomp",
            "xcomp",
            "dative",
            "attr",
            "oprd",
            "acomp",
        ]
        for c in span
    )
    argless_span = all(
        c.dep_
        not in [
            "nsubj",
            "nsubjpass",
            "csubj",
            "csubjpass",
            "dobj",
            "ccomp",
            "xcomp",
            "dative",
            "attr",
            "oprd",
            "acomp",
        ]
        for c in span
    )

    ## nesting classifiers
    if spanroot.dep_ == "conj":
        while spanroot.dep_ == "conj":
            spanroot = spanroot.head

    if spanroot.dep_ == "poss":
        head = spanroot.head
        if head.dep_ in ["pobj", "dobj", "obj", "iobj", "dative"]:
            category = "Posessive Noun (Object)"
        elif head.dep_ in ["nsubj", "nsubjpass"]:
            category = "Posessive Noun (Subject)"
        else:
            category = "Posessive Noun (Other)"

    ## Conjunctions
    # Preconjunctions
    if spanroot.dep_ in ["preconj", "cc"]:
        category = "Conjunction"

    ## NOUN PHRASES
    # adverbial phrases
    if spanroot.dep_ in ["amod"]:
        category = "Adjectival modifier"
        # adverbial phrases
    if spanroot.dep_ in ["compound"]:
        category = "Compound noun"

    ## Nominal category
    if spanroot.dep_ in ["pobj", "dobj", "obj", "iobj", "dative"]:
        if "acl" in c_dep:
            category = "Noun + Complement (Object)"
        else:
            category = "Object"

    if spanroot.dep_ in ["nsubj", "nsubjpass"]:
        if "acl" in c_dep:
            category = "Noun + Complement (Subject)"
        else:
            category = "Subject"

    ## ADJUNCTS
    # prep phrases
    if spanroot.dep_ in ["prep", "agent"]:
        category = "Prepositional phrase"

    # adverbial phrases
    if spanroot.dep_ in ["advmod", "npadvmod", "nmod", "npmod", "quantmod", "nummod"]:
        category = "Adverbial phrase"

    ## Predication patterns
    if spanroot.dep_ in ["acomp", "oprd"]:
        if "xcomp" in c_dep:
            category = "Subject predicate to-cl"
        else:
            category = "Adjectival complement"

    if spanroot.dep_ in ["attr"]:
        subjless = all(
            c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
            for c in spanroot.children
        )

        c_head = [c.dep_ for c in spanroot.head.children]
        if "expl" in c_head and "no_det" in span_t_dep_:
            category = "There is/are no NOUN"
        elif "expl" in c_head and spanroot.pos_ in ["NOUN"]:
            category = "There is/are + Noun complement"
        elif "expl" in c_head and spanroot.tag_ in ["NN", "NNS"]:
            category = "There is/are + Noun complement"

        elif spanroot.pos_ in ["NOUN", "PRON"]:
            if "acl" in c_dep:
                category = "Noun + Complement (attr)"
            else:
                category = "Nominal complement"

        elif not subjless and spanroot.pos_ in ["VERB", "AUX"]:
            category = "Main verb 4"

        elif spanroot.tag_ in ["NNP"]:
            category = "Nominal complement"

    ## External comp
    if spanroot.dep_ in ["xcomp"]:
        if spanroot.head.pos_ == "ADJ" and "to_aux" in c_t_dep_:
            category = "Adjective complement to-cl"
        if spanroot.head.pos_ == "VERB" and "to_aux" in c_t_dep_:
            category = "Verb complement to-cl"

    if spanroot.dep_ in ["pcomp"]:
        if (
            str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]
            and "ccomp" in c_dep
        ):
            category = "Participle + that-cl"
        elif str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
            category = "Participle"

    ## Simple classifier
    # if spanroot.dep_ in ['pcomp']:
    #     if str(spanroot.morph) in ["Aspect=Prog|Tense=Pres|VerbForm=Part"]:
    #         category = "Gerund"

    if spanroot.dep_ in ["neg"]:
        category = "Negative particle"
    if spanroot.dep_ in ["aux", "auxpass"]:
        category = "Auxiliary"

    # Modal verbs
    if spanroot.tag_ == "MD":
        category = "Modal auxiliary"

    ####################################
    ### clausal ####
    ####################################
    if spanroot.dep_ in ["ROOT", "advcl", "ccomp", "acl", "pcomp", "relcl", "punct"]:
        _check_to = [
            c.dep_
            for c in spanroot.subtree
            if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"])
            and c.head.dep_ == "xcomp"
        ]
        _check_ing = [
            c.dep_
            for c in spanroot.subtree
            if "Prog" in str(c.morph) and c.dep_ == "xcomp"
        ]
        root_before_ccomp = [
            c.i > spanroot.i for c in spanroot.children if c.dep_ == "ccomp"
        ]

        _check_for_to = [
            "_".join([c.norm_, c.dep_])
            for c in spanroot.subtree
            if c.head.dep_ == "advcl" and (c.dep_ == "mark" or c.dep_ == "aux")
        ]
        entire_cl = (
            spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
        )

        ## Start with broad category, which is then re-evaluated for specific constructions.
        if spanroot.dep_ in ["advcl", "acl", "punct", "pcomp"]:  #'mark',
            ## Adverbial clauses
            subjless = all(
                c.dep_ not in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
                for c in spanroot.children
            )
            entire_cl = (
                spanroot.left_edge.i == span.start and spanroot.right_edge.i == span.end
            )

            ### Finite-adverbial clauses
            if "mark" in span_dep and (
                spanroot.pos_ in ["VERB", "AUX"] or "aux" in span_dep
            ):
                category = "Finite adverbial clause"

            elif "mark" in span_dep and "aux" in span_dep:
                category = "Finite adverbial clause"

            elif (
                "mark" in span_dep
                and spanroot.pos_ in ["VERB", "AUX"]
                and "expl" in c_dep
            ):
                category = "Finite adverbial clause"

            elif "advmod" in span_dep and ("WRB" in span_tag or "WDT" in span_tag):
                if spanroot.pos_ in ["VERB", "AUX"]:
                    category = "Finite adverbial clause"

                elif spanroot.pos_ not in ["VERB", "AUX"] and subjless:
                    category = "Non-finite adv clause 1"

                elif not argmentless:
                    category = "Finite adverbial clause"

            ## non-finite
            elif (
                str(spanroot.morph)
                in [
                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
                    "Aspect=Perf|Tense=Past|VerbForm=Part",
                ]
                and "aux" not in c_dep
            ):
                # he doing his job
                if argmentless:
                    # e.g., frankly speaking, strictly speaking
                    category = "Adverbial Phrase"
                else:
                    category = "Non-finite adv clause 2"

            elif (
                spanroot.pos_ not in ["VERB", "AUX"] and "mark" in span_dep and subjless
            ):
                category = "Non-finite adv clause 3"

            elif "aux" in c_dep and "TO" in c_tag:
                category = "Adverbial Phrase"

            elif "mark" not in span_dep and spanroot.pos_ in ["VERB", "AUX"]:
                category = "Dependent Verb phrase"

            elif not argmentless:
                category = "Adverbial clause"

            elif spanroot.dep_ == "advcl":
                category = "Adverbial phrase"

            else:
                category = "Finite adverbial clause "

        if spanroot.dep_ in ["relcl", "ccomp", "acl", "punct", "pcomp"]:
            head = spanroot.head
            if ";" in [t.norm_ for t in head.children]:
                category = "Main verb 3"

            elif "nsubj" not in span_dep:
                category = "Dependent verb 1"

            elif "mark" in span_dep:
                category = "Complement clause"
            elif (
                str(spanroot.morph)
                in [
                    "Aspect=Prog|Tense=Pres|VerbForm=Part",
                    "Aspect=Perf|Tense=Past|VerbForm=Part",
                ]
                and "aux" not in c_dep
            ):
                category = "Non-finite complement clause"
            elif spanroot.dep_ in ["relcl"]:
                category = "Relative clause"
            elif spanroot.dep_ in ["ccomp"]:
                category = "Complement clause"
            elif spanroot.dep_ in ["acl"]:
                category = "Noun Complement clause"

        ## Specific constructions
        # Extraposed that-clause or to-infinitives
        if ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and spanroot.pos_ in [
            "VERB",
            "AUX",
        ]:
            # print(c_dep)
            if ("acomp" in c_dep or "oprd" in c_dep) and "ccomp" in c_dep:
                # eg it seems odd (oprd) that X.
                # eg it is certain (acomp) that X.
                category = (
                    "Extraposed that-cl (adj-complement)"  # e.g., it is certain that X.
                )

            elif "xcomp" in c_dep or ("advcl" in c_dep):
                if "for_mark" in _check_for_to:
                    category = (
                        "Extraposed to-cl (explicit subj)"  # eg It is possible to .
                    )
                elif _check_to:
                    category = "Extraposed to-cl 1"  # eg It is possible to .
                elif _check_ing:
                    category = "Extraposed -ing 1"  # eg It is possible to .
            elif (
                ("prep" in right_dep or "npadvmod" in right_dep)
                and "ccomp" in right_dep
                and spanroot.lemma_ == "be"
            ):
                category = "Cleft construction"

            elif "attr" in c_dep:
                category = "Extraposed that-cl (copula)"  # eg It is a wonder that X.

            else:
                category = "Extraposed that-cl (VERB)"

        # if "ccomp" in c_dep and "auxpass" in c_dep and ("it_nsubjpass" in span_t_dep_ or "it_nsubj" in span_t_dep_):
        #     category = "Extraposed that-cl (VERB)1" #e.g., it has been shown that X.
        elif (
            "it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_
        ) and "acomp" in c_dep:
            if "xcomp" in c_dep:
                if _check_to:
                    category = "Extraposed to-cl 2"  # eg it is difficult to decide.
                elif _check_ing:
                    category = "Extraposed -ing 2"  # eg it is difficult to decide.

            else:
                category = "Extraposed that-cl (adj-complement) 2"

        elif ("it_nsubjpass" in c_t_dep_ or "it_nsubj" in c_t_dep_) and "oprd" in c_dep:
            category = (
                "Extraposed that-cl (adj-complement) 3"  # eg it seems odd that X.
            )

        # something without dummy subject "it"
        elif (
            (("nsubj" in c_dep and spanroot.lemma_ in ["be"]) or "nsubjpass" in c_dep)
            and spanroot.pos_ in ["AUX", "VERB"]
            and "it" not in c_norm
        ):
            # store xcomp, if the head of the xcomp is acomp
            _check_xcomp = [
                c.dep_
                for c in spanroot.subtree
                if c.dep_ in ["xcomp"] and c.head.dep_ == "acomp"
            ]
            _check_ccomp = [
                c.dep_
                for c in spanroot.subtree
                if c.dep_ in ["ccomp"] and c.head.dep_ == "acomp"
            ]
            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]

            if ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in c_dep:
                if any(root_before_ccomp):
                    category = "Post-predicate that-cl"
                else:
                    category = "Comment clause"

            elif ("attr" in c_dep or "acomp" in c_dep) and "ccomp" in _check_ccomp:
                category = "Post-predicate that-cl 2"

            elif ("attr" in c_dep or "acomp" in c_dep) and "xcomp" in _check_xcomp:
                category = "Post-predicate to-cl"

            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_to:
                category = "Subject predicate to-cl"

            elif "xcomp" in c_dep and "auxpass" in c_dep and _check_to:
                category = "Subject predicate to-cl (passive)"

            elif "xcomp" in c_dep and spanroot.lemma_ in ["be"] and _check_ing:
                category = "Subject predicate -ing"
            elif "ccomp" in c_dep:
                category = "Subject predicate that-cl"
            elif "acomp" in c_dep:
                category = "Adjectival predicate"

            elif "mark" in c_dep and ("nsubj" in c_dep or "nsubjpass" in c_dep):
                category = "Finite-adverbial clause"
            elif not argmentless and "SCONJ" in c_pos:
                category = "Finite-adverbial clause"
            else:
                category = "Main verb 1"

        ## without dummy subject it, and lexical verbs
        elif (
            ("nsubj" in c_dep or "nsubjpass" in c_dep) in c_dep
            and spanroot.pos_ in ["AUX", "VERB"]
            and "it" not in c_norm
            and spanroot.lemma_ not in ["be"]
        ):
            _check_wh = [
                c.dep_
                for c in spanroot.subtree
                if (
                    c.dep_ in ["attr", "advmod", "dobj", "nsubj"]
                    and c.tag_ in ["WP", "WRB", "WDT", "WP$"]
                )
                and c.head.dep_ == "ccomp"
            ]
            _check_if = [
                c.dep_
                for c in spanroot.subtree
                if (c.dep_ in ["mark"] and c.norm_ in ["whether", "if"])
                and c.head.dep_ == "ccomp"
            ]

            # _check_to = [c.dep_ for c in spanroot.subtree if (c.dep_ in ["aux"] and c.pos_ in ["PART", "SCONJ"]) and c.head.dep_ == "xcomp"]
            # _check_ing = [c.dep_ for c in spanroot.subtree if "Prog" in str(c.morph) and c.dep_ == "xcomp"]

            if "ccomp" in c_dep and (_check_wh or _check_if):
                category = "Post-predicate wh-cl"

            elif "ccomp" in c_dep:
                if any(root_before_ccomp):
                    category = "Post-predicate that-cl"
                else:
                    category = "Comment clause"

            elif "xcomp" in c_dep:
                if _check_to:
                    category = "Post-predicate to-cl"
                elif _check_ing:
                    category = "Post-predicate -ing"

        # Existential
        elif "expl" in c_dep and "NOUN" in c_pos and "mark" not in c_dep:
            category = "There is/are NOUN"

        elif (
            "ccomp" in c_dep and "it_nsubj" in span_t_dep_ and spanroot.pos_ in ["AUX"]
        ):
            category = "Cleft construction"

        ### The end of clausal analysis

    if spanroot.dep_ in ["parataxis"]:
        if "_".join(span_dep) in [
            "nsubj_parataxis",
            "aux_parataxis",
            "nsubj_aux_parataxis",
        ]:
            category = "Comment clause"
        else:
            category = "Parataxis"

    if spanroot.dep_ in ["dep", "csubj", "csubjpass"]:
        if (
            spanroot.head.dep_ in ["ROOT", "ccomp"]
            and spanroot.head.pos_ in ["AUX", "VERB"]
            and spanroot.pos_ in ["AUX", "VERB"]
        ):
            if spanroot.morph == spanroot.head.morph:
                category = "Main verb 4"
            else:
                category = "Dependent verb 2"
        elif str(spanroot.morph) == "Aspect=Prog|Tense=Pres|VerbForm=Part":
            category = "Gerund"
        elif "VerbForm=Fin" in str(spanroot.morph) or "VerbForm=Inf" in str(
            spanroot.morph
        ):
            category = "Dependent verb 2"
        elif spanroot.dep_ in ["csubj", "csubjpass"]:
            category = "Dependent verb (csubj)"

    # Appositive phrases
    if spanroot.dep_ in ["appos"]:
        if "nummod" in c_dep:
            category = "Apposition"
        if spanroot.pos_ in ["PROPN"]:
            category = "Appositive Proper Nouns"
        elif spanroot.pos_ in ["NOUN"]:
            category = "Appositive Noun Phrase"
        elif spanroot.pos_ in ["VERB", "AUX"]:
            _check = any(
                c.dep_ in ["nsubj", "nsubjpass", "csubj", "csubjpass"]
                for c in spanroot.children
            )
            if _check:
                category = "Appositive Finite-clause"

    if spanroot.dep_ in ["appos", "dep", "attr"]:
        if not subjless and spanroot.pos_ in ["VERB", "AUX"]:
            category = "Main verb (likely parsing error)"

    # sometimes the dep are on the conjunctions
    if spanroot.dep_ in ["dep", "mark"]:
        if spanroot.tag_ in ["RB", "IN", "CC"]:
            category = "Conjunction"

    if spanroot.dep_ in ["intj"]:
        category = "Introjection"

    # sometimes the extra-clausal links are not accurate
    if (
        spanroot.dep_
        in ["aux", "auxpass", "oprd", "appos", "xcomp", "attr", "dep", "meta", "prt"]
        and category == None
    ):
        if spanroot.head.dep_ == "ROOT":
            category = "Main verb"
        else:
            category = "dependent verb 5"

    if span.label_ == "CITATION":
        if "NNP" in span_tag or "NNPS" in span_tag:
            if span_dep[0] == "punct" and span_dep[-1] == "punct":
                category = "Parenthetical Citation"
            elif span_tag[0] in ["NNP", "NNPS"]:
                category = "Narrative Citation"
        else:
            category = "Other Citation"

    if category == None:
        category = spanroot.dep_

    return category


def const_table(
    doc: Union[spacy.tokens.Doc, Dict[str, str]],
    spans_key: str = "sc",
    attrs: List[str] = SPAN_ATTRS,
):
    columns = attrs + [
        "Conf. score",
        "sent no.",
        "grammatical realization",
        "span dep",
        "ner",
        "POS",
        "span dep seq",
        "TAG sequence",
        "POS sequence",
        "head",
        "head dep",
        "children",
        "morphology",
        "sent",
    ]
    data = []
    # data = span_info_aggregator(doc, columns)
    sentences = {s: i for i, s in enumerate(doc.sents)}

    for span, score in zip(doc.spans[spans_key], doc.spans[spans_key].attrs["scores"]):
        span_info = []
        span_info.extend([str(getattr(span, attr)) for attr in attrs])

        span_info.append(score)
        span_info.append(int(sentences[span.sent]))
        span_info.append(construction_classifier2(doc, span))
        span_info.append(span.root.dep_)
        span_info.append(span.root.ent_type_)
        span_info.append(span.root.tag_)
        span_info.append("_".join([t.dep_ for t in span]))
        span_info.append("_".join([t.tag_ for t in span]))
        span_info.append("_".join([t.pos_ for t in span]))
        span_info.append(span.root.head.norm_)
        span_info.append(span.root.head.dep_)
        span_info.append("_".join([c.dep_ for c in span.root.children]))
        span_info.append(str(span.root.morph))
        span_info.append(span.sent.text.strip())

        data.append(span_info)

    return data, columns


def ngrammar(seq: list, n=2, concat=False, sep="-"):
    result = []
    n_item = len(seq)
    for idx, item in enumerate(seq):
        if idx + n <= n_item:
            if concat:
                result.append(sep.join(seq[idx : idx + n]))
            else:
                result.append(seq[idx : idx + n])
    return result


def diversity_values(count_vec: list):
    result = {}
    if len(count_vec) == 0:
        count_vec = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

    result["shannon"] = dv.alpha.shannon(list(count_vec), base=2)
    result["brillouin_d"] = dv.alpha.brillouin_d(list(count_vec))
    result["simpson_d"] = 1 - dv.alpha.simpson(list(count_vec))
    result["simpson_e"] = dv.alpha.simpson_e(list(count_vec))
    # result['gini_index'] = dv.alpha.gini_index(list(count_vec))
    # result['faith_pd'] = dv.alpha.faith_pd(list(count_vec))

    return result