(doc: Doc)
Extract direct quotations with an attributable speaker from a document
    using simple rules and patterns. Does not extract indirect or mixed quotations!

    Args:
        doc

    Yields:
        Next direct quotation in ``doc`` as a (speaker, cue, content) triple.

    Notes:
        Loosely inspired by Krestel, Bergler, Witte. "Minding the Source: Automatic
        Tagging of Reported Speech in Newspaper Articles".
Extract direct quotations with an attributable speaker from a document
    using simple rules and patterns. Does not extract indirect or mixed quotations!
Extract direct quotations with an attributable speaker from a document using simple rules and patterns . Does not extract indirect or mixed quotations!
def direct_quotations(doc: Doc) -> Iterable[DQTriple]:
    """
    Extract direct quotations with an attributable speaker from a document
    using simple rules and patterns. Does not extract indirect or mixed quotations!

    Args:
        doc

    Yields:
        Next direct quotation in ``doc`` as a (speaker, cue, content) triple.

    Notes:
        Loosely inspired by Krestel, Bergler, Witte. "Minding the Source: Automatic
        Tagging of Reported Speech in Newspaper Articles".
    """
    # TODO: train a model to do this instead, maybe similar to entity recognition
    try:
        _reporting_verbs = constants.REPORTING_VERBS[doc.lang_]
    except KeyError:
        raise ValueError(
            f"direct quotation extraction is not implemented for lang='{doc.lang_}', "
            f"only {sorted(constants.REPORTING_VERBS.keys())}"
        )
    qtok_idxs = [tok.i for tok in doc if tok.is_quote]
    if len(qtok_idxs) % 2 != 0:
        raise ValueError(
            f"{len(qtok_idxs)} quotation marks found, indicating an unclosed quotation; "
            "given the limitations of this method, it's safest to bail out "
            "rather than guess which quotation is unclosed"
        )
    qtok_pair_idxs = list(itertoolz.partition(2, qtok_idxs))
    for qtok_start_idx, qtok_end_idx in qtok_pair_idxs:
        content = doc[qtok_start_idx : qtok_end_idx + 1]
        cue = None
        speaker = None
        # filter quotations by content
        if (
            # quotations should have at least a couple tokens
            # excluding the first/last quotation mark tokens
            len(content) < 4
            # filter out titles of books and such, if possible
            or all(
                tok.is_title
                for tok in content
                # if tok.pos in {NOUN, PROPN}
                if not (tok.is_punct or tok.is_stop)
            )
            # TODO: require closing punctuation before the quotation mark?
            # content[-2].is_punct is False
        ):
            continue
        # get window of adjacent/overlapping sentences
        window_sents = (
            sent
            for sent in doc.sents
            # these boundary cases are a subtle bit of work...
            if (
                (sent.start < qtok_start_idx and sent.end >= qtok_start_idx - 1)
                or (sent.start <= qtok_end_idx + 1 and sent.end > qtok_end_idx)
            )
        )
        # get candidate cue verbs in window
        cue_cands = [
            tok
            for sent in window_sents
            for tok in sent
            if (
                tok.pos == VERB
                and tok.lemma_ in _reporting_verbs
                # cue verbs must occur *outside* any quotation content
                and not any(
                    qts_idx <= tok.i <= qte_idx
                    for qts_idx, qte_idx in qtok_pair_idxs
                )
            )
        ]
        # sort candidates by proximity to quote content
        cue_cands = sorted(
            cue_cands,
            key=lambda cc: min(abs(cc.i - qtok_start_idx), abs(cc.i - qtok_end_idx)),
        )
        for cue_cand in cue_cands:
            if cue is not None:
                break
            for speaker_cand in cue_cand.children:
                if speaker_cand.dep in _ACTIVE_SUBJ_DEPS:
                    cue = expand_verb(cue_cand)
                    speaker = expand_noun(speaker_cand)
                    break
        if content and cue and speaker:
            yield DQTriple(
                speaker=sorted(speaker, key=attrgetter("i")),
                cue=sorted(cue, key=attrgetter("i")),
                content=content,
            )
def direct_quotations ( doc : Doc ) -> Iterable [ DQTriple ] : # TODO: train a model to do this instead, maybe similar to entity recognition try : _reporting_verbs = constants . REPORTING_VERBS [ doc . lang_ ] except KeyError : raise ValueError ( f"direct quotation extraction is not implemented for lang='{doc.lang_}', " f"only {sorted(constants.REPORTING_VERBS.keys())}" ) qtok_idxs = [ tok . i for tok in doc if tok . is_quote ] if len ( qtok_idxs ) % 2 != 0 : raise ValueError ( f"{len(qtok_idxs)} quotation marks found, indicating an unclosed quotation; " "given the limitations of this method, it's safest to bail out " "rather than guess which quotation is unclosed" ) qtok_pair_idxs = list ( itertoolz . partition ( 2 , qtok_idxs ) ) for qtok_start_idx , qtok_end_idx in qtok_pair_idxs : content = doc [ qtok_start_idx : qtok_end_idx + 1 ] cue = None speaker = None # filter quotations by content if ( # quotations should have at least a couple tokens # excluding the first/last quotation mark tokens len ( content ) < 4 # filter out titles of books and such, if possible or all ( tok . is_title for tok in content # if tok.pos in {NOUN, PROPN} if not ( tok . is_punct or tok . is_stop ) ) # TODO: require closing punctuation before the quotation mark? # content[-2].is_punct is False ) : continue # get window of adjacent/overlapping sentences window_sents = ( sent for sent in doc . sents # these boundary cases are a subtle bit of work... if ( ( sent . start < qtok_start_idx and sent . end >= qtok_start_idx - 1 ) or ( sent . start <= qtok_end_idx + 1 and sent . end > qtok_end_idx ) ) ) # get candidate cue verbs in window cue_cands = [ tok for sent in window_sents for tok in sent if ( tok . pos == VERB and tok . lemma_ in _reporting_verbs # cue verbs must occur *outside* any quotation content and not any ( qts_idx <= tok . i <= qte_idx for qts_idx , qte_idx in qtok_pair_idxs ) ) ] # sort candidates by proximity to quote content cue_cands = sorted ( cue_cands , key = lambda cc : min ( abs ( cc . i - qtok_start_idx ) , abs ( cc . i - qtok_end_idx ) ) , ) for cue_cand in cue_cands : if cue is not None : break for speaker_cand in cue_cand . children : if speaker_cand . dep in _ACTIVE_SUBJ_DEPS : cue = expand_verb ( cue_cand ) speaker = expand_noun ( speaker_cand ) break if content and cue and speaker : yield DQTriple ( speaker = sorted ( speaker , key = attrgetter ( "i" ) ) , cue = sorted ( cue , key = attrgetter ( "i" ) ) , content = content , )