(s: Sentence, replace_entities=True, mark_boundaries=False)
return sentence_tokens
Get the list of sentence tokens.

    :param s: a sentence object
    :param replace_entities: optionally replace the entities with a placeholder.
    :param mark_boundaries: optionally add sentence start and end markers.
    :return: a list of tokens
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "Who killed Lora Palmer ?".split()], entities=[{"type": "NNP", 'linkings': [], 'token_ids': [2,3]}]))
    ['Who', 'killed', '<e>', '?']
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "Who killed Lora Palmer ?".split()], entities=[{"type": "NN", 'linkings': [], 'token_ids': [0]}]))
    ['Who', 'killed', 'Lora', 'Palmer', '?']
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "where are the nfl redskins from ?".split()], entities=[{'linkings': [['Q212654', None]], 'token_ids': [4], 'type': 'NNP'}, {'linkings': [['Q1215884', None]], 'token_ids': [3], 'type': 'NNP'}, {'linkings': [['Q618123', 'geographical object']], 'token_ids': [0], 'type': 'NN'}]))
    ['where', 'are', 'the', '<e>', '<e>', 'from', '?']
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "who won the prise 2009 ?".split()], entities=[{'linkings': [['2009', '2009']], 'token_ids': [4], 'type': 'YEAR'}]))
    ['who', 'won', 'the', 'prise', '<year>', '?']
Get the list of sentence tokens.
Get the list of sentence tokens .
def _get_sentence_tokens(s: Sentence, replace_entities=True, mark_boundaries=False):
    """
    Get the list of sentence tokens.

    :param s: a sentence object
    :param replace_entities: optionally replace the entities with a placeholder.
    :param mark_boundaries: optionally add sentence start and end markers.
    :return: a list of tokens
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "Who killed Lora Palmer ?".split()], entities=[{"type": "NNP", 'linkings': [], 'token_ids': [2,3]}]))
    ['Who', 'killed', '<e>', '?']
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "Who killed Lora Palmer ?".split()], entities=[{"type": "NN", 'linkings': [], 'token_ids': [0]}]))
    ['Who', 'killed', 'Lora', 'Palmer', '?']
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "where are the nfl redskins from ?".split()], entities=[{'linkings': [['Q212654', None]], 'token_ids': [4], 'type': 'NNP'}, {'linkings': [['Q1215884', None]], 'token_ids': [3], 'type': 'NNP'}, {'linkings': [['Q618123', 'geographical object']], 'token_ids': [0], 'type': 'NN'}]))
    ['where', 'are', 'the', '<e>', '<e>', 'from', '?']
    >>> _get_sentence_tokens(Sentence(tagged=[{'originalText': k, 'pos': 'O', 'ner': 'O'} for k in "who won the prise 2009 ?".split()], entities=[{'linkings': [['2009', '2009']], 'token_ids': [4], 'type': 'YEAR'}]))
    ['who', 'won', 'the', 'prise', '<year>', '?']
    """
    sentence_tokens = s.tokens
    if replace_entities:
        updated_tokens = []
        right = 0
        entities = sorted(s.entities, key=lambda x: x['token_ids'][0] if x['token_ids'] else 0)
        for e in entities:
            if e['token_ids'] and e['type'] != "NN":
                left = min(e['token_ids'])
                entity_token = SPECIAL_TOKENS['YEAR'] if e['type'] == "YEAR" else ENTITY_TOKEN
                updated_tokens += sentence_tokens[right:left] + [entity_token]
                right = max(e['token_ids']) + 1
        if right < len(sentence_tokens):
            updated_tokens += sentence_tokens[right:]
        sentence_tokens = updated_tokens
    if mark_boundaries:
        sentence_tokens = SENT_TOKENS[0:1] + sentence_tokens + SENT_TOKENS[1:2]
    return sentence_tokens
def _get_sentence_tokens ( s : Sentence , replace_entities = True , mark_boundaries = False ) : sentence_tokens = s . tokens if replace_entities : updated_tokens = [ ] right = 0 entities = sorted ( s . entities , key = lambda x : x [ 'token_ids' ] [ 0 ] if x [ 'token_ids' ] else 0 ) for e in entities : if e [ 'token_ids' ] and e [ 'type' ] != "NN" : left = min ( e [ 'token_ids' ] ) entity_token = SPECIAL_TOKENS [ 'YEAR' ] if e [ 'type' ] == "YEAR" else ENTITY_TOKEN updated_tokens += sentence_tokens [ right : left ] + [ entity_token ] right = max ( e [ 'token_ids' ] ) + 1 if right < len ( sentence_tokens ) : updated_tokens += sentence_tokens [ right : ] sentence_tokens = updated_tokens if mark_boundaries : sentence_tokens = SENT_TOKENS [ 0 : 1 ] + sentence_tokens + SENT_TOKENS [ 1 : 2 ] return sentence_tokens