(tokens)
return tokens
A function to split possessive markers at the end of alphanumeric (and hyphenated) tokens.

    Takes the output of any of the tokenizer functions and produces and updated list.
    To use it, simply wrap the tokenizer function, for example::

    >>> my_sentence = "This is Fred's latest book."
    >>> split_possessive_markers(word_tokenizer(my_sentence))
    ['This', 'is', 'Fred', "'s", 'latest', 'book', '.']

    :param tokens: a list of tokens
    :returns: an updated list if a split was made or the original list otherwise
A function to split possessive markers at the end of alphanumeric (and hyphenated) tokens.
A function to split possessive markers at the end of alphanumeric ( and hyphenated ) tokens .
def split_possessive_markers(tokens):
    """
    A function to split possessive markers at the end of alphanumeric (and hyphenated) tokens.

    Takes the output of any of the tokenizer functions and produces and updated list.
    To use it, simply wrap the tokenizer function, for example::

    >>> my_sentence = "This is Fred's latest book."
    >>> split_possessive_markers(word_tokenizer(my_sentence))
    ['This', 'is', 'Fred', "'s", 'latest', 'book', '.']

    :param tokens: a list of tokens
    :returns: an updated list if a split was made or the original list otherwise
    """
    idx = -1

    for token in list(tokens):
        idx += 1

        if IS_POSSESSIVE.match(token) is not None:
            if token[-1].lower() == 's' and token[-2] in APOSTROPHES:
                tokens.insert(idx, token[:-2])
                idx += 1
                tokens[idx] = token[-2:]
            elif token[-2].lower() == 's' and token[-1] in APOSTROPHES:
                tokens.insert(idx, token[:-1])
                idx += 1
                tokens[idx] = token[-1:]

    return tokens
def split_possessive_markers ( tokens ) : idx = - 1 for token in list ( tokens ) : idx += 1 if IS_POSSESSIVE . match ( token ) is not None : if token [ - 1 ] . lower ( ) == 's' and token [ - 2 ] in APOSTROPHES : tokens . insert ( idx , token [ : - 2 ] ) idx += 1 tokens [ idx ] = token [ - 2 : ] elif token [ - 2 ] . lower ( ) == 's' and token [ - 1 ] in APOSTROPHES : tokens . insert ( idx , token [ : - 1 ] ) idx += 1 tokens [ idx ] = token [ - 1 : ] return tokens