(sentence)
return tokens
This tokenizer extends the alphanumeric :func:`symbol_tokenizer` by splitting fewer cases:

    1. Dots appearing after a letter are maintained as part of the word, except for the last word
       in a sentence if that dot is the sentence terminal. Therefore, abbreviation marks (words
       containing or ending in a ``.``, like "i.e.") remain intact and URL or ID segments remain
       complete ("www.ex-ample.com", "EC1.2.3.4.5", etc.). The only dots that never are attached
       are triple dots (``...``; ellipsis).
    2. Commas surrounded by alphanumeric characters are maintained in the word, too, e.g. ``a,b``.
       Colons surrounded by digits are maintained, e.g., 'at 12:30pm' or 'Isaiah 12:3'.
       Commas, semi-colons, and colons dangling at the end of a token are always spliced off.
    3. Any two alphanumeric letters that are separated by a single hyphen are joined together;
       Those "inner" hyphens may optionally be followed by a linebreak surrounded by spaces;
       The spaces will be removed, however. For example, ``Hel- \\r\\n \t lo`` contains a (Windows)
       linebreak and will be returned as ``Hel-lo``.
    4. Apostrophes are always allowed in words as long as they are not repeated; The single quote
       ASCII letter ``'`` is only allowed as a terminal apostrophe after the letter ``s``,
       otherwise it must be surrounded by letters. To support DNA and chemicals, a apostrophe
       (prime) may be located before the hyphen, as in the single token "5'-ACGT-3'" (if any
       non-ASCII hyphens are used instead of the shown single quote).
    5. Superscript 1, 2, and 3, optionally prefixed with a superscript minus, are attached to a
       word if it is no longer than 3 letters (optionally 4 if the first letter is a power prefix
       in the range from yocto, y (10^-24) to yotta, Y (10^+24)).
    6. Subscript digits are attached if prefixed with letters that look like a chemical formula.
This tokenizer extends the alphanumeric :func:`symbol_tokenizer` by splitting fewer cases:
This tokenizer extends the alphanumeric : func : symbol_tokenizer by splitting fewer cases :
def word_tokenizer(sentence):
    """
    This tokenizer extends the alphanumeric :func:`symbol_tokenizer` by splitting fewer cases:

    1. Dots appearing after a letter are maintained as part of the word, except for the last word
       in a sentence if that dot is the sentence terminal. Therefore, abbreviation marks (words
       containing or ending in a ``.``, like "i.e.") remain intact and URL or ID segments remain
       complete ("www.ex-ample.com", "EC1.2.3.4.5", etc.). The only dots that never are attached
       are triple dots (``...``; ellipsis).
    2. Commas surrounded by alphanumeric characters are maintained in the word, too, e.g. ``a,b``.
       Colons surrounded by digits are maintained, e.g., 'at 12:30pm' or 'Isaiah 12:3'.
       Commas, semi-colons, and colons dangling at the end of a token are always spliced off.
    3. Any two alphanumeric letters that are separated by a single hyphen are joined together;
       Those "inner" hyphens may optionally be followed by a linebreak surrounded by spaces;
       The spaces will be removed, however. For example, ``Hel- \\r\\n \t lo`` contains a (Windows)
       linebreak and will be returned as ``Hel-lo``.
    4. Apostrophes are always allowed in words as long as they are not repeated; The single quote
       ASCII letter ``'`` is only allowed as a terminal apostrophe after the letter ``s``,
       otherwise it must be surrounded by letters. To support DNA and chemicals, a apostrophe
       (prime) may be located before the hyphen, as in the single token "5'-ACGT-3'" (if any
       non-ASCII hyphens are used instead of the shown single quote).
    5. Superscript 1, 2, and 3, optionally prefixed with a superscript minus, are attached to a
       word if it is no longer than 3 letters (optionally 4 if the first letter is a power prefix
       in the range from yocto, y (10^-24) to yotta, Y (10^+24)).
    6. Subscript digits are attached if prefixed with letters that look like a chemical formula.
    """
    pruned = HYPHENATED_LINEBREAK.sub(r'\1\2', sentence)
    tokens = [token for span in space_tokenizer(pruned) for
              token in word_tokenizer.split(span) if token]

    # splice the sentence terminal off the last word/token if it has any at its borders
    # only look for the sentence terminal in the last three tokens
    for idx, word in enumerate(reversed(tokens[-3:]), 1):
        if (word_tokenizer.match(word) and not APO_MATCHER.match(word)) or \
                any(t in word for t in SENTENCE_TERMINALS):
            last = len(word) - 1

            if 0 == last or u'...' == word:
                # any case of "..." or any single char (last == 0)
                pass  # leave the token as it is
            elif any(word.rfind(t) == last for t in SENTENCE_TERMINALS):
                # "stuff."
                tokens[-idx] = word[:-1]
                tokens.insert(len(tokens) - idx + 1, word[-1])
            elif any(word.find(t) == 0 for t in SENTENCE_TERMINALS):
                # ".stuff"
                tokens[-idx] = word[0]
                tokens.insert(len(tokens) - idx + 1, word[1:])

            break

    # keep splicing off any dangling commas and (semi-) colons
    dirty = True
    while dirty:
        dirty = False

        for idx, word in enumerate(reversed(tokens), 1):
            while len(word) > 1 and word[-1] in u',;:':
                char = word[-1]  # the dangling comma/colon
                word = word[:-1]
                tokens[-idx] = word
                tokens.insert(len(tokens) - idx + 1, char)
                idx += 1
                dirty = True
            if dirty:
                break  # restart check to avoid index errors

    return tokens
def word_tokenizer ( sentence ) : pruned = HYPHENATED_LINEBREAK . sub ( r'\1\2' , sentence ) tokens = [ token for span in space_tokenizer ( pruned ) for token in word_tokenizer . split ( span ) if token ] # splice the sentence terminal off the last word/token if it has any at its borders # only look for the sentence terminal in the last three tokens for idx , word in enumerate ( reversed ( tokens [ - 3 : ] ) , 1 ) : if ( word_tokenizer . match ( word ) and not APO_MATCHER . match ( word ) ) or any ( t in word for t in SENTENCE_TERMINALS ) : last = len ( word ) - 1 if 0 == last or u'...' == word : # any case of "..." or any single char (last == 0) pass # leave the token as it is elif any ( word . rfind ( t ) == last for t in SENTENCE_TERMINALS ) : # "stuff." tokens [ - idx ] = word [ : - 1 ] tokens . insert ( len ( tokens ) - idx + 1 , word [ - 1 ] ) elif any ( word . find ( t ) == 0 for t in SENTENCE_TERMINALS ) : # ".stuff" tokens [ - idx ] = word [ 0 ] tokens . insert ( len ( tokens ) - idx + 1 , word [ 1 : ] ) break # keep splicing off any dangling commas and (semi-) colons dirty = True while dirty : dirty = False for idx , word in enumerate ( reversed ( tokens ) , 1 ) : while len ( word ) > 1 and word [ - 1 ] in u',;:' : char = word [ - 1 ] # the dangling comma/colon word = word [ : - 1 ] tokens [ - idx ] = word tokens . insert ( len ( tokens ) - idx + 1 , char ) idx += 1 dirty = True if dirty : break # restart check to avoid index errors return tokens