(s)
return s.replace('\n', '<eos>').split()
Tokenizer which breaks on newlines.

        Arguments:
            s (str): String to tokenize.

        Returns:
            str: String with "<eos>" in place of newlines.
Tokenizer which breaks on newlines.
Tokenizer which breaks on newlines .
def newline_tokenizer(s):
        """
        Tokenizer which breaks on newlines.

        Arguments:
            s (str): String to tokenize.

        Returns:
            str: String with "<eos>" in place of newlines.

        """
        # replace newlines with '<eos>' so that
        # the newlines count as words
        return s.replace('\n', '<eos>').split()
def newline_tokenizer ( s ) : # replace newlines with '<eos>' so that # the newlines count as words return s . replace ( '\n' , '<eos>' ) . split ( )