()
Preprocess the text by selecting only max n-grams
    Alternative split.
Preprocess the text by selecting only max n-grams
    Alternative split.
Preprocess the text by selecting only max n - grams Alternative split .
def preprocesses_split_lines3():
    """Preprocess the text by selecting only max n-grams
    Alternative split.
    """
    LOGGER.info("Reading filtered data:")
    answers = set()
    for encoded_line in open(NEWS_FILE_NAME_FILTERED):
        line = encoded_line.decode('utf-8')
        if line.count(" ") < 5:
            answers.add(line)
    LOGGER.info("There are %s 'answers' (sub-sentences)", len(answers))
    LOGGER.info("Here are some examples:")
    for answer in itertools.islice(answers, 10):
        LOGGER.info(answer)
    with open(NEWS_FILE_NAME_SPLIT, "wb") as output_file:
        output_file.write("".join(answers).encode('utf-8'))
def preprocesses_split_lines3 ( ) : LOGGER . info ( "Reading filtered data:" ) answers = set ( ) for encoded_line in open ( NEWS_FILE_NAME_FILTERED ) : line = encoded_line . decode ( 'utf-8' ) if line . count ( " " ) < 5 : answers . add ( line ) LOGGER . info ( "There are %s 'answers' (sub-sentences)" , len ( answers ) ) LOGGER . info ( "Here are some examples:" ) for answer in itertools . islice ( answers , 10 ) : LOGGER . info ( answer ) with open ( NEWS_FILE_NAME_SPLIT , "wb" ) as output_file : output_file . write ( "" . join ( answers ) . encode ( 'utf-8' ) )