()
Preprocess the text by splitting the lines between min-length and max_length
    I don't like this step:
      I think the start-of-sentence is important.
      I think the end-of-sentence is important.
      Sometimes the stripped down sub-sentence is missing crucial context.
      Important NGRAMs are cut (though given enough data, that might be moot).
    I do this to enable batch-learning by padding to a fixed length.
Preprocess the text by splitting the lines between min-length and max_length
    I don't like this step:
      I think the start-of-sentence is important.
      I think the end-of-sentence is important.
      Sometimes the stripped down sub-sentence is missing crucial context.
      Important NGRAMs are cut (though given enough data, that might be moot).
    I do this to enable batch-learning by padding to a fixed length.
Preprocess the text by splitting the lines between min - length and max_length I don t like this step : I think the start - of - sentence is important . I think the end - of - sentence is important . Sometimes the stripped down sub - sentence is missing crucial context . Important NGRAMs are cut ( though given enough data that might be moot ) . I do this to enable batch - learning by padding to a fixed length .
def preprocesses_split_lines():
    """Preprocess the text by splitting the lines between min-length and max_length
    I don't like this step:
      I think the start-of-sentence is important.
      I think the end-of-sentence is important.
      Sometimes the stripped down sub-sentence is missing crucial context.
      Important NGRAMs are cut (though given enough data, that might be moot).
    I do this to enable batch-learning by padding to a fixed length.
    """
    LOGGER.info("Reading filtered data:")
    answers = set()
    with open(NEWS_FILE_NAME_SPLIT, "wb") as output_file:
        for _line in open(NEWS_FILE_NAME_FILTERED):
            line = _line.decode('utf-8')
            while len(line) > MIN_INPUT_LEN:
                if len(line) <= CONFIG.max_input_len:
                    answer = line
                    line = ""
                else:
                    space_location = line.rfind(" ", MIN_INPUT_LEN, CONFIG.max_input_len - 1)
                    if space_location > -1:
                        answer = line[:space_location]
                        line = line[len(answer) + 1:]
                    else:
                        space_location = line.rfind(" ") # no limits this time
                        if space_location == -1:
                            break # we are done with this line
                        else:
                            line = line[space_location + 1:]
                            continue
                answers.add(answer)
                output_file.write(answer.encode('utf-8') + b"\n")
def preprocesses_split_lines ( ) : LOGGER . info ( "Reading filtered data:" ) answers = set ( ) with open ( NEWS_FILE_NAME_SPLIT , "wb" ) as output_file : for _line in open ( NEWS_FILE_NAME_FILTERED ) : line = _line . decode ( 'utf-8' ) while len ( line ) > MIN_INPUT_LEN : if len ( line ) <= CONFIG . max_input_len : answer = line line = "" else : space_location = line . rfind ( " " , MIN_INPUT_LEN , CONFIG . max_input_len - 1 ) if space_location > - 1 : answer = line [ : space_location ] line = line [ len ( answer ) + 1 : ] else : space_location = line . rfind ( " " ) # no limits this time if space_location == - 1 : break # we are done with this line else : line = line [ space_location + 1 : ] continue answers . add ( answer ) output_file . write ( answer . encode ( 'utf-8' ) + b"\n" )