()
Pre-process the data - step 3 - filter only sentences with the right chars
Pre-process the data - step 3 - filter only sentences with the right chars
Pre - process the data - step 3 - filter only sentences with the right chars
def preprocesses_data_filter():
    """Pre-process the data - step 3 - filter only sentences with the right chars"""
    most_popular_chars = read_top_chars()
    LOGGER.info("Reading and filtering data:")
    with open(NEWS_FILE_NAME_FILTERED, "wb") as output_file:
        for line in open(NEWS_FILE_NAME_CLEAN):
            decoded_line = line.decode('utf-8')
            if decoded_line and not bool(set(decoded_line) - most_popular_chars):
                output_file.write(line)
    LOGGER.info("Done.")
def preprocesses_data_filter ( ) : most_popular_chars = read_top_chars ( ) LOGGER . info ( "Reading and filtering data:" ) with open ( NEWS_FILE_NAME_FILTERED , "wb" ) as output_file : for line in open ( NEWS_FILE_NAME_CLEAN ) : decoded_line = line . decode ( 'utf-8' ) if decoded_line and not bool ( set ( decoded_line ) - most_popular_chars ) : output_file . write ( line ) LOGGER . info ( "Done." )