()
Pre-process the data - step 2 - analyze the characters
Pre-process the data - step 2 - analyze the characters
Pre - process the data - step 2 - analyze the characters
def preprocesses_data_analyze_chars():
    """Pre-process the data - step 2 - analyze the characters"""
    counter = Counter()
    LOGGER.info("Reading data:")
    for line in open(NEWS_FILE_NAME_CLEAN):
        decoded_line = line.decode('utf-8')
        counter.update(decoded_line)
#     data = open(NEWS_FILE_NAME_CLEAN).read().decode('utf-8')
#     LOGGER.info("Read.\nCounting characters:")
#     counter = Counter(data.replace("\n", ""))
    LOGGER.info("Done.\nWriting to file:")
    with open(CHAR_FREQUENCY_FILE_NAME, 'wb') as output_file:
        output_file.write(json.dumps(counter))
    most_popular_chars = {key for key, _value in counter.most_common(CONFIG.number_of_chars)}
    LOGGER.info("The top %s chars are:", CONFIG.number_of_chars)
    LOGGER.info("".join(sorted(most_popular_chars)))
def preprocesses_data_analyze_chars ( ) : counter = Counter ( ) LOGGER . info ( "Reading data:" ) for line in open ( NEWS_FILE_NAME_CLEAN ) : decoded_line = line . decode ( 'utf-8' ) counter . update ( decoded_line ) #     data = open(NEWS_FILE_NAME_CLEAN).read().decode('utf-8') #     LOGGER.info("Read.\nCounting characters:") #     counter = Counter(data.replace("\n", "")) LOGGER . info ( "Done.\nWriting to file:" ) with open ( CHAR_FREQUENCY_FILE_NAME , 'wb' ) as output_file : output_file . write ( json . dumps ( counter ) ) most_popular_chars = { key for key , _value in counter . most_common ( CONFIG . number_of_chars ) } LOGGER . info ( "The top %s chars are:" , CONFIG . number_of_chars ) LOGGER . info ( "" . join ( sorted ( most_popular_chars ) ) )