<kaggle_start><data_title>Large Movie review<data_name>large-movie-review
<code># Run this cell! It sets some things up for you.
import matplotlib.pyplot as plt
import os
import math
import zipfile
import time
import operator
from collections import defaultdict, Counter

plt.rcParams["figure.figsize"] = (5, 4)  # set default size of plots
if not os.path.isdir("data"):
    os.mkdir("data")  # make the data directory
if not os.path.isdir("./checkpoints"):
    os.mkdir("./checkpoints")  # directory to save checkpoints
PATH_TO_DATA = "/kaggle/input/large-movie-review/large_movie_review_dataset"  # path to the data directory
POS_LABEL = "pos"
NEG_LABEL = "neg"
TRAIN_DIR = os.path.join(PATH_TO_DATA, "train")
TEST_DIR = os.path.join(PATH_TO_DATA, "test")
for label in [POS_LABEL, NEG_LABEL]:
    if len(os.listdir(TRAIN_DIR + "/" + label)) == 12500:
        print(
            "Great! You have 12500 {} reviews in {}".format(
                label, TRAIN_DIR + "/" + label
            )
        )
    else:
        print("Oh no! Something is wrong. Check your code which loads the reviews")


###### PREPROCESSING BLOCK ######
###### DO NOT MODIFY THIS FUNCTION #####
def tokenize_doc(doc):
    """
    Tokenize a document and return its bag-of-words representation.
    doc - a string representing a document.
    returns a dictionary mapping each word to the number of times it appears in doc.
    """
    bow = defaultdict(float)
    tokens = doc.split()
    lowered_tokens = map(lambda t: t.lower(), tokens)
    for token in lowered_tokens:
        bow[token] += 1.0
    return dict(bow)


###### END FUNCTION #####
def n_word_types(word_counts):
    """
    Implement Me!
    return a count of all word types in the corpus
    using information from word_counts
    """
    pass


def n_word_tokens(word_counts):
    """
    Implement Me!
    return a count of all word tokens in the corpus
    using information from word_counts
    """
    pass


###### NAIVE BAYES BLOCK ######
class NaiveBayes:
    """A Naive Bayes model for text classification."""

    def __init__(self, path_to_data, tokenizer):
        # Vocabulary is a set that stores every word seen in the training data
        self.vocab = set()
        self.path_to_data = path_to_data
        self.tokenize_doc = tokenizer
        self.train_dir = os.path.join(path_to_data, "train")
        self.test_dir = os.path.join(path_to_data, "test")
        # class_total_doc_counts is a dictionary that maps a class (i.e., pos/neg) to
        # the number of documents in the trainning set of that class
        self.class_total_doc_counts = {POS_LABEL: 0.0, NEG_LABEL: 0.0}
        # class_total_word_counts is a dictionary that maps a class (i.e., pos/neg) to
        # the number of words in the training set in documents of that class
        self.class_total_word_counts = {POS_LABEL: 0.0, NEG_LABEL: 0.0}
        # class_word_counts is a dictionary of dictionaries. It maps a class (i.e.,
        # pos/neg) to a dictionary of word counts. For example:
        #    self.class_word_counts[POS_LABEL]['awesome']
        # stores the number of times the word 'awesome' appears in documents
        # of the positive class in the training documents.
        self.class_word_counts = {
            POS_LABEL: defaultdict(float),
            NEG_LABEL: defaultdict(float),
        }

    def train_model(self):
        """
        This function processes the entire training set using the global PATH
        variable above.  It makes use of the tokenize_doc and update_model
        functions you will implement.
        """
        pos_path = os.path.join(self.train_dir, POS_LABEL)
        neg_path = os.path.join(self.train_dir, NEG_LABEL)
        for p, label in [(pos_path, POS_LABEL), (neg_path, NEG_LABEL)]:
            for f in os.listdir(p):
                with open(os.path.join(p, f), "r") as doc:
                    content = doc.read()
                    self.tokenize_and_update_model(content, label)
        self.report_statistics_after_training()

    def report_statistics_after_training(self):
        """
        Report a number of statistics after training.
        """
        print("REPORTING CORPUS STATISTICS")
        print(
            "NUMBER OF DOCUMENTS IN POSITIVE CLASS:",
            self.class_total_doc_counts[POS_LABEL],
        )
        print(
            "NUMBER OF DOCUMENTS IN NEGATIVE CLASS:",
            self.class_total_doc_counts[NEG_LABEL],
        )
        print(
            "NUMBER OF TOKENS IN POSITIVE CLASS:",
            self.class_total_word_counts[POS_LABEL],
        )
        print(
            "NUMBER OF TOKENS IN NEGATIVE CLASS:",
            self.class_total_word_counts[NEG_LABEL],
        )
        print(
            "VOCABULARY SIZE: NUMBER OF UNIQUE WORDTYPES IN TRAINING CORPUS:",
            len(self.vocab),
        )

    def update_model(self, bow, label):
        """
        IMPLEMENT ME!
        Update internal statistics given a document represented as a bag-of-words
        bow - a map from words to their counts
        label - the class of the document whose bag-of-words representation was input
        This function doesn't return anything but should update a number of internal
        statistics. Specifically, it updates:
          - the internal map the counts, per class, how many times each word was
            seen (self.class_word_counts)
          - the number of words seen for each label (self.class_total_word_counts)
          - the vocabulary seen so far (self.vocab)
          - the number of documents seen of each label (self.class_total_doc_counts)
        """
        pass

    def tokenize_and_update_model(self, doc, label):
        """
        Implement me!
        Tokenizes a document doc and updates internal count statistics.
        doc - a string representing a document.
        label - the sentiment of the document (either postive or negative)
        stop_word - a boolean flag indicating whether to stop word or not
        Make sure when tokenizing to lower case all of the tokens!
        """
        pass

    def top_n(self, label, n):
        """
        Implement me!
        Returns the most frequent n tokens for documents with class 'label'.
        """
        pass

    def p_word_given_label(self, word, label):
        """
        Implement me!
        Returns the probability of word given label
        according to this NB model.
        """
        pass

    def p_word_given_label_and_alpha(self, word, label, alpha):
        """
        Implement me!
        Returns the probability of word given label wrt psuedo counts.
        alpha - pseudocount parameter
        """
        pass

    def log_likelihood(self, bow, label, alpha):
        """
        Implement me!
        Computes the log likelihood of a set of words given a label and pseudocount.
        bow - a bag of words (i.e., a tokenized document)
        label - either the positive or negative label
        alpha - float; pseudocount parameter
        """
        pass

    def log_prior(self, label):
        """
        Implement me!
        Returns the log prior of a document having the class 'label'.
        """
        pass

    def unnormalized_log_posterior(self, bow, label, alpha):
        """
        Implement me!
        Computes the unnormalized log posterior (of doc being of class 'label').
        bow - a bag of words (i.e., a tokenized document)
        """
        pass

    def classify(self, bow, alpha):
        """
        Implement me!
        Compares the unnormalized log posterior for doc for both the positive
        and negative classes and returns the either POS_LABEL or NEG_LABEL
        (depending on which resulted in the higher unnormalized log posterior)
        bow - a bag of words (i.e., a tokenized document)
        """
        pass

    def likelihood_ratio(self, word, alpha):
        """
        Implement me!
        Returns the ratio of P(word|pos) to P(word|neg).
        """
        pass

    def evaluate_classifier_accuracy(self, alpha):
        """
        DO NOT MODIFY THIS FUNCTION
        alpha - pseudocount parameter.
        This function should go through the test data, classify each instance and
        compute the accuracy of the classifier (the fraction of classifications
        the classifier gets right.
        """
        correct = 0.0
        total = 0.0
        pos_path = os.path.join(self.test_dir, POS_LABEL)
        neg_path = os.path.join(self.test_dir, NEG_LABEL)
        for p, label in [(pos_path, POS_LABEL), (neg_path, NEG_LABEL)]:
            for f in os.listdir(p):
                with open(os.path.join(p, f), "r") as doc:
                    content = doc.read()
                    bow = self.tokenize_doc(content)
                    if self.classify(bow, alpha) == label:
                        correct += 1.0
                    total += 1.0
        return 100 * correct / total


# ### Question 1.1 (5 points)
# Complete the cell below to fill out the `word_counts` dictionary variable. `word_counts` keeps track of how many times a word type appears across the corpus. For instance, `word_counts["movie"]` should store the number 61492, the count of how many times the word `movie` appears in the corpus.
import glob
import codecs

word_counts = Counter()  # Counters are often useful for NLP in python
for label in [POS_LABEL, NEG_LABEL]:
    for directory in [TRAIN_DIR, TEST_DIR]:
        for fn in glob.glob(directory + "/" + label + "/*txt"):
            doc = codecs.open(fn, "r", "utf8")  # Open the file with UTF-8 encoding
            # IMPLEMENT ME
            pass
if word_counts["movie"] == 61492:
    print(
        "yay! there are {} total instances of the word type movie in the corpus".format(
            word_counts["movie"]
        )
    )
else:
    print("hmm. Something seems off. Double check your code")

# ### Question 1.2 (5 points)
# Fill out the functions `n_word_types`, `n_word_tokens` in the [Preprocessing Block](#Preprocessing-Block).
# ***Note: you will have to rerun the `Preprocessing Block` cell every time you change its code for it to have any effect!***
print("there are {} word types in the corpus".format(n_word_types(word_counts)))
print("there are {} word tokens in the corpus".format(n_word_tokens(word_counts)))

# What is the difference between word types and tokens? Why are the number of tokens much higher than the number of types?
# ***Answer in one or two lines here.***
# ### Question 1.3 (5 points)
# Using the `word_counts` dictionary you just created, make a new dictionary called `sorted_dict` where the words are sorted according to their counts, in decending order:
sorted_dict = dict(sorted(word_counts.items(), key=lambda item: item[1], reverse=True))

# Now print the first 30 values from sorted_dict.
for i, (word, count) in enumerate(sorted_dict.items()):
    if i < 30:
        print(f"{word}: {count}")
    else:
        break

# ## Zipf's Law
# ### Question 1.4 (10 points)
# In this section, you will verify a key statistical property of text: [Zipf's Law](https://en.wikipedia.org/wiki/Zipf%27s_law).
# Zipf's Law describes the relations between the frequency rank of words and frequency value of words.  For a word $w$, its frequency is inversely proportional to its rank:
# $$count_w = K \frac{1}{rank_w}$$
# $K$ is a constant, specific to the corpus and how words are being defined.
# What would this look like if you took the log of both sides of the equation?
# ***Answer in one or two lines here.***
# The log of word frequency and log of word rank have a linear connection when we take the log of both sides of the equation, giving us log(count w) = log(K) - log(rank w).
# Therefore, if Zipf's Law holds, after sorting the words descending on frequency, word frequency decreases in an approximately linear fashion under a log-log scale.
# Now, please make such a log-log plot by plotting the rank versus frequency
# *Hint: Make use of the sorted dictionary you just created.*
# Use a scatter plot where the x-axis is the *log(rank)*, and y-axis is *log(frequency)*.  You should get this information from `word_counts`; for example, you can take the individual word counts and sort them.  dict methods `.items()` and/or `values()` may be useful.  (Note that it doesn't really matter whether ranks start at 1 or 0 in terms of how the plot comes out.) You can check your results by comparing your plots to ones on Wikipedia; they should look qualitatively similar.
# *Please remember to label the meaning of the x-axis and y-axis.*
import math
import operator

x = []
y = []
X_LABEL = "log(rank)"
Y_LABEL = "log(frequency)"
# Add your code here
# You should fill the x and y arrays.
# Running this cell should produce your plot below.
plt.scatter(x, y)
plt.xlabel(X_LABEL)
plt.ylabel(Y_LABEL)

# # Part Two: Naive Bayes
# This section of the homework will walk you through coding a Naive Bayes classifier that can distinguish between positive and negative reviews (with some level of accuracy).
# ## Question 2.1 (10 pts)
# To start, implement the `update_model` and `tokenize_and_update_model` functions in the [Naive Bayes Block](#Naive-Bayes-Block). Make sure to read the functions' comments so you know what to update. Also review the `NaiveBayes` class variables in the `def __init__` method of the `NaiveBayes class` to get a sense of which statistics are important to keep track of. Once you have implemented `update_model`, run the train model function using the code below.

nb = NaiveBayes(PATH_TO_DATA, tokenizer=tokenize_doc)
nb.train_model()
if len(nb.vocab) == 251637:
    print("Great! The vocabulary size is {}".format(251637))
else:
    print(
        "Oh no! Something seems off. Double check your code before continuing. Maybe a mistake in update_model?"
    )

# ## Exploratory analysis
# Let’s begin to explore the count statistics stored by the update model function. Implement the provided `top_n` function in the [Naive Bayes Block](#Naive-Bayes-Block) to find the top 10 most common words in the positive class and top 10 most common words in the negative class.
print("TOP 10 WORDS FOR CLASS " + POS_LABEL + ":")
for tok, count in nb.top_n(POS_LABEL, 10):
    print("", tok, count)
print()
print("TOP 10 WORDS FOR CLASS " + NEG_LABEL + ":")
for tok, count in nb.top_n(NEG_LABEL, 10):
    print("", tok, count)
print()

# ### Question 2.2 (5 points)
# What is the first thing that you notice when you look at the top 10 words for the 2 classes? Are these words helpful for discriminating between the two classes? Do you imagine that processing other English text will result in a similar phenomenon? What about other languages?
# ***Answer in one or two lines here.***
# The top words for both classes are stop words, which make it difficult to distinguish between the two classes. This is the first thing to note. Both favorable and bad evaluations may have a lot of these exclamation points. It's likely that reading other English work would cause a similar occurrence, although it may depend on the text's particular subject matter or genre. Depending on the syntax and vocabulary of the respective language, the top words in other languages may also vary.
# ### Question 2.3 (5 points)
# The Naive Bayes model assumes that all features are conditionally independent given the class label. For our purposes, this means that the probability of seeing a particular word in a document with class label $y$ is independent of the rest of the words in that document. Implement the `p_word_given_label` function in the [Naive Bayes Block](#Naive-Bayes-Block). This function calculates P (w|y) (i.e., the probability of seeing word w in a document given the label of that document is y).
# Use your `p_word_given_label` function to compute the probability of seeing the word “amazing” given each sentiment label. Repeat the computation for the word “dull.”
print("P('amazing'|pos):", nb.p_word_given_label("amazing", POS_LABEL))
print("P('amazing'|neg):", nb.p_word_given_label("amazing", NEG_LABEL))
print("P('dull'|pos):", nb.p_word_given_label("dull", POS_LABEL))
print("P('dull'|neg):", nb.p_word_given_label("dull", NEG_LABEL))

# Which word has a higher probability, given the positive class? Which word has a higher probability, given the negative class? Is this behavior expected?
# ***Answer in one or two lines here.***
# ### Question 2.4 (5 points)
# In the next cell, compute the probability of the word "car-thievery" in the positive training data and negative training data.
print("P('car-thievery'|pos):", nb.p_word_given_label("car-thievery", POS_LABEL))
print("P('car-thievery'|neg):", nb.p_word_given_label("car-thievery", NEG_LABEL))

# What is unusual about P('car-thievery'|neg)? What would happen if we took the log of "P('car-thievery'|neg)"? What would happen if we multiplied "P('car-thievery'|neg)" by "P('dull'|neg)"? Why might these operations cause problems for a Naive Bayes classifier?
# ***Answer in one or two lines here.***
# ### Question 2.5 (5 points)
# We can address the issues from question 2.4 with add-$\alpha$ smoothing (like add-1 smoothing except instead of adding 1 we add $\alpha$). Implement
# `p_word_given_label_and_alpha` in the [Naive Bayes Block](#Naive-Bayes-Block) and then run the next cell.
# **Hint:** look at the slides from the lecture on add-1 smoothing.
print(
    "P('stop-sign.'|pos):",
    nb.p_word_given_label_and_alpha("stop-sign.", POS_LABEL, 0.2),
)

# ### Question 2.6 (5 points)
# *Prior and Likelihood*
# As noted before, the Naive Bayes model assumes that all words in a document are independent of one another given the document’s label. Because of this we can write the likelihood of a document as:
# $P(w_{d1},\cdots,w_{dn}|y_d) = \prod_{i=1}^{n}P(w_{di}|y_d)$
# However, if a document has a lot of words, the likelihood will become extremely small and we’ll encounter numerical underflow. Underflow is a common problem when dealing with probabilistic models; if you are unfamiliar with it, you can get a brief overview on [Wikipedia](https:/en.wikipedia.org/wiki/Arithmetic_underflow). To deal with underflow, a common transformation is to work in log-space.
# $\log[P(w_{d1},\cdots,w_{dn}|y_d)] = \sum_{i=1}^{n}\log[P(w_{di}|y_d)]$
# Implement the `log_likelihood` function in the [Naive Bayes Block](#Naive-Bayes-Block). **Hint:** it should make calls to the p word given label and alpha function.
# Implement the `log_prior` function in the [Naive Bayes Block](#Naive-Bayes-Block). This function takes a class label and returns the log of the fraction of the training documents that are of that label.
# ### Question 2.7 (5 points)
# Naive Bayes is a model that tells us how to compute the posterior
# probability of a document being of some label (i.e.,
# $P(y_d|\mathbf{w_d})$).  Specifically, we do so using bayes rule:
# $P(y_d|\mathbf{w_d}) = \frac{P(y_d)P(\mathbf{w_d}|y_d)}{P(\mathbf{w_d})}$
# In the previous section you implemented functions to compute both
# the log prior ($\log[P(y_d)]$) and the log likelihood
# ($\log[P( \mathbf{w_d} |y_d)]$ ). Now, all you're missing is the
# *normalizer*, $P(\mathbf{w_d})$.
# Derive the normalizer by expanding $P(\mathbf{w_d})$.
# ***Answer in one or two lines here. Provide the formula and define each term in this formula.***
# ### Question 2.8 (5 points)
# One way to classify a document is to compute the unnormalized log posterior for both labels and take the argmax (i.e., the label that yields the higher unnormalized log posterior). The unnormalized log posterior is the sum of the log prior and the log likelihood of the document. Why don’t we need to compute the log normalizer here?
# ***Answer in one or two lines here.***
# ### Question 2.9 (10 points)
# As we saw earlier, the top 10 words from each class do not give us much to go on when classifying a document. A much more powerful metric is the likelihood ratio, which is defined as
# $LR(w)=\frac{P(w|y=\mathrm{pos})}{P(w|y=\mathrm{neg})}$
# A word with LR 3 is 3 times more likely to appear in the positive class than in the negative. A word with LR 0.3 is one-third as likely to appear in the positive class as opposed to the negative class.
# Implement the nb.likelihod_ratio function and use it to investigate the likelihood ratio of "amazing" and "dull"
print("LIKELIHOOD RATIO OF 'amazing':", nb.likelihood_ratio("amazing", 0.2))
print("LIKELIHOOD RATIO OF 'dull':", nb.likelihood_ratio("dull", 0.2))
print("LIKELIHOOD RATIO OF 'and':", nb.likelihood_ratio("and", 0.2))
print("LIKELIHOOD RATIO OF 'to':", nb.likelihood_ratio("to", 0.2))

# What is the minimum and maximum possible values the likelihood ratio can take? Does it make sense that $LR('amazing') > LR('to')$?
# ***Answer in one or two lines here.***
# Find the word in the vocabulary with the highest liklihood ratio below.
# Add your code here

# ### Question 2.10 (5 points)
# The unnormalized log posterior is the sum of the log prior and the log likelihood of the document. Implement the `unnormalized_log_posterior` function and the `classify` function in the [Naive Bayes Block](#Naive-Bayes-Block). The `classify` function should use the unnormalized log posteriors but should not compute the normalizer. Once you implement the `classify` function, we'd like to evaluate its accuracy.
print(nb.evaluate_classifier_accuracy(0.2))

# ### Question 2.11 (5 points)
# Try evaluating your model again with a smoothing parameter of 1000.
print(nb.evaluate_classifier_accuracy(1000.0))

# Does the accuracy go up or down when alpha is raised to 1000? Why do you think this is?
# ***Answer in one or two lines here.***
# ### Question 2.12 (5 points)
# Find a review that your classifier got wrong.
# In this cell, print out a review your classifier got wrong, along with its label.


# What are two reasons your system might have misclassified this example? What improvements could you make that may help your system classify this example correctly?
# ***Answer in one or two lines here.***
# ### Question 2.13 (5 points)
# Often times we care about multi-class classification rather than binary classification.
# How many counts would we need to keep track of if the model were modified to support 5-class classification?
# ***Answer in one or two lines here.***
#
# Use your own code or an external library such as nltk to perform tokenization,
# text normalization, word filtering, etc.
# Fill out your work in def tokenize_doc_and_more (below) and then show improvement by
# running the cells below.
def tokenize_doc_and_more(doc):
    """
    Return some representation of a document.
    At a minimum, you need to perform tokenization, the rest is up to you.
    """
    # Implement me!
    bow = defaultdict(float)
    # your code goes here
    return bow


nb = NaiveBayes(PATH_TO_DATA, tokenizer=tokenize_doc_and_more)
nb.train_model()
nb.evaluate_classifier_accuracy(1.0)
<|endoftext|>
[{"Id": 119553423, "ScriptId": 35192466, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 7676061, "CreationDate": "02/18/2023 12:39:05", "VersionNumber": 1.0, "Title": "This one", "EvaluationDate": "02/18/2023", "IsChange": true, "TotalLines": 554.0, "LinesInsertedFromPrevious": 554.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 0.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
[{"Id": 169373332, "KernelVersionId": 119553423, "SourceDatasetVersionId": 5019693}]
[{"Id": 5019693, "DatasetId": 2912825, "DatasourceVersionId": 5089756, "CreatorUserId": 7676061, "LicenseName": "Unknown", "CreationDate": "02/18/2023 08:51:26", "VersionNumber": 1.0, "Title": "Large Movie review", "Slug": "large-movie-review", "Subtitle": NaN, "Description": NaN, "VersionNotes": "Initial release", "TotalCompressedBytes": 0.0, "TotalUncompressedBytes": 0.0}]
[{"Id": 2912825, "CreatorUserId": 7676061, "OwnerUserId": 7676061.0, "OwnerOrganizationId": NaN, "CurrentDatasetVersionId": 5019693.0, "CurrentDatasourceVersionId": 5089756.0, "ForumId": 2950169, "Type": 2, "CreationDate": "02/18/2023 08:51:26", "LastActivityDate": "02/18/2023", "TotalViews": 124, "TotalDownloads": 3, "TotalVotes": 0, "TotalKernels": 0}]
# Run this cell! It sets some things up for you.
import matplotlib.pyplot as plt
import os
import math
import zipfile
import time
import operator
from collections import defaultdict, Counter

plt.rcParams["figure.figsize"] = (5, 4)  # set default size of plots
if not os.path.isdir("data"):
    os.mkdir("data")  # make the data directory
if not os.path.isdir("./checkpoints"):
    os.mkdir("./checkpoints")  # directory to save checkpoints
PATH_TO_DATA = "/kaggle/input/large-movie-review/large_movie_review_dataset"  # path to the data directory
POS_LABEL = "pos"
NEG_LABEL = "neg"
TRAIN_DIR = os.path.join(PATH_TO_DATA, "train")
TEST_DIR = os.path.join(PATH_TO_DATA, "test")
for label in [POS_LABEL, NEG_LABEL]:
    if len(os.listdir(TRAIN_DIR + "/" + label)) == 12500:
        print(
            "Great! You have 12500 {} reviews in {}".format(
                label, TRAIN_DIR + "/" + label
            )
        )
    else:
        print("Oh no! Something is wrong. Check your code which loads the reviews")


###### PREPROCESSING BLOCK ######
###### DO NOT MODIFY THIS FUNCTION #####
def tokenize_doc(doc):
    """
    Tokenize a document and return its bag-of-words representation.
    doc - a string representing a document.
    returns a dictionary mapping each word to the number of times it appears in doc.
    """
    bow = defaultdict(float)
    tokens = doc.split()
    lowered_tokens = map(lambda t: t.lower(), tokens)
    for token in lowered_tokens:
        bow[token] += 1.0
    return dict(bow)


###### END FUNCTION #####
def n_word_types(word_counts):
    """
    Implement Me!
    return a count of all word types in the corpus
    using information from word_counts
    """
    pass


def n_word_tokens(word_counts):
    """
    Implement Me!
    return a count of all word tokens in the corpus
    using information from word_counts
    """
    pass


###### NAIVE BAYES BLOCK ######
class NaiveBayes:
    """A Naive Bayes model for text classification."""

    def __init__(self, path_to_data, tokenizer):
        # Vocabulary is a set that stores every word seen in the training data
        self.vocab = set()
        self.path_to_data = path_to_data
        self.tokenize_doc = tokenizer
        self.train_dir = os.path.join(path_to_data, "train")
        self.test_dir = os.path.join(path_to_data, "test")
        # class_total_doc_counts is a dictionary that maps a class (i.e., pos/neg) to
        # the number of documents in the trainning set of that class
        self.class_total_doc_counts = {POS_LABEL: 0.0, NEG_LABEL: 0.0}
        # class_total_word_counts is a dictionary that maps a class (i.e., pos/neg) to
        # the number of words in the training set in documents of that class
        self.class_total_word_counts = {POS_LABEL: 0.0, NEG_LABEL: 0.0}
        # class_word_counts is a dictionary of dictionaries. It maps a class (i.e.,
        # pos/neg) to a dictionary of word counts. For example:
        #    self.class_word_counts[POS_LABEL]['awesome']
        # stores the number of times the word 'awesome' appears in documents
        # of the positive class in the training documents.
        self.class_word_counts = {
            POS_LABEL: defaultdict(float),
            NEG_LABEL: defaultdict(float),
        }

    def train_model(self):
        """
        This function processes the entire training set using the global PATH
        variable above.  It makes use of the tokenize_doc and update_model
        functions you will implement.
        """
        pos_path = os.path.join(self.train_dir, POS_LABEL)
        neg_path = os.path.join(self.train_dir, NEG_LABEL)
        for p, label in [(pos_path, POS_LABEL), (neg_path, NEG_LABEL)]:
            for f in os.listdir(p):
                with open(os.path.join(p, f), "r") as doc:
                    content = doc.read()
                    self.tokenize_and_update_model(content, label)
        self.report_statistics_after_training()

    def report_statistics_after_training(self):
        """
        Report a number of statistics after training.
        """
        print("REPORTING CORPUS STATISTICS")
        print(
            "NUMBER OF DOCUMENTS IN POSITIVE CLASS:",
            self.class_total_doc_counts[POS_LABEL],
        )
        print(
            "NUMBER OF DOCUMENTS IN NEGATIVE CLASS:",
            self.class_total_doc_counts[NEG_LABEL],
        )
        print(
            "NUMBER OF TOKENS IN POSITIVE CLASS:",
            self.class_total_word_counts[POS_LABEL],
        )
        print(
            "NUMBER OF TOKENS IN NEGATIVE CLASS:",
            self.class_total_word_counts[NEG_LABEL],
        )
        print(
            "VOCABULARY SIZE: NUMBER OF UNIQUE WORDTYPES IN TRAINING CORPUS:",
            len(self.vocab),
        )

    def update_model(self, bow, label):
        """
        IMPLEMENT ME!
        Update internal statistics given a document represented as a bag-of-words
        bow - a map from words to their counts
        label - the class of the document whose bag-of-words representation was input
        This function doesn't return anything but should update a number of internal
        statistics. Specifically, it updates:
          - the internal map the counts, per class, how many times each word was
            seen (self.class_word_counts)
          - the number of words seen for each label (self.class_total_word_counts)
          - the vocabulary seen so far (self.vocab)
          - the number of documents seen of each label (self.class_total_doc_counts)
        """
        pass

    def tokenize_and_update_model(self, doc, label):
        """
        Implement me!
        Tokenizes a document doc and updates internal count statistics.
        doc - a string representing a document.
        label - the sentiment of the document (either postive or negative)
        stop_word - a boolean flag indicating whether to stop word or not
        Make sure when tokenizing to lower case all of the tokens!
        """
        pass

    def top_n(self, label, n):
        """
        Implement me!
        Returns the most frequent n tokens for documents with class 'label'.
        """
        pass

    def p_word_given_label(self, word, label):
        """
        Implement me!
        Returns the probability of word given label
        according to this NB model.
        """
        pass

    def p_word_given_label_and_alpha(self, word, label, alpha):
        """
        Implement me!
        Returns the probability of word given label wrt psuedo counts.
        alpha - pseudocount parameter
        """
        pass

    def log_likelihood(self, bow, label, alpha):
        """
        Implement me!
        Computes the log likelihood of a set of words given a label and pseudocount.
        bow - a bag of words (i.e., a tokenized document)
        label - either the positive or negative label
        alpha - float; pseudocount parameter
        """
        pass

    def log_prior(self, label):
        """
        Implement me!
        Returns the log prior of a document having the class 'label'.
        """
        pass

    def unnormalized_log_posterior(self, bow, label, alpha):
        """
        Implement me!
        Computes the unnormalized log posterior (of doc being of class 'label').
        bow - a bag of words (i.e., a tokenized document)
        """
        pass

    def classify(self, bow, alpha):
        """
        Implement me!
        Compares the unnormalized log posterior for doc for both the positive
        and negative classes and returns the either POS_LABEL or NEG_LABEL
        (depending on which resulted in the higher unnormalized log posterior)
        bow - a bag of words (i.e., a tokenized document)
        """
        pass

    def likelihood_ratio(self, word, alpha):
        """
        Implement me!
        Returns the ratio of P(word|pos) to P(word|neg).
        """
        pass

    def evaluate_classifier_accuracy(self, alpha):
        """
        DO NOT MODIFY THIS FUNCTION
        alpha - pseudocount parameter.
        This function should go through the test data, classify each instance and
        compute the accuracy of the classifier (the fraction of classifications
        the classifier gets right.
        """
        correct = 0.0
        total = 0.0
        pos_path = os.path.join(self.test_dir, POS_LABEL)
        neg_path = os.path.join(self.test_dir, NEG_LABEL)
        for p, label in [(pos_path, POS_LABEL), (neg_path, NEG_LABEL)]:
            for f in os.listdir(p):
                with open(os.path.join(p, f), "r") as doc:
                    content = doc.read()
                    bow = self.tokenize_doc(content)
                    if self.classify(bow, alpha) == label:
                        correct += 1.0
                    total += 1.0
        return 100 * correct / total


# ### Question 1.1 (5 points)
# Complete the cell below to fill out the `word_counts` dictionary variable. `word_counts` keeps track of how many times a word type appears across the corpus. For instance, `word_counts["movie"]` should store the number 61492, the count of how many times the word `movie` appears in the corpus.
import glob
import codecs

word_counts = Counter()  # Counters are often useful for NLP in python
for label in [POS_LABEL, NEG_LABEL]:
    for directory in [TRAIN_DIR, TEST_DIR]:
        for fn in glob.glob(directory + "/" + label + "/*txt"):
            doc = codecs.open(fn, "r", "utf8")  # Open the file with UTF-8 encoding
            # IMPLEMENT ME
            pass
if word_counts["movie"] == 61492:
    print(
        "yay! there are {} total instances of the word type movie in the corpus".format(
            word_counts["movie"]
        )
    )
else:
    print("hmm. Something seems off. Double check your code")

# ### Question 1.2 (5 points)
# Fill out the functions `n_word_types`, `n_word_tokens` in the [Preprocessing Block](#Preprocessing-Block).
# ***Note: you will have to rerun the `Preprocessing Block` cell every time you change its code for it to have any effect!***
print("there are {} word types in the corpus".format(n_word_types(word_counts)))
print("there are {} word tokens in the corpus".format(n_word_tokens(word_counts)))

# What is the difference between word types and tokens? Why are the number of tokens much higher than the number of types?
# ***Answer in one or two lines here.***
# ### Question 1.3 (5 points)
# Using the `word_counts` dictionary you just created, make a new dictionary called `sorted_dict` where the words are sorted according to their counts, in decending order:
sorted_dict = dict(sorted(word_counts.items(), key=lambda item: item[1], reverse=True))

# Now print the first 30 values from sorted_dict.
for i, (word, count) in enumerate(sorted_dict.items()):
    if i < 30:
        print(f"{word}: {count}")
    else:
        break

# ## Zipf's Law
# ### Question 1.4 (10 points)
# In this section, you will verify a key statistical property of text: [Zipf's Law](https://en.wikipedia.org/wiki/Zipf%27s_law).
# Zipf's Law describes the relations between the frequency rank of words and frequency value of words.  For a word $w$, its frequency is inversely proportional to its rank:
# $$count_w = K \frac{1}{rank_w}$$
# $K$ is a constant, specific to the corpus and how words are being defined.
# What would this look like if you took the log of both sides of the equation?
# ***Answer in one or two lines here.***
# The log of word frequency and log of word rank have a linear connection when we take the log of both sides of the equation, giving us log(count w) = log(K) - log(rank w).
# Therefore, if Zipf's Law holds, after sorting the words descending on frequency, word frequency decreases in an approximately linear fashion under a log-log scale.
# Now, please make such a log-log plot by plotting the rank versus frequency
# *Hint: Make use of the sorted dictionary you just created.*
# Use a scatter plot where the x-axis is the *log(rank)*, and y-axis is *log(frequency)*.  You should get this information from `word_counts`; for example, you can take the individual word counts and sort them.  dict methods `.items()` and/or `values()` may be useful.  (Note that it doesn't really matter whether ranks start at 1 or 0 in terms of how the plot comes out.) You can check your results by comparing your plots to ones on Wikipedia; they should look qualitatively similar.
# *Please remember to label the meaning of the x-axis and y-axis.*
import math
import operator

x = []
y = []
X_LABEL = "log(rank)"
Y_LABEL = "log(frequency)"
# Add your code here
# You should fill the x and y arrays.
# Running this cell should produce your plot below.
plt.scatter(x, y)
plt.xlabel(X_LABEL)
plt.ylabel(Y_LABEL)

# # Part Two: Naive Bayes
# This section of the homework will walk you through coding a Naive Bayes classifier that can distinguish between positive and negative reviews (with some level of accuracy).
# ## Question 2.1 (10 pts)
# To start, implement the `update_model` and `tokenize_and_update_model` functions in the [Naive Bayes Block](#Naive-Bayes-Block). Make sure to read the functions' comments so you know what to update. Also review the `NaiveBayes` class variables in the `def __init__` method of the `NaiveBayes class` to get a sense of which statistics are important to keep track of. Once you have implemented `update_model`, run the train model function using the code below.

nb = NaiveBayes(PATH_TO_DATA, tokenizer=tokenize_doc)
nb.train_model()
if len(nb.vocab) == 251637:
    print("Great! The vocabulary size is {}".format(251637))
else:
    print(
        "Oh no! Something seems off. Double check your code before continuing. Maybe a mistake in update_model?"
    )

# ## Exploratory analysis
# Let’s begin to explore the count statistics stored by the update model function. Implement the provided `top_n` function in the [Naive Bayes Block](#Naive-Bayes-Block) to find the top 10 most common words in the positive class and top 10 most common words in the negative class.
print("TOP 10 WORDS FOR CLASS " + POS_LABEL + ":")
for tok, count in nb.top_n(POS_LABEL, 10):
    print("", tok, count)
print()
print("TOP 10 WORDS FOR CLASS " + NEG_LABEL + ":")
for tok, count in nb.top_n(NEG_LABEL, 10):
    print("", tok, count)
print()

# ### Question 2.2 (5 points)
# What is the first thing that you notice when you look at the top 10 words for the 2 classes? Are these words helpful for discriminating between the two classes? Do you imagine that processing other English text will result in a similar phenomenon? What about other languages?
# ***Answer in one or two lines here.***
# The top words for both classes are stop words, which make it difficult to distinguish between the two classes. This is the first thing to note. Both favorable and bad evaluations may have a lot of these exclamation points. It's likely that reading other English work would cause a similar occurrence, although it may depend on the text's particular subject matter or genre. Depending on the syntax and vocabulary of the respective language, the top words in other languages may also vary.
# ### Question 2.3 (5 points)
# The Naive Bayes model assumes that all features are conditionally independent given the class label. For our purposes, this means that the probability of seeing a particular word in a document with class label $y$ is independent of the rest of the words in that document. Implement the `p_word_given_label` function in the [Naive Bayes Block](#Naive-Bayes-Block). This function calculates P (w|y) (i.e., the probability of seeing word w in a document given the label of that document is y).
# Use your `p_word_given_label` function to compute the probability of seeing the word “amazing” given each sentiment label. Repeat the computation for the word “dull.”
print("P('amazing'|pos):", nb.p_word_given_label("amazing", POS_LABEL))
print("P('amazing'|neg):", nb.p_word_given_label("amazing", NEG_LABEL))
print("P('dull'|pos):", nb.p_word_given_label("dull", POS_LABEL))
print("P('dull'|neg):", nb.p_word_given_label("dull", NEG_LABEL))

# Which word has a higher probability, given the positive class? Which word has a higher probability, given the negative class? Is this behavior expected?
# ***Answer in one or two lines here.***
# ### Question 2.4 (5 points)
# In the next cell, compute the probability of the word "car-thievery" in the positive training data and negative training data.
print("P('car-thievery'|pos):", nb.p_word_given_label("car-thievery", POS_LABEL))
print("P('car-thievery'|neg):", nb.p_word_given_label("car-thievery", NEG_LABEL))

# What is unusual about P('car-thievery'|neg)? What would happen if we took the log of "P('car-thievery'|neg)"? What would happen if we multiplied "P('car-thievery'|neg)" by "P('dull'|neg)"? Why might these operations cause problems for a Naive Bayes classifier?
# ***Answer in one or two lines here.***
# ### Question 2.5 (5 points)
# We can address the issues from question 2.4 with add-$\alpha$ smoothing (like add-1 smoothing except instead of adding 1 we add $\alpha$). Implement
# `p_word_given_label_and_alpha` in the [Naive Bayes Block](#Naive-Bayes-Block) and then run the next cell.
# **Hint:** look at the slides from the lecture on add-1 smoothing.
print(
    "P('stop-sign.'|pos):",
    nb.p_word_given_label_and_alpha("stop-sign.", POS_LABEL, 0.2),
)

# ### Question 2.6 (5 points)
# *Prior and Likelihood*
# As noted before, the Naive Bayes model assumes that all words in a document are independent of one another given the document’s label. Because of this we can write the likelihood of a document as:
# $P(w_{d1},\cdots,w_{dn}|y_d) = \prod_{i=1}^{n}P(w_{di}|y_d)$
# However, if a document has a lot of words, the likelihood will become extremely small and we’ll encounter numerical underflow. Underflow is a common problem when dealing with probabilistic models; if you are unfamiliar with it, you can get a brief overview on [Wikipedia](https:/en.wikipedia.org/wiki/Arithmetic_underflow). To deal with underflow, a common transformation is to work in log-space.
# $\log[P(w_{d1},\cdots,w_{dn}|y_d)] = \sum_{i=1}^{n}\log[P(w_{di}|y_d)]$
# Implement the `log_likelihood` function in the [Naive Bayes Block](#Naive-Bayes-Block). **Hint:** it should make calls to the p word given label and alpha function.
# Implement the `log_prior` function in the [Naive Bayes Block](#Naive-Bayes-Block). This function takes a class label and returns the log of the fraction of the training documents that are of that label.
# ### Question 2.7 (5 points)
# Naive Bayes is a model that tells us how to compute the posterior
# probability of a document being of some label (i.e.,
# $P(y_d|\mathbf{w_d})$).  Specifically, we do so using bayes rule:
# $P(y_d|\mathbf{w_d}) = \frac{P(y_d)P(\mathbf{w_d}|y_d)}{P(\mathbf{w_d})}$
# In the previous section you implemented functions to compute both
# the log prior ($\log[P(y_d)]$) and the log likelihood
# ($\log[P( \mathbf{w_d} |y_d)]$ ). Now, all you're missing is the
# *normalizer*, $P(\mathbf{w_d})$.
# Derive the normalizer by expanding $P(\mathbf{w_d})$.
# ***Answer in one or two lines here. Provide the formula and define each term in this formula.***
# ### Question 2.8 (5 points)
# One way to classify a document is to compute the unnormalized log posterior for both labels and take the argmax (i.e., the label that yields the higher unnormalized log posterior). The unnormalized log posterior is the sum of the log prior and the log likelihood of the document. Why don’t we need to compute the log normalizer here?
# ***Answer in one or two lines here.***
# ### Question 2.9 (10 points)
# As we saw earlier, the top 10 words from each class do not give us much to go on when classifying a document. A much more powerful metric is the likelihood ratio, which is defined as
# $LR(w)=\frac{P(w|y=\mathrm{pos})}{P(w|y=\mathrm{neg})}$
# A word with LR 3 is 3 times more likely to appear in the positive class than in the negative. A word with LR 0.3 is one-third as likely to appear in the positive class as opposed to the negative class.
# Implement the nb.likelihod_ratio function and use it to investigate the likelihood ratio of "amazing" and "dull"
print("LIKELIHOOD RATIO OF 'amazing':", nb.likelihood_ratio("amazing", 0.2))
print("LIKELIHOOD RATIO OF 'dull':", nb.likelihood_ratio("dull", 0.2))
print("LIKELIHOOD RATIO OF 'and':", nb.likelihood_ratio("and", 0.2))
print("LIKELIHOOD RATIO OF 'to':", nb.likelihood_ratio("to", 0.2))

# What is the minimum and maximum possible values the likelihood ratio can take? Does it make sense that $LR('amazing') > LR('to')$?
# ***Answer in one or two lines here.***
# Find the word in the vocabulary with the highest liklihood ratio below.
# Add your code here

# ### Question 2.10 (5 points)
# The unnormalized log posterior is the sum of the log prior and the log likelihood of the document. Implement the `unnormalized_log_posterior` function and the `classify` function in the [Naive Bayes Block](#Naive-Bayes-Block). The `classify` function should use the unnormalized log posteriors but should not compute the normalizer. Once you implement the `classify` function, we'd like to evaluate its accuracy.
print(nb.evaluate_classifier_accuracy(0.2))

# ### Question 2.11 (5 points)
# Try evaluating your model again with a smoothing parameter of 1000.
print(nb.evaluate_classifier_accuracy(1000.0))

# Does the accuracy go up or down when alpha is raised to 1000? Why do you think this is?
# ***Answer in one or two lines here.***
# ### Question 2.12 (5 points)
# Find a review that your classifier got wrong.
# In this cell, print out a review your classifier got wrong, along with its label.


# What are two reasons your system might have misclassified this example? What improvements could you make that may help your system classify this example correctly?
# ***Answer in one or two lines here.***
# ### Question 2.13 (5 points)
# Often times we care about multi-class classification rather than binary classification.
# How many counts would we need to keep track of if the model were modified to support 5-class classification?
# ***Answer in one or two lines here.***
#
# Use your own code or an external library such as nltk to perform tokenization,
# text normalization, word filtering, etc.
# Fill out your work in def tokenize_doc_and_more (below) and then show improvement by
# running the cells below.
def tokenize_doc_and_more(doc):
    """
    Return some representation of a document.
    At a minimum, you need to perform tokenization, the rest is up to you.
    """
    # Implement me!
    bow = defaultdict(float)
    # your code goes here
    return bow


nb = NaiveBayes(PATH_TO_DATA, tokenizer=tokenize_doc_and_more)
nb.train_model()
nb.evaluate_classifier_accuracy(1.0)
