docs: def extract_graphs_and_lemmas_from_tweets(tweet_generator):
      """
      Given a tweet python generator, we encode the information into mention and retweet graphs and a lemma matrix.
  
      We assume that the tweets are given in increasing timestamp.
  
      Inputs:  - tweet_generator: A python generator of tweets in python dictionary (json) format.
  
      Outputs: - mention_graph: The mention graph as a SciPy sparse matrix.
               - retweet_graph: The retweet graph as a SciPy sparse matrix.
               - user_lemma_matrix: The user lemma vector representation matrix as a SciPy sparse matrix.
               - tweet_id_set: A python set containing the Twitter ids for all the dataset tweets.
               - user_id_set: A python set containing the Twitter ids for all the dataset users.
               - lemma_to_attribute: A map from lemmas to numbers in python dictionary format.
      """
      ####################################################################################################################
      # Prepare for iterating over tweets.
      ####################################################################################################################
      # These are initialized as lists for incremental extension.
      tweet_id_set = set()
      user_id_set = list()
  
      add_tweet_id = tweet_id_set.add
      append_user_id = user_id_set.append
  
      # Initialize sparse matrix arrays.
      mention_graph_row = list()
      mention_graph_col = list()
  
      retweet_graph_row = list()
      retweet_graph_col = list()
  
      user_lemma_matrix_row = list()
      user_lemma_matrix_col = list()
      user_lemma_matrix_data = list()
  
      append_mention_graph_row = mention_graph_row.append
      append_mention_graph_col = mention_graph_col.append
  
      append_retweet_graph_row = retweet_graph_row.append
      append_retweet_graph_col = retweet_graph_col.append
  
      extend_user_lemma_matrix_row = user_lemma_matrix_row.extend
      extend_user_lemma_matrix_col = user_lemma_matrix_col.extend
      extend_user_lemma_matrix_data = user_lemma_matrix_data.extend
  
      # Initialize dictionaries.
      id_to_node = dict()
      id_to_name = dict()
      id_to_username = dict()
      id_to_listedcount = dict()
      lemma_to_attribute = dict()
  
      sent_tokenize, _treebank_word_tokenize = get_tokenizer()
      # tagger = HunposTagger('hunpos-1.0-linux/english.model', 'hunpos-1.0-linux/hunpos-tag')
      # tagger = PerceptronTagger()
      tagger = get_braupt_tagger()
      lemmatizer, lemmatize = get_lemmatizer("wordnet")
      stopset = get_stopset()
      first_cap_re, all_cap_re = get_camel_case_regexes()
      digits_punctuation_whitespace_re = get_digits_punctuation_whitespace_regex()
      pos_set = get_pos_set()
  
      ####################################################################################################################
      # Iterate over tweets.
      ####################################################################################################################
      counter = 0
      for tweet in tweet_generator:
          # print(tweet)
          # Increment tweet counter.
          counter += 1
          # if counter % 10000 == 0:
          #     print(counter)
          # print(counter)
  
          # Extract base tweet's values.
          try:
              tweet_id = tweet["id"]
              user_id = tweet["user"]["id"]
              user_name = tweet["user"]["name"]
              user_screen_name = tweet["user"]["screen_name"]
  
              listed_count_raw = tweet["user"]["listed_count"]
  
              tweet_text = tweet["text"]
  
              tweet_in_reply_to_user_id = tweet["in_reply_to_user_id"]
              tweet_in_reply_to_screen_name = tweet["in_reply_to_screen_name"]
              tweet_entities_user_mentions = tweet["entities"]["user_mentions"]
  
              if "retweeted_status" not in tweet.keys():
                  user_mention_id_list = list()
                  user_mention_screen_name_list = list()
                  for user_mention in tweet_entities_user_mentions:
                      user_mention_id_list.append(user_mention["id"])
                      user_mention_screen_name_list.append(user_mention["screen_name"])
              else:
                  # Extract base tweet's values.
                  original_tweet = tweet["retweeted_status"]
  
                  original_tweet_id = original_tweet["id"]
                  original_tweet_user_id = original_tweet["user"]["id"]
                  original_tweet_user_name = original_tweet["user"]["name"]
                  original_tweet_user_screen_name = original_tweet["user"]["screen_name"]
  
                  listed_count_raw = original_tweet["user"]["listed_count"]
  
                  original_tweet_text = original_tweet["text"]
  
                  original_tweet_in_reply_to_user_id = original_tweet["in_reply_to_user_id"]
                  original_tweet_in_reply_to_screen_name = original_tweet["in_reply_to_screen_name"]
                  original_tweet_entities_user_mentions = original_tweet["entities"]["user_mentions"]
  
                  original_tweet_user_mention_id_list = list()
                  original_tweet_user_mention_screen_name_list = list()
                  for user_mention in original_tweet_entities_user_mentions:
                      original_tweet_user_mention_id_list.append(user_mention["id"])
                      original_tweet_user_mention_screen_name_list.append(user_mention["screen_name"])
          except KeyError:
              continue
          # Map users to distinct integer numbers.
          graph_size = len(id_to_node)
          source_node = id_to_node.setdefault(user_id, graph_size)
  
          if listed_count_raw is None:
              id_to_listedcount[user_id] = 0
          else:
              id_to_listedcount[user_id] = int(listed_count_raw)
  
          # Update sets, lists and dictionaries.
          add_tweet_id(tweet_id)
          id_to_name[user_id] = user_screen_name
          id_to_username[user_id] = user_name
          append_user_id(user_id)
  
          ################################################################################################################
          # We are dealing with an original tweet.
          ################################################################################################################
          if "retweeted_status" not in tweet.keys():
              ############################################################################################################
              # Update user-lemma frequency matrix.
              ############################################################################################################
              # Extract lemmas from the text.
              tweet_lemmas, lemma_to_keywordbag = clean_document(tweet_text, sent_tokenize, _treebank_word_tokenize,
                                                                 tagger, lemmatizer, lemmatize, stopset,
                                                                 first_cap_re, all_cap_re, digits_punctuation_whitespace_re,
                                                                 pos_set)
  
              number_of_lemmas = len(tweet_lemmas)
  
              # Update the user-lemma frequency matrix one-by-one.
              attribute_list = list()
              append_attribute = attribute_list.append
              for lemma in tweet_lemmas:
                  # Map lemmas to distinct integer numbers.
                  vocabulary_size = len(lemma_to_attribute)
                  attribute = lemma_to_attribute.setdefault(lemma, vocabulary_size)
                  append_attribute(attribute)
  
              # Add values to the sparse matrix arrays.
              extend_user_lemma_matrix_row(number_of_lemmas*[source_node])
              extend_user_lemma_matrix_col(attribute_list)
              extend_user_lemma_matrix_data(number_of_lemmas*[1.0])
  
              ############################################################################################################
              # Update mention matrix.
              ############################################################################################################
              # Get mentioned user ids.
              mentioned_user_id_set = list()
              if tweet_in_reply_to_user_id is not None:
                  mentioned_user_id_set.append(tweet_in_reply_to_user_id)
  
                  id_to_name[tweet_in_reply_to_user_id] = tweet_in_reply_to_screen_name
              for user_mention, mentioned_user_id, mentioned_user_screen_name in zip(tweet_entities_user_mentions,
                                                                                     user_mention_id_list,
                                                                                     user_mention_screen_name_list):
                  mentioned_user_id_set.append(mentioned_user_id)
  
                  id_to_name[mentioned_user_id] = mentioned_user_screen_name
  
              # We remove duplicates.
              mentioned_user_id_set = set(mentioned_user_id_set)
  
              # Update the mention graph one-by-one.
              for mentioned_user_id in mentioned_user_id_set:
                  # Map users to distinct integer numbers.
                  graph_size = len(id_to_node)
                  mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)
  
                  append_user_id(mentioned_user_id)
  
                  # Add values to the sparse matrix arrays.
                  append_mention_graph_row(source_node)
                  append_mention_graph_col(mention_target_node)
  
          ################################################################################################################
          # We are dealing with a retweet.
          ################################################################################################################
          else:
              # Map users to distinct integer numbers.
              graph_size = len(id_to_node)
              original_tweet_node = id_to_node.setdefault(original_tweet_user_id, graph_size)
  
              if listed_count_raw is None:
                  id_to_listedcount[user_id] = 0
              else:
                  id_to_listedcount[user_id] = int(listed_count_raw)
  
              # Update retweet graph.
              append_retweet_graph_row(source_node)
              append_retweet_graph_col(original_tweet_node)
  
              # Extract lemmas from the text.
              tweet_lemmas, lemma_to_keywordbag = clean_document(original_tweet_text, sent_tokenize, _treebank_word_tokenize,
                                                                 tagger, lemmatizer, lemmatize, stopset,
                                                                 first_cap_re, all_cap_re, digits_punctuation_whitespace_re,
                                                                 pos_set)
  
              number_of_lemmas = len(tweet_lemmas)
  
              # Update the user-lemma frequency matrix one-by-one.
              attribute_list = list()
              append_attribute = attribute_list.append
              for lemma in tweet_lemmas:
                  # Map lemmas to distinct integer numbers.
                  vocabulary_size = len(lemma_to_attribute)
                  attribute = lemma_to_attribute.setdefault(lemma, vocabulary_size)
                  append_attribute(attribute)
  
              # Add values to the sparse matrix arrays.
              extend_user_lemma_matrix_row(number_of_lemmas*[source_node])
              extend_user_lemma_matrix_col(attribute_list)
              extend_user_lemma_matrix_data(number_of_lemmas*[1.0])
  
              # Get mentioned user ids.
              mentioned_user_id_set = list()
              if original_tweet_in_reply_to_user_id is not None:
                  mentioned_user_id_set.append(original_tweet_in_reply_to_user_id)
  
                  id_to_name[original_tweet_in_reply_to_user_id] = original_tweet_in_reply_to_screen_name
              for user_mention, mentioned_user_id, mentioned_user_screen_name in zip(original_tweet_entities_user_mentions,
                                                                                     original_tweet_user_mention_id_list,
                                                                                     original_tweet_user_mention_screen_name_list):
                  mentioned_user_id_set.append(mentioned_user_id)
  
                  id_to_name[mentioned_user_id] = mentioned_user_screen_name
  
              # We remove duplicates.
              mentioned_user_id_set = set(mentioned_user_id_set)
  
              # Get mentioned user ids.
              retweet_mentioned_user_id_set = list()
              if original_tweet_in_reply_to_user_id is not None:
                  retweet_mentioned_user_id_set.append(original_tweet_in_reply_to_user_id)
  
                  id_to_name[original_tweet_in_reply_to_user_id] = original_tweet_in_reply_to_screen_name
              for user_mention, mentioned_user_id, mentioned_user_screen_name in zip(original_tweet_entities_user_mentions,
                                                                                     original_tweet_user_mention_id_list,
                                                                                     original_tweet_user_mention_screen_name_list):
                  retweet_mentioned_user_id_set.append(mentioned_user_id)
  
                  id_to_name[mentioned_user_id] = mentioned_user_screen_name
  
              # We remove duplicates.
              retweet_mentioned_user_id_set = set(retweet_mentioned_user_id_set)
  
              mentioned_user_id_set.update(retweet_mentioned_user_id_set)
  
              # Update the mention graph one-by-one.
              for mentioned_user_id in mentioned_user_id_set:
                  # Map users to distinct integer numbers.
                  graph_size = len(id_to_node)
                  mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)
  
                  append_user_id(mentioned_user_id)
  
                  # Add values to the sparse matrix arrays.
                  append_mention_graph_row(source_node)
                  append_mention_graph_col(mention_target_node)
  
              # This is the first time we deal with this tweet.
              if original_tweet_id not in tweet_id_set:
                  # Update sets, lists and dictionaries.
                  add_tweet_id(original_tweet_id)
                  id_to_name[original_tweet_user_id] = original_tweet_user_screen_name
                  id_to_username[original_tweet_user_id] = original_tweet_user_name
                  append_user_id(original_tweet_user_id)
  
                  ########################################################################################################
                  # Update user-lemma frequency matrix.
                  ########################################################################################################
                  # Update the user-lemma frequency matrix one-by-one.
                  attribute_list = list()
                  append_attribute = attribute_list.append
                  for lemma in tweet_lemmas:
                      # Map lemmas to distinct integer numbers.
                      vocabulary_size = len(lemma_to_attribute)
                      attribute = lemma_to_attribute.setdefault(lemma, vocabulary_size)
                      append_attribute(attribute)
  
                  # Add values to the sparse matrix arrays.
                  extend_user_lemma_matrix_row(number_of_lemmas*[source_node])
                  extend_user_lemma_matrix_col(attribute_list)
                  extend_user_lemma_matrix_data(number_of_lemmas*[1.0])
  
                  ########################################################################################################
                  # Update mention matrix.
                  ########################################################################################################
                  # Update the mention graph one-by-one.
                  for mentioned_user_id in retweet_mentioned_user_id_set:
                      # Map users to distinct integer numbers.
                      graph_size = len(id_to_node)
                      mention_target_node = id_to_node.setdefault(mentioned_user_id, graph_size)
  
                      append_user_id(mentioned_user_id)
  
                      # Add values to the sparse matrix arrays.
                      append_mention_graph_row(original_tweet_node)
                      append_mention_graph_col(mention_target_node)
              else:
                  pass
  
      ####################################################################################################################
      # Final steps of preprocessing tweets.
      ####################################################################################################################
      # Discard any duplicates.
      user_id_set = set(user_id_set)
      number_of_users = len(user_id_set)
      # min_number_of_users = max(user_id_set) + 1
  
      # Form mention graph adjacency matrix.
      mention_graph_row = np.array(mention_graph_row, dtype=np.int64)
      mention_graph_col = np.array(mention_graph_col, dtype=np.int64)
      mention_graph_data = np.ones_like(mention_graph_row, dtype=np.float64)
  
      mention_graph = spsp.coo_matrix((mention_graph_data, (mention_graph_row, mention_graph_col)),
                                      shape=(number_of_users, number_of_users))
      mention_graph = spsp.coo_matrix(spsp.csr_matrix(mention_graph))
  
      # Form retweet graph adjacency matrix.
      retweet_graph_row = np.array(retweet_graph_row, dtype=np.int64)
      retweet_graph_col = np.array(retweet_graph_col, dtype=np.int64)
      retweet_graph_data = np.ones_like(retweet_graph_row, dtype=np.float64)
  
      retweet_graph = spsp.coo_matrix((retweet_graph_data, (retweet_graph_row, retweet_graph_col)),
                                      shape=(number_of_users, number_of_users))
      retweet_graph = spsp.coo_matrix(spsp.csr_matrix(retweet_graph))
  
      # Form user-lemma matrix.
      number_of_lemmas = len(lemma_to_attribute)
  
      user_lemma_matrix_row = np.array(user_lemma_matrix_row, dtype=np.int64)
      user_lemma_matrix_col = np.array(user_lemma_matrix_col, dtype=np.int64)
      user_lemma_matrix_data = np.array(user_lemma_matrix_data, dtype=np.float64)
  
      user_lemma_matrix = spsp.coo_matrix((user_lemma_matrix_data, (user_lemma_matrix_row, user_lemma_matrix_col)),
                                          shape=(number_of_users, number_of_lemmas))
      user_lemma_matrix = spsp.coo_matrix(spsp.csr_matrix(user_lemma_matrix))
  
      node_to_id = dict(zip(id_to_node.values(), id_to_node.keys()))
  
      # tagger.close()
  
      return mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, node_to_id, lemma_to_attribute, id_to_name, id_to_username, id_to_listedcount