
import nltk
import re
import sys
import math
import matplotlib.pyplot as plt

from nltk import ne_chunk
from nltk.util import ngrams
from wordcloud import WordCloud, STOPWORDS
from collections import defaultdict, Counter
from pos_tagger import POSTagger


sys.path.insert(
    0, '/Users/diakite_w/Documents/Dev/ExperimentationsACA/FrenchLefffLemmatizer')
from FrenchLefffLemmatizer import FrenchLefffLemmatizer

import math
import matplotlib.pyplot as plt


def read_txt(textfile):
    with open(textfile, 'r') as f:
        text = f.read()
        text = text.replace('\n', ' ')
        text = text.replace('- ', '')
        text = text.replace('.', '')
        text = text.replace('-', '')
        text = text.replace("‘l'", 'ï')
        	
    return text

def display_wordcloud(tokens):
    '''
        Display a simple wordcloup from 
        processed tokens
    '''
    # Join all tokens to make one big string
    join_text = ' '.join(tokens)

    wordcloud = WordCloud(background_color='white',
                          width=1200,
                          height=1000
                          ).generate(join_text)
    plt.imshow(wordcloud)
    plt.axis('off')
    plt.show()



def link_splitted_words(texte):
    next_elt = texte[1:]


def tokenize(text):

    fll = FrenchLefffLemmatizer()
    splck = SpellChecker()

    contracted_pronouns = ["l'", "m'", "n'", "d'", "c'", "j'", "qu'", "s'"]
    dictionnary = []



def link_splitted_words(texte):
	next_elt = texte[1:]


def tokenizer(text):

    fll = FrenchLefffLemmatizer()
    #splck = SpellChecker()


    # Put everything to lower case
    text = text.lower()

    # Tokenize text
    tokens = nltk.tokenize.word_tokenize(text)
    print('Nombre de tokens dans le texte :', len(tokens))

    # Remove contacted pronous from tokens
    contracted_pronouns = ["l'", "m'", "n'", "d'", "c'", "j'", "qu'", "s'"]
    tokens = [t[2:] if t[:2] in contracted_pronouns else t for t in tokens]

    # Spell check every tokens
    #tokens = [splck.correct(t) for t in tokens]

    # Remove all words with len <= 2
    tokens = [t for t in tokens if len(t) > 2]

    #tokens = [splck.correct(t) if t not in dictionnary else t for t in tokens]

    tokens = [t for t in tokens if len(t) > 2]

    tokens = [t for t in tokens if t not in stopwords]

    tokens = [fll.lemmatize(t) for t in tokens]

    print('Nombre de tokens apres traitement :', len(tokens), '\n')
    return tokens


def compute_tf(document, word):
    '''
            Compute the TF of a document 
    '''
    with open(document, 'r') as f:
        text = f.read()
        text = text.replace('\n', ' ')
        text = text.replace('- ', '')
        text = text.replace('.', '')
        text = text.replace('-', '')
        text = text.replace("‘l'", 'ï')

    tokens = tokenize(text)


def extract_ngrams(tokens, n):
    '''
            Return list of n-grams
    '''
    return Counter(ngrams(tokens, n))


# Load stop words
with open('stopwords-fr.txt', 'r') as f:
	stopwords = [l[:-1] for l in f.readlines()]
	stopwords = list(set(stopwords))
	#print(stopwords)

# Load stop words
stopwords = list(set(w.rstrip() for w in open('stopwords-fr.txt')))
# print(stopwords)

# Read text file
textfile = 'data/simple.txt'
with open(textfile, 'r') as f:
	text = f.read()
	text = text.replace('\n', ' ')
	text = text.replace('- ', '')
	text = text.replace('.', '')
	text = text.replace('-', '')
	#print(text)

tokens = tokenizer(text)
vocabulary = list(set(tokens))

# Compute dictionnary of word (key) and index
word_idx = {w: idx for w, idx in enumerate(vocabulary)}
#print(word_idx)
# DO YOU REALLY NEED THIS ?

# Count nb of word appearance
word_idf = defaultdict(lambda: 0)
for token in tokens:
	word_idf[token] += 1

# Compute idf
for word in vocabulary:
	word_idf[word] = math.log(1 / float(1))

# Compute tf
word_tf = defaultdict(lambda: 0)
for word in tokens:
	word_tf[word] += 1

for word in vocabulary:
	word_tf[word] /= len(tokens)


join_text = ' '.join(tokens)	

wordcloud = WordCloud(background_color='white',
                          width=1200,
                          height=1000
                         ).generate(join_text)

plt.imshow(wordcloud)
plt.axis('off')
plt.show()


# Extracting n-grams
bigrams = extract_ngrams(tokens, 2)
trigrams = extract_ngrams(tokens, 3)
[print(t) for t in trigrams.most_common(3)]
[print(t) for t in bigrams.most_common(3)]


