from dotenv import load_dotenv
print('loading the enviroment variable form .env file')
load_dotenv()
auto_encoder/__init__.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder import utils
import os,sys
import pandas as pd
import numpy  as np
import re

class DataCleaning:
    def __init__(self,datacleaningconfig:config_entity.DataCleaningConfig,
                 dataingestionartifact:artifact_entity.DataIngestionArtifact,
                 datavalidationartifact:artifact_entity.DataValidationArtifact):
        try:
            self.datacleaningconfig = datacleaningconfig
            self.dataingestionartifact = dataingestionartifact
            self.datavalidationartifact = datavalidationartifact
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def drop_duplicate(self,df:pd.DataFrame)->pd.DataFrame:
        try:
            logging.info(f"dropping the duplicate history from history")
            unique_titles = df.drop_duplicates()
            unique_titles.reset_index(inplace=True)
            unique_titles = unique_titles.drop('index',axis=1)
            unique_titles['title'] = unique_titles['title'].astype('str')
            logging.info(f"successfully droped duplicacy, and now size :- {unique_titles.shape}")
            return unique_titles
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def remove_less_legth_history_and_linc(self,df:pd.DataFrame)->pd.DataFrame:
        try:
            new_titles = []
            logging.info(f"remove less lenght and linc history !")
            for i in range(len(df)): 
                if  (2 < len(df['title'][i].split())) and (0 == df['title'][i].count('/')):
                    new_titles.append(df['title'][i])
            df1 = pd.DataFrame({'title':new_titles})
            logging.info(f"successfully removed less length and linc, and now size :- {df1.shape}")
            return df1
        except Exception as e:
            raise AutoencoderException(e,sys)


    def text_cleaning(self,df:pd.DataFrame)->str:
        try:
            logging.info(f"cleaning unnecessay words and lowerise the sentences")
            for i in range(len(df)):
                lowered = df['title'][i].lower()
                cleaned_content = re.sub("[^a-zA-Z]"," ",lowered)  
                df['title'][i] = cleaned_content
            logging.info(f"successfully cleaned the text!")


            logging.info(f"concatenating entire titles as a single str!")
            text1 = ""
            for i in range(len(df)):
                text1 += f"{df['title'][i]}"
            logging.info(f"successfully concated entire text!")


            logging.info(f"removing unneccessary extra spaces.")
            text2 = []
            for i in [''.join(i) for i in text1.split(' ')]:
                if i == '':
                    pass
                else:
                    text2.append(i)

            logging.info(f"preparing final text data!")
            final_text = ""
            for i in text2:
                final_text += f" {i}"
            logging.info(f'successfully prepared the final text with length :- {len(final_text)}')

            return final_text
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def initiate_datacleaning(self)->artifact_entity.DataCleaningArtifact:
        try:
            logging.info(f"initiating the datacleaning instance methods!")
            df_file_path = self.dataingestionartifact.merge_df_path
            dataframe = pd.read_csv(df_file_path)

            dataframe = self.drop_duplicate(df=dataframe)
            dataframe = self.remove_less_legth_history_and_linc(df=dataframe)
            final_text = self.text_cleaning(df=dataframe)

            cleaned_file_path = self.datacleaningconfig.cleaned_file_path
            utils.save_text_file(file_path=cleaned_file_path,text=final_text)

            datacleaningArtifact = artifact_entity.DataCleaningArtifact(cleaned_file_path=cleaned_file_path)
            return datacleaningArtifact
        except Exception as e:
            raise AutoencoderException(e,sys)

auto_encoder/component/DataCleaning.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder.entity import config_entity,artifact_entity
import os,sys
import pandas as pd
import numpy as np

class DataIngestion:
    def __init__(self,dataingestionconfig:config_entity.DataingestionConfig,):
        try:
            self.dataingestionconfig = dataingestionconfig
        except Exception as e:
            raise AutoencoderException(e,sys)
    

    def merge_files(self)->pd.DataFrame:
        try:
            logging.info('Merging all the csv files in one dataframe!')
            list_of_file_path = self.dataingestionconfig.history_file_path
            one = pd.read_csv(list_of_file_path[0])
            for i,path in enumerate(list_of_file_path):
                if i == 0:
                    continue
                one = pd.concat([one,pd.read_csv(path)])
            dataframe = one[['title']]
            logging.info(f"file size after merged :- {dataframe.shape}")
            return dataframe
        except Exception as e:
            raise AutoencoderException(e,sys)

            


    def initiateingestion(self)->artifact_entity.DataCleaningArtifact:
        try:
            logging.info(f"initiating DataIngestion")
            directory_path = self.dataingestionconfig.data_dir_path
            os.makedirs(directory_path,exist_ok=True)
            titles = self.merge_files()
            ingestion_file_path = self.dataingestionconfig.data_file_path
            titles.to_csv(ingestion_file_path,index=False,header=True)
            dataingestionartifact = artifact_entity.DataIngestionArtifact(
                merge_df_path=self.dataingestionconfig.data_file_path
            )
            logging.info(f"initiated DataIngestion")
            return dataingestionartifact
        except Exception as e:
            raise AutoencoderException(e,sys)

    
auto_encoder/component/DataIngestion.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder import utils
import os,sys,re
import pandas as pd
import numpy as np
import nltk
import re                                                       # to regular expression
from nltk.corpus import stopwords                               # to stopwords
from nltk.stem.porter  import PorterStemmer                     # to stemming
from nltk.stem import WordNetLemmatizer                         # to lemmatizer
from sklearn.feature_extraction.text import CountVectorizer     # to BOW
from sklearn.feature_extraction.text import TfidfVectorizer     # to  TF-IDF
from gensim.models import Word2Vec                              # to word2vec
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.utils import to_categorical
import pickle
import warnings
warnings.filterwarnings('ignore')

class DataTransformation:
    def __init__(self,datacleaningartifact:artifact_entity.DataCleaningArtifact,
                 datatransformationconfig:config_entity.DataTransformationConfig):
        try:
            self.datacleaningartifact = datacleaningartifact
            self.datatransformationconfig = datatransformationconfig
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def load_text_file(self):
        try:
            file_path = self.datacleaningartifact.cleaned_file_path
            file = open(file_path,'r')
            data = file.read()
            logging.info(f"read text file from {file_path}")
            return data
        except Exception as e:
            raise AutoencoderException(e,sys)

    def lemmetized(self,text_data:str):
        try:
            lemmetizer = WordNetLemmatizer()
            words = nltk.word_tokenize(text_data)            
            lemmetized = [lemmetizer.lemmatize(word) for word in words]
            lemmetized_data = ' '.join(lemmetized)

            logging.info(f"text_data successfully lemmetized !")
            lemmetized_file_path = self.datatransformationconfig.lemmetized_file_path
            lemmetizer_obj_path = self.datatransformationconfig.lemmetizer_obj_path
            logging.info(f"saving the lemmetized file at this position :- {lemmetized_file_path}")

            utils.save_text_file(file_path=lemmetized_file_path,text=lemmetized_data)
            utils.save_object(file_path=lemmetizer_obj_path,obj=lemmetizer)

            
            return lemmetized_file_path
        
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def Tokenizer(self):
        try:
            tokenizer = Tokenizer()
            lemmetized_file_path = self.datatransformationconfig.lemmetized_file_path
            data = open(lemmetized_file_path,'r')
            Data  = data.read()
            tokenizer.fit_on_texts([Data])
            vocab_size = len(tokenizer.word_index)+1
            word_index = tokenizer.word_index
            word_index_file_path = self.datatransformationconfig.word_index_file_path
            utils.save_object(file_path=word_index_file_path,
                              obj=word_index)
            logging.info(f"word index successfully save at this position :- {word_index_file_path}")
            logging.info(f"no. of unique words :- {vocab_size}")


            tokenizer_path = self.datatransformationconfig.tokenizer_model_path
            directory = os.path.dirname(tokenizer_path)
            os.makedirs(directory,exist_ok=True)
            pickle.dump(tokenizer,open(tokenizer_path,'wb'))
            logging.info(f"tokenizer successfully fit and saved on this position :- {tokenizer_path}")

            sequence_data = tokenizer.texts_to_sequences([Data])[0]

            return vocab_size,sequence_data
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def transform_sequences(self,sequence_data):
        try:
            sequences = []
            for i in range(3,len(sequence_data)):
                words = sequence_data[i-3:i+1]
                sequences.append(words)
            sequences = np.array(sequences)

            sequence_data_textfile_path = self.datatransformationconfig.sequence_data_textfile_path
            utils.save_text_file(file_path=sequence_data_textfile_path,text=str(sequence_data[0:]))
            logging.info(f"sequence data textfile successfully saved at this position :- {sequence_data_textfile_path}")

            numpy_sequence_file_path = self.datatransformationconfig.sequence_data_npFile
            utils.save_numpy_array_data(file_path=numpy_sequence_file_path,array=sequences)
            logging.info(f'successfully saved sequence numpy array !')
            return sequences
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def set_independent_and_depen(self,sequences):
        try:
            x = []
            y = []
            for i in sequences:
                x.append(i[0:3])
                y.append(i[3])
            x = np.array(x)
            y = np.array(y)
            return x,y
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def transform_dependent_var(self,y_data,vocab_size):
        try:
            y = to_categorical(y_data,num_classes=vocab_size)
            logging.info(f"successfully transformed the y_data !")
            return y
        except Exception as e:
            raise AutoencoderException(e,sys)




    def InitiateTransformation(self)->artifact_entity.DataTransformationArtifact:
        try:
            text_data = self.load_text_file()
            lemmetized_file_path = self.lemmetized(text_data=text_data)
            vocab_size,sequence_data = self.Tokenizer()
            sequences = self.transform_sequences(sequence_data=sequence_data)
            x_data,y_data = self.set_independent_and_depen(sequences=sequences)
            y_dat = self.transform_dependent_var(y_data=y_data,vocab_size=vocab_size)

            x_train_file_path = self.datatransformationconfig.x_train_file_path
            y_train_file_path = self.datatransformationconfig.y_train_file_path
            word_index_file_path = self.datatransformationconfig.word_index_file_path

            utils.save_object(file_path=x_train_file_path,obj=x_data)
            utils.save_object(file_path=y_train_file_path,obj=y_dat)

            train_data_dir_path = self.datatransformationconfig.Train_data_dir
            logging.info(f"succesfully saved your trained data at this position :- {train_data_dir_path}")


            

            transformation_artifact = artifact_entity.DataTransformationArtifact(
                transformed_data_file_path=self.datatransformationconfig.transformed_data_file_path,
                lemmetized_data_file_path=lemmetized_file_path,
                vocab_size=vocab_size,
                x_train_data_path=x_train_file_path,
                y_train_data_path=y_train_file_path ,
                word_index_path = word_index_file_path,
                tokenizer_model_path=self.datatransformationconfig.tokenizer_model_path,
                lemmetizer_model_path=self.datatransformationconfig.lemmetizer_obj_path
                )
            return transformation_artifact
        except Exception as e:
            raise AutoencoderException(e,sys)
        





auto_encoder/component/DataTransformation.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder import utils
import os,sys
import pandas as pd
import numpy  as np
import re

class DataValidation:
    def __init__(self,dataingestionartifact:artifact_entity.DataIngestionArtifact,
                 datavalidationconfig:config_entity.DataValidationConfig):
        try:
            self.dataingestionartifact = dataingestionartifact
            self.datavalidationconfig = datavalidationconfig
            self.validation_data = dict()
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def file_type(self,report_key:str):
        try:
            file_name = os.path.basename(self.dataingestionartifact.merge_df_path)
            if file_name.endswith('.csv'):
                self.validation_data[report_key] = '.csv file'
            else:
                self.validation_data[report_key] = 'unknown'
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def duplicate_record(self,report_key:str):
        try:
            df_file_path = self.dataingestionartifact.merge_df_path
            dataframe = pd.read_csv(df_file_path)
            no_of_sum = dataframe.duplicated().sum()
            self.validation_data[report_key] = str(no_of_sum)
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    
    def columns(self,report_key:str):
        try:
            df_file_path = self.dataingestionartifact.merge_df_path
            df = pd.read_csv(df_file_path)
            list_of_columns = list(df.columns)
            self.validation_data[report_key] = list_of_columns
        except Exception as e:
            raise AutoencoderException(e,sys)
    
    def shape_of_df(self,report_key:str):
        try:
            df_file_path = self.dataingestionartifact.merge_df_path
            df = pd.read_csv(df_file_path)
            a = dict()
            a['rows'] = df.shape[0]
            a['columns'] = df.shape[1]
            self.validation_data[report_key] = a
        except Exception as e:
            raise AutoencoderException(e,sys)
        
    def initiate_validation(self)->artifact_entity.DataValidationArtifact:
        try:
            logging.info(f"initiating datavalidation class")
            self.file_type(report_key='file_extension')
            self.duplicate_record(report_key='no_of_duplicate_record')
            self.columns(report_key='name_of_columns')
            self.shape_of_df(report_key='shape_of_df')

            utils.write_yaml_file(file_path=self.datavalidationconfig.validation_report_file_path,
                                  data=self.validation_data)
            data_validation_artifact = artifact_entity.DataValidationArtifact(validation_report_file_path=self.datavalidationconfig.validation_report_file_path)
            return data_validation_artifact
        except Exception as e:
            raise AutoencoderException(e,sys)
    
        


    

auto_encoder/component/DataValidation.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder import utils
import os,sys,re,pickle
import pandas as pd
import numpy as np
import tensorflow as tf
class ModelEvaluation:
    def __init__(self,modeltrainerartifact:artifact_entity.ModelTrainerArtifact,
                 modeltrainerconfig:config_entity.ModelTrainerConfig):
        try:
            self.modeltrainerartifact = modeltrainerartifact
            self.modeltrainerconfig = modeltrainerconfig
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def load_model(self,file_path:str):
        try:
            model = tf.keras.models.load_model(file_path)
            return model
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def InitiateModelEvaluation(self):
        try:
            model_file_path = self.modeltrainerartifact.model_file_path
            x_train_file_path = self.modeltrainerartifact.x_train_file_path
            y_train_file_path = self.modeltrainerartifact.y_train_file_path

            model = self.load_model(file_path=model_file_path)
            x_train = utils.load_obj(file_path=x_train_file_path)
            y_train = utils.load_obj(file_path=y_train_file_path)

            evaluation_result = model.evaluate(x_train,y_train)

            loss = evaluation_result[0] * 10
            accuracy = evaluation_result[1] * 100
            loss = round(loss,2)
            accuracy = round(accuracy,2)
            logging.info(f"model evaluted with this accuracy :-{accuracy} and loss is :- {loss}")

            difference= accuracy - loss

            if (difference >20) or (difference < 0):
                logging.info(f"Model is underfit !")
                print(f"model is underfit Try to improve the accuracy of your model!")

            elif difference < 4:
                logging.info(f"model is overfitted ! Try to regularize your model !")
                print(f"model is overfitted !")

            elif (difference > 5) and (difference < 15):
                logging.info(f"Hurray , model is generelized !")
                print(f'model is generailized !')
            print(f"model successfully evaluated !")

        except Exception as e:
            raise AutoencoderException(e,sys)

auto_encoder/component/ModelEvaluatin.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
import numpy as np
import pandas as pd
import os,sys
from auto_encoder import utils
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder.exception import AutoencoderException
from auto_encoder.logger import logging
from auto_encoder.predictor import ModelResolver
import tensorflow as tf

class ModelPusher:
    def __init__(self,modelpusherconfig:config_entity.ModelPusherConfig,
                 transformationArtifact:artifact_entity.DataTransformationArtifact,
                 modeltrainerArtifact:artifact_entity.ModelTrainerArtifact):
        
        try:
            self.modelpusherconfig = modelpusherconfig
            self.transformationArtifact = transformationArtifact
            self.modeltrainerArtifact = modeltrainerArtifact
            self.model_resolver = ModelResolver()
        except Exception as e:
            raise AutoencoderException(e,sys)


    def initiate_Modelpusher(self,)->artifact_entity.ModelpusherArtifact:
        try:

            tokenizer = utils.load_obj(file_path=self.transformationArtifact.tokenizer_model_path)
            lemmitizer = utils.load_obj(file_path=self.transformationArtifact.lemmetizer_model_path)
            model = tf.keras.models.load_model(self.modeltrainerArtifact.model_file_path)            
            wordIndex = utils.load_obj(file_path=self.transformationArtifact.word_index_path)
            
            # saved in artifact --> modelpusher
            # utils.save_object(file_path=self.modelpusherconfig.pusher_model_path,model_obj=model)
            utils.save_object(file_path=self.modelpusherconfig.pusher_lemmetizer_path,obj=lemmitizer)
            utils.save_object(file_path=self.modelpusherconfig.pusher_Toknizer_path,obj=tokenizer)
            utils.save_object(file_path=self.modelpusherconfig.pusher_wordIndex_path,obj=wordIndex)
            model.save(self.modelpusherconfig.pusher_model_path)
            logging.info(f"saved models in artifact -> Modelpusher")

            # get the path to save in saved_models
            tokenizer_path = self.model_resolver.get_latest_save_tokenizer_path()
            lemmitizer_path = self.model_resolver.get_latest_save_lemmetizer_path()
            model_path = self.model_resolver.get_latest_save_model_path()
            wordindex_path = self.model_resolver.get_latest_save_wordindex_path()

            # to save the models in saved_models
            utils.save_object(file_path=tokenizer_path,obj=tokenizer)
            utils.save_object(file_path=lemmitizer_path,obj=lemmitizer)
            utils.save_object(file_path=wordindex_path,obj=wordIndex)
            model.save(model_path)
            logging.info(f"saved models in saved_models")
            logging.info(f"Training Pipeline executed Successfully !")
            print(f"Training Pipeline executed Successfully !")


            modelPusherArtifact = artifact_entity.ModelpusherArtifact(pusher_model_dir=self.modelpusherconfig.pusher_model_dir,
                                                                      saved_model_dir=self.modelpusherconfig.saved_model,
                                                                      lemmetizer_path=lemmitizer_path,
                                                                      tokenizer_path=tokenizer_path,
                                                                      wordindex_path=wordindex_path,
                                                                      vocab_size=self.transformationArtifact.vocab_size,
                                                                      trained_model_path=model_path)
            return modelPusherArtifact

        except Exception as e:
            raise AutoencoderException(e,sys)
auto_encoder/component/ModelPusher.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder import utils
import os,sys,re,pickle
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding,LSTM,Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint


class ModelTrainer:
    def __init__(self,transformerartifact:artifact_entity.DataTransformationArtifact,
                 modeltrainerconfig:config_entity.ModelTrainerConfig):
        try:
            self.transformerartifact = transformerartifact
            self.modeltrainerconfig = modeltrainerconfig
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def get_model(self,vocab_size,n_input):
        try:
            model = Sequential()
            model.add(Embedding(vocab_size,10,input_length=3))
            model.add(LSTM(1000,return_sequences=True)) # to use next LSTM
            model.add(LSTM(1000))
            model.add(Dense(850,activation='relu'))
            model.add(Dense(vocab_size,activation='softmax'))
            model.compile(loss='categorical_crossentropy',optimizer=Adam(learning_rate=0.001),metrics=['accuracy'])
            logging.info(f"Model has been declared successfully !")
            return model
        except Exception as e:
            raise AutoencoderException(e,sys)
    

    def InitiateModelTrainer(self)->artifact_entity.ModelTrainerArtifact:
        try:
            x_train_data_path = self.transformerartifact.x_train_data_path
            y_train_data_path = self.transformerartifact.y_train_data_path
            model_file_path = self.modeltrainerconfig.model_file_path
            vocab_size = self.transformerartifact.vocab_size
            

            x_train = utils.load_obj(file_path=x_train_data_path)
            y_train = utils.load_obj(file_path=y_train_data_path)
            logging.info(f"successfully loaded data for the training !")


            model = self.get_model(vocab_size=vocab_size,n_input=3)
            checkpoint = ModelCheckpoint(model_file_path,monitor='loss',verbose=1,save_best_only=True)
            model_history = model.fit(x_train,y_train,epochs=1,batch_size=50,callbacks=[checkpoint])

            logging.info(f"model trained successfully !")
            history = model_history.history
            epoch_history_file_path = self.modeltrainerconfig.epoch_history_file_path
            utils.save_object(file_path=epoch_history_file_path,obj=history)


            model_trainer_artifact = artifact_entity.ModelTrainerArtifact(
                model_file_path=model_file_path,
                epoch_history_file_path=epoch_history_file_path,
                x_train_file_path=x_train_data_path,
                y_train_file_path=y_train_data_path
            )
            return model_trainer_artifact
        except Exception as e:
            raise AutoencoderException(e,sys)


auto_encoder/component/ModelTrainer.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from dataclasses import dataclass

@dataclass
class DataIngestionArtifact:
    merge_df_path:str

@dataclass
class DataValidationArtifact:
    validation_report_file_path:str

@dataclass
class DataCleaningArtifact:
    cleaned_file_path:str



@dataclass
class DataTransformationArtifact:
    transformed_data_file_path:str
    lemmetized_data_file_path:str
    vocab_size:int
    x_train_data_path:str
    y_train_data_path:str
    word_index_path:str
    tokenizer_model_path:str
    lemmetizer_model_path:str


@dataclass
class ModelTrainerArtifact:
    model_file_path:str
    epoch_history_file_path:str
    x_train_file_path:str
    y_train_file_path:str

@dataclass
class ModelEvaluationArtifact:
    pass

@dataclass
class ModelpusherArtifact:
    pusher_model_dir:str
    saved_model_dir:str
    lemmetizer_path:str
    tokenizer_path:str
    wordindex_path:str
    vocab_size:str
    trained_model_path:str
auto_encoder/entity/artifact_entity.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
import os,sys
from auto_encoder.exception import AutoencoderException
from auto_encoder.logger import  logging
from datetime import datetime
from auto_encoder import utils
from typing import List

## Loading all necessary enviroment variable
FILE_NAME = os.getenv('FILE_NAME')
CLEANED_FILE_NAME = os.getenv('CLEANED_FILE_NAME')
VALIDATION_REPORT_FILE_NAME = os.getenv('VALIDATION_REPORT_FILE_NAME')
LEMMATIZED_FILE_NAME = os.getenv('LEMMATIZED_FILE_NAME')
TRANSFORM_FILE_NAME = os.getenv('TRANSFORM_FILE_NAME')
TOKENIZER_NAME = os.getenv('TOKENIZER_NAME')
SEQUENCE_DATA_FILE = os.getenv('SEQUENCE_DATA_FILE')
SEQUENCED_NP_ARRAY = os.getenv('SEQUENCED_NP_ARRAY')
WORD_INDEX_FIL = os.getenv('WORD_INDEX_FIL')
X_TRAIN_DATA_FILE_NAME = os.getenv('X_TRAIN_DATA_FILE_NAME')
Y_TRAIN_DATA_FILE_NAME = os.getenv('Y_TRAIN_DATA_FILE_NAME')
MODEL_NAME = os.getenv('MODEL_NAME')
LEMMETIZER_NAME = os.getenv('LEMMETIZER_NAME')



class TrainingPipelineConfig:
    def __init__(self):
        try:
            self.artifact_dir = os.path.join(os.getcwd(),'artifacts',f"{datetime.now().strftime('%m%d%Y__%H%M%S')}")
        except Exception as e:
            raise AutoencoderException(e,sys)
    

class DataingestionConfig:
    def __init__(self,Training_pipeline_config:TrainingPipelineConfig): 
        try:
            logging.info('configuring all the dataIngestion path variable')
            self.history_file_path:List[str] = utils.get_data_path()
            self.data_ingestion_dir:str = os.path.join(Training_pipeline_config.artifact_dir,'DataIngestion')
            self.data_dir_path:str = os.path.join(self.data_ingestion_dir,'Dataset')
            self.data_file_path:str = os.path.join(self.data_dir_path,FILE_NAME)
            logging.info('successfully configured')
        except Exception as e:
            raise AutoencoderException(e,sys)


class DataCleaningConfig:
    def __init__(self,Training_pipeline_config:TrainingPipelineConfig):
        try:
            self.data_cleainig_dir = os.path.join(Training_pipeline_config.artifact_dir,'datacleaning')
            self.data_dir = os.path.join(self.data_cleainig_dir,'cleaned_file')
            self.cleaned_file_path = os.path.join(self.data_dir,CLEANED_FILE_NAME)
        except Exception as e:
            raise AutoencoderException(e,sys)
        
class DataValidationConfig:
    def __init__(self,Training_pipeline_config:TrainingPipelineConfig):
        self.validatin_Dir = os.path.join(Training_pipeline_config.artifact_dir,'datavalidation')
        self.validation_report_file_path = os.path.join(self.validatin_Dir,'VALIDATION_REPORT_FILE_NAME')


class DataTransformationConfig:
    def __init__(self,Training_pipeline_config:TrainingPipelineConfig):
        try:
            self.transformation_dir = os.path.join(Training_pipeline_config.artifact_dir,'DataTransformation')
            self.transformed_data_file_path = os.path.join(self.transformation_dir,TRANSFORM_FILE_NAME)
            self.lemmetized_dir = os.path.join(self.transformation_dir,'lemmetized')
            self.lemmetized_file_path = os.path.join(self.lemmetized_dir,LEMMATIZED_FILE_NAME)
            self.tokenizer_dir = os.path.join(self.transformation_dir,'Tokenizer')
            self.tokenizer_model_path = os.path.join(self.tokenizer_dir,TOKENIZER_NAME)
            self.sequence_data_dir = os.path.join(self.transformation_dir,'SequenceData')
            self.sequence_data_textfile_path = os.path.join(self.sequence_data_dir,SEQUENCE_DATA_FILE)
            self.sequence_data_npFile = os.path.join(self.sequence_data_dir,SEQUENCED_NP_ARRAY)
            self.word_index_dir = os.path.join(self.transformation_dir,'wordindex')
            self.word_index_file_path = os.path.join(self.word_index_dir,WORD_INDEX_FIL)
            self.Train_data_dir  = os.path.join(self.transformation_dir,'Trained_data')
            self.x_train_file_path = os.path.join(self.Train_data_dir,X_TRAIN_DATA_FILE_NAME)
            self.y_train_file_path = os.path.join(self.Train_data_dir,Y_TRAIN_DATA_FILE_NAME)
            self.lemmetizer_obj_path = os.path.join(self.lemmetized_dir,LEMMETIZER_NAME)
        except Exception as e:
            raise AutoencoderException(e,sys)


class ModelTrainerConfig:
        def __init__(self,Training_pipeline_config:TrainingPipelineConfig):
            try:
                self.model_trainer_dir = os.path.join(Training_pipeline_config.artifact_dir,'Model')
                self.model_file_path = os.path.join(self.model_trainer_dir,MODEL_NAME)
                self.epoch_history_dir = os.path.join(self.model_trainer_dir,"epoch_history")
                self.epoch_history_file_path = os.path.join(self.epoch_history_dir,'epoch_history.pkl')                
            except Exception as e:
                raise AutoencoderException(e,sys)


class ModelPusherConfig:
    def __init__(self,Training_pipeline_config:TrainingPipelineConfig):
        self.model_Pusher_dir  = os.path.join(Training_pipeline_config.artifact_dir,'modelPusher')
        try:
            self.saved_model = os.path.join('saved_models')
            self.pusher_model_dir = os.path.join(self.model_Pusher_dir,'saved_models')
            self.pusher_model_path = os.path.join(self.pusher_model_dir,MODEL_NAME)
            self.pusher_Toknizer_path = os.path.join(self.pusher_model_dir,TOKENIZER_NAME)
            self.pusher_lemmetizer_path = os.path.join(self.pusher_model_dir,LEMMETIZER_NAME)
            self.pusher_wordIndex_path = os.path.join(self.pusher_model_dir,WORD_INDEX_FIL)

        except Exception as e:
            raise AutoencoderException(e,sys)

auto_encoder/entity/config_entity.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
# exception file. to control execution of our program
import sys

class AutoencoderException(Exception):
    def __init__(self, error_message:Exception, error_detail:sys):
        super().__init__(error_message)
        self.error_message = AutoencoderException.error_message_detail(error_message,error_detail=error_detail)
    
    @staticmethod
    def error_message_detail(error:Exception, error_detail:sys)->str:
        error_class,error_mesage,exc_tb = error_detail.exc_info()
        line_number = exc_tb.tb_frame.f_lineno  # to get line number
        file_name = exc_tb.tb_frame.f_code.co_filename
        # formating of error message
        error_message = f" error occured in this {file_name} file, at the line no. [{line_number}], Error message :- [{error_mesage}]"
        return error_message
    
    def __str__(self):
        return AutoencoderException.__name__.__str__()
auto_encoder/exception/__init__.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
# code for generate the logs
# logs will help to us, traverse the execution of our project.
import os
import logging
from datetime import datetime

LOG_DIRECTORY_NAME = "autoencoder_logs"
CURRENT_TIME_STAMP = f"{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}"
FILE_NAME =f"log_{CURRENT_TIME_STAMP}.log"
FILE_PATH = os.path.join(LOG_DIRECTORY_NAME,FILE_NAME)
os.makedirs(LOG_DIRECTORY_NAME,exist_ok=True)
logging.basicConfig(filename=FILE_PATH,
                    filemode="w",
                    level=logging.DEBUG,
                    format="[%(asctime)s]  - %(levelname)s - %(message)s")

# for the testing
if __name__ == "__main__": 
    logging.info(f"logging file testing !")
auto_encoder/logger/__init__.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
from auto_encoder.entity import config_entity,artifact_entity
from auto_encoder.component.DataIngestion import DataIngestion
from auto_encoder.component.DataCleaning import DataCleaning
from auto_encoder.component.DataValidation import DataValidation
from auto_encoder.component.DataTransformation import DataTransformation
from auto_encoder.component.ModelTrainer import ModelTrainer
from auto_encoder.component.ModelEvaluatin import ModelEvaluation
from auto_encoder.component.ModelPusher import ModelPusher
import os,sys
from auto_encoder import utils

def RunTrainingPipeline():
    try:
        TrainingPipelineObj = config_entity.TrainingPipelineConfig()
        DataingestionConfigObj = config_entity.DataingestionConfig(
            Training_pipeline_config=TrainingPipelineObj
        )
        DataIngestionObj = DataIngestion(dataingestionconfig=DataingestionConfigObj)
        DataIngestionArtifact = DataIngestionObj.initiateingestion()


        DataValidationConfig = config_entity.DataValidationConfig(Training_pipeline_config=TrainingPipelineObj)
        datavalidationObj = DataValidation(dataingestionartifact=DataIngestionArtifact,
                       datavalidationconfig=DataValidationConfig)
        dataValidationArtifact = datavalidationObj.initiate_validation()
        

        DataCleaningConfig = config_entity.DataCleaningConfig(Training_pipeline_config=TrainingPipelineObj)
        DataCleaningObj = DataCleaning(datacleaningconfig=DataCleaningConfig,
                                       dataingestionartifact=DataIngestionArtifact,
                                       datavalidationartifact=dataValidationArtifact)
        datacleaningArtifact = DataCleaningObj.initiate_datacleaning()


        TransformatonConfigObj = config_entity.DataTransformationConfig(Training_pipeline_config=TrainingPipelineObj)
        DataTransformationObj = DataTransformation(datacleaningartifact=datacleaningArtifact,
                                                   datatransformationconfig=TransformatonConfigObj)
        DataTransformationArtifact =  DataTransformationObj.InitiateTransformation()


        ModelTrainerconfigObj = config_entity.ModelTrainerConfig(Training_pipeline_config=TrainingPipelineObj)
        ModelTrainerObj = ModelTrainer(transformerartifact=DataTransformationArtifact,
                                       modeltrainerconfig=ModelTrainerconfigObj) 
        model_trainer_artifact = ModelTrainerObj.InitiateModelTrainer()


        ModelEvaluation_obj = ModelEvaluation(modeltrainerartifact=model_trainer_artifact,
                                          modeltrainerconfig=ModelTrainerconfigObj)
        ModelEvaluation_obj.InitiateModelEvaluation()


        modelpusherConfig = config_entity.ModelPusherConfig(Training_pipeline_config=TrainingPipelineObj)
        obj_of_model_pusher = ModelPusher(modelpusherconfig=modelpusherConfig,
                                                transformationArtifact=DataTransformationArtifact,
                                                modeltrainerArtifact=model_trainer_artifact)
        obj_of_modelPusherArtifact = obj_of_model_pusher.initiate_Modelpusher()
        
        return obj_of_modelPusherArtifact                  
    except Exception as e:
        raise AutoencoderException(e,sys)
auto_encoder/pipeline/Training_Pipeline.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
import numpy as  np
import os,sys
from auto_encoder.exception import AutoencoderException
from auto_encoder.logger import logging

################ lemmetizer

def word_suggetions(model,tokenizer,lemmetizer ,text):
    try:
        lemmetized_text = [lemmetizer.lemmatize(word) for word in text]
        sequence = tokenizer.texts_to_sequences([lemmetized_text])
        sequence = np.array(sequence)
        y_pred = np.argmax(model.predict(sequence))

        predicted_word = ""

        for key,value in tokenizer.word_index.items():
            if value == y_pred:
                predicted_word = key
                break
        return predicted_word
    except Exception as e:
        raise AutoencoderException(e,sys)
    
def auto_suggestion(word_suggetions,model,tokenizer,lemmetizer):
    while True:
        text = input('Search Here .. ')

        if text == 'stop':
            print(f"Thanks for the testing !")
            break
        elif len(text.split(' '))<3:
            print(f"please inseart minumum 3 words, your text length is low !!")
            text = input('search again here ...')
        else:
            words = text.split(" ")
            words = words[-3:]
            print(f"i am suggesting to you this word on the bases of these words :- {words}")
            predicted_word = word_suggetions(model=model,tokenizer=tokenizer,lemmetizer=lemmetizer,text=words)
            search_paragraph = text+" "+predicted_word
        print(predicted_word)
        print(f"your search paragraph :- {search_paragraph}")
auto_encoder/pipeline/suggestion.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
import os,sys
from typing import Optional
from auto_encoder.exception import AutoencoderException

MODEL_NAME = os.getenv('MODEL_NAME')
TOKENIZER_NAME = os.getenv('TOKENIZER_NAME')
LEMMETIZER_NAME = os.getenv('LEMMETIZER_NAME')
WORD_INDEX_FIL = os.getenv('WORD_INDEX_FIL')

class ModelResolver:
    # all are default instance variable name
    def __init__(self, model_registry:str = "saved_models",
                lemmetizer:str = 'lemmetizer',
                tokenizer:str = 'tokenizer',
                word_index:str = 'wordIndex',
                model_dir_name = "model"  
                ):  

        self.model_registry = model_registry
        os.makedirs(self.model_registry, exist_ok=True)
        self.lemmetizer = lemmetizer
        self.tokenizer = tokenizer
        self.word_index = word_index
        self.model_dir_name = model_dir_name

# 1
    def get_latest_dir_path(self)->Optional[str]:
        try:
            dir_name = os.listdir(self.model_registry)
            if len(dir_name) == 0:
                return None
            
            dir_name = list(map(int, dir_name))
            latest_dir_name = max(dir_name)
            return os.path.join(self.model_registry, f"{latest_dir_name}")

        except Exception as e:
            raise AutoencoderException(e,sys)
# 2

    def get_latest_model_path(self):
        try:
            latest_dir = self.get_latest_dir_path()
            if latest_dir is None:
                raise Exception(f"Model is Not avaliable")

            return os.path.join(latest_dir, self.model_dir_name, MODEL_NAME)
        except Exception as e:
            raise AutoencoderException(e,sys)
# 3

    def get_latest_tokenizer_path(self):
        try:
            latest_dir = self.get_latest_dir_path()
            if latest_dir is None:
                raise Exception(f"Transform data is not avaliable")

            return os.path.join(latest_dir, self.tokenizer,TOKENIZER_NAME)
        except Exception as e:
            raise AutoencoderException(e,sys)

# 4
    def get_latest_lemmetizer_path(self):
        try:
            latest_dir = self.get_latest_dir_path()
            if latest_dir is None:
                raise Exception(f"Traget encoder data is not avaliable")

            return os.path.join(latest_dir, self.lemmetizer, LEMMETIZER_NAME)
        except Exception as e:
            raise AutoencoderException(e,sys)
        

# 4
    def get_latest_wordindex_path(self):
        try:
            latest_dir = self.get_latest_dir_path()
            if latest_dir is None:
                raise Exception(f"Traget encoder data is not avaliable")

            return os.path.join(latest_dir, self.word_index,WORD_INDEX_FIL )
        except Exception as e:
            raise AutoencoderException(e,sys)
        

    def get_latest_save_dir_path(self)->str:

        try:
            latest_dir = self.get_latest_dir_path()
            if latest_dir ==  None:
                return os.path.join(self.model_registry, f"{0}")
            
            latest_dir_num = int(os.path.basename(self.get_latest_dir_path())) #basename --> last location of this path
            latest_dir  = os.path.join(self.model_registry, f"{latest_dir_num+1}") ## add 1 so that it will increase everytime
            return latest_dir 
        except Exception as e:
            raise AutoencoderException(e,sys)
# 6

    def get_latest_save_model_path(self):
        try:
            latest_dir = self.get_latest_save_dir_path()
            return os.path.join(latest_dir, self.model_dir_name, MODEL_NAME)
        except Exception as e:
            raise AutoencoderException(e,sys)
# 7
    def get_latest_save_tokenizer_path(self):
        try:
            latest_dir = self.get_latest_save_dir_path()
            return os.path.join(latest_dir, self.tokenizer, TOKENIZER_NAME) 
        except Exception as e:
            raise AutoencoderException(e,sys)

# 8
    def get_latest_save_lemmetizer_path(self):
        try:
            latest_dir = self.get_latest_save_dir_path()
            return os.path.join(latest_dir, self.lemmetizer, LEMMETIZER_NAME) 
        except Exception as e:
            raise AutoencoderException(e,sys)
        
# 9

    def get_latest_save_wordindex_path(self):
        try:
            latest_dir = self.get_latest_save_dir_path()
            return os.path.join(latest_dir, self.word_index, WORD_INDEX_FIL) 
        except Exception as e:
            raise AutoencoderException(e,sys)
auto_encoder/predictor.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.logger import logging
from auto_encoder.exception import AutoencoderException
import os,sys
import yaml
import numpy as np
import pickle
from typing import List


def get_data_path()->List[str]:
    try:
        logging.info(f'arranging file path of all chrome history csv files in a list!')
        file_paths = []
        data_dir_path = 'C:/Users/Ranjit Singh/Desktop/Data'
        elemnts = os.listdir(data_dir_path)
        for item in elemnts:
            if item.endswith('.csv'):
                file_path = os.path.join(data_dir_path,item)
                file_paths.append(file_path)
        return file_paths    
    except Exception as e:
        raise AutoencoderException(e,sys)
    

def save_text_file(file_path:str, text:str):
    try:
        logging.info(f"saving the cleaned text file")
        dirNames  = os.path.dirname(file_path)
        os.makedirs(dirNames,exist_ok=True)
        with open(file_path,'w') as file:
            file.write(text)
        logging.info(f"successfully saved the cleaned text file !")
    except Exception as e:
        raise AutoencoderException(e,sys)
    

    
def write_yaml_file(file_path:str,data:dict):
    try:
        file_dir = os.path.dirname(file_path) 
        os.makedirs(file_dir,exist_ok=True)  
        logging.info('writing Validationroport.yaml file')
        with open(file_path,"w") as file_writer:
            yaml.dump(data,file_writer)
    except Exception as e:
        raise AutoencoderException(e,sys)
    
# to save the transform data    
def save_numpy_array_data(file_path:str,array):
    try:
        logging.info('saving transformed data !')
        file_dir = os.path.dirname(file_path)
        os.makedirs(file_dir,exist_ok=True)
        np.save(open(file_path,'wb'),array)
    except Exception as e:
        raise AutoencoderException(e, sys)
    
def load_numpy_array(file_path:str)->np.array:
    try:
        logging.info(f"loading numpy array from {file_path}")
        Data = np.load(open(file_path,'rb'))
        return Data
    except Exception as e:
        raise AutoencoderException(e,sys)
    
def save_object(file_path:str,obj):
    try:
        directory_path = os.path.dirname(file_path)
        os.makedirs(directory_path,exist_ok=True)
        pickle.dump(obj,open(file_path,'wb'))
        logging.info(f"saved object file path :- {file_path}")
    except Exception as e:
        raise AutoencoderException(e,sys)
  

def load_obj(file_path:str):
    try:
        if not os.path.exists(file_path):
            raise Exception(f"file not found error {file_path}")
        else:
            model_obj = pickle.load(open(file_path,'rb'))
            logging.info(f"successfully load the obj :- {file_path}")
        return model_obj
    except Exception as e:
        raise AutoencoderException(e,sys)
auto_encoder/utils.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
import pandas as pd
import numpy as np
import re
import os

## code to clean the chrome history
files = os.listdir('Data')
for i in files:
    if i.endswith('.csv'):
        file_path = os.path.join(f'Data/{i}')
        df = pd.read_csv(file_path)
        title = df[['title']]


        unique_titles = title.drop_duplicates()
        unique_titles.reset_index(inplace=True)
        unique_titles = unique_titles.drop('index',axis=1)
        unique_titles['title'] = unique_titles['title'].astype('str')

        # to get only all those history which has length > 2
        new_titles = []
        for i in range(len(unique_titles)): 
            if  2 < len(unique_titles['title'][i].split()):
                new_titles.append(unique_titles['title'][i])

        df2 = pd.DataFrame({'title':new_titles})

        # to remove all those linc history from my search history
        removed_lincs = []
        for i in range(len(df2)):
            if 0 == df2['title'][i].count('/'):
                removed_lincs.append(df2['title'][i])

        df3 = pd.DataFrame({'title':removed_lincs})


# to clean the search history text
        for i in range(len(df3)):
            lowered = df3['title'][i].lower()
            cleaned_content = re.sub("[^a-zA-Z]"," ",lowered)  
            df3['title'][i] = cleaned_content

        # to concatenate all the history as a paragraph
        text = ""
        for i in range(len(df3)):
            text += f"{df3['title'][i]}"

        # to remove out the extra space from the text
        tex2 = []
        for i in [''.join(i) for i in text.split(' ')]:
            if i == '':
                pass
            else:
                tex2.append(i)

        # to get final data as a paragraph
        final_text = ""
        for i in tex2:
            final_text += f" {i}"
        
        # to save the cleaned history files
        base_file  = os.path.basename(file_path)
        cleaned_file_name = base_file.replace('.csv','.txt')
        # to save final data as a textfile
        with open(f'Data/cleaned_History_files/{cleaned_file_name}','w') as file:
            file.write(final_text)
        print(f"{cleaned_file_name} is successfully in directory")
    else:
        pass
   
chrome_history_cleaner.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from setuptools import find_packages,setup
from typing import List


requirement_filePath = 'requirements.txt'
editable_indiacator = '-e .'
def get_requirements()->list[str]:
    with open(requirement_filePath) as requirementFile:
        requirement_list = requirementFile.readlines()
    requirement_list = [requirement.replace("\n","") for requirement in requirement_list]
    # removing the editable_indicator from the requirements
    if editable_indiacator in requirement_list:
        requirement_list.remove(editable_indiacator)
    return requirement_list


setup(name='auto-encoder',
      version='0.0.1', # every time when you will release your next time your project u wil have to change the version.
      description='auto generate the search text whenever you will search any text in search bar.',
      author='Ranjit Singh',
      author_email='jiradhey402@gmail.com',    # mail must be associated with git
      packages=find_packages(),     # it will find all the packages from your project.
      install_reqires =get_requirements()  # varaible assigned by itself. to give the idea about dependencies.
    )
setup.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
import os,sys
from auto_encoder.exception import AutoencoderException
from auto_encoder.pipeline.suggestion import word_suggetions,auto_suggestion
from auto_encoder.pipeline.Training_Pipeline import RunTrainingPipeline
from auto_encoder.exception import AutoencoderException
from auto_encoder.logger import logging
from auto_encoder import utils
import tensorflow as tf
import os,sys



if __name__=="__main__":
    try:
        logging.info(f"suggestion started !!")
        Pusher_Artifact = RunTrainingPipeline()
        permission = input('Do you want to test the suggestion ? press Y/N  >>  ')
        if permission == 'Y':
            tokenizer_path = Pusher_Artifact.tokenizer_path
            model_path = Pusher_Artifact.trained_model_path
            lemmetizer_path = Pusher_Artifact.lemmetizer_path

            logging.info(f"loading all the models to get suggestions ..")
            tokenizer = utils.load_obj(file_path=tokenizer_path)
            lemmetizer = utils.load_obj(file_path=lemmetizer_path)
            model = tf.keras.models.load_model(model_path)
            logging.info(f"successfully loaded all the model to get the suggestion !!")

            logging.info(f"calling the suggestion functions!!")
            auto_suggestion(word_suggetions=word_suggetions,model=model,tokenizer=tokenizer,lemmetizer=lemmetizer)

        else:
            print(f'Thanks for confirmation !')

    except Exception as e:
        raise AutoencoderException(e,sys)






test.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP
from auto_encoder.pipeline.Training_Pipeline import RunTrainingPipeline
from auto_encoder.exception import AutoencoderException
import os,sys

if __name__=="__main__":
    try:
        Pusher_Artifact = RunTrainingPipeline()
    except Exception as e:
        raise AutoencoderException(e,sys)
train.py
aniruddhakasar/Chrome_SearchBar-autoencoder-NLP