(self, instruction, drop=None,
                              preprocess=True,
                              label_column=None,
                              test_size=0.2,
                              random_state=49,
                              learning_rate=1e-2,
                              epochs=5,
                              monitor="val_loss",
                              batch_size=32,
                              max_text_length=20,
                              generate_plots=True,
                              save_model=False,
                              save_path=os.getcwd())
return self.models["text_classification"]
function to apply text_classification algorithm for sentiment analysis
    :param many params: used to hyperparametrize the function.
    :return a dictionary object with all of the information for the algorithm.
function to apply text_classification algorithm for sentiment analysis
    :param many params: used to hyperparametrize the function.
    :return a dictionary object with all of the information for the algorithm.
function to apply text_classification algorithm for sentiment analysis : param many params : used to hyperparametrize the function . : return a dictionary object with all of the information for the algorithm .
def text_classification_query(self, instruction, drop=None,
                              preprocess=True,
                              label_column=None,
                              test_size=0.2,
                              random_state=49,
                              learning_rate=1e-2,
                              epochs=5,
                              monitor="val_loss",
                              batch_size=32,
                              max_text_length=20,
                              generate_plots=True,
                              save_model=False,
                              save_path=os.getcwd()):
    """
    function to apply text_classification algorithm for sentiment analysis
    :param many params: used to hyperparametrize the function.
    :return a dictionary object with all of the information for the algorithm.
    """

    if test_size < 0:
        raise Exception("Test size must be a float between 0 and 1")

    if test_size >= 1:
        raise Exception(
            "Test size must be a float between 0 and 1 (a test size greater than or equal to 1 results in no training "
            "data)")

    if epochs < 1:
        raise Exception("Epoch number is less than 1 (model will not be trained)")

    if max_text_length <= 1:
        raise Exception("Max text length should be larger than 1")

    if batch_size < 1:
        raise Exception("Batch size must be equal to or greater than 1")

    if save_model:
        if not os.path.exists(save_path):
            raise Exception("Save path does not exists")

    if test_size == 0:
        testing = False
    else:
        testing = True

    data = DataReader(self.dataset)
    data = data.data_generator()

    if preprocess:
        data.fillna(0, inplace=True)

    if drop is not None:
        data.drop(drop, axis=1, inplace=True)

    if label_column is None:
        label = "label"
    else:
        label = label_column

    X, Y, target = get_target_values(data, instruction, label)
    Y = np.array(Y)
    classes = np.unique(Y)

    logger("->", "Target Column Found: {}".format(target))

    vocab = {}
    if preprocess:
        logger("Preprocessing data")
        X = lemmatize_text(text_clean_up(X.array))
        vocab = X
        X = encode_text(X, X)

    X = np.array(X)
    model = get_keras_text_class(len(vocab), len(classes), learning_rate)
    logger("Building Keras LSTM model dynamically")

    X_train, X_test, y_train, y_test = train_test_split(
        X, Y, test_size=test_size, random_state=random_state)

    X_train = sequence.pad_sequences(X_train, maxlen=max_text_length)
    X_test = sequence.pad_sequences(X_test, maxlen=max_text_length)

    y_vals = np.unique(np.append(y_train, y_test))
    label_mappings = {}
    for i in range(len(y_vals)):
        label_mappings[y_vals[i]] = i
    map_func = np.vectorize(lambda x: label_mappings[x])
    y_train = map_func(y_train)
    y_test = map_func(y_test)

    logger("Training initial model")

    # early stopping callback
    es = EarlyStopping(
        monitor=monitor,
        mode='auto',
        verbose=0,
        patience=5)


    history = model.fit(X_train, y_train, validation_data=(X_test, y_test),
                            batch_size=batch_size,
                            epochs=epochs, callbacks=[es], verbose=0)

    logger("->", "Final training loss: {}".format(history.history["loss"][len(history.history["loss"]) - 1]))
    if testing:
        logger("->",
               "Final validation loss: {}".format(history.history["val_loss"][len(history.history["val_loss"]) - 1]))
        logger("->", "Final validation accuracy: {}".format(
            history.history["val_accuracy"][len(history.history["val_accuracy"]) - 1]))
        losses = {'training_loss': history.history['loss'], 'val_loss': history.history['val_loss']}
        accuracy = {'training_accuracy': history.history['accuracy'],
                    'validation_accuracy': history.history['val_accuracy']}
    else:
        logger("->", "Final validation loss: {}".format("0, No validation done"))
        losses = {'training_loss': history.history['loss']}
        accuracy = {'training_accuracy': history.history['accuracy']}

    plots = {}
    if generate_plots:
        # generates appropriate classification plots by feeding all
        # information
        logger("Generating plots")
        plots = generate_classification_plots(history)

    if save_model:
        save(model, save_model, save_path=save_path)

    logger("Storing information in client object under key 'text_classification'")
    # storing values the model dictionary

    self.models["text_classification"] = {"model": model,
                                          "classes": classes,
                                          "plots": plots,
                                          "target": Y,
                                          "vocabulary": vocab,
                                          "interpreter": label_mappings,
                                          # "max_text_length": max_text_length,
                                          'test_data': {'X': X_test, 'y': y_test},
                                          'losses': losses,
                                          'accuracy': accuracy}
    clearLog()
    return self.models["text_classification"]
def text_classification_query ( self , instruction , drop = None , preprocess = True , label_column = None , test_size = 0.2 , random_state = 49 , learning_rate = 1e-2 , epochs = 5 , monitor = "val_loss" , batch_size = 32 , max_text_length = 20 , generate_plots = True , save_model = False , save_path = os . getcwd ( ) ) : if test_size < 0 : raise Exception ( "Test size must be a float between 0 and 1" ) if test_size >= 1 : raise Exception ( "Test size must be a float between 0 and 1 (a test size greater than or equal to 1 results in no training " "data)" ) if epochs < 1 : raise Exception ( "Epoch number is less than 1 (model will not be trained)" ) if max_text_length <= 1 : raise Exception ( "Max text length should be larger than 1" ) if batch_size < 1 : raise Exception ( "Batch size must be equal to or greater than 1" ) if save_model : if not os . path . exists ( save_path ) : raise Exception ( "Save path does not exists" ) if test_size == 0 : testing = False else : testing = True data = DataReader ( self . dataset ) data = data . data_generator ( ) if preprocess : data . fillna ( 0 , inplace = True ) if drop is not None : data . drop ( drop , axis = 1 , inplace = True ) if label_column is None : label = "label" else : label = label_column X , Y , target = get_target_values ( data , instruction , label ) Y = np . array ( Y ) classes = np . unique ( Y ) logger ( "->" , "Target Column Found: {}" . format ( target ) ) vocab = { } if preprocess : logger ( "Preprocessing data" ) X = lemmatize_text ( text_clean_up ( X . array ) ) vocab = X X = encode_text ( X , X ) X = np . array ( X ) model = get_keras_text_class ( len ( vocab ) , len ( classes ) , learning_rate ) logger ( "Building Keras LSTM model dynamically" ) X_train , X_test , y_train , y_test = train_test_split ( X , Y , test_size = test_size , random_state = random_state ) X_train = sequence . pad_sequences ( X_train , maxlen = max_text_length ) X_test = sequence . pad_sequences ( X_test , maxlen = max_text_length ) y_vals = np . unique ( np . append ( y_train , y_test ) ) label_mappings = { } for i in range ( len ( y_vals ) ) : label_mappings [ y_vals [ i ] ] = i map_func = np . vectorize ( lambda x : label_mappings [ x ] ) y_train = map_func ( y_train ) y_test = map_func ( y_test ) logger ( "Training initial model" ) # early stopping callback es = EarlyStopping ( monitor = monitor , mode = 'auto' , verbose = 0 , patience = 5 ) history = model . fit ( X_train , y_train , validation_data = ( X_test , y_test ) , batch_size = batch_size , epochs = epochs , callbacks = [ es ] , verbose = 0 ) logger ( "->" , "Final training loss: {}" . format ( history . history [ "loss" ] [ len ( history . history [ "loss" ] ) - 1 ] ) ) if testing : logger ( "->" , "Final validation loss: {}" . format ( history . history [ "val_loss" ] [ len ( history . history [ "val_loss" ] ) - 1 ] ) ) logger ( "->" , "Final validation accuracy: {}" . format ( history . history [ "val_accuracy" ] [ len ( history . history [ "val_accuracy" ] ) - 1 ] ) ) losses = { 'training_loss' : history . history [ 'loss' ] , 'val_loss' : history . history [ 'val_loss' ] } accuracy = { 'training_accuracy' : history . history [ 'accuracy' ] , 'validation_accuracy' : history . history [ 'val_accuracy' ] } else : logger ( "->" , "Final validation loss: {}" . format ( "0, No validation done" ) ) losses = { 'training_loss' : history . history [ 'loss' ] } accuracy = { 'training_accuracy' : history . history [ 'accuracy' ] } plots = { } if generate_plots : # generates appropriate classification plots by feeding all # information logger ( "Generating plots" ) plots = generate_classification_plots ( history ) if save_model : save ( model , save_model , save_path = save_path ) logger ( "Storing information in client object under key 'text_classification'" ) # storing values the model dictionary self . models [ "text_classification" ] = { "model" : model , "classes" : classes , "plots" : plots , "target" : Y , "vocabulary" : vocab , "interpreter" : label_mappings , # "max_text_length": max_text_length, 'test_data' : { 'X' : X_test , 'y' : y_test } , 'losses' : losses , 'accuracy' : accuracy } clearLog ( ) return self . models [ "text_classification" ]