(self, instruction, label_column=None,
                        drop=None,
                        epochs=10,
                        preprocess=True,
                        random_state=49,
                        test_size=0.2,
                        top_k=5000,
                        batch_size=32,
                        buffer_size=1000,
                        embedding_dim=256,
                        units=512,
                        gpu=False,
                        generate_plots=True,
                        save_model_decoder=False,
                        save_path_decoder=os.getcwd(),
                        save_model_encoder=False,
                        save_path_encoder=os.getcwd())
return self.models["image_caption"]
function to apply predictive algorithm for image_caption generation
    :param many params: used to hyperparametrize the function.
    :return a dictionary object with all of the information for the algorithm.
function to apply predictive algorithm for image_caption generation
    :param many params: used to hyperparametrize the function.
    :return a dictionary object with all of the information for the algorithm.
function to apply predictive algorithm for image_caption generation : param many params : used to hyperparametrize the function . : return a dictionary object with all of the information for the algorithm .
def image_caption_query(self, instruction, label_column=None,
                        drop=None,
                        epochs=10,
                        preprocess=True,
                        random_state=49,
                        test_size=0.2,
                        top_k=5000,
                        batch_size=32,
                        buffer_size=1000,
                        embedding_dim=256,
                        units=512,
                        gpu=False,
                        generate_plots=True,
                        save_model_decoder=False,
                        save_path_decoder=os.getcwd(),
                        save_model_encoder=False,
                        save_path_encoder=os.getcwd()):
    '''
    function to apply predictive algorithm for image_caption generation
    :param many params: used to hyperparametrize the function.
    :return a dictionary object with all of the information for the algorithm.
    '''

    if test_size < 0:
        raise Exception("Test size must be a float between 0 and 1")

    if test_size >= 1:
        raise Exception(
            "Test size must be a float between 0 and 1 (a test size greater than or equal to 1 results in no training "
            "data)")

    if top_k < 1:
        raise Exception("Top_k value must be equal to or greater than 1")

    if batch_size < 1:
        raise Exception("Batch size must be equal to or greater than 1")

    if buffer_size < 1:
        raise Exception("Buffer size must be equal to or greater than 1")

    if embedding_dim < 1:
        raise Exception("Embedding dimension must be equal to or greater than 1")

    if units < 1:
        raise Exception("Units must be equal to or greater than 1")

    if epochs < 1:
        raise Exception("Epoch number is less than 1 (model will not be trained)")

    if save_model_decoder:
        if not os.path.exists(save_path_decoder):
            raise Exception("Decoder save path does not exists")

    if save_model_encoder:
        if not os.path.exists(save_path_encoder):
            raise Exception("Encoder save path does not exists")

    if test_size == 0:
        testing = False
    else:
        testing = True

    if gpu:
        if tf.test.gpu_device_name():
            print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
        else:
            raise Exception("Please install GPU version of Tensorflow")

        device = '/device:GPU:0'
    else:
        device = '/device:CPU:0'

    np.random.seed(random_state)
    tf.random.set_seed(random_state)

    data = DataReader(self.dataset)
    df = data.data_generator()

    if preprocess:
        df.fillna(0, inplace=True)
    if drop is not None:
        df.drop(drop, axis=1, inplace=True)

    logger("Preprocessing data")

    train_captions = []
    img_name_vector = []

    if label_column is None:
        label = instruction
    else:
        label = label_column

    x = get_path_column(df)
    y = get_similar_column(get_value_instruction(label), df)
    logger("->", "Target Column Found: {}".format(y))

    for row in df.iterrows():
        if preprocess:
            caption = '<start> ' + row[1][y] + ' <end>'
        image_id = row[1][x]
        image_path = image_id

        img_name_vector.append(image_path)
        train_captions.append(caption)
    with NoStdStreams():
        image_model = tf.keras.applications.InceptionV3(include_top=False,
                                                        weights='imagenet')
    new_input = image_model.input
    hidden_layer = image_model.layers[-1].output
    logger("Extracting features from model")
    image_features_extract_model = tf.keras.Model(new_input, hidden_layer)

    image_dataset = tf.data.Dataset.from_tensor_slices(
        sorted(set(img_name_vector)))
    image_dataset = image_dataset.map(
        load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(16)

    for img, path in image_dataset:
        batch_features = image_features_extract_model(img)
        batch_features = tf.reshape(
            batch_features, (batch_features.shape[0], -1, batch_features.shape[3]))

        for bf, p in zip(batch_features, path):
            path_of_feature = p.numpy().decode("utf-8")
            np.save(path_of_feature, bf.numpy())
    logger("->", "Tokenizing top {} words".format(top_k))
    tokenizer = tf.keras.preprocessing.text.Tokenizer(
        num_words=top_k, oov_token="<unk>", filters='!"#$%&()*+.,-/:;=?@[\]^_`{|}~ ')
    tokenizer.fit_on_texts(train_captions)
    tokenizer.word_index['<pad>'] = 0
    tokenizer.index_word[0] = '<pad>'
    train_seqs = tokenizer.texts_to_sequences(train_captions)
    cap_vector = tf.keras.preprocessing.sequence.pad_sequences(
        train_seqs, padding='post')

    vocab_size = top_k + 1
    # num_steps = len(img_name_vector) // batch_size

    if testing:
        img_name_train, img_name_val, cap_train, cap_val = train_test_split(
            img_name_vector, cap_vector, test_size=test_size, random_state=0)
    else:
        img_name_train = img_name_vector
        cap_train = cap_vector

    dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))

    dataset = dataset.map(lambda item1, item2: tf.numpy_function(
        map_func, [item1, item2], [tf.float32, tf.int32]),
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)

    # Shuffle and batch
    logger("Shuffling dataset")
    dataset = dataset.shuffle(buffer_size).batch(batch_size)
    dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)

    if testing:
        dataset_val = tf.data.Dataset.from_tensor_slices((img_name_val, cap_val))

        dataset_val = dataset_val.map(lambda item1, item2: tf.numpy_function(
            map_func, [item1, item2], [tf.float32, tf.int32]),
                                      num_parallel_calls=tf.data.experimental.AUTOTUNE)

        # Shuffle and batch
        dataset_val = dataset_val.shuffle(buffer_size).batch(batch_size)
        dataset_val = dataset_val.prefetch(
            buffer_size=tf.data.experimental.AUTOTUNE)

    logger("Establishing encoder decoder framework")
    encoder = CNN_Encoder(embedding_dim)
    decoder = RNN_Decoder(embedding_dim, units, vocab_size)

    optimizer = tf.keras.optimizers.Adam()
    loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
        from_logits=True, reduction='none')

    def loss_function(real, pred):
        mask = tf.math.logical_not(tf.math.equal(real, 0))
        loss_ = loss_object(real, pred)

        mask = tf.cast(mask, dtype=loss_.dtype)
        loss_ *= mask

        return tf.reduce_mean(loss_)

    @tf.function
    def train_step(img_tensor, target):
        with tf.device(device):
            loss = 0

            # initializing the hidden state for each batch
            # because the captions are not related from image to image
            hidden = decoder.reset_state(batch_size=target.shape[0])

            dec_input = tf.expand_dims(
                [tokenizer.word_index['<start>']] * target.shape[0], 1)

            with tf.GradientTape() as tape:
                features = encoder(img_tensor)

                for i in range(1, target.shape[1]):
                    # passing the features through the decoder
                    predictions, hidden, _ = decoder(dec_input, features, hidden)

                    loss += loss_function(target[:, i], predictions)

                    # using teacher forcing
                    dec_input = tf.expand_dims(target[:, i], 1)

            total_loss = (loss / int(target.shape[1]))

            trainable_variables = encoder.trainable_variables + decoder.trainable_variables

            gradients = tape.gradient(loss, trainable_variables)

            optimizer.apply_gradients(zip(gradients, trainable_variables))

            return loss, total_loss

    @tf.function
    def val_step(img_tensor, target):
        with tf.device(device):
            loss = 0

            # initializing the hidden state for each batch
            # because the captions are not related from image to image
            hidden = decoder.reset_state(batch_size=target.shape[0])

            dec_input = tf.expand_dims(
                [tokenizer.word_index['<start>']] * target.shape[0], 1)

            with tf.GradientTape() as tape:
                features = encoder(img_tensor)

                for i in range(1, target.shape[1]):
                    # passing the features through the decoder
                    predictions, hidden, _ = decoder(dec_input, features, hidden)

                    loss += loss_function(target[:, i], predictions)

                    # using teacher forcing
                    dec_input = tf.expand_dims(target[:, i], 1)

            total_loss = (loss / int(target.shape[1]))
            return total_loss

    logger("Training model...")
    with tf.device(device):
        loss_plot_train = []
        loss_plot_val = []
        for epoch in range(epochs):
            total_loss = 0
            total_loss_val = 0

            for (batch, (img_tensor, target)) in enumerate(dataset):
                batch_loss, t_loss = train_step(img_tensor, target)
                total_loss += t_loss

            loss_plot_train.append(total_loss.numpy())

            if testing:
                for (batch, (img_tensor, target)) in enumerate(dataset_val):
                    batch_loss, t_loss = train_step(img_tensor, target)
                    total_loss_val += t_loss

                loss_plot_val.append(total_loss_val.numpy())

    dir_name = os.path.dirname(img_name_vector[0])
    files = os.listdir(dir_name)

    for item in files:
        if item.endswith(".npy"):
            os.remove(os.path.join(dir_name, item))

    plots = {}
    if generate_plots:
        logger("Generating plots")
        plots.update({"loss": libra.plotting.nonkeras_generate_plots.plot_loss(loss_plot_train, loss_plot_val)})

    logger("->", "Final training loss: {}".format(str(total_loss.numpy())))
    total_loss = total_loss.numpy()
    if testing:
        total_loss_val = total_loss_val.numpy()
        total_loss_val_str = str(total_loss_val)
    else:
        total_loss_val = 0
        total_loss_val_str = str("0, No validation done")

    logger("->", "Final validation loss: {}".format(total_loss_val_str))

    if save_model_decoder:
        logger("Saving decoder checkpoint...")
        encoder.save_weights(save_path_decoder + "decoderImgCap.ckpt")

    if save_model_encoder:
        logger("Saving encoder checkpoint...")
        encoder.save_weights(save_path_encoder + "encoderImgCap.ckpt")

    logger("Storing information in client object under key 'image_caption'")

    self.models["image_caption"] = {
        "decoder": decoder,
        "encoder": encoder,
        "tokenizer": tokenizer,
        "feature_extraction": image_features_extract_model,
        "plots": plots,
        'losses': {
            'Training loss': total_loss,
            'Validation loss': total_loss_val
        }
    }
    clearLog()
    return self.models["image_caption"]
def image_caption_query ( self , instruction , label_column = None , drop = None , epochs = 10 , preprocess = True , random_state = 49 , test_size = 0.2 , top_k = 5000 , batch_size = 32 , buffer_size = 1000 , embedding_dim = 256 , units = 512 , gpu = False , generate_plots = True , save_model_decoder = False , save_path_decoder = os . getcwd ( ) , save_model_encoder = False , save_path_encoder = os . getcwd ( ) ) : if test_size < 0 : raise Exception ( "Test size must be a float between 0 and 1" ) if test_size >= 1 : raise Exception ( "Test size must be a float between 0 and 1 (a test size greater than or equal to 1 results in no training " "data)" ) if top_k < 1 : raise Exception ( "Top_k value must be equal to or greater than 1" ) if batch_size < 1 : raise Exception ( "Batch size must be equal to or greater than 1" ) if buffer_size < 1 : raise Exception ( "Buffer size must be equal to or greater than 1" ) if embedding_dim < 1 : raise Exception ( "Embedding dimension must be equal to or greater than 1" ) if units < 1 : raise Exception ( "Units must be equal to or greater than 1" ) if epochs < 1 : raise Exception ( "Epoch number is less than 1 (model will not be trained)" ) if save_model_decoder : if not os . path . exists ( save_path_decoder ) : raise Exception ( "Decoder save path does not exists" ) if save_model_encoder : if not os . path . exists ( save_path_encoder ) : raise Exception ( "Encoder save path does not exists" ) if test_size == 0 : testing = False else : testing = True if gpu : if tf . test . gpu_device_name ( ) : print ( 'Default GPU Device: {}' . format ( tf . test . gpu_device_name ( ) ) ) else : raise Exception ( "Please install GPU version of Tensorflow" ) device = '/device:GPU:0' else : device = '/device:CPU:0' np . random . seed ( random_state ) tf . random . set_seed ( random_state ) data = DataReader ( self . dataset ) df = data . data_generator ( ) if preprocess : df . fillna ( 0 , inplace = True ) if drop is not None : df . drop ( drop , axis = 1 , inplace = True ) logger ( "Preprocessing data" ) train_captions = [ ] img_name_vector = [ ] if label_column is None : label = instruction else : label = label_column x = get_path_column ( df ) y = get_similar_column ( get_value_instruction ( label ) , df ) logger ( "->" , "Target Column Found: {}" . format ( y ) ) for row in df . iterrows ( ) : if preprocess : caption = '<start> ' + row [ 1 ] [ y ] + ' <end>' image_id = row [ 1 ] [ x ] image_path = image_id img_name_vector . append ( image_path ) train_captions . append ( caption ) with NoStdStreams ( ) : image_model = tf . keras . applications . InceptionV3 ( include_top = False , weights = 'imagenet' ) new_input = image_model . input hidden_layer = image_model . layers [ - 1 ] . output logger ( "Extracting features from model" ) image_features_extract_model = tf . keras . Model ( new_input , hidden_layer ) image_dataset = tf . data . Dataset . from_tensor_slices ( sorted ( set ( img_name_vector ) ) ) image_dataset = image_dataset . map ( load_image , num_parallel_calls = tf . data . experimental . AUTOTUNE ) . batch ( 16 ) for img , path in image_dataset : batch_features = image_features_extract_model ( img ) batch_features = tf . reshape ( batch_features , ( batch_features . shape [ 0 ] , - 1 , batch_features . shape [ 3 ] ) ) for bf , p in zip ( batch_features , path ) : path_of_feature = p . numpy ( ) . decode ( "utf-8" ) np . save ( path_of_feature , bf . numpy ( ) ) logger ( "->" , "Tokenizing top {} words" . format ( top_k ) ) tokenizer = tf . keras . preprocessing . text . Tokenizer ( num_words = top_k , oov_token = "<unk>" , filters = '!"#$%&()*+.,-/:;=?@[\]^_`{|}~ ' ) tokenizer . fit_on_texts ( train_captions ) tokenizer . word_index [ '<pad>' ] = 0 tokenizer . index_word [ 0 ] = '<pad>' train_seqs = tokenizer . texts_to_sequences ( train_captions ) cap_vector = tf . keras . preprocessing . sequence . pad_sequences ( train_seqs , padding = 'post' ) vocab_size = top_k + 1 # num_steps = len(img_name_vector) // batch_size if testing : img_name_train , img_name_val , cap_train , cap_val = train_test_split ( img_name_vector , cap_vector , test_size = test_size , random_state = 0 ) else : img_name_train = img_name_vector cap_train = cap_vector dataset = tf . data . Dataset . from_tensor_slices ( ( img_name_train , cap_train ) ) dataset = dataset . map ( lambda item1 , item2 : tf . numpy_function ( map_func , [ item1 , item2 ] , [ tf . float32 , tf . int32 ] ) , num_parallel_calls = tf . data . experimental . AUTOTUNE ) # Shuffle and batch logger ( "Shuffling dataset" ) dataset = dataset . shuffle ( buffer_size ) . batch ( batch_size ) dataset = dataset . prefetch ( buffer_size = tf . data . experimental . AUTOTUNE ) if testing : dataset_val = tf . data . Dataset . from_tensor_slices ( ( img_name_val , cap_val ) ) dataset_val = dataset_val . map ( lambda item1 , item2 : tf . numpy_function ( map_func , [ item1 , item2 ] , [ tf . float32 , tf . int32 ] ) , num_parallel_calls = tf . data . experimental . AUTOTUNE ) # Shuffle and batch dataset_val = dataset_val . shuffle ( buffer_size ) . batch ( batch_size ) dataset_val = dataset_val . prefetch ( buffer_size = tf . data . experimental . AUTOTUNE ) logger ( "Establishing encoder decoder framework" ) encoder = CNN_Encoder ( embedding_dim ) decoder = RNN_Decoder ( embedding_dim , units , vocab_size ) optimizer = tf . keras . optimizers . Adam ( ) loss_object = tf . keras . losses . SparseCategoricalCrossentropy ( from_logits = True , reduction = 'none' ) def loss_function ( real , pred ) : mask = tf . math . logical_not ( tf . math . equal ( real , 0 ) ) loss_ = loss_object ( real , pred ) mask = tf . cast ( mask , dtype = loss_ . dtype ) loss_ *= mask return tf . reduce_mean ( loss_ ) @ tf . function def train_step ( img_tensor , target ) : with tf . device ( device ) : loss = 0 # initializing the hidden state for each batch # because the captions are not related from image to image hidden = decoder . reset_state ( batch_size = target . shape [ 0 ] ) dec_input = tf . expand_dims ( [ tokenizer . word_index [ '<start>' ] ] * target . shape [ 0 ] , 1 ) with tf . GradientTape ( ) as tape : features = encoder ( img_tensor ) for i in range ( 1 , target . shape [ 1 ] ) : # passing the features through the decoder predictions , hidden , _ = decoder ( dec_input , features , hidden ) loss += loss_function ( target [ : , i ] , predictions ) # using teacher forcing dec_input = tf . expand_dims ( target [ : , i ] , 1 ) total_loss = ( loss / int ( target . shape [ 1 ] ) ) trainable_variables = encoder . trainable_variables + decoder . trainable_variables gradients = tape . gradient ( loss , trainable_variables ) optimizer . apply_gradients ( zip ( gradients , trainable_variables ) ) return loss , total_loss @ tf . function def val_step ( img_tensor , target ) : with tf . device ( device ) : loss = 0 # initializing the hidden state for each batch # because the captions are not related from image to image hidden = decoder . reset_state ( batch_size = target . shape [ 0 ] ) dec_input = tf . expand_dims ( [ tokenizer . word_index [ '<start>' ] ] * target . shape [ 0 ] , 1 ) with tf . GradientTape ( ) as tape : features = encoder ( img_tensor ) for i in range ( 1 , target . shape [ 1 ] ) : # passing the features through the decoder predictions , hidden , _ = decoder ( dec_input , features , hidden ) loss += loss_function ( target [ : , i ] , predictions ) # using teacher forcing dec_input = tf . expand_dims ( target [ : , i ] , 1 ) total_loss = ( loss / int ( target . shape [ 1 ] ) ) return total_loss logger ( "Training model..." ) with tf . device ( device ) : loss_plot_train = [ ] loss_plot_val = [ ] for epoch in range ( epochs ) : total_loss = 0 total_loss_val = 0 for ( batch , ( img_tensor , target ) ) in enumerate ( dataset ) : batch_loss , t_loss = train_step ( img_tensor , target ) total_loss += t_loss loss_plot_train . append ( total_loss . numpy ( ) ) if testing : for ( batch , ( img_tensor , target ) ) in enumerate ( dataset_val ) : batch_loss , t_loss = train_step ( img_tensor , target ) total_loss_val += t_loss loss_plot_val . append ( total_loss_val . numpy ( ) ) dir_name = os . path . dirname ( img_name_vector [ 0 ] ) files = os . listdir ( dir_name ) for item in files : if item . endswith ( ".npy" ) : os . remove ( os . path . join ( dir_name , item ) ) plots = { } if generate_plots : logger ( "Generating plots" ) plots . update ( { "loss" : libra . plotting . nonkeras_generate_plots . plot_loss ( loss_plot_train , loss_plot_val ) } ) logger ( "->" , "Final training loss: {}" . format ( str ( total_loss . numpy ( ) ) ) ) total_loss = total_loss . numpy ( ) if testing : total_loss_val = total_loss_val . numpy ( ) total_loss_val_str = str ( total_loss_val ) else : total_loss_val = 0 total_loss_val_str = str ( "0, No validation done" ) logger ( "->" , "Final validation loss: {}" . format ( total_loss_val_str ) ) if save_model_decoder : logger ( "Saving decoder checkpoint..." ) encoder . save_weights ( save_path_decoder + "decoderImgCap.ckpt" ) if save_model_encoder : logger ( "Saving encoder checkpoint..." ) encoder . save_weights ( save_path_encoder + "encoderImgCap.ckpt" ) logger ( "Storing information in client object under key 'image_caption'" ) self . models [ "image_caption" ] = { "decoder" : decoder , "encoder" : encoder , "tokenizer" : tokenizer , "feature_extraction" : image_features_extract_model , "plots" : plots , 'losses' : { 'Training loss' : total_loss , 'Validation loss' : total_loss_val } } clearLog ( ) return self . models [ "image_caption" ]