(client, model=None, save=True, save_model=False)
the body of the analyze function in queries.py. Used to generate ROC, confusion matrix etc.
    :param model: is the actual model that you want to analyze for and against
    :param client: is the whole client object :)
    :param save: whether to save into client dictionary
    :param save_model: whether to save as .png
    :return model dictionary for the specified model
the body of the analyze function in queries.py. Used to generate ROC, confusion matrix etc.
    :param model: is the actual model that you want to analyze for and against
    :param client: is the whole client object :)
    :param save: whether to save into client dictionary
    :param save_model: whether to save as .png
    :return model dictionary for the specified model
the body of the analyze function in queries . py . Used to generate ROC confusion matrix etc . : param model : is the actual model that you want to analyze for and against : param client : is the whole client object : ) : param save : whether to save into client dictionary : param save_model : whether to save as . png : return model dictionary for the specified model
def analyze(client, model=None, save=True, save_model=False):
    '''
    the body of the analyze function in queries.py. Used to generate ROC, confusion matrix etc.
    :param model: is the actual model that you want to analyze for and against
    :param client: is the whole client object :)
    :param save: whether to save into client dictionary
    :param save_model: whether to save as .png
    :return model dictionary for the specified model
    '''

    if model is None:
        model = client.latest_model

    if not model in client.models:
        raise NameError('Model name is not valid or has not been created yet.')

    logger("Analyzing {} for further understanding".format(model))

    modeldict = client.models[model]

    if 'test_data' in modeldict:
        logger("Making predictions for test data")
        data = modeldict['test_data']['X']
        real = modeldict['test_data']['y']
        preds = modeldict['model'].predict(data)
        if model == 'text_classification':
            logger("Identifying textual relationships and numerical correspondence")
            preds = np.argmax(preds, axis=-1)
        if model == 'classification_ANN':  # formats labels column
            enc = sklearn.preprocessing.LabelEncoder()
            real = modeldict['interpreter'].inverse_transform(
                real).reshape(1, -1)[0]
            preds = modeldict['interpreter'].inverse_transform(
                preds).reshape(1, -1)[0]
            real = enc.fit_transform(real)
            preds = enc.transform(preds)
            logger("Transforming dataset for plotting module")

    if model == 'k_means_clustering':
        logger("Gathering metrics for display: ")
        inertia = modeldict['model'].inertia_
        centers = modeldict['model'].cluster_centers_
        logger(" ", ("Optimal number of clusters: {}".format(str(len(centers)))))
        logger(
            "->",
            ("Sum of squared distance or inertia: {}".format(
                str(inertia))))
        if save:
            modeldict['n_centers'] = len(centers)
            modeldict['centroids'] = centers
            modeldict['inertia'] = inertia
    elif model == 'regression_ANN':
        logger("Gathering metrics for display: ")
        MSE = sklearn.metrics.mean_squared_error(real, preds)
        MAE = sklearn.metrics.mean_absolute_error(real, preds)
        logger(" ", ("MSE on test set: {}".format(str(MSE))))
        logger("->", ("MAE on test set: {}".format(str(MAE))))
        if save:
            modeldict['MSE'] = MSE
            modeldict['MAE'] = MAE
    # classification models
    elif model in ['svm', 'nearest_neighbor', 'decision_tree', 'classification_ANN', 'text_classification', 'xgboost']:
        logger("->", "Plotting ROC curves and creating confusion matrix...")
        if model in ['svm', 'nearest_neighbor',
                     'decision_tree', 'text_classification', 'xgboost']:
            label_source = modeldict['interpreter']
            labels = []
            for num in np.unique(np.concatenate((real, preds))):
                labels.append(label_source.inverse_transform([[num]])[0])
        else:
            label_source = enc
            labels = enc.classes_

        # create roc plots
        roc = plot_mc_roc(real, preds, label_source)

        # create confusion matrices
        cm = confusion_matrix(real, preds)
        cm = ConfusionMatrixDisplay(
            confusion_matrix=cm,
            display_labels=labels).plot()
        cm = cm.figure_

        logger('Investigating potential issues with calculations')
        logger("Gathering metrics for display: ")
        # get accuracy from modeldict
        if model in ['svm', 'nearest_neighbor',
                     'decision_tree','xgboost']:
            accuracy = modeldict['accuracy']['accuracy_score']
        else:
            accuracy = modeldict['accuracy']['validation_accuracy']
        recall = recall_score(real, preds, average='micro')
        precision = precision_score(real, preds, average='micro')
        f1 = f1_score(real, preds, average='micro')

        logger("->",
               ("Accuracy on test set: {}".format(str((accuracy[-1] if isinstance(accuracy, list) else accuracy)))))
        logger("->", ("Recall on test set: {}".format(str(recall))))
        logger("->", ("Precision on test set: {}".format(str(precision))))
        logger("->", ("F1 Score on test set: {}".format(str(f1))))
        if save:
            if 'plots' not in modeldict:
                modeldict['plots'] = {}
            modeldict['plots']['roc_curve'] = roc
            modeldict['plots']['confusion_matrix'] = cm
            if 'scores' not in modeldict:
                modeldict['scores'] = {}
            modeldict['scores']['recall_score'] = recall
            modeldict['scores']['precision_score'] = precision
            modeldict['scores']['f1_score'] = f1

    elif model == 'convolutional_NN':

        logger("Making predictions for test data")

        data = modeldict['data']['test']
        real = modeldict['res']['real']
        ans = modeldict['res']['ans']
        labels = []
        for k,v in data.class_indices.items():
            labels.append(k)
        
        # create roc plots
        # roc = plot_mc_roc(data, ans)

        # create confusion matrices
        cm = confusion_matrix(real, ans)
        cm = ConfusionMatrixDisplay(
            confusion_matrix=cm,
            display_labels=labels).plot()
        cm = cm.figure_

        logger("Gathering metrics for display: ")

        print(classification_report(real, ans, target_names=labels))
        accuracy = modeldict['accuracy']['validation_accuracy']

        logger("->",
               ("Accuracy on test set: {}".format(str((accuracy[-1] if isinstance(accuracy, list) else accuracy)))))

        logger("->", "Creating confusion matrix...")

        # if save:
        #     if 'plots' not in modeldict:
        #         modeldict['plots'] = {}
        #     # modeldict['plots']['roc_curve'] = roc
        #     modeldict['plots']['confusion_matrix'] = cm
        #     modeldict['plots']['classification_report'] = classification_report(real, ans, target_names=labels)


    else:
        print("further analysis is not supported for {}".format(model))

    client.plots(model=model, save=save_model)

    clearLog()
def analyze ( client , model = None , save = True , save_model = False ) : if model is None : model = client . latest_model if not model in client . models : raise NameError ( 'Model name is not valid or has not been created yet.' ) logger ( "Analyzing {} for further understanding" . format ( model ) ) modeldict = client . models [ model ] if 'test_data' in modeldict : logger ( "Making predictions for test data" ) data = modeldict [ 'test_data' ] [ 'X' ] real = modeldict [ 'test_data' ] [ 'y' ] preds = modeldict [ 'model' ] . predict ( data ) if model == 'text_classification' : logger ( "Identifying textual relationships and numerical correspondence" ) preds = np . argmax ( preds , axis = - 1 ) if model == 'classification_ANN' : # formats labels column enc = sklearn . preprocessing . LabelEncoder ( ) real = modeldict [ 'interpreter' ] . inverse_transform ( real ) . reshape ( 1 , - 1 ) [ 0 ] preds = modeldict [ 'interpreter' ] . inverse_transform ( preds ) . reshape ( 1 , - 1 ) [ 0 ] real = enc . fit_transform ( real ) preds = enc . transform ( preds ) logger ( "Transforming dataset for plotting module" ) if model == 'k_means_clustering' : logger ( "Gathering metrics for display: " ) inertia = modeldict [ 'model' ] . inertia_ centers = modeldict [ 'model' ] . cluster_centers_ logger ( " " , ( "Optimal number of clusters: {}" . format ( str ( len ( centers ) ) ) ) ) logger ( "->" , ( "Sum of squared distance or inertia: {}" . format ( str ( inertia ) ) ) ) if save : modeldict [ 'n_centers' ] = len ( centers ) modeldict [ 'centroids' ] = centers modeldict [ 'inertia' ] = inertia elif model == 'regression_ANN' : logger ( "Gathering metrics for display: " ) MSE = sklearn . metrics . mean_squared_error ( real , preds ) MAE = sklearn . metrics . mean_absolute_error ( real , preds ) logger ( " " , ( "MSE on test set: {}" . format ( str ( MSE ) ) ) ) logger ( "->" , ( "MAE on test set: {}" . format ( str ( MAE ) ) ) ) if save : modeldict [ 'MSE' ] = MSE modeldict [ 'MAE' ] = MAE # classification models elif model in [ 'svm' , 'nearest_neighbor' , 'decision_tree' , 'classification_ANN' , 'text_classification' , 'xgboost' ] : logger ( "->" , "Plotting ROC curves and creating confusion matrix..." ) if model in [ 'svm' , 'nearest_neighbor' , 'decision_tree' , 'text_classification' , 'xgboost' ] : label_source = modeldict [ 'interpreter' ] labels = [ ] for num in np . unique ( np . concatenate ( ( real , preds ) ) ) : labels . append ( label_source . inverse_transform ( [ [ num ] ] ) [ 0 ] ) else : label_source = enc labels = enc . classes_ # create roc plots roc = plot_mc_roc ( real , preds , label_source ) # create confusion matrices cm = confusion_matrix ( real , preds ) cm = ConfusionMatrixDisplay ( confusion_matrix = cm , display_labels = labels ) . plot ( ) cm = cm . figure_ logger ( 'Investigating potential issues with calculations' ) logger ( "Gathering metrics for display: " ) # get accuracy from modeldict if model in [ 'svm' , 'nearest_neighbor' , 'decision_tree' , 'xgboost' ] : accuracy = modeldict [ 'accuracy' ] [ 'accuracy_score' ] else : accuracy = modeldict [ 'accuracy' ] [ 'validation_accuracy' ] recall = recall_score ( real , preds , average = 'micro' ) precision = precision_score ( real , preds , average = 'micro' ) f1 = f1_score ( real , preds , average = 'micro' ) logger ( "->" , ( "Accuracy on test set: {}" . format ( str ( ( accuracy [ - 1 ] if isinstance ( accuracy , list ) else accuracy ) ) ) ) ) logger ( "->" , ( "Recall on test set: {}" . format ( str ( recall ) ) ) ) logger ( "->" , ( "Precision on test set: {}" . format ( str ( precision ) ) ) ) logger ( "->" , ( "F1 Score on test set: {}" . format ( str ( f1 ) ) ) ) if save : if 'plots' not in modeldict : modeldict [ 'plots' ] = { } modeldict [ 'plots' ] [ 'roc_curve' ] = roc modeldict [ 'plots' ] [ 'confusion_matrix' ] = cm if 'scores' not in modeldict : modeldict [ 'scores' ] = { } modeldict [ 'scores' ] [ 'recall_score' ] = recall modeldict [ 'scores' ] [ 'precision_score' ] = precision modeldict [ 'scores' ] [ 'f1_score' ] = f1 elif model == 'convolutional_NN' : logger ( "Making predictions for test data" ) data = modeldict [ 'data' ] [ 'test' ] real = modeldict [ 'res' ] [ 'real' ] ans = modeldict [ 'res' ] [ 'ans' ] labels = [ ] for k , v in data . class_indices . items ( ) : labels . append ( k ) # create roc plots # roc = plot_mc_roc(data, ans) # create confusion matrices cm = confusion_matrix ( real , ans ) cm = ConfusionMatrixDisplay ( confusion_matrix = cm , display_labels = labels ) . plot ( ) cm = cm . figure_ logger ( "Gathering metrics for display: " ) print ( classification_report ( real , ans , target_names = labels ) ) accuracy = modeldict [ 'accuracy' ] [ 'validation_accuracy' ] logger ( "->" , ( "Accuracy on test set: {}" . format ( str ( ( accuracy [ - 1 ] if isinstance ( accuracy , list ) else accuracy ) ) ) ) ) logger ( "->" , "Creating confusion matrix..." ) # if save: #     if 'plots' not in modeldict: #         modeldict['plots'] = {} #     # modeldict['plots']['roc_curve'] = roc #     modeldict['plots']['confusion_matrix'] = cm #     modeldict['plots']['classification_report'] = classification_report(real, ans, target_names=labels) else : print ( "further analysis is not supported for {}" . format ( model ) ) client . plots ( model = model , save = save_model ) clearLog ( )