(self, fit_params={})
Perform K-fold cross validation on the model for the dataset provided in the request.
Perform K-fold cross validation on the model for the dataset provided in the request.
Perform K - fold cross validation on the model for the dataset provided in the request .
def _cross_validate(self, fit_params={}):
        """
        Perform K-fold cross validation on the model for the dataset provided in the request.
        """

        # Flatten the true labels for the training data
        y_train = self.y_train.values if self.y_train.shape[1] > 1 else self.y_train.values.ravel()

        if self.model.estimator_type == "classifier":

            # Get unique labels for classification
            labels = np.unique(y_train)

            # Set up a dictionary for the scoring metrics
            scoring = {'accuracy':'accuracy'}

            # Prepare arguments for the scorers
            metric_args = self.model.metric_args
            
            if 'average' in metric_args and metric_args['average'] is not None:
                # If the score is being averaged over classes a single scorer per metric is sufficient
                scoring['precision'] = metrics.make_scorer(metrics.precision_score, **metric_args)
                scoring['recall'] = metrics.make_scorer(metrics.recall_score, **metric_args)
                scoring['fscore'] = metrics.make_scorer(metrics.f1_score, **metric_args)

                output_format = "clf_overall"
            else:
                # If there is no averaging we will need multiple scorers; one for each class
                for label in labels:
                    metric_args['pos_label'] = label
                    metric_args['labels'] = [label]
                    scoring['precision_'+str(label)] = metrics.make_scorer(metrics.precision_score, **metric_args)
                    scoring['recall_'+str(label)] = metrics.make_scorer(metrics.recall_score, **metric_args)
                    scoring['fscore_'+str(label)] = metrics.make_scorer(metrics.f1_score, **metric_args)
                
                output_format = "clf_classes"

        elif self.model.estimator_type == "regressor":
            scoring = ['r2', 'neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_median_absolute_error', 'explained_variance']
        
        # Perform cross validation using the training data and the model pipeline
        scores = cross_validate(self.model.pipe, self.X_train, y_train, scoring=scoring, cv=self.model.cv, fit_params=fit_params, return_train_score=False)

        # Prepare the metrics data frame according to the output format
        if self.model.estimator_type == "classifier":           
            # Get cross validation predictions for the confusion matrix
            y_pred = cross_val_predict(self.model.pipe, self.X_train, y_train, cv=self.model.cv, fit_params=fit_params)

            # Prepare the confusion matrix and add it to the model
            self._prep_confusion_matrix(y_train, y_pred, labels)

            # Create an empty data frame to set the structure
            metrics_df = pd.DataFrame(columns=["class", "accuracy", "accuracy_std", "precision", "precision_std", "recall",\
            "recall_std", "fscore", "fscore_std"])

            if output_format == "clf_overall":               
                # Add the overall metrics to the data frame
                metrics_df.loc[0] = ["overall", np.average(scores["test_accuracy"]), np.std(scores["test_accuracy"]),\
                np.average(scores["test_precision"]), np.std(scores["test_precision"]),\
                np.average(scores["test_recall"]), np.std(scores["test_recall"]),\
                np.average(scores["test_fscore"]), np.std(scores["test_fscore"])]

            elif output_format == "clf_classes":
                # Add accuracy which is calculated at an overall level
                metrics_df.loc[0] = ["overall", np.average(scores["test_accuracy"]), np.std(scores["test_accuracy"]),\
                np.NaN, np.NaN, np.NaN, np.NaN, np.NaN, np.NaN]

                # Add the metrics for each class to the data frame
                for i, label in enumerate(labels):
                    metrics_df.loc[i+1] = [label, np.NaN, np.NaN, np.average(scores["test_precision_"+str(label)]),\
                    np.std(scores["test_precision_"+str(label)]), np.average(scores["test_recall_"+str(label)]),\
                    np.std(scores["test_recall_"+str(label)]), np.average(scores["test_fscore_"+str(label)]),\
                    np.std(scores["test_fscore_"+str(label)])]
            
            # Finalize the structure of the result DataFrame
            metrics_df.loc[:,"model_name"] = self.model.name
            metrics_df = metrics_df.loc[:,["model_name", "class", "accuracy", "accuracy_std", "precision", "precision_std", "recall",\
            "recall_std", "fscore", "fscore_std"]]

            # Add the score to the model
            self.model.score =  metrics_df["accuracy"].values[0]

        elif self.model.estimator_type == "regressor":
            # Create an empty data frame to set the structure
            metrics_df = pd.DataFrame(columns=["r2_score", "r2_score_std", "mean_squared_error", "mean_squared_error_std",\
            "mean_absolute_error", "mean_absolute_error_std", "median_absolute_error", "median_absolute_error_std",\
            "explained_variance_score", "explained_variance_score_std"])
            
            # Add the overall metrics to the data frame
            metrics_df.loc[0] = [np.average(scores["test_r2"]), np.std(scores["test_r2"]),\
            np.average(scores["test_neg_mean_squared_error"]), np.std(scores["test_neg_mean_squared_error"]),\
            np.average(scores["test_neg_mean_absolute_error"]), np.std(scores["test_neg_mean_absolute_error"]),\
            np.average(scores["test_neg_median_absolute_error"]), np.std(scores["test_neg_median_absolute_error"]),\
            np.average(scores["test_explained_variance"]), np.std(scores["test_explained_variance"])]
        
            # Finalize the structure of the result DataFrame
            metrics_df.loc[:,"model_name"] = self.model.name
            metrics_df = metrics_df.loc[:,["model_name", "r2_score", "r2_score_std", "mean_squared_error", "mean_squared_error_std",\
            "mean_absolute_error", "mean_absolute_error_std", "median_absolute_error", "median_absolute_error_std",\
            "explained_variance_score", "explained_variance_score_std"]]

            # Add the score to the model
            self.model.score =  metrics_df["r2_score"].values[0]

        # Save the metrics_df to the model
        self.model.metrics_df = metrics_df
def _cross_validate ( self , fit_params = { } ) : # Flatten the true labels for the training data y_train = self . y_train . values if self . y_train . shape [ 1 ] > 1 else self . y_train . values . ravel ( ) if self . model . estimator_type == "classifier" : # Get unique labels for classification labels = np . unique ( y_train ) # Set up a dictionary for the scoring metrics scoring = { 'accuracy' : 'accuracy' } # Prepare arguments for the scorers metric_args = self . model . metric_args if 'average' in metric_args and metric_args [ 'average' ] is not None : # If the score is being averaged over classes a single scorer per metric is sufficient scoring [ 'precision' ] = metrics . make_scorer ( metrics . precision_score , * * metric_args ) scoring [ 'recall' ] = metrics . make_scorer ( metrics . recall_score , * * metric_args ) scoring [ 'fscore' ] = metrics . make_scorer ( metrics . f1_score , * * metric_args ) output_format = "clf_overall" else : # If there is no averaging we will need multiple scorers; one for each class for label in labels : metric_args [ 'pos_label' ] = label metric_args [ 'labels' ] = [ label ] scoring [ 'precision_' + str ( label ) ] = metrics . make_scorer ( metrics . precision_score , * * metric_args ) scoring [ 'recall_' + str ( label ) ] = metrics . make_scorer ( metrics . recall_score , * * metric_args ) scoring [ 'fscore_' + str ( label ) ] = metrics . make_scorer ( metrics . f1_score , * * metric_args ) output_format = "clf_classes" elif self . model . estimator_type == "regressor" : scoring = [ 'r2' , 'neg_mean_squared_error' , 'neg_mean_absolute_error' , 'neg_median_absolute_error' , 'explained_variance' ] # Perform cross validation using the training data and the model pipeline scores = cross_validate ( self . model . pipe , self . X_train , y_train , scoring = scoring , cv = self . model . cv , fit_params = fit_params , return_train_score = False ) # Prepare the metrics data frame according to the output format if self . model . estimator_type == "classifier" : # Get cross validation predictions for the confusion matrix y_pred = cross_val_predict ( self . model . pipe , self . X_train , y_train , cv = self . model . cv , fit_params = fit_params ) # Prepare the confusion matrix and add it to the model self . _prep_confusion_matrix ( y_train , y_pred , labels ) # Create an empty data frame to set the structure metrics_df = pd . DataFrame ( columns = [ "class" , "accuracy" , "accuracy_std" , "precision" , "precision_std" , "recall" , "recall_std" , "fscore" , "fscore_std" ] ) if output_format == "clf_overall" : # Add the overall metrics to the data frame metrics_df . loc [ 0 ] = [ "overall" , np . average ( scores [ "test_accuracy" ] ) , np . std ( scores [ "test_accuracy" ] ) , np . average ( scores [ "test_precision" ] ) , np . std ( scores [ "test_precision" ] ) , np . average ( scores [ "test_recall" ] ) , np . std ( scores [ "test_recall" ] ) , np . average ( scores [ "test_fscore" ] ) , np . std ( scores [ "test_fscore" ] ) ] elif output_format == "clf_classes" : # Add accuracy which is calculated at an overall level metrics_df . loc [ 0 ] = [ "overall" , np . average ( scores [ "test_accuracy" ] ) , np . std ( scores [ "test_accuracy" ] ) , np . NaN , np . NaN , np . NaN , np . NaN , np . NaN , np . NaN ] # Add the metrics for each class to the data frame for i , label in enumerate ( labels ) : metrics_df . loc [ i + 1 ] = [ label , np . NaN , np . NaN , np . average ( scores [ "test_precision_" + str ( label ) ] ) , np . std ( scores [ "test_precision_" + str ( label ) ] ) , np . average ( scores [ "test_recall_" + str ( label ) ] ) , np . std ( scores [ "test_recall_" + str ( label ) ] ) , np . average ( scores [ "test_fscore_" + str ( label ) ] ) , np . std ( scores [ "test_fscore_" + str ( label ) ] ) ] # Finalize the structure of the result DataFrame metrics_df . loc [ : , "model_name" ] = self . model . name metrics_df = metrics_df . loc [ : , [ "model_name" , "class" , "accuracy" , "accuracy_std" , "precision" , "precision_std" , "recall" , "recall_std" , "fscore" , "fscore_std" ] ] # Add the score to the model self . model . score = metrics_df [ "accuracy" ] . values [ 0 ] elif self . model . estimator_type == "regressor" : # Create an empty data frame to set the structure metrics_df = pd . DataFrame ( columns = [ "r2_score" , "r2_score_std" , "mean_squared_error" , "mean_squared_error_std" , "mean_absolute_error" , "mean_absolute_error_std" , "median_absolute_error" , "median_absolute_error_std" , "explained_variance_score" , "explained_variance_score_std" ] ) # Add the overall metrics to the data frame metrics_df . loc [ 0 ] = [ np . average ( scores [ "test_r2" ] ) , np . std ( scores [ "test_r2" ] ) , np . average ( scores [ "test_neg_mean_squared_error" ] ) , np . std ( scores [ "test_neg_mean_squared_error" ] ) , np . average ( scores [ "test_neg_mean_absolute_error" ] ) , np . std ( scores [ "test_neg_mean_absolute_error" ] ) , np . average ( scores [ "test_neg_median_absolute_error" ] ) , np . std ( scores [ "test_neg_median_absolute_error" ] ) , np . average ( scores [ "test_explained_variance" ] ) , np . std ( scores [ "test_explained_variance" ] ) ] # Finalize the structure of the result DataFrame metrics_df . loc [ : , "model_name" ] = self . model . name metrics_df = metrics_df . loc [ : , [ "model_name" , "r2_score" , "r2_score_std" , "mean_squared_error" , "mean_squared_error_std" , "mean_absolute_error" , "mean_absolute_error_std" , "median_absolute_error" , "median_absolute_error_std" , "explained_variance_score" , "explained_variance_score_std" ] ] # Add the score to the model self . model . score = metrics_df [ "r2_score" ] . values [ 0 ] # Save the metrics_df to the model self . model . metrics_df = metrics_df