(self, caller="external", ordered_data=False)
Return key metrics based on a test dataset.
        Metrics returned for a classifier are: accuracy, precision, recall, fscore, support
        Metrics returned for a regressor are: r2_score, mean_squared_error, mean_absolute_error, median_absolute_error, explained_variance_score
Return key metrics based on a test dataset.
        Metrics returned for a classifier are: accuracy, precision, recall, fscore, support
        Metrics returned for a regressor are: r2_score, mean_squared_error, mean_absolute_error, median_absolute_error, explained_variance_score
Return key metrics based on a test dataset . Metrics returned for a classifier are : accuracy precision recall fscore support Metrics returned for a regressor are : r2_score mean_squared_error mean_absolute_error median_absolute_error explained_variance_score
def calculate_metrics(self, caller="external", ordered_data=False): 
        """
        Return key metrics based on a test dataset.
        Metrics returned for a classifier are: accuracy, precision, recall, fscore, support
        Metrics returned for a regressor are: r2_score, mean_squared_error, mean_absolute_error, median_absolute_error, explained_variance_score
        """
        
        # If the function call was made externally, process the request
        if caller == "external":
            # Open an existing model and get the training & test dataset and targets based on the request
            self.X_test, self.y_test = self._get_model_and_data(target=True, ordered_data=ordered_data)    

        # Keep a copy of the y_test before any transformations
        y_test_copy = self.y_test.copy()

        # Scale the targets and increase stationarity if required
        if self.model.scale_target or self.model.make_stationary:
            # If using differencing, we assume sufficient lag values for inversing the transformation later
            y_orig = self.y_test.values.ravel() if self.model.make_stationary=='difference' else None
            # Apply the transformer to the test targets
            self.y_test = self.model.target_transformer.transform(self.y_test)  
            # Drop samples where self.y_test cannot be transformed due to insufficient lags
            self.X_test = self.X_test.iloc[len(self.X_test)-len(self.y_test):]

        # Refresh the keras model to avoid tensorflow errors
        if self.model.using_keras:
            self._keras_refresh()
        
        # Get predictions based on the samples
        if ordered_data:
            self.y_pred = self.sequence_predict(variant="internal")

            # Handle possible null values where a prediction could not be generated
            self.y_pred = self.y_pred[self.rows_per_pred - self.first_pred_modifier:]
            self.y_test = y_test_copy.iloc[-1*len(self.y_pred):]

            # Inverse transformations predictions if required 
            if self.model.scale_target or self.model.make_stationary:
                # Set up indices for differencing
                end = self.placeholders
                start = end - self.diff_lags

                # Add lags for inversing differencing
                self.y_pred = self.y_pred if y_orig is None else np.append(y_orig[start : end], self.y_pred)

                # Apply the transformer to the test targets
                self.y_pred = self.model.target_transformer.inverse_transform(self.y_pred) 

                # Remove lags used for making the series stationary in case of differencing
                if self.model.make_stationary == 'difference':
                    self.y_pred = self.y_pred[self.diff_lags:]
        else:
            self.y_pred = self.model.pipe.predict(self.X_test)

            # Inverse transformations on the predictions if required
            if self.model.scale_target or self.model.make_stationary:
                # Apply the transformer to the predictions
                self.y_pred = self.model.target_transformer.inverse_transform(self.y_pred)
                # Reset y_test to orginal values
                self.y_test = y_test_copy
        
        # Flatten the y_test DataFrame
        self.y_test = self.y_test.values.ravel()

        # Try getting the metric_args from the model
        try:
            metric_args = self.model.metric_args
        except AttributeError: 
            metric_args = {}
               
        if self.model.estimator_type == "classifier":
            labels = self.model.pipe.named_steps['estimator'].classes_
            
            # Check if the average parameter is specified
            if len(metric_args) > 0  and "average" in metric_args:
                # Metrics are returned as an overall average
                metric_rows = ["overall"]
            else:
                # Get the class labels to be used as rows for the result DataFrame
                metric_rows = labels
            
            # Get key classifier metrics
            metrics_df = pd.DataFrame([x for x in metrics.precision_recall_fscore_support\
                                       (self.y_test, self.y_pred, **metric_args)],\
                                      index=["precision", "recall", "fscore", "support"], columns=metric_rows).transpose()
            # Add accuracy
            self.model.score = metrics.accuracy_score(self.y_test, self.y_pred)
            metrics_df.loc["overall", "accuracy"] = self.model.score
            # Finalize the structure of the result DataFrame
            metrics_df.loc[:,"model_name"] = self.model.name
            metrics_df.loc[:,"class"] = metrics_df.index
            metrics_df = metrics_df.loc[:,["model_name", "class", "accuracy", "precision", "recall", "fscore", "support"]]
            
            # Prepare the confusion matrix and add it to the model
            self._prep_confusion_matrix(self.y_test, self.y_pred, labels)
            
        elif self.model.estimator_type == "regressor":
            # Get the r2 score
            self.model.score = metrics.r2_score(self.y_test, self.y_pred, **metric_args)
            metrics_df = pd.DataFrame([[self.model.score]], columns=["r2_score"])
                        
            # Get the mean squared error
            metrics_df.loc[:,"mean_squared_error"] = metrics.mean_squared_error(self.y_test, self.y_pred, **metric_args)
            
            # Get the mean absolute error
            metrics_df.loc[:,"mean_absolute_error"] = metrics.mean_absolute_error(self.y_test, self.y_pred, **metric_args)
            
            # Get the median absolute error
            metrics_df.loc[:,"median_absolute_error"] = metrics.median_absolute_error(self.y_test, self.y_pred)
            
            # Get the explained variance score
            metrics_df.loc[:,"explained_variance_score"] = metrics.explained_variance_score(self.y_test, self.y_pred, **metric_args)

            # If the target was scaled we need to inverse transform certain metrics to the original scale
            # However, if we used the sequence prediction function, the inverse transform has already been performed
            if not ordered_data and (self.model.scale_target or self.model.make_stationary):
                for m in ["mean_squared_error", "mean_absolute_error", "median_absolute_error"]:
                    metrics_df.loc[:, m] = self.model.target_transformer.inverse_transform(metrics_df.loc[:, [m]], array_like=False).values.ravel()
            
            # Finalize the structure of the result DataFrame
            metrics_df.loc[:,"model_name"] = self.model.name
            metrics_df = metrics_df.loc[:,["model_name", "r2_score", "mean_squared_error", "mean_absolute_error",\
                                           "median_absolute_error", "explained_variance_score"]]
           
        if caller == "external":
            
            if self.model.calc_feature_importances:
                # Calculate model agnostic feature importances
                self._calc_importances(X = self.X_test, y = self.y_test)

            self.response = metrics_df

            # Send the reponse table description to Qlik
            if self.model.estimator_type == "classifier":
                self._send_table_description("metrics_clf")
            elif self.model.estimator_type == "regressor":
                self._send_table_description("metrics_reg")

            # Debug information is printed to the terminal and logs if the paramater debug = true
            if self.model.debug:
                self._print_log(4)

            # Finally send the response
            return self.response
        else:
            # Save the metrics_df to the model
            self.model.metrics_df = metrics_df
def calculate_metrics ( self , caller = "external" , ordered_data = False ) : # If the function call was made externally, process the request if caller == "external" : # Open an existing model and get the training & test dataset and targets based on the request self . X_test , self . y_test = self . _get_model_and_data ( target = True , ordered_data = ordered_data ) # Keep a copy of the y_test before any transformations y_test_copy = self . y_test . copy ( ) # Scale the targets and increase stationarity if required if self . model . scale_target or self . model . make_stationary : # If using differencing, we assume sufficient lag values for inversing the transformation later y_orig = self . y_test . values . ravel ( ) if self . model . make_stationary == 'difference' else None # Apply the transformer to the test targets self . y_test = self . model . target_transformer . transform ( self . y_test ) # Drop samples where self.y_test cannot be transformed due to insufficient lags self . X_test = self . X_test . iloc [ len ( self . X_test ) - len ( self . y_test ) : ] # Refresh the keras model to avoid tensorflow errors if self . model . using_keras : self . _keras_refresh ( ) # Get predictions based on the samples if ordered_data : self . y_pred = self . sequence_predict ( variant = "internal" ) # Handle possible null values where a prediction could not be generated self . y_pred = self . y_pred [ self . rows_per_pred - self . first_pred_modifier : ] self . y_test = y_test_copy . iloc [ - 1 * len ( self . y_pred ) : ] # Inverse transformations predictions if required  if self . model . scale_target or self . model . make_stationary : # Set up indices for differencing end = self . placeholders start = end - self . diff_lags # Add lags for inversing differencing self . y_pred = self . y_pred if y_orig is None else np . append ( y_orig [ start : end ] , self . y_pred ) # Apply the transformer to the test targets self . y_pred = self . model . target_transformer . inverse_transform ( self . y_pred ) # Remove lags used for making the series stationary in case of differencing if self . model . make_stationary == 'difference' : self . y_pred = self . y_pred [ self . diff_lags : ] else : self . y_pred = self . model . pipe . predict ( self . X_test ) # Inverse transformations on the predictions if required if self . model . scale_target or self . model . make_stationary : # Apply the transformer to the predictions self . y_pred = self . model . target_transformer . inverse_transform ( self . y_pred ) # Reset y_test to orginal values self . y_test = y_test_copy # Flatten the y_test DataFrame self . y_test = self . y_test . values . ravel ( ) # Try getting the metric_args from the model try : metric_args = self . model . metric_args except AttributeError : metric_args = { } if self . model . estimator_type == "classifier" : labels = self . model . pipe . named_steps [ 'estimator' ] . classes_ # Check if the average parameter is specified if len ( metric_args ) > 0 and "average" in metric_args : # Metrics are returned as an overall average metric_rows = [ "overall" ] else : # Get the class labels to be used as rows for the result DataFrame metric_rows = labels # Get key classifier metrics metrics_df = pd . DataFrame ( [ x for x in metrics . precision_recall_fscore_support ( self . y_test , self . y_pred , * * metric_args ) ] , index = [ "precision" , "recall" , "fscore" , "support" ] , columns = metric_rows ) . transpose ( ) # Add accuracy self . model . score = metrics . accuracy_score ( self . y_test , self . y_pred ) metrics_df . loc [ "overall" , "accuracy" ] = self . model . score # Finalize the structure of the result DataFrame metrics_df . loc [ : , "model_name" ] = self . model . name metrics_df . loc [ : , "class" ] = metrics_df . index metrics_df = metrics_df . loc [ : , [ "model_name" , "class" , "accuracy" , "precision" , "recall" , "fscore" , "support" ] ] # Prepare the confusion matrix and add it to the model self . _prep_confusion_matrix ( self . y_test , self . y_pred , labels ) elif self . model . estimator_type == "regressor" : # Get the r2 score self . model . score = metrics . r2_score ( self . y_test , self . y_pred , * * metric_args ) metrics_df = pd . DataFrame ( [ [ self . model . score ] ] , columns = [ "r2_score" ] ) # Get the mean squared error metrics_df . loc [ : , "mean_squared_error" ] = metrics . mean_squared_error ( self . y_test , self . y_pred , * * metric_args ) # Get the mean absolute error metrics_df . loc [ : , "mean_absolute_error" ] = metrics . mean_absolute_error ( self . y_test , self . y_pred , * * metric_args ) # Get the median absolute error metrics_df . loc [ : , "median_absolute_error" ] = metrics . median_absolute_error ( self . y_test , self . y_pred ) # Get the explained variance score metrics_df . loc [ : , "explained_variance_score" ] = metrics . explained_variance_score ( self . y_test , self . y_pred , * * metric_args ) # If the target was scaled we need to inverse transform certain metrics to the original scale # However, if we used the sequence prediction function, the inverse transform has already been performed if not ordered_data and ( self . model . scale_target or self . model . make_stationary ) : for m in [ "mean_squared_error" , "mean_absolute_error" , "median_absolute_error" ] : metrics_df . loc [ : , m ] = self . model . target_transformer . inverse_transform ( metrics_df . loc [ : , [ m ] ] , array_like = False ) . values . ravel ( ) # Finalize the structure of the result DataFrame metrics_df . loc [ : , "model_name" ] = self . model . name metrics_df = metrics_df . loc [ : , [ "model_name" , "r2_score" , "mean_squared_error" , "mean_absolute_error" , "median_absolute_error" , "explained_variance_score" ] ] if caller == "external" : if self . model . calc_feature_importances : # Calculate model agnostic feature importances self . _calc_importances ( X = self . X_test , y = self . y_test ) self . response = metrics_df # Send the reponse table description to Qlik if self . model . estimator_type == "classifier" : self . _send_table_description ( "metrics_clf" ) elif self . model . estimator_type == "regressor" : self . _send_table_description ( "metrics_reg" ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 4 ) # Finally send the response return self . response else : # Save the metrics_df to the model self . model . metrics_df = metrics_df