(self)
return self.response
Train and test the model based on the provided dataset
Train and test the model based on the provided dataset
Train and test the model based on the provided dataset
def fit(self):
        """
        Train and test the model based on the provided dataset
        """
        
        # Open an existing model and get the training & test dataset and targets
        train_test_df, target_df = self._get_model_and_data(target=True, set_feature_def=True)
        
        # Check that the estimator is an supervised ML algorithm
        if self.model.estimator_type not in ["classifier", "regressor"]:
            err = "Incorrect usage. The estimator specified is not a known classifier or regressor: {0}".format(self.model.estimator)
            raise Exception(err)
        
        # Check which validation strategy is to be used, if any
        # For an explanation of cross validation in scikit-learn see: http://scikit-learn.org/stable/modules/cross_validation.html#multimetric-cross-validation
        if self.model.time_series_split > 0:
            self.model.validation = "timeseries"
            # Set up cross validation to be performed using TimeSeriesSplit
            self.model.cv = TimeSeriesSplit(n_splits=self.model.time_series_split, max_train_size=self.model.max_train_size)
        elif self.model.cv > 0:
            self.model.validation = "k-fold"
        elif self.model.test_size > 0:
            self.model.validation = "hold-out"
        else:
            self.model.validation = "external"

        if self.model.validation == "hold-out":        
            # Split the data into training and testing subsets
            self.X_train, self.X_test, self.y_train, self.y_test = \
            train_test_split(train_test_df, target_df, test_size=self.model.test_size, random_state=self.model.random_state)
        else:
            self.X_train = train_test_df
            self.y_train = target_df
        
        # Add the training and test data to the model if required
        if self.model.retain_data:
            self.model.X_train = self.X_train
            self.model.y_train = self.y_train
            
            try:
                self.model.X_test = self.X_test
                self.model.y_test = self.y_test
            except AttributeError:
                pass
        
        # Scale the targets and increase stationarity if required
        if self.model.scale_target or self.model.make_stationary:
            # Set up the target transformer
            self.model.target_transformer = TargetTransformer(scale=self.model.scale_target, make_stationary=self.model.make_stationary, stationarity_lags=self.model.stationarity_lags,\
                missing=self.model.missing, scaler=self.model.scaler, logfile=self.logfile, **self.model.scaler_kwargs)

            # Fit the transformer to the training targets
            self.model.target_transformer = self.model.target_transformer.fit(self.y_train)

            # Apply the transformer to the training targets
            self.y_train = self.model.target_transformer.transform(self.y_train)
            # Drop samples where the target cannot be transformed due to insufficient lags
            self.X_train = self.X_train.iloc[len(self.X_train)-len(self.y_train):] 
        
        # Add lag observations to the samples if required
        if self.model.lags or self.model.lag_target:
            # Check if the current sample will be included as an input, or whether we only use lag observations for predictions
            extrapolate = 1 if self.model.current_sample_as_input else 0
            # Add the lag observations
            self.X_train = self._add_lags(self.X_train, self.y_train, extrapolate=extrapolate, update_features_df=True)
            # Drop targets for samples which were dropped due to null values after adding lags.
            if len(self.y_train) > len(self.X_train):
                self.y_train = self.y_train.iloc[len(self.y_train)-len(self.X_train):]

        # If this is a Keras estimator, we require the preprocessing to return a data frame instead of a numpy array
        prep_return = 'df' if self.model.using_keras else 'np'

        # Construct the preprocessor
        prep = Preprocessor(self.model.features_df, return_type=prep_return, scale_hashed=self.model.scale_hashed, scale_vectors=self.model.scale_vectors,\
        missing=self.model.missing, scaler=self.model.scaler, logfile=self.logfile, **self.model.scaler_kwargs)

        # Setup a list to store steps for the sklearn pipeline
        pipe_steps = [('preprocessor', prep)]

        if self.model.dim_reduction:
            # Construct the dimensionality reduction object
            reduction = self.decomposers[self.model.reduction](**self.model.dim_reduction_args)
            
            # Include dimensionality reduction in the pipeline steps
            pipe_steps.append(('reduction', reduction))
            self.model.estimation_step = 2
        else:
            self.model.estimation_step = 1      

        # If this is a Keras estimator, update the input shape and reshape the data if required
        if self.model.using_keras:
            # Update the input shape based on the final number of features after preprocessing
            self._keras_update_shape(prep)

            # Add the Keras build function, architecture and prediction_periods to the estimator keyword arguments
            self.model.estimator_kwargs['build_fn'] = self._keras_build_fn
            self.model.estimator_kwargs['architecture'] = self.model.architecture
            self.model.estimator_kwargs['prediction_periods'] = self.model.prediction_periods

            # Debug information is printed to the terminal and logs if the paramater debug = true
            if self.model.debug:
                self._print_log(10)
            
            # Check than an identifier has been provided for sorting data if this is a sequence prediction problem
            if self.model.lags or len(self.model.first_layer_kwargs["input_shape"]) > 1:
                assert len(self.model.original_features_df[self.model.original_features_df['variable_type'].isin(["identifier"])]) == 1, \
                    "An identifier is mandatory when using lags or with sequence prediction problems. Define this field in your feature definitions."

            # Cater for multi-step predictions
            if self.model.prediction_periods > 1:
                # Transform y to a vector of values equal to prediction_periods
                self.y_train = utils.vectorize_array(self.y_train, steps=self.model.prediction_periods)
                # Drop values from x for which we don't have sufficient y values
                self.X_train = self.X_train.iloc[:-len(self.X_train)+len(self.y_train)]

            # Add a pipeline step to update the input shape and reshape the data if required
            # This transform will also add lag observations if specified through the lags parameter
            # If lag_target is True, an additional feature will be created for each sample using the previous value of y 
            reshape = Reshaper(first_layer_kwargs=self.model.first_layer_kwargs, logfile=self.logfile)
            pipe_steps.append(('reshape', reshape))
            self.model.estimation_step += self.model.estimation_step

            # Avoid tensorflow error for keras models
            # https://github.com/tensorflow/tensorflow/issues/14356
            # https://stackoverflow.com/questions/40785224/tensorflow-cannot-interpret-feed-dict-key-as-tensor
            kerasbackend.clear_session()
        
        # Try assuming the pipeline involves a grid search
        try:
            # Construct an estimator
            estimator = self.algorithms[self.model.estimator](**self.model.estimator_kwargs)

            # Prepare the grid search using the previously set parameter grid
            grid_search = GridSearchCV(estimator=estimator, param_grid=self.model.param_grid, **self.model.grid_search_args)
            
            # Add grid search to the pipeline steps
            pipe_steps.append(('grid_search', grid_search))

            # Construct the sklearn pipeline using the list of steps
            self.model.pipe = Pipeline(pipe_steps)

            if self.model.validation in ["k-fold", "timeseries"]:
                # Perform K-fold cross validation
                self._cross_validate()

            # Fit the training data to the pipeline
            if self.model.using_keras:
                # https://stackoverflow.com/questions/54652536/keras-tensorflow-backend-error-tensor-input-10-specified-in-either-feed-de
                session = tf.Session()
                kerasbackend.set_session(session)
                with session.as_default():
                    with session.graph.as_default():
                        sys.stdout.write("\nMODEL: {}, INPUT SHAPE: {}\n\n".format(self.model.name, self.model.first_layer_kwargs['input_shape']))
                        y = self.y_train.values if self.y_train.shape[1] > 1 else self.y_train.values.ravel()
                        self.model.pipe.fit(self.X_train, y)
            else:
                self.model.pipe.fit(self.X_train, self.y_train.values.ravel())

            # Get the best parameters and the cross validation results
            grid_search = self.model.pipe.named_steps['grid_search']
            self.model.best_params = grid_search.best_params_
            self.model.cv_results = grid_search.cv_results_

            # Get the best estimator to add to the final pipeline
            estimator = grid_search.best_estimator_

            # Update the pipeline with the best estimator
            self.model.pipe.steps[self.model.estimation_step] = ('estimator', estimator)

        except AttributeError:
            # Construct an estimator
            estimator = self.algorithms[self.model.estimator](**self.model.estimator_kwargs)

            # Add the estimator to the pipeline steps
            pipe_steps.append(('estimator', estimator))

            # Construct the sklearn pipeline using the list of steps
            self.model.pipe = Pipeline(pipe_steps)

            if self.model.validation in ["k-fold", "timeseries"]:
                # Perform K-fold cross validation
                self._cross_validate()

            # Fit the training data to the pipeline
            if self.model.using_keras:
                # https://stackoverflow.com/questions/54652536/keras-tensorflow-backend-error-tensor-input-10-specified-in-either-feed-de
                session = tf.Session()
                kerasbackend.set_session(session)
                with session.as_default():
                    with session.graph.as_default():
                        sys.stdout.write("\nMODEL: {}, INPUT SHAPE: {}\n\n".format(self.model.name, self.model.first_layer_kwargs['input_shape']))
                        y = self.y_train.values if self.y_train.shape[1] > 1 else self.y_train.values.ravel()
                        self.model.pipe.fit(self.X_train, y)
            else:
                self.model.pipe.fit(self.X_train, self.y_train.values.ravel())
        
        if self.model.validation == "hold-out":       
            # Evaluate the model using the test data            
            self.calculate_metrics(caller="internal")
        
        if self.model.calc_feature_importances:
            # Select the dataset for calculating importances
            if self.model.validation == "hold-out":
                X = self.X_test
                y = self.y_test # Already a numpy array after calculate_metrics
            else:
                X = self.X_train
                y = self.y_train.values.ravel()
            
            # Calculate model agnostic feature importances
            self._calc_importances(X = X, y = y)

        # Persist the model to disk
        self.model = self.model.save(self.model.name, self.path, overwrite=self.model.overwrite, compress=self.model.compress)
                
        # Update the cache to keep this model in memory
        self._update_cache()
        
        # Prepare the output
        if self.model.validation != "external": 
            message = [[self.model.name, 'Model successfully trained, tested and saved to disk.',\
                        time.strftime('%X %x %Z', time.localtime(self.model.state_timestamp)),\
                        "{0} model has a score of {1:.3f} against the test data."\
                        .format(self.model.estimator, self.model.score), self.model.score]]
        else:
            message = [[self.model.name, 'Model successfully trained and saved to disk.',\
                        time.strftime('%X %x %Z', time.localtime(self.model.state_timestamp)),\
                        "{0} model score unknown as test_size was <= 0."\
                        .format(self.model.estimator), np.NaN]]
            
        self.response = pd.DataFrame(message, columns=['model_name', 'result', 'time_stamp', 'score_result', 'score'])
        
        # Send the reponse table description to Qlik
        self._send_table_description("fit")
        
        # Debug information is printed to the terminal and logs if the paramater debug = true
        if self.model.debug:
            self._print_log(4)
        
        # Finally send the response
        return self.response
def fit ( self ) : # Open an existing model and get the training & test dataset and targets train_test_df , target_df = self . _get_model_and_data ( target = True , set_feature_def = True ) # Check that the estimator is an supervised ML algorithm if self . model . estimator_type not in [ "classifier" , "regressor" ] : err = "Incorrect usage. The estimator specified is not a known classifier or regressor: {0}" . format ( self . model . estimator ) raise Exception ( err ) # Check which validation strategy is to be used, if any # For an explanation of cross validation in scikit-learn see: http://scikit-learn.org/stable/modules/cross_validation.html#multimetric-cross-validation if self . model . time_series_split > 0 : self . model . validation = "timeseries" # Set up cross validation to be performed using TimeSeriesSplit self . model . cv = TimeSeriesSplit ( n_splits = self . model . time_series_split , max_train_size = self . model . max_train_size ) elif self . model . cv > 0 : self . model . validation = "k-fold" elif self . model . test_size > 0 : self . model . validation = "hold-out" else : self . model . validation = "external" if self . model . validation == "hold-out" : # Split the data into training and testing subsets self . X_train , self . X_test , self . y_train , self . y_test = train_test_split ( train_test_df , target_df , test_size = self . model . test_size , random_state = self . model . random_state ) else : self . X_train = train_test_df self . y_train = target_df # Add the training and test data to the model if required if self . model . retain_data : self . model . X_train = self . X_train self . model . y_train = self . y_train try : self . model . X_test = self . X_test self . model . y_test = self . y_test except AttributeError : pass # Scale the targets and increase stationarity if required if self . model . scale_target or self . model . make_stationary : # Set up the target transformer self . model . target_transformer = TargetTransformer ( scale = self . model . scale_target , make_stationary = self . model . make_stationary , stationarity_lags = self . model . stationarity_lags , missing = self . model . missing , scaler = self . model . scaler , logfile = self . logfile , * * self . model . scaler_kwargs ) # Fit the transformer to the training targets self . model . target_transformer = self . model . target_transformer . fit ( self . y_train ) # Apply the transformer to the training targets self . y_train = self . model . target_transformer . transform ( self . y_train ) # Drop samples where the target cannot be transformed due to insufficient lags self . X_train = self . X_train . iloc [ len ( self . X_train ) - len ( self . y_train ) : ] # Add lag observations to the samples if required if self . model . lags or self . model . lag_target : # Check if the current sample will be included as an input, or whether we only use lag observations for predictions extrapolate = 1 if self . model . current_sample_as_input else 0 # Add the lag observations self . X_train = self . _add_lags ( self . X_train , self . y_train , extrapolate = extrapolate , update_features_df = True ) # Drop targets for samples which were dropped due to null values after adding lags. if len ( self . y_train ) > len ( self . X_train ) : self . y_train = self . y_train . iloc [ len ( self . y_train ) - len ( self . X_train ) : ] # If this is a Keras estimator, we require the preprocessing to return a data frame instead of a numpy array prep_return = 'df' if self . model . using_keras else 'np' # Construct the preprocessor prep = Preprocessor ( self . model . features_df , return_type = prep_return , scale_hashed = self . model . scale_hashed , scale_vectors = self . model . scale_vectors , missing = self . model . missing , scaler = self . model . scaler , logfile = self . logfile , * * self . model . scaler_kwargs ) # Setup a list to store steps for the sklearn pipeline pipe_steps = [ ( 'preprocessor' , prep ) ] if self . model . dim_reduction : # Construct the dimensionality reduction object reduction = self . decomposers [ self . model . reduction ] ( * * self . model . dim_reduction_args ) # Include dimensionality reduction in the pipeline steps pipe_steps . append ( ( 'reduction' , reduction ) ) self . model . estimation_step = 2 else : self . model . estimation_step = 1 # If this is a Keras estimator, update the input shape and reshape the data if required if self . model . using_keras : # Update the input shape based on the final number of features after preprocessing self . _keras_update_shape ( prep ) # Add the Keras build function, architecture and prediction_periods to the estimator keyword arguments self . model . estimator_kwargs [ 'build_fn' ] = self . _keras_build_fn self . model . estimator_kwargs [ 'architecture' ] = self . model . architecture self . model . estimator_kwargs [ 'prediction_periods' ] = self . model . prediction_periods # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 10 ) # Check than an identifier has been provided for sorting data if this is a sequence prediction problem if self . model . lags or len ( self . model . first_layer_kwargs [ "input_shape" ] ) > 1 : assert len ( self . model . original_features_df [ self . model . original_features_df [ 'variable_type' ] . isin ( [ "identifier" ] ) ] ) == 1 , "An identifier is mandatory when using lags or with sequence prediction problems. Define this field in your feature definitions." # Cater for multi-step predictions if self . model . prediction_periods > 1 : # Transform y to a vector of values equal to prediction_periods self . y_train = utils . vectorize_array ( self . y_train , steps = self . model . prediction_periods ) # Drop values from x for which we don't have sufficient y values self . X_train = self . X_train . iloc [ : - len ( self . X_train ) + len ( self . y_train ) ] # Add a pipeline step to update the input shape and reshape the data if required # This transform will also add lag observations if specified through the lags parameter # If lag_target is True, an additional feature will be created for each sample using the previous value of y  reshape = Reshaper ( first_layer_kwargs = self . model . first_layer_kwargs , logfile = self . logfile ) pipe_steps . append ( ( 'reshape' , reshape ) ) self . model . estimation_step += self . model . estimation_step # Avoid tensorflow error for keras models # https://github.com/tensorflow/tensorflow/issues/14356 # https://stackoverflow.com/questions/40785224/tensorflow-cannot-interpret-feed-dict-key-as-tensor kerasbackend . clear_session ( ) # Try assuming the pipeline involves a grid search try : # Construct an estimator estimator = self . algorithms [ self . model . estimator ] ( * * self . model . estimator_kwargs ) # Prepare the grid search using the previously set parameter grid grid_search = GridSearchCV ( estimator = estimator , param_grid = self . model . param_grid , * * self . model . grid_search_args ) # Add grid search to the pipeline steps pipe_steps . append ( ( 'grid_search' , grid_search ) ) # Construct the sklearn pipeline using the list of steps self . model . pipe = Pipeline ( pipe_steps ) if self . model . validation in [ "k-fold" , "timeseries" ] : # Perform K-fold cross validation self . _cross_validate ( ) # Fit the training data to the pipeline if self . model . using_keras : # https://stackoverflow.com/questions/54652536/keras-tensorflow-backend-error-tensor-input-10-specified-in-either-feed-de session = tf . Session ( ) kerasbackend . set_session ( session ) with session . as_default ( ) : with session . graph . as_default ( ) : sys . stdout . write ( "\nMODEL: {}, INPUT SHAPE: {}\n\n" . format ( self . model . name , self . model . first_layer_kwargs [ 'input_shape' ] ) ) y = self . y_train . values if self . y_train . shape [ 1 ] > 1 else self . y_train . values . ravel ( ) self . model . pipe . fit ( self . X_train , y ) else : self . model . pipe . fit ( self . X_train , self . y_train . values . ravel ( ) ) # Get the best parameters and the cross validation results grid_search = self . model . pipe . named_steps [ 'grid_search' ] self . model . best_params = grid_search . best_params_ self . model . cv_results = grid_search . cv_results_ # Get the best estimator to add to the final pipeline estimator = grid_search . best_estimator_ # Update the pipeline with the best estimator self . model . pipe . steps [ self . model . estimation_step ] = ( 'estimator' , estimator ) except AttributeError : # Construct an estimator estimator = self . algorithms [ self . model . estimator ] ( * * self . model . estimator_kwargs ) # Add the estimator to the pipeline steps pipe_steps . append ( ( 'estimator' , estimator ) ) # Construct the sklearn pipeline using the list of steps self . model . pipe = Pipeline ( pipe_steps ) if self . model . validation in [ "k-fold" , "timeseries" ] : # Perform K-fold cross validation self . _cross_validate ( ) # Fit the training data to the pipeline if self . model . using_keras : # https://stackoverflow.com/questions/54652536/keras-tensorflow-backend-error-tensor-input-10-specified-in-either-feed-de session = tf . Session ( ) kerasbackend . set_session ( session ) with session . as_default ( ) : with session . graph . as_default ( ) : sys . stdout . write ( "\nMODEL: {}, INPUT SHAPE: {}\n\n" . format ( self . model . name , self . model . first_layer_kwargs [ 'input_shape' ] ) ) y = self . y_train . values if self . y_train . shape [ 1 ] > 1 else self . y_train . values . ravel ( ) self . model . pipe . fit ( self . X_train , y ) else : self . model . pipe . fit ( self . X_train , self . y_train . values . ravel ( ) ) if self . model . validation == "hold-out" : # Evaluate the model using the test data             self . calculate_metrics ( caller = "internal" ) if self . model . calc_feature_importances : # Select the dataset for calculating importances if self . model . validation == "hold-out" : X = self . X_test y = self . y_test # Already a numpy array after calculate_metrics else : X = self . X_train y = self . y_train . values . ravel ( ) # Calculate model agnostic feature importances self . _calc_importances ( X = X , y = y ) # Persist the model to disk self . model = self . model . save ( self . model . name , self . path , overwrite = self . model . overwrite , compress = self . model . compress ) # Update the cache to keep this model in memory self . _update_cache ( ) # Prepare the output if self . model . validation != "external" : message = [ [ self . model . name , 'Model successfully trained, tested and saved to disk.' , time . strftime ( '%X %x %Z' , time . localtime ( self . model . state_timestamp ) ) , "{0} model has a score of {1:.3f} against the test data." . format ( self . model . estimator , self . model . score ) , self . model . score ] ] else : message = [ [ self . model . name , 'Model successfully trained and saved to disk.' , time . strftime ( '%X %x %Z' , time . localtime ( self . model . state_timestamp ) ) , "{0} model score unknown as test_size was <= 0." . format ( self . model . estimator ) , np . NaN ] ] self . response = pd . DataFrame ( message , columns = [ 'model_name' , 'result' , 'time_stamp' , 'score_result' , 'score' ] ) # Send the reponse table description to Qlik self . _send_table_description ( "fit" ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 4 ) # Finally send the response return self . response