(self, estimator_args, scaler_args, execution_args, metric_args=None, dim_reduction_args=None)
Set input parameters based on the request.
        :
        :Refer to the sklearn API Reference for parameters avaialble for specific algorithms and scalers
        :http://scikit-learn.org/stable/modules/classes.html#api-reference
        :
        :Additional parameters used by this SSE are: 
        :overwrite, test_size, randon_state, compress, retain_data, debug
        :For details refer to the GitHub project: https://github.com/nabeel-oz/qlik-py-tools
Set input parameters based on the request.
        :
        :Refer to the sklearn API Reference for parameters avaialble for specific algorithms and scalers
        :http://scikit-learn.org/stable/modules/classes.html#api-reference
        :
        :Additional parameters used by this SSE are: 
        :overwrite, test_size, randon_state, compress, retain_data, debug
        :For details refer to the GitHub project: https://github.com/nabeel-oz/qlik-py-tools
Set input parameters based on the request . : : Refer to the sklearn API Reference for parameters avaialble for specific algorithms and scalers : http : // scikit - learn . org / stable / modules / classes . html#api - reference : : Additional parameters used by this SSE are : : overwrite test_size randon_state compress retain_data debug : For details refer to the GitHub project : https : // github . com / nabeel - oz / qlik - py - tools
def _set_params(self, estimator_args, scaler_args, execution_args, metric_args=None, dim_reduction_args=None):
        """
        Set input parameters based on the request.
        :
        :Refer to the sklearn API Reference for parameters avaialble for specific algorithms and scalers
        :http://scikit-learn.org/stable/modules/classes.html#api-reference
        :
        :Additional parameters used by this SSE are: 
        :overwrite, test_size, randon_state, compress, retain_data, debug
        :For details refer to the GitHub project: https://github.com/nabeel-oz/qlik-py-tools
        """
        
        # Set default values which will be used if execution arguments are not passed
        
        # Default parameters:
        self.model.overwrite = True
        self.model.debug = False
        self.model.test_size = 0.33
        self.model.cv = 0
        self.model.time_series_split = 0
        self.model.max_train_size = None
        self.model.random_state = 42
        self.model.compress = 3
        self.model.retain_data = False
        self.model.scale_hashed = True
        self.model.scale_vectors = True
        self.model.scaler = "StandardScaler"
        self.model.scaler_kwargs = {}
        self.model.estimator_kwargs = {}
        self.model.missing = "zeros"
        self.model.calc_feature_importances = False
        self.model.importances_n_repeats = 30
        self.model.lags= None
        self.model.lag_target = False
        self.model.scale_target = False
        self.model.scale_lag_target= True
        self.model.make_stationary = None
        self.model.stationarity_lags = [1]
        self.model.using_keras = False
        self.model.current_sample_as_input = True
        self.model.prediction_periods = 1
        
        # Default metric parameters:
        if metric_args is None:
            self.model.metric_args = {}
        
        # Set execution parameters
                
        # If the execution key word arguments were included in the request, get the parameters and values
        if len(execution_args) > 0:
            
            # Transform the string of arguments into a dictionary
            execution_args = utils.get_kwargs(execution_args)
            
            # Set the overwite parameter if any existing model with the specified name should be overwritten
            if 'overwrite' in execution_args:
                self.model.overwrite = 'true' == execution_args['overwrite'].lower()
            
            # Set the test_size parameter that will be used to split the samples into training and testing data sets
            # Default value is 0.33, i.e. we use 66% of the samples for training and 33% for testing
            if 'test_size' in execution_args:
                self.model.test_size = utils.atof(execution_args['test_size'])

            # Enable K-fold cross validation. For more information see: http://scikit-learn.org/stable/modules/cross_validation.html#multimetric-cross-validation
            # Default value is 0 in which case a simple holdout strategy based on the test_size parameter is used.
            # If cv > 0 then the model is validated used K = cv folds and the test_size parameter is ignored.
            if 'cv' in execution_args:
                self.model.cv = utils.atoi(execution_args['cv'])
            
            # Enable timeseries backtesting using TimeSeriesSplit. https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.TimeSeriesSplit.html
            # This will select the a validation strategy appropriate for time series and sequential data.
            # The feature definitions must include an 'identifier' field which can be used to sort the series into the correct order.
            # The integer supplied in this parameter will split the data into the given number of subsets for training and testing.
            if 'time_series_split' in execution_args:
                self.model.time_series_split = utils.atoi(execution_args['time_series_split'])

            # This parameter can be used together with time_series_split.
            # It specifies the maximum samples to be used for training in each split, which allows for rolling/ walk forward validation.
            if 'max_train_size' in execution_args:
                self.model.max_train_size = utils.atoi(execution_args['max_train_size'])

            # Add lag observations to the feature matrix. Only applicable for Keras models.
            # An identifier field must be included in the feature definitions to correctly sort the data for this capability.
            # For e.g. if lags=2, features from the previous two samples will be concatenated as input features for the current sample.
            # This is useful for framing timeseries and sequence prediction problems into 3D or 4D data required for deep learning.
            if 'lags' in execution_args:
                self.model.lags = utils.atoi(execution_args['lags'])

            # Include targets in the lag observations
            # If True an additional feature will be created for each sample using the previous value of y 
            if 'lag_target' in execution_args:
                self.model.lag_target = 'true' == execution_args['lag_target'].lower()
            
            # Scale the target before fitting
            # The scaling will be inversed before predictions so they are returned in the original scale 
            if 'scale_target' in execution_args:
                self.model.scale_target = 'true' == execution_args['scale_target'].lower()

            # Scale lag values of the targets before fitting
            # Even if scale_target is set to false, the lag values of targets being used as features can be scaled by setting this to true 
            if 'scale_lag_target' in execution_args:
                self.model.scale_lag_target = 'true' == execution_args['scale_lag_target'].lower()

            # Make the target series more stationary. This only applies to sequence prediction problems.
            # Valid values are 'log' in which case we apply a logarithm to the target values,
            # or 'difference' in which case we transform the targets into variance from the previous value.
            # The transformation will be reversed before returning predictions.
            if 'make_stationary' in execution_args:
                self.model.make_stationary = execution_args['make_stationary'].lower()

                # Provide lags periods for differencing
                # By default the difference will be done with lag = 1. Alternate lags can be provided by passing a list of lags as a list.
                # e.g. 'stationarity_lags=1;12|list|int'
                if 'stationarity_lags' in execution_args:
                    self.model.stationarity_lags = utils.get_kwargs_by_type({'stationarity_lags': execution_args['stationarity_lags']})['stationarity_lags']

            # Specify if the current sample should be used as input to the model
            # This is to allow for models that only use lag observations to make future predictions
            if 'current_sample_as_input' in execution_args:
                self.model.current_sample_as_input = 'true' == execution_args['current_sample_as_input'].lower()

            # Specify the number of predictions expected from the model
            # This can be used to get a model to predict the next m periods given inputs for the previous n periods.
            # This is only valid for Keras models which have a final output layer with more than one node
            if 'prediction_periods' in execution_args:
                self.model.prediction_periods = utils.atoi(execution_args['prediction_periods'])
            
            # Seed used by the random number generator when generating the training testing split
            if 'random_state' in execution_args:
                self.model.random_state = utils.atoi(execution_args['random_state'])
            
            # Compression level between 1-9 used by joblib when saving the model
            if 'compress' in execution_args:
                self.model.compress = utils.atoi(execution_args['compress'])
                
            # Flag to determine if the training and test data should be saved in the model
            if 'retain_data' in execution_args:
                self.model.retain_data = 'true' == execution_args['retain_data'].lower()

            # Flag to determine if feature importances should be calculated when the fit method is called
            if 'calculate_importances' in execution_args:
                self.model.calc_feature_importances = 'true' == execution_args['calculate_importances'].lower()

            # Sets the number of times a feature is randomly shuffled during the feature importance calculation
            if 'importances_n_repeats' in execution_args:
                self.model.importances_n_repeats = utils.atoi(execution_args['importances_n_repeats'])
                       
            # Set the debug option for generating execution logs
            # Valid values are: true, false
            if 'debug' in execution_args:
                self.model.debug = 'true' == execution_args['debug'].lower()
                
                # Additional information is printed to the terminal and logs if the paramater debug = true
                if self.model.debug:
                    # Increment log counter for the class. Each instance of the class generates a new log.
                    self.__class__.log_no += 1

                    # Create a log file for the instance
                    # Logs will be stored in ..\logs\SKLearn Log <n>.txt
                    self.logfile = os.path.join(os.getcwd(), 'logs', 'SKLearn Log {}.txt'.format(self.log_no))
                    
                    # Create dictionary of parameters to display for debug
                    self.exec_params = {"overwrite":self.model.overwrite, "test_size":self.model.test_size, "cv":self.model.cv,\
                    "time_series_split": self.model.time_series_split, "max_train_size":self.model.max_train_size, "lags":self.model.lags,\
                    "lag_target":self.model.lag_target, "scale_target":self.model.scale_target, "make_stationary":self.model.make_stationary,\
                    "random_state":self.model.random_state, "compress":self.model.compress, "retain_data":self.model.retain_data,\
                    "calculate_importances": self.model.calc_feature_importances, "importances_n_repeats": self.model.importances_n_repeats,\
                    "debug":self.model.debug}

                    self._print_log(1)
        
        # If the scaler key word arguments were included in the request, get the parameters and values
        if len(scaler_args) > 0:
            
            # Transform the string of arguments into a dictionary
            scaler_args = utils.get_kwargs(scaler_args)
                   
            # Set scaler arguments that will be used when preprocessing the data
            # Valid values are: StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler and QuantileTransformer
            # More information here: http://scikit-learn.org/stable/modules/preprocessing.html
            if 'scaler' in scaler_args:
                self.model.scaler = scaler_args.pop('scaler')
                
                if 'missing' in scaler_args:
                    self.model.missing = scaler_args.pop('missing').lower()
                
                if 'scale_hashed' in scaler_args:
                    self.model.scale_hashed = 'true' == scaler_args.pop('scale_hashed').lower()
                
                if 'scale_vectors' in scaler_args:
                    self.model.scale_vectors = 'true' == scaler_args.pop('scale_vectors').lower()
                
                # Get the rest of the scaler parameters, converting values to the correct data type
                self.model.scaler_kwargs = utils.get_kwargs_by_type(scaler_args) 
            else:
                err = "Arguments for scaling did not include the scaler name e.g StandardScaler"
                raise Exception(err)
            
        # If the estimator key word arguments were included in the request, get the parameters and values
        if len(estimator_args) > 0:
            
            # Transform the string of arguments into a dictionary
            estimator_args = utils.get_kwargs(estimator_args)
                   
            # Set estimator arguments that will be used when preprocessing the data
            # The parameters available will depend on the selected estimator
            # More information here: http://scikit-learn.org/stable/modules/classes.html#api-reference
            if 'estimator' in estimator_args:
                self.model.estimator = estimator_args.pop('estimator')
                
                # Set the estimator type for the model
                if self.model.estimator in self.classifiers:
                    self.model.estimator_type = "classifier"
                elif self.model.estimator in self.regressors:
                    self.model.estimator_type = "regressor"
                elif self.model.estimator in self.decomposers:
                    self.model.estimator_type = "decomposer"
                elif self.model.estimator in self.clusterers:
                    self.model.estimator_type = "clusterer"
                else:
                    err = "Unknown estimator class: {0}".format(self.model.estimator)
                    raise Exception(err)

                # Get the rest of the estimator parameters, converting values to the correct data type
                self.model.estimator_kwargs = utils.get_kwargs_by_type(estimator_args)  
            else:
                err = "Arguments for estimator did not include the estimator class e.g. RandomForestClassifier"
                raise Exception(err)
        
        # If key word arguments for model evaluation metrics are included in the request, get the parameters and values
        if metric_args is not None and len(metric_args) > 0:
            # Transform the string of arguments into a dictionary
            metric_args = utils.get_kwargs(metric_args)
            
            # Get the metric parameters, converting values to the correct data type
            self.model.metric_args = utils.get_kwargs_by_type(metric_args)  
        
        # If key word arguments for dimensionality reduction are included in the request, get the parameters and values
        if dim_reduction_args is not None and len(dim_reduction_args) > 0:
            # Transform the string of arguments into a dictionary
            dim_reduction_args = utils.get_kwargs(dim_reduction_args)
                   
            # Set dim_reduction arguments that will be used after preprocessing the data
            # The parameters available will depend on the selected dimensionality reduction method
            # Acceptable classes are PCA, KernelPCA, IncrementalPCA, TruncatedSVD
            # More information here: http://scikit-learn.org/stable/modules/classes.html#api-reference
            if 'reduction' in dim_reduction_args:
                self.model.reduction = dim_reduction_args.pop('reduction')
                
                # Get the rest of the dim_reduction parameters, converting values to the correct data type
                self.model.dim_reduction_args = utils.get_kwargs_by_type(dim_reduction_args)  
            else:
                err = "Arguments for dimensionality reduction did not include the class e.g. PCA"
                raise Exception(err)
        
        # Debug information is printed to the terminal and logs if the paramater debug = true
        if self.model.debug:
            self._print_log(2)
def _set_params ( self , estimator_args , scaler_args , execution_args , metric_args = None , dim_reduction_args = None ) : # Set default values which will be used if execution arguments are not passed # Default parameters: self . model . overwrite = True self . model . debug = False self . model . test_size = 0.33 self . model . cv = 0 self . model . time_series_split = 0 self . model . max_train_size = None self . model . random_state = 42 self . model . compress = 3 self . model . retain_data = False self . model . scale_hashed = True self . model . scale_vectors = True self . model . scaler = "StandardScaler" self . model . scaler_kwargs = { } self . model . estimator_kwargs = { } self . model . missing = "zeros" self . model . calc_feature_importances = False self . model . importances_n_repeats = 30 self . model . lags = None self . model . lag_target = False self . model . scale_target = False self . model . scale_lag_target = True self . model . make_stationary = None self . model . stationarity_lags = [ 1 ] self . model . using_keras = False self . model . current_sample_as_input = True self . model . prediction_periods = 1 # Default metric parameters: if metric_args is None : self . model . metric_args = { } # Set execution parameters # If the execution key word arguments were included in the request, get the parameters and values if len ( execution_args ) > 0 : # Transform the string of arguments into a dictionary execution_args = utils . get_kwargs ( execution_args ) # Set the overwite parameter if any existing model with the specified name should be overwritten if 'overwrite' in execution_args : self . model . overwrite = 'true' == execution_args [ 'overwrite' ] . lower ( ) # Set the test_size parameter that will be used to split the samples into training and testing data sets # Default value is 0.33, i.e. we use 66% of the samples for training and 33% for testing if 'test_size' in execution_args : self . model . test_size = utils . atof ( execution_args [ 'test_size' ] ) # Enable K-fold cross validation. For more information see: http://scikit-learn.org/stable/modules/cross_validation.html#multimetric-cross-validation # Default value is 0 in which case a simple holdout strategy based on the test_size parameter is used. # If cv > 0 then the model is validated used K = cv folds and the test_size parameter is ignored. if 'cv' in execution_args : self . model . cv = utils . atoi ( execution_args [ 'cv' ] ) # Enable timeseries backtesting using TimeSeriesSplit. https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.TimeSeriesSplit.html # This will select the a validation strategy appropriate for time series and sequential data. # The feature definitions must include an 'identifier' field which can be used to sort the series into the correct order. # The integer supplied in this parameter will split the data into the given number of subsets for training and testing. if 'time_series_split' in execution_args : self . model . time_series_split = utils . atoi ( execution_args [ 'time_series_split' ] ) # This parameter can be used together with time_series_split. # It specifies the maximum samples to be used for training in each split, which allows for rolling/ walk forward validation. if 'max_train_size' in execution_args : self . model . max_train_size = utils . atoi ( execution_args [ 'max_train_size' ] ) # Add lag observations to the feature matrix. Only applicable for Keras models. # An identifier field must be included in the feature definitions to correctly sort the data for this capability. # For e.g. if lags=2, features from the previous two samples will be concatenated as input features for the current sample. # This is useful for framing timeseries and sequence prediction problems into 3D or 4D data required for deep learning. if 'lags' in execution_args : self . model . lags = utils . atoi ( execution_args [ 'lags' ] ) # Include targets in the lag observations # If True an additional feature will be created for each sample using the previous value of y  if 'lag_target' in execution_args : self . model . lag_target = 'true' == execution_args [ 'lag_target' ] . lower ( ) # Scale the target before fitting # The scaling will be inversed before predictions so they are returned in the original scale  if 'scale_target' in execution_args : self . model . scale_target = 'true' == execution_args [ 'scale_target' ] . lower ( ) # Scale lag values of the targets before fitting # Even if scale_target is set to false, the lag values of targets being used as features can be scaled by setting this to true  if 'scale_lag_target' in execution_args : self . model . scale_lag_target = 'true' == execution_args [ 'scale_lag_target' ] . lower ( ) # Make the target series more stationary. This only applies to sequence prediction problems. # Valid values are 'log' in which case we apply a logarithm to the target values, # or 'difference' in which case we transform the targets into variance from the previous value. # The transformation will be reversed before returning predictions. if 'make_stationary' in execution_args : self . model . make_stationary = execution_args [ 'make_stationary' ] . lower ( ) # Provide lags periods for differencing # By default the difference will be done with lag = 1. Alternate lags can be provided by passing a list of lags as a list. # e.g. 'stationarity_lags=1;12|list|int' if 'stationarity_lags' in execution_args : self . model . stationarity_lags = utils . get_kwargs_by_type ( { 'stationarity_lags' : execution_args [ 'stationarity_lags' ] } ) [ 'stationarity_lags' ] # Specify if the current sample should be used as input to the model # This is to allow for models that only use lag observations to make future predictions if 'current_sample_as_input' in execution_args : self . model . current_sample_as_input = 'true' == execution_args [ 'current_sample_as_input' ] . lower ( ) # Specify the number of predictions expected from the model # This can be used to get a model to predict the next m periods given inputs for the previous n periods. # This is only valid for Keras models which have a final output layer with more than one node if 'prediction_periods' in execution_args : self . model . prediction_periods = utils . atoi ( execution_args [ 'prediction_periods' ] ) # Seed used by the random number generator when generating the training testing split if 'random_state' in execution_args : self . model . random_state = utils . atoi ( execution_args [ 'random_state' ] ) # Compression level between 1-9 used by joblib when saving the model if 'compress' in execution_args : self . model . compress = utils . atoi ( execution_args [ 'compress' ] ) # Flag to determine if the training and test data should be saved in the model if 'retain_data' in execution_args : self . model . retain_data = 'true' == execution_args [ 'retain_data' ] . lower ( ) # Flag to determine if feature importances should be calculated when the fit method is called if 'calculate_importances' in execution_args : self . model . calc_feature_importances = 'true' == execution_args [ 'calculate_importances' ] . lower ( ) # Sets the number of times a feature is randomly shuffled during the feature importance calculation if 'importances_n_repeats' in execution_args : self . model . importances_n_repeats = utils . atoi ( execution_args [ 'importances_n_repeats' ] ) # Set the debug option for generating execution logs # Valid values are: true, false if 'debug' in execution_args : self . model . debug = 'true' == execution_args [ 'debug' ] . lower ( ) # Additional information is printed to the terminal and logs if the paramater debug = true if self . model . debug : # Increment log counter for the class. Each instance of the class generates a new log. self . __class__ . log_no += 1 # Create a log file for the instance # Logs will be stored in ..\logs\SKLearn Log <n>.txt self . logfile = os . path . join ( os . getcwd ( ) , 'logs' , 'SKLearn Log {}.txt' . format ( self . log_no ) ) # Create dictionary of parameters to display for debug self . exec_params = { "overwrite" : self . model . overwrite , "test_size" : self . model . test_size , "cv" : self . model . cv , "time_series_split" : self . model . time_series_split , "max_train_size" : self . model . max_train_size , "lags" : self . model . lags , "lag_target" : self . model . lag_target , "scale_target" : self . model . scale_target , "make_stationary" : self . model . make_stationary , "random_state" : self . model . random_state , "compress" : self . model . compress , "retain_data" : self . model . retain_data , "calculate_importances" : self . model . calc_feature_importances , "importances_n_repeats" : self . model . importances_n_repeats , "debug" : self . model . debug } self . _print_log ( 1 ) # If the scaler key word arguments were included in the request, get the parameters and values if len ( scaler_args ) > 0 : # Transform the string of arguments into a dictionary scaler_args = utils . get_kwargs ( scaler_args ) # Set scaler arguments that will be used when preprocessing the data # Valid values are: StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler and QuantileTransformer # More information here: http://scikit-learn.org/stable/modules/preprocessing.html if 'scaler' in scaler_args : self . model . scaler = scaler_args . pop ( 'scaler' ) if 'missing' in scaler_args : self . model . missing = scaler_args . pop ( 'missing' ) . lower ( ) if 'scale_hashed' in scaler_args : self . model . scale_hashed = 'true' == scaler_args . pop ( 'scale_hashed' ) . lower ( ) if 'scale_vectors' in scaler_args : self . model . scale_vectors = 'true' == scaler_args . pop ( 'scale_vectors' ) . lower ( ) # Get the rest of the scaler parameters, converting values to the correct data type self . model . scaler_kwargs = utils . get_kwargs_by_type ( scaler_args ) else : err = "Arguments for scaling did not include the scaler name e.g StandardScaler" raise Exception ( err ) # If the estimator key word arguments were included in the request, get the parameters and values if len ( estimator_args ) > 0 : # Transform the string of arguments into a dictionary estimator_args = utils . get_kwargs ( estimator_args ) # Set estimator arguments that will be used when preprocessing the data # The parameters available will depend on the selected estimator # More information here: http://scikit-learn.org/stable/modules/classes.html#api-reference if 'estimator' in estimator_args : self . model . estimator = estimator_args . pop ( 'estimator' ) # Set the estimator type for the model if self . model . estimator in self . classifiers : self . model . estimator_type = "classifier" elif self . model . estimator in self . regressors : self . model . estimator_type = "regressor" elif self . model . estimator in self . decomposers : self . model . estimator_type = "decomposer" elif self . model . estimator in self . clusterers : self . model . estimator_type = "clusterer" else : err = "Unknown estimator class: {0}" . format ( self . model . estimator ) raise Exception ( err ) # Get the rest of the estimator parameters, converting values to the correct data type self . model . estimator_kwargs = utils . get_kwargs_by_type ( estimator_args ) else : err = "Arguments for estimator did not include the estimator class e.g. RandomForestClassifier" raise Exception ( err ) # If key word arguments for model evaluation metrics are included in the request, get the parameters and values if metric_args is not None and len ( metric_args ) > 0 : # Transform the string of arguments into a dictionary metric_args = utils . get_kwargs ( metric_args ) # Get the metric parameters, converting values to the correct data type self . model . metric_args = utils . get_kwargs_by_type ( metric_args ) # If key word arguments for dimensionality reduction are included in the request, get the parameters and values if dim_reduction_args is not None and len ( dim_reduction_args ) > 0 : # Transform the string of arguments into a dictionary dim_reduction_args = utils . get_kwargs ( dim_reduction_args ) # Set dim_reduction arguments that will be used after preprocessing the data # The parameters available will depend on the selected dimensionality reduction method # Acceptable classes are PCA, KernelPCA, IncrementalPCA, TruncatedSVD # More information here: http://scikit-learn.org/stable/modules/classes.html#api-reference if 'reduction' in dim_reduction_args : self . model . reduction = dim_reduction_args . pop ( 'reduction' ) # Get the rest of the dim_reduction parameters, converting values to the correct data type self . model . dim_reduction_args = utils . get_kwargs_by_type ( dim_reduction_args ) else : err = "Arguments for dimensionality reduction did not include the class e.g. PCA" raise Exception ( err ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 2 )