(self, load_script=False)
Fit the data to the model and then transform.
        This method is meant to be used for unsupervised learning models for clustering and dimensionality reduction.
        The models can be fit and tranformed through the load script or through chart expressions in Qlik.
        The load_script flag needs to be set accordingly for the correct response.
Fit the data to the model and then transform.
        This method is meant to be used for unsupervised learning models for clustering and dimensionality reduction.
        The models can be fit and tranformed through the load script or through chart expressions in Qlik.
        The load_script flag needs to be set accordingly for the correct response.
Fit the data to the model and then transform . This method is meant to be used for unsupervised learning models for clustering and dimensionality reduction . The models can be fit and tranformed through the load script or through chart expressions in Qlik . The load_script flag needs to be set accordingly for the correct response .
def fit_transform(self, load_script=False):
        """
        Fit the data to the model and then transform.
        This method is meant to be used for unsupervised learning models for clustering and dimensionality reduction.
        The models can be fit and tranformed through the load script or through chart expressions in Qlik.
        The load_script flag needs to be set accordingly for the correct response.
        """

        # Interpret the request data based on the expected row and column structure
        row_template = ['strData', 'strData']
        col_headers = ['model_name', 'n_features']
        feature_col_num = 1
        
        # An additional key field column is expected if the call is made through the load script
        if load_script:
            row_template = ['strData', 'strData', 'strData']
            col_headers = ['model_name', 'key', 'n_features']
            feature_col_num = 2
        
        # Create a Pandas Data Frame for the request data
        self.request_df = utils.request_df(self.request, row_template, col_headers)
               
        # Initialize the persistent model
        self.model = PersistentModel()
        
        # Get the model name from the request dataframe
        self.model.name = self.request_df.loc[0, 'model_name']
        
        # Get the model from cache or disk
        self._get_model()
        
        # Debug information is printed to the terminal and logs if the paramater debug = true
        if self.model.debug:
            self._print_log(3)
        
        # Check that the estimator is an unsupervised ML algorithm
        if self.model.estimator_type not in ["decomposer", "clusterer"]:
            err = "Incorrect usage. The estimator specified is not a known decompostion or clustering algorithm: {0}".format(self.model.estimator)
            raise Exception(err)

        if load_script:
            # Set the key column as the index
            self.request_df.set_index("key", drop=False, inplace=True)

        # Split the features provided as a string into individual columns
        self.X = pd.DataFrame([x[feature_col_num].split("|") for x in self.request_df.values.tolist()], columns=self.model.features_df.loc[:,"name"].tolist(),\
        index=self.request_df.index)
        
        # Convert the data types based on feature definitions 
        self.X = utils.convert_types(self.X, self.model.features_df)

        # Construct the preprocessor
        prep = Preprocessor(self.model.features_df, scale_hashed=self.model.scale_hashed, scale_vectors=self.model.scale_vectors,\
        missing=self.model.missing, scaler=self.model.scaler, logfile=self.logfile, **self.model.scaler_kwargs)
        
        # Create a chache for the pipeline's transformers
        # https://scikit-learn.org/stable/modules/compose.html#caching-transformers-avoid-repeated-computation
        # cachedir = mkdtemp()

        # Construct a sklearn pipeline
        self.model.pipe = Pipeline([('preprocessor', prep)]) #, memory=cachedir)

        if self.model.dim_reduction:
            # Construct the dimensionality reduction object
            reduction = self.decomposers[self.model.reduction](**self.model.dim_reduction_args)
            
            # Include dimensionality reduction in the sklearn pipeline
            self.model.pipe.steps.insert(1, ('reduction', reduction))
            self.model.estimation_step = 2
        else:
            self.model.estimation_step = 1  

        # Construct an estimator
        estimator = self.algorithms[self.model.estimator](**self.model.estimator_kwargs)

        # Add the estimator to the sklearn pipeline
        self.model.pipe.steps.append(('estimator', estimator))  

        # Fit the data to the pipeline
        if self.model.estimator_type == "decomposer":
            # If the estimator is a decomposer we apply the fit_transform method at the end of the pipeline
            self.y = self.model.pipe.fit_transform(self.X)

            # Prepare the response
            self.response = pd.DataFrame(self.y, index=self.X.index)

        elif self.model.estimator_type == "clusterer":
            # If the estimator is a decomposer we apply the fit_predict method at the end of the pipeline
            self.y = self.model.pipe.fit_predict(self.X)

            # Prepare the response
            self.response = pd.DataFrame(self.y, columns=["result"], index=self.X.index)
                
        # Clear the cache directory setup for the pipeline's transformers
        # rmtree(cachedir)
        
        # Update the cache to keep this model in memory
        self._update_cache()
        
        if load_script:
            # Add the key field column to the response
            self.response = self.request_df.join(self.response).drop(['n_features'], axis=1)
        
            # If the function was called through the load script we return a Data Frame
            if self.model.estimator_type == "decomposer":
                self._send_table_description("reduce")
            elif self.model.estimator_type == "clusterer":
                self._send_table_description("cluster")
            
            # Debug information is printed to the terminal and logs if the paramater debug = true
            if self.model.debug:
                self._print_log(4)
            
            return self.response
            
        # If the function was called through a chart expression we return a Series
        else:
            # Dimensionality reduction is only possible through the load script
            if self.model.estimator_type == "decomposer":
                err = "Dimensionality reduction is only possible through the load script."
                raise Exception(err)
            
            # Debug information is printed to the terminal and logs if the paramater debug = true
            if self.model.debug:
                self._print_log(4)
            
            return self.response.loc[:,'result']
def fit_transform ( self , load_script = False ) : # Interpret the request data based on the expected row and column structure row_template = [ 'strData' , 'strData' ] col_headers = [ 'model_name' , 'n_features' ] feature_col_num = 1 # An additional key field column is expected if the call is made through the load script if load_script : row_template = [ 'strData' , 'strData' , 'strData' ] col_headers = [ 'model_name' , 'key' , 'n_features' ] feature_col_num = 2 # Create a Pandas Data Frame for the request data self . request_df = utils . request_df ( self . request , row_template , col_headers ) # Initialize the persistent model self . model = PersistentModel ( ) # Get the model name from the request dataframe self . model . name = self . request_df . loc [ 0 , 'model_name' ] # Get the model from cache or disk self . _get_model ( ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 3 ) # Check that the estimator is an unsupervised ML algorithm if self . model . estimator_type not in [ "decomposer" , "clusterer" ] : err = "Incorrect usage. The estimator specified is not a known decompostion or clustering algorithm: {0}" . format ( self . model . estimator ) raise Exception ( err ) if load_script : # Set the key column as the index self . request_df . set_index ( "key" , drop = False , inplace = True ) # Split the features provided as a string into individual columns self . X = pd . DataFrame ( [ x [ feature_col_num ] . split ( "|" ) for x in self . request_df . values . tolist ( ) ] , columns = self . model . features_df . loc [ : , "name" ] . tolist ( ) , index = self . request_df . index ) # Convert the data types based on feature definitions  self . X = utils . convert_types ( self . X , self . model . features_df ) # Construct the preprocessor prep = Preprocessor ( self . model . features_df , scale_hashed = self . model . scale_hashed , scale_vectors = self . model . scale_vectors , missing = self . model . missing , scaler = self . model . scaler , logfile = self . logfile , * * self . model . scaler_kwargs ) # Create a chache for the pipeline's transformers # https://scikit-learn.org/stable/modules/compose.html#caching-transformers-avoid-repeated-computation # cachedir = mkdtemp() # Construct a sklearn pipeline self . model . pipe = Pipeline ( [ ( 'preprocessor' , prep ) ] ) #, memory=cachedir) if self . model . dim_reduction : # Construct the dimensionality reduction object reduction = self . decomposers [ self . model . reduction ] ( * * self . model . dim_reduction_args ) # Include dimensionality reduction in the sklearn pipeline self . model . pipe . steps . insert ( 1 , ( 'reduction' , reduction ) ) self . model . estimation_step = 2 else : self . model . estimation_step = 1 # Construct an estimator estimator = self . algorithms [ self . model . estimator ] ( * * self . model . estimator_kwargs ) # Add the estimator to the sklearn pipeline self . model . pipe . steps . append ( ( 'estimator' , estimator ) ) # Fit the data to the pipeline if self . model . estimator_type == "decomposer" : # If the estimator is a decomposer we apply the fit_transform method at the end of the pipeline self . y = self . model . pipe . fit_transform ( self . X ) # Prepare the response self . response = pd . DataFrame ( self . y , index = self . X . index ) elif self . model . estimator_type == "clusterer" : # If the estimator is a decomposer we apply the fit_predict method at the end of the pipeline self . y = self . model . pipe . fit_predict ( self . X ) # Prepare the response self . response = pd . DataFrame ( self . y , columns = [ "result" ] , index = self . X . index ) # Clear the cache directory setup for the pipeline's transformers # rmtree(cachedir) # Update the cache to keep this model in memory self . _update_cache ( ) if load_script : # Add the key field column to the response self . response = self . request_df . join ( self . response ) . drop ( [ 'n_features' ] , axis = 1 ) # If the function was called through the load script we return a Data Frame if self . model . estimator_type == "decomposer" : self . _send_table_description ( "reduce" ) elif self . model . estimator_type == "clusterer" : self . _send_table_description ( "cluster" ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 4 ) return self . response # If the function was called through a chart expression we return a Series else : # Dimensionality reduction is only possible through the load script if self . model . estimator_type == "decomposer" : err = "Dimensionality reduction is only possible through the load script." raise Exception ( err ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . model . debug : self . _print_log ( 4 ) return self . response . loc [ : , 'result' ]