(self)
Prepare the data for retraining the model.
        This transforms the data into spaCy's training data format with tuples of text and entity offsets.
Prepare the data for retraining the model.
        This transforms the data into spaCy's training data format with tuples of text and entity offsets.
Prepare the data for retraining the model . This transforms the data into spaCy s training data format with tuples of text and entity offsets .
def _prep_data(self):
        """
        Prepare the data for retraining the model.
        This transforms the data into spaCy's training data format with tuples of text and entity offsets.
        """

        # Firstly, we transform the dataframe which has repeated texts with one row per entity...
        # to a list of each text with its corresponding dictionary of entities.

        prev_text = ''
        self.train = []
        entities = {"entities": []}

        # For each sample in the dataframe
        for i in self.request_df.index:
            # Extract the text for the current index
            text = self.request_df.loc[i, 'text']    

            # If this is not the first record and we have reached a new text
            if i > 0 and text != prev_text:
                # Add the text and dictionary of entities to the training set
                self.train.append((prev_text, entities))

                # Reset variables
                entities = {"entities": []}
                prev_text = text
            # For the first record we set previous text to this text
            elif i == 0:
                prev_text = text

            # Extract the entity and entity type for the current index
            entity = (self.request_df.loc[i, 'entity'], self.request_df.loc[i, 'entity_type'])

            # Add entity to the entities dictionary 
            entities["entities"].append(entity)

        # Add the final text and dictionary of entities to the training set
        self.train.append((prev_text, entities))
        
        # Print the semi-transformed data to the logs
        if self.debug:
            self._print_log(6)

        # Complete the data prep by calculating entity offsets and finalizing the format for spaCy

        # Format the training data for spaCy
        for sample in self.train:
            # Get the text and named entities for the current sample
            text = sample[0]
            entities = sample[1]["entities"]
            entity_boundaries = []
            
            # Structure the entities and types into a DataFrame
            entities_df = pd.DataFrame(zip(*entities)).T
            entities_df.columns = ['ents', 'types']
            
            # For each unique entity
            for entity in entities_df.ents.unique():
                
                # Set up a regex pattern to look for the entity w.r.t. word boundaries 
                pattern = re.compile(r"\b" + entity + r"\b")

                # Get entity types for the entity. This may be a series of values if the entity appears more than once.
                types = entities_df[entities_df.ents == entity].types.reset_index(drop=True)
                has_multiple_types = True if len(types.unique()) > 1 else False
                i = 0
                
                # Find all occurrences of the entity in the text
                for match in re.finditer(pattern, text):
                    entity_boundaries.append((match.start(), match.end(), types[i]))

                    # Assign types according to the series
                    if has_multiple_types:
                        i += 1
                        
            if len(entity_boundaries)  > 0:
                # Prepare variables to check for overlapping entity boundaries
                start, stop, entity_type = map(list, zip(*entity_boundaries))

                # Drop overlapping entities, i.e. where an entity is a subset of a longer entity
                for i in range(len(start)):
                    other_start, other_stop = copy(start), copy(stop)
                    del other_start[i]
                    del other_stop[i]
                    
                    for j in range(len(other_start)):
                        if start[i] >= other_start[j] and stop[i] <= other_stop[j]:
                            entity_boundaries.remove((start[i], stop[i], entity_type[i]))
            
            # Add the entity boundaries to the sample
            sample[1]["entities"] = entity_boundaries

        # If required, split the data into training and testing sets
        if self.test > 0:
            self.train, self.validation = train_test_split(self.train, test_size=self.test)
        # Otherwise use the entire dataset for training
        else:
            self.validation = None

        # Print the final training data to the logs
        if self.debug:
            self._print_log(6)
def _prep_data ( self ) : # Firstly, we transform the dataframe which has repeated texts with one row per entity... # to a list of each text with its corresponding dictionary of entities. prev_text = '' self . train = [ ] entities = { "entities" : [ ] } # For each sample in the dataframe for i in self . request_df . index : # Extract the text for the current index text = self . request_df . loc [ i , 'text' ] # If this is not the first record and we have reached a new text if i > 0 and text != prev_text : # Add the text and dictionary of entities to the training set self . train . append ( ( prev_text , entities ) ) # Reset variables entities = { "entities" : [ ] } prev_text = text # For the first record we set previous text to this text elif i == 0 : prev_text = text # Extract the entity and entity type for the current index entity = ( self . request_df . loc [ i , 'entity' ] , self . request_df . loc [ i , 'entity_type' ] ) # Add entity to the entities dictionary  entities [ "entities" ] . append ( entity ) # Add the final text and dictionary of entities to the training set self . train . append ( ( prev_text , entities ) ) # Print the semi-transformed data to the logs if self . debug : self . _print_log ( 6 ) # Complete the data prep by calculating entity offsets and finalizing the format for spaCy # Format the training data for spaCy for sample in self . train : # Get the text and named entities for the current sample text = sample [ 0 ] entities = sample [ 1 ] [ "entities" ] entity_boundaries = [ ] # Structure the entities and types into a DataFrame entities_df = pd . DataFrame ( zip ( * entities ) ) . T entities_df . columns = [ 'ents' , 'types' ] # For each unique entity for entity in entities_df . ents . unique ( ) : # Set up a regex pattern to look for the entity w.r.t. word boundaries  pattern = re . compile ( r"\b" + entity + r"\b" ) # Get entity types for the entity. This may be a series of values if the entity appears more than once. types = entities_df [ entities_df . ents == entity ] . types . reset_index ( drop = True ) has_multiple_types = True if len ( types . unique ( ) ) > 1 else False i = 0 # Find all occurrences of the entity in the text for match in re . finditer ( pattern , text ) : entity_boundaries . append ( ( match . start ( ) , match . end ( ) , types [ i ] ) ) # Assign types according to the series if has_multiple_types : i += 1 if len ( entity_boundaries ) > 0 : # Prepare variables to check for overlapping entity boundaries start , stop , entity_type = map ( list , zip ( * entity_boundaries ) ) # Drop overlapping entities, i.e. where an entity is a subset of a longer entity for i in range ( len ( start ) ) : other_start , other_stop = copy ( start ) , copy ( stop ) del other_start [ i ] del other_stop [ i ] for j in range ( len ( other_start ) ) : if start [ i ] >= other_start [ j ] and stop [ i ] <= other_stop [ j ] : entity_boundaries . remove ( ( start [ i ] , stop [ i ] , entity_type [ i ] ) ) # Add the entity boundaries to the sample sample [ 1 ] [ "entities" ] = entity_boundaries # If required, split the data into training and testing sets if self . test > 0 : self . train , self . validation = train_test_split ( self . train , test_size = self . test ) # Otherwise use the entire dataset for training else : self . validation = None # Print the final training data to the logs if self . debug : self . _print_log ( 6 )