(self, locked_timeout=2)
Update an existing spaCy model with labelled training data.
        The model is stored to disk using spaCy's to_disk method.
        If the model is found to be locked (based on the existance of a lock file) this function will fail after 'locked_timeout' seconds.
Update an existing spaCy model with labelled training data.
        The model is stored to disk using spaCy's to_disk method.
        If the model is found to be locked (based on the existance of a lock file) this function will fail after 'locked_timeout' seconds.
Update an existing spaCy model with labelled training data . The model is stored to disk using spaCy s to_disk method . If the model is found to be locked ( based on the existance of a lock file ) this function will fail after locked_timeout seconds .
def _retrain_model(self, locked_timeout=2):
        """
        Update an existing spaCy model with labelled training data.
        The model is stored to disk using spaCy's to_disk method.
        If the model is found to be locked (based on the existance of a lock file) this function will fail after 'locked_timeout' seconds.
        """

        # Load the model, set up the pipeline and train the entity recognizer:
        
        # Load existing spaCy model
        if not self.blank:
            # If this is a custom model, set the path to the directory
            if self.custom:
                self.base_model = self.path + self.base_model + "/"
            
            nlp = spacy.load(self.base_model)  
        # If the parameter blank=true is passed we start with a blank Language class, e.g. en
        else:
            nlp = spacy.blank(self.base_model)  

        # Debug information is printed to the terminal and logs if the paramater debug = true
        if self.debug:
            self._print_log(7)

        # create the built-in pipeline components and add them to the pipeline

        # nlp.create_pipe works for built-ins that are registered with spaCy
        if "ner" not in nlp.pipe_names:
            ner = nlp.create_pipe("ner")
            nlp.add_pipe(ner, last=True)
        # otherwise, get it so we can add labels
        else:
            ner = nlp.get_pipe("ner")

        # add labels
        for _, annotations in self.train:
            for ent in annotations.get("entities"):
                ner.add_label(ent[2])

        # Retrain the model:
        
        # get names of other pipes to disable them during training
        other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
        with nlp.disable_pipes(*other_pipes):  # only train NER
           
            # Setup lists to store the loss for each epoch
            self.losses_train = []
            self.losses_test = []
            
            # reset and initialize the weights randomly – but only if we're
            # training a new model
            if self.blank:
                nlp.begin_training()
            for epoch in range(self.epochs): 
                random.shuffle(self.train)
                losses = {}
                # batch up the examples using spaCy's minibatch
                batches = minibatch(self.train, size=self.batch_size)
                for batch in batches:
                    texts, annotations = zip(*batch)
                    nlp.update(
                        texts,  # batch of texts
                        annotations,  # batch of annotations
                        drop=self.drop,  # dropout - make it harder to memorise data
                        losses=losses,
                    )
                # Store loss for the epoch to a list
                self.losses_train.append(('Epoch {}'.format(epoch+1), losses['ner']))

                # Debug information is printed to the terminal and logs if the paramater debug = true
                if self.debug:
                    self._print_log(8)
                
                # If a test dataset is available, calculate losses for it as well
                if self.validation is not None:
                    losses = {}

                    # batch up the examples using spaCy's minibatch
                    batches = minibatch(self.validation, size=self.batch_size)
                    for batch in batches:
                        texts, annotations = zip(*batch)
                        # Get losses for the test data without updating the model 
                        nlp.update(
                            texts,  # batch of texts
                            annotations,  # batch of annotations
                            sgd = None,  # do not update model weights
                            losses=losses,
                        )
                    # Store loss for the epoch to a list
                    self.losses_test.append(('Epoch {}'.format(epoch+1), losses['ner']))

                    # Debug information is printed to the terminal and logs if the paramater debug = true
                    if self.debug:
                        self._print_log(9)

        # Save model to output directory:
        
        output_dir = pathlib.Path(self.path + self.model + '/')
        lock_file = pathlib.Path(self.path + self.model + '.lock')
        
        # Create the output directory if required
        if not output_dir.exists():
            output_dir.mkdir(parents=True, exist_ok=False)
        # If the model is locked wait a few seconds
        elif lock_file.exists():
            time.sleep(locked_timeout)
            # If the model is still locked raise an exception
            if lock_file.exists():
                raise TimeoutError("The specified model is locked. If you think this is wrong, please delete file {0}".format(lock_file))

        try:
            # Create the lock file
            joblib.dump(self.path + self.model, filename=lock_file)

            # Store the spaCy model to disk
            nlp.to_disk(output_dir)
        finally:
            # Delete the lock file
            lock_file.unlink()

        # Debug information is printed to the terminal and logs if the paramater debug = true
        if self.debug:
            self._print_log(10)

        # Evaluate the model:

        # Prepare spaCy docs and golds for getting evaluation metrics
        docs_golds = []
        for sample in self.train:
            doc = nlp.make_doc(sample[0])
            gold = GoldParse(doc, entities=sample[1]["entities"])
            docs_golds.append((doc, gold))
        
        # Get scores for training data
        scorer_train = nlp.evaluate(docs_golds)
        # Add the training scores to evaluation metrics
        self.metrics = self._prep_scores(scorer_train)

        # Get scores for testing data and add to the evaluation metrics
        if self.validation is not None:
            docs_golds = []
            for sample in self.validation:
                doc = nlp.make_doc(sample[0])
                gold = GoldParse(doc, entities=sample[1]["entities"])
                docs_golds.append((doc, gold))
            
            scorer_test = nlp.evaluate(docs_golds)
            self.metrics = pd.concat([self.metrics, self._prep_scores(scorer_test, subset='test')], ignore_index=True)

        # Add loss metrics
        self.metrics = pd.concat([self.metrics, self._prep_losses(self.losses_train)], ignore_index=True)
        if self.validation is not None:
            self.metrics = pd.concat([self.metrics, self._prep_losses(self.losses_test, subset='test')], ignore_index=True)
def _retrain_model ( self , locked_timeout = 2 ) : # Load the model, set up the pipeline and train the entity recognizer: # Load existing spaCy model if not self . blank : # If this is a custom model, set the path to the directory if self . custom : self . base_model = self . path + self . base_model + "/" nlp = spacy . load ( self . base_model ) # If the parameter blank=true is passed we start with a blank Language class, e.g. en else : nlp = spacy . blank ( self . base_model ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . debug : self . _print_log ( 7 ) # create the built-in pipeline components and add them to the pipeline # nlp.create_pipe works for built-ins that are registered with spaCy if "ner" not in nlp . pipe_names : ner = nlp . create_pipe ( "ner" ) nlp . add_pipe ( ner , last = True ) # otherwise, get it so we can add labels else : ner = nlp . get_pipe ( "ner" ) # add labels for _ , annotations in self . train : for ent in annotations . get ( "entities" ) : ner . add_label ( ent [ 2 ] ) # Retrain the model: # get names of other pipes to disable them during training other_pipes = [ pipe for pipe in nlp . pipe_names if pipe != "ner" ] with nlp . disable_pipes ( * other_pipes ) : # only train NER # Setup lists to store the loss for each epoch self . losses_train = [ ] self . losses_test = [ ] # reset and initialize the weights randomly – but only if we're # training a new model if self . blank : nlp . begin_training ( ) for epoch in range ( self . epochs ) : random . shuffle ( self . train ) losses = { } # batch up the examples using spaCy's minibatch batches = minibatch ( self . train , size = self . batch_size ) for batch in batches : texts , annotations = zip ( * batch ) nlp . update ( texts , # batch of texts annotations , # batch of annotations drop = self . drop , # dropout - make it harder to memorise data losses = losses , ) # Store loss for the epoch to a list self . losses_train . append ( ( 'Epoch {}' . format ( epoch + 1 ) , losses [ 'ner' ] ) ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . debug : self . _print_log ( 8 ) # If a test dataset is available, calculate losses for it as well if self . validation is not None : losses = { } # batch up the examples using spaCy's minibatch batches = minibatch ( self . validation , size = self . batch_size ) for batch in batches : texts , annotations = zip ( * batch ) # Get losses for the test data without updating the model  nlp . update ( texts , # batch of texts annotations , # batch of annotations sgd = None , # do not update model weights losses = losses , ) # Store loss for the epoch to a list self . losses_test . append ( ( 'Epoch {}' . format ( epoch + 1 ) , losses [ 'ner' ] ) ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . debug : self . _print_log ( 9 ) # Save model to output directory: output_dir = pathlib . Path ( self . path + self . model + '/' ) lock_file = pathlib . Path ( self . path + self . model + '.lock' ) # Create the output directory if required if not output_dir . exists ( ) : output_dir . mkdir ( parents = True , exist_ok = False ) # If the model is locked wait a few seconds elif lock_file . exists ( ) : time . sleep ( locked_timeout ) # If the model is still locked raise an exception if lock_file . exists ( ) : raise TimeoutError ( "The specified model is locked. If you think this is wrong, please delete file {0}" . format ( lock_file ) ) try : # Create the lock file joblib . dump ( self . path + self . model , filename = lock_file ) # Store the spaCy model to disk nlp . to_disk ( output_dir ) finally : # Delete the lock file lock_file . unlink ( ) # Debug information is printed to the terminal and logs if the paramater debug = true if self . debug : self . _print_log ( 10 ) # Evaluate the model: # Prepare spaCy docs and golds for getting evaluation metrics docs_golds = [ ] for sample in self . train : doc = nlp . make_doc ( sample [ 0 ] ) gold = GoldParse ( doc , entities = sample [ 1 ] [ "entities" ] ) docs_golds . append ( ( doc , gold ) ) # Get scores for training data scorer_train = nlp . evaluate ( docs_golds ) # Add the training scores to evaluation metrics self . metrics = self . _prep_scores ( scorer_train ) # Get scores for testing data and add to the evaluation metrics if self . validation is not None : docs_golds = [ ] for sample in self . validation : doc = nlp . make_doc ( sample [ 0 ] ) gold = GoldParse ( doc , entities = sample [ 1 ] [ "entities" ] ) docs_golds . append ( ( doc , gold ) ) scorer_test = nlp . evaluate ( docs_golds ) self . metrics = pd . concat ( [ self . metrics , self . _prep_scores ( scorer_test , subset = 'test' ) ] , ignore_index = True ) # Add loss metrics self . metrics = pd . concat ( [ self . metrics , self . _prep_losses ( self . losses_train ) ] , ignore_index = True ) if self . validation is not None : self . metrics = pd . concat ( [ self . metrics , self . _prep_losses ( self . losses_test , subset = 'test' ) ] , ignore_index = True )