(self, sentence)
return ' '.join(output)
segment single sentence (whitespace-tokenized string) with BPE encoding
segment single sentence (whitespace-tokenized string) with BPE encoding
segment single sentence ( whitespace - tokenized string ) with BPE encoding
def segment(self, sentence):
        """segment single sentence (whitespace-tokenized string) with BPE encoding"""
        output = []
        for word in sentence.split():
            new_word = [out for segment in self._isolate_glossaries(word)
                        for out in encode(segment,
                                          self.bpe_codes,
                                          self.bpe_codes_reverse,
                                          self.vocab,
                                          self.separator,
                                          self.version,
                                          self.cache,
                                          self.glossaries)]

            for item in new_word[:-1]:
                output.append(item + self.separator)
            output.append(new_word[-1])

        return ' '.join(output)
def segment ( self , sentence ) : output = [ ] for word in sentence . split ( ) : new_word = [ out for segment in self . _isolate_glossaries ( word ) for out in encode ( segment , self . bpe_codes , self . bpe_codes_reverse , self . vocab , self . separator , self . version , self . cache , self . glossaries ) ] for item in new_word [ : - 1 ] : output . append ( item + self . separator ) output . append ( new_word [ - 1 ] ) return ' ' . join ( output )