(self, sentences, oov_way='avg', filter_spec_tokens=True)
return self.oov(batches, oov_way, filter_spec_tokens)
Get tokens, tokens embedding

        Parameters
        ----------
        sentences : List[str]
            sentences for encoding.
        oov_way : str, default avg.
            use **avg**, **sum** or **last** to get token embedding for those out of
            vocabulary words
        filter_spec_tokens : bool
            filter [CLS], [SEP] tokens.

        Returns
        -------
        List[(List[str], List[ndarray])]
            List of tokens, and tokens embedding
Get tokens, tokens embedding
Get tokens tokens embedding
def embedding(self, sentences, oov_way='avg', filter_spec_tokens=True):
        """
        Get tokens, tokens embedding

        Parameters
        ----------
        sentences : List[str]
            sentences for encoding.
        oov_way : str, default avg.
            use **avg**, **sum** or **last** to get token embedding for those out of
            vocabulary words
        filter_spec_tokens : bool
            filter [CLS], [SEP] tokens.

        Returns
        -------
        List[(List[str], List[ndarray])]
            List of tokens, and tokens embedding
        """
        data_iter = self.data_loader(sentences=sentences)
        batches = []
        for token_ids, valid_length, token_types in data_iter:
            token_ids = token_ids.as_in_context(self.ctx)
            valid_length = valid_length.as_in_context(self.ctx)
            token_types = token_types.as_in_context(self.ctx)
            sequence_outputs = self.bert(token_ids, token_types,
                                         valid_length.astype(self.dtype))
            for token_id, sequence_output in zip(token_ids.asnumpy(),
                                                 sequence_outputs.asnumpy()):
                batches.append((token_id, sequence_output))
        return self.oov(batches, oov_way, filter_spec_tokens)
def embedding ( self , sentences , oov_way = 'avg' , filter_spec_tokens = True ) : data_iter = self . data_loader ( sentences = sentences ) batches = [ ] for token_ids , valid_length , token_types in data_iter : token_ids = token_ids . as_in_context ( self . ctx ) valid_length = valid_length . as_in_context ( self . ctx ) token_types = token_types . as_in_context ( self . ctx ) sequence_outputs = self . bert ( token_ids , token_types , valid_length . astype ( self . dtype ) ) for token_id , sequence_output in zip ( token_ids . asnumpy ( ) , sequence_outputs . asnumpy ( ) ) : batches . append ( ( token_id , sequence_output ) ) return self . oov ( batches , oov_way , filter_spec_tokens )