(self, batches, oov_way='avg', filter_spec_tokens=True)
return sentences
How to handle oov. Also filter out [CLS], [SEP] tokens.

        Parameters
        ----------
        batches : List[(tokens_id,
                        sequence_outputs,
                        pooled_output].
            batch   token_ids (max_seq_length, ),
                    sequence_outputs (max_seq_length, dim, ),
                    pooled_output (dim, )
        oov_way : str
            use **avg**, **sum** or **last** to get token embedding for those out of
            vocabulary words
        filter_spec_tokens : bool
            filter [CLS], [SEP] tokens.

        Returns
        -------
        List[(List[str], List[ndarray])]
            List of tokens, and tokens embedding
How to handle oov. Also filter out [CLS], [SEP] tokens.
How to handle oov . Also filter out [ CLS ] [ SEP ] tokens .
def oov(self, batches, oov_way='avg', filter_spec_tokens=True):
        """
        How to handle oov. Also filter out [CLS], [SEP] tokens.

        Parameters
        ----------
        batches : List[(tokens_id,
                        sequence_outputs,
                        pooled_output].
            batch   token_ids (max_seq_length, ),
                    sequence_outputs (max_seq_length, dim, ),
                    pooled_output (dim, )
        oov_way : str
            use **avg**, **sum** or **last** to get token embedding for those out of
            vocabulary words
        filter_spec_tokens : bool
            filter [CLS], [SEP] tokens.

        Returns
        -------
        List[(List[str], List[ndarray])]
            List of tokens, and tokens embedding
        """
        sentences = []
        for token_ids, sequence_outputs in batches:
            tokens = []
            tensors = []
            oov_len = 1
            for token_id, sequence_output in zip(token_ids, sequence_outputs):
                if token_id == 1:
                    # [PAD] token, sequence is finished.
                    break
                if (token_id in (2, 3)) and filter_spec_tokens:
                    # [CLS], [SEP]
                    continue
                token = self.vocab.idx_to_token[token_id]
                if token.startswith('##'):
                    token = token[2:]
                    tokens[-1] += token
                    if oov_way == 'last':
                        tensors[-1] = sequence_output
                    else:
                        tensors[-1] += sequence_output
                    if oov_way == 'avg':
                        oov_len += 1
                else:  # iv, avg last oov
                    if oov_len > 1:
                        tensors[-1] /= oov_len
                        oov_len = 1
                    tokens.append(token)
                    tensors.append(sequence_output)
            if oov_len > 1:  # if the whole sentence is one oov, handle this special case
                tensors[-1] /= oov_len
            sentences.append((tokens, tensors))
        return sentences
def oov ( self , batches , oov_way = 'avg' , filter_spec_tokens = True ) : sentences = [ ] for token_ids , sequence_outputs in batches : tokens = [ ] tensors = [ ] oov_len = 1 for token_id , sequence_output in zip ( token_ids , sequence_outputs ) : if token_id == 1 : # [PAD] token, sequence is finished. break if ( token_id in ( 2 , 3 ) ) and filter_spec_tokens : # [CLS], [SEP] continue token = self . vocab . idx_to_token [ token_id ] if token . startswith ( '##' ) : token = token [ 2 : ] tokens [ - 1 ] += token if oov_way == 'last' : tensors [ - 1 ] = sequence_output else : tensors [ - 1 ] += sequence_output if oov_way == 'avg' : oov_len += 1 else : # iv, avg last oov if oov_len > 1 : tensors [ - 1 ] /= oov_len oov_len = 1 tokens . append ( token ) tensors . append ( sequence_output ) if oov_len > 1 : # if the whole sentence is one oov, handle this special case tensors [ - 1 ] /= oov_len sentences . append ( ( tokens , tensors ) ) return sentences