(examples, label_list, max_seq_length, tokenizer)
return features
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of InputBatch s .
def convert_examples_to_features(examples, label_list, max_seq_length, tokenizer):
    """Loads a data file into a list of `InputBatch`s."""

    label_map = {label: i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[: (max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        label_id = label_map[example.label]
        if ex_index < 0:
            logger.info("*** Example ***")
            logger.info("guid: {}".format(example.guid))
            logger.info("tokens: {}".format(" ".join([str(x) for x in tokens])))
            logger.info("input_ids: {}".format(" ".join([str(x) for x in input_ids])))
            logger.info("input_mask: {}".format(" ".join([str(x) for x in input_mask])))
            logger.info(
                "segment_ids: {}".format(" ".join([str(x) for x in segment_ids]))
            )
            logger.info("label: {} (id = {})".format(example.label, label_id))

        features.append(
            InputFeatures(
                input_ids=input_ids,
                input_mask=input_mask,
                segment_ids=segment_ids,
                label_id=label_id,
            )
        )
    return features
def convert_examples_to_features ( examples , label_list , max_seq_length , tokenizer ) : label_map = { label : i for i , label in enumerate ( label_list ) } features = [ ] for ( ex_index , example ) in enumerate ( examples ) : tokens_a = tokenizer . tokenize ( example . text_a ) tokens_b = None if example . text_b : tokens_b = tokenizer . tokenize ( example . text_b ) # Modifies `tokens_a` and `tokens_b` in place so that the total # length is less than the specified length. # Account for [CLS], [SEP], [SEP] with "- 3" _truncate_seq_pair ( tokens_a , tokens_b , max_seq_length - 3 ) else : # Account for [CLS] and [SEP] with "- 2" if len ( tokens_a ) > max_seq_length - 2 : tokens_a = tokens_a [ : ( max_seq_length - 2 ) ] # The convention in BERT is: # (a) For sequence pairs: #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1 # (b) For single sequences: #  tokens:   [CLS] the dog is hairy . [SEP] #  type_ids: 0   0   0   0  0     0 0 # # Where "type_ids" are used to indicate whether this is the first # sequence or the second sequence. The embedding vectors for `type=0` and # `type=1` were learned during pre-training and are added to the wordpiece # embedding vector (and position vector). This is not *strictly* necessary # since the [SEP] token unambigiously separates the sequences, but it makes # it easier for the model to learn the concept of sequences. # # For classification tasks, the first vector (corresponding to [CLS]) is # used as as the "sentence vector". Note that this only makes sense because # the entire model is fine-tuned. tokens = [ "[CLS]" ] + tokens_a + [ "[SEP]" ] segment_ids = [ 0 ] * len ( tokens ) if tokens_b : tokens += tokens_b + [ "[SEP]" ] segment_ids += [ 1 ] * ( len ( tokens_b ) + 1 ) input_ids = tokenizer . convert_tokens_to_ids ( tokens ) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. input_mask = [ 1 ] * len ( input_ids ) # Zero-pad up to the sequence length. padding = [ 0 ] * ( max_seq_length - len ( input_ids ) ) input_ids += padding input_mask += padding segment_ids += padding assert len ( input_ids ) == max_seq_length assert len ( input_mask ) == max_seq_length assert len ( segment_ids ) == max_seq_length label_id = label_map [ example . label ] if ex_index < 0 : logger . info ( "*** Example ***" ) logger . info ( "guid: {}" . format ( example . guid ) ) logger . info ( "tokens: {}" . format ( " " . join ( [ str ( x ) for x in tokens ] ) ) ) logger . info ( "input_ids: {}" . format ( " " . join ( [ str ( x ) for x in input_ids ] ) ) ) logger . info ( "input_mask: {}" . format ( " " . join ( [ str ( x ) for x in input_mask ] ) ) ) logger . info ( "segment_ids: {}" . format ( " " . join ( [ str ( x ) for x in segment_ids ] ) ) ) logger . info ( "label: {} (id = {})" . format ( example . label , label_id ) ) features . append ( InputFeatures ( input_ids = input_ids , input_mask = input_mask , segment_ids = segment_ids , label_id = label_id , ) ) return features