(examples, label_list, max_seq_length, tokenizer)
return features
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of InputBatch s .
def convert_examples_to_features(examples, label_list, max_seq_length, tokenizer):
    """Loads a data file into a list of `InputBatch`s."""

    print("#examples", len(examples))

    label_map = {}
    for (i, label) in enumerate(label_list):
        label_map[label] = i

    features = [[]]
    for (ex_index, example) in enumerate(tqdm(examples)):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = tokenizer.tokenize(example.text_b)

        tokens_c = tokenizer.tokenize(example.text_c)

        _truncate_seq_tuple(tokens_a, tokens_b, tokens_c, max_seq_length - 4)
        tokens_b = tokens_c + ["[SEP]"] + tokens_b

        tokens = []
        segment_ids = []
        tokens.append("[CLS]")
        segment_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            segment_ids.append(0)
        tokens.append("[SEP]")
        segment_ids.append(0)

        if tokens_b:
            for token in tokens_b:
                tokens.append(token)
                segment_ids.append(1)
            tokens.append("[SEP]")
            segment_ids.append(1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        while len(input_ids) < max_seq_length:
            input_ids.append(0)
            input_mask.append(0)
            segment_ids.append(0)

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        label_id = label_map[example.label]
        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                [tokenization.printable_text(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features[-1].append(
            InputFeatures(
                input_ids=input_ids,
                input_mask=input_mask,
                segment_ids=segment_ids,
                label_id=label_id))
        if len(features[-1]) == n_class:
            features.append([])

    if len(features[-1]) == 0:
        features = features[:-1]
    print('#features', len(features))
    return features
def convert_examples_to_features ( examples , label_list , max_seq_length , tokenizer ) : print ( "#examples" , len ( examples ) ) label_map = { } for ( i , label ) in enumerate ( label_list ) : label_map [ label ] = i features = [ [ ] ] for ( ex_index , example ) in enumerate ( tqdm ( examples ) ) : tokens_a = tokenizer . tokenize ( example . text_a ) tokens_b = tokenizer . tokenize ( example . text_b ) tokens_c = tokenizer . tokenize ( example . text_c ) _truncate_seq_tuple ( tokens_a , tokens_b , tokens_c , max_seq_length - 4 ) tokens_b = tokens_c + [ "[SEP]" ] + tokens_b tokens = [ ] segment_ids = [ ] tokens . append ( "[CLS]" ) segment_ids . append ( 0 ) for token in tokens_a : tokens . append ( token ) segment_ids . append ( 0 ) tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) if tokens_b : for token in tokens_b : tokens . append ( token ) segment_ids . append ( 1 ) tokens . append ( "[SEP]" ) segment_ids . append ( 1 ) input_ids = tokenizer . convert_tokens_to_ids ( tokens ) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. input_mask = [ 1 ] * len ( input_ids ) # Zero-pad up to the sequence length. while len ( input_ids ) < max_seq_length : input_ids . append ( 0 ) input_mask . append ( 0 ) segment_ids . append ( 0 ) assert len ( input_ids ) == max_seq_length assert len ( input_mask ) == max_seq_length assert len ( segment_ids ) == max_seq_length label_id = label_map [ example . label ] if ex_index < 5 : logger . info ( "*** Example ***" ) logger . info ( "guid: %s" % ( example . guid ) ) logger . info ( "tokens: %s" % " " . join ( [ tokenization . printable_text ( x ) for x in tokens ] ) ) logger . info ( "input_ids: %s" % " " . join ( [ str ( x ) for x in input_ids ] ) ) logger . info ( "input_mask: %s" % " " . join ( [ str ( x ) for x in input_mask ] ) ) logger . info ( "segment_ids: %s" % " " . join ( [ str ( x ) for x in segment_ids ] ) ) logger . info ( "label: %s (id = %d)" % ( example . label , label_id ) ) features [ - 1 ] . append ( InputFeatures ( input_ids = input_ids , input_mask = input_mask , segment_ids = segment_ids , label_id = label_id ) ) if len ( features [ - 1 ] ) == n_class : features . append ( [ ] ) if len ( features [ - 1 ] ) == 0 : features = features [ : - 1 ] print ( '#features' , len ( features ) ) return features