(persona, history, reply, tokenizer, lm_labels=False, with_eos=True)
return instance
Build a sequence of input from 3 segments: persona, history and last reply.
Build a sequence of input from 3 segments: persona, history and last reply.
Build a sequence of input from 3 segments : persona history and last reply .
def build_input_from_segments(persona, history, reply, tokenizer, lm_labels=False, with_eos=True):
    """ Build a sequence of input from 3 segments: persona, history and last reply. """
    bos, eos, speaker1, speaker2 = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[:-1])
    sequence = [[bos] + list(chain(*persona))] + history + [reply + ([eos] if with_eos else [])]
    sequence = [sequence[0]] + [[speaker2 if (len(sequence)-i) % 2 else speaker1] + s for i, s in enumerate(sequence[1:])]
    instance = {}
    instance["input_ids"] = list(chain(*sequence))
    instance["token_type_ids"] = [speaker2 if i % 2 else speaker1 for i, s in enumerate(sequence) for _ in s]
    instance["mc_token_ids"] = len(instance["input_ids"]) - 1
    instance["lm_labels"] = [-100] * len(instance["input_ids"])
    if lm_labels:
        instance["lm_labels"] = ([-100] * sum(len(s) for s in sequence[:-1])) + [-100] + sequence[-1][1:]
    return instance
def build_input_from_segments ( persona , history , reply , tokenizer , lm_labels = False , with_eos = True ) : bos , eos , speaker1 , speaker2 = tokenizer . convert_tokens_to_ids ( SPECIAL_TOKENS [ : - 1 ] ) sequence = [ [ bos ] + list ( chain ( * persona ) ) ] + history + [ reply + ( [ eos ] if with_eos else [ ] ) ] sequence = [ sequence [ 0 ] ] + [ [ speaker2 if ( len ( sequence ) - i ) % 2 else speaker1 ] + s for i , s in enumerate ( sequence [ 1 : ] ) ] instance = { } instance [ "input_ids" ] = list ( chain ( * sequence ) ) instance [ "token_type_ids" ] = [ speaker2 if i % 2 else speaker1 for i , s in enumerate ( sequence ) for _ in s ] instance [ "mc_token_ids" ] = len ( instance [ "input_ids" ] ) - 1 instance [ "lm_labels" ] = [ - 100 ] * len ( instance [ "input_ids" ] ) if lm_labels : instance [ "lm_labels" ] = ( [ - 100 ] * sum ( len ( s ) for s in sequence [ : - 1 ] ) ) + [ - 100 ] + sequence [ - 1 ] [ 1 : ] return instance