(model, config, gpt2_checkpoint_path)
return model
Load tf checkpoints in a pytorch model
Load tf checkpoints in a pytorch model
Load tf checkpoints in a pytorch model
def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path):
    """ Load tf checkpoints in a pytorch model
    """
    try:
        import re
        import numpy as np
        import tensorflow as tf
    except ImportError:
        logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
            "https://www.tensorflow.org/install/ for installation instructions.")
        raise
    tf_path = os.path.abspath(gpt2_checkpoint_path)
    logger.info("Converting TensorFlow checkpoint from {}".format(tf_path))
    # Load weights from TF model
    init_vars = tf.train.list_variables(tf_path)
    names = []
    arrays = []
    for name, shape in init_vars:
        logger.info("Loading TF weight {} with shape {}".format(name, shape))
        array = tf.train.load_variable(tf_path, name)
        names.append(name)
        arrays.append(array.squeeze())

    for name, array in zip(names, arrays):
        name = name[6:]  # skip "model/"
        name = name.split('/')
        pointer = model
        for m_name in name:
            if re.fullmatch(r'[A-Za-z]+\d+', m_name):
                l = re.split(r'(\d+)', m_name)
            else:
                l = [m_name]
            if l[0] == 'w' or l[0] == 'g':
                pointer = getattr(pointer, 'weight')
            elif l[0] == 'b':
                pointer = getattr(pointer, 'bias')
            elif l[0] == 'wpe' or l[0] == 'wte':
                pointer = getattr(pointer, l[0])
                pointer = getattr(pointer, 'weight')
            else:
                pointer = getattr(pointer, l[0])
            if len(l) >= 2:
                num = int(l[1])
                pointer = pointer[num]
        try:
            assert pointer.shape == array.shape
        except AssertionError as e:
            e.args += (pointer.shape, array.shape)
            raise
        logger.info("Initialize PyTorch weight {}".format(name))
        pointer.data = torch.from_numpy(array)
    return model
def load_tf_weights_in_gpt2 ( model , config , gpt2_checkpoint_path ) : try : import re import numpy as np import tensorflow as tf except ImportError : logger . error ( "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " "https://www.tensorflow.org/install/ for installation instructions." ) raise tf_path = os . path . abspath ( gpt2_checkpoint_path ) logger . info ( "Converting TensorFlow checkpoint from {}" . format ( tf_path ) ) # Load weights from TF model init_vars = tf . train . list_variables ( tf_path ) names = [ ] arrays = [ ] for name , shape in init_vars : logger . info ( "Loading TF weight {} with shape {}" . format ( name , shape ) ) array = tf . train . load_variable ( tf_path , name ) names . append ( name ) arrays . append ( array . squeeze ( ) ) for name , array in zip ( names , arrays ) : name = name [ 6 : ] # skip "model/" name = name . split ( '/' ) pointer = model for m_name in name : if re . fullmatch ( r'[A-Za-z]+\d+' , m_name ) : l = re . split ( r'(\d+)' , m_name ) else : l = [ m_name ] if l [ 0 ] == 'w' or l [ 0 ] == 'g' : pointer = getattr ( pointer , 'weight' ) elif l [ 0 ] == 'b' : pointer = getattr ( pointer , 'bias' ) elif l [ 0 ] == 'wpe' or l [ 0 ] == 'wte' : pointer = getattr ( pointer , l [ 0 ] ) pointer = getattr ( pointer , 'weight' ) else : pointer = getattr ( pointer , l [ 0 ] ) if len ( l ) >= 2 : num = int ( l [ 1 ] ) pointer = pointer [ num ] try : assert pointer . shape == array . shape except AssertionError as e : e . args += ( pointer . shape , array . shape ) raise logger . info ( "Initialize PyTorch weight {}" . format ( name ) ) pointer . data = torch . from_numpy ( array ) return model