(
    source: Union[Path, str], tok_config: Dict, trf_config: Dict
)
return HFObjects(tokenizer, transformer, vocab_file_contents)
Create a Huggingface transformer model from pretrained weights. Will
    download the model if it is not already downloaded.

    source (Union[str, Path]): The name of the model or a path to it, such as
        'bert-base-cased'.
    tok_config (dict): Settings to pass to the tokenizer.
    trf_config (dict): Settings to pass to the transformer.
Create a Huggingface transformer model from pretrained weights. Will
    download the model if it is not already downloaded.
Create a Huggingface transformer model from pretrained weights . Will download the model if it is not already downloaded .
def huggingface_from_pretrained(
    source: Union[Path, str], tok_config: Dict, trf_config: Dict
) -> HFObjects:
    """Create a Huggingface transformer model from pretrained weights. Will
    download the model if it is not already downloaded.

    source (Union[str, Path]): The name of the model or a path to it, such as
        'bert-base-cased'.
    tok_config (dict): Settings to pass to the tokenizer.
    trf_config (dict): Settings to pass to the transformer.
    """
    if hasattr(source, "absolute"):
        str_path = str(source.absolute())
    else:
        str_path = source
    tokenizer = AutoTokenizer.from_pretrained(str_path, **tok_config)
    vocab_file_contents = None
    if hasattr(tokenizer, "vocab_file"):
        with open(tokenizer.vocab_file, "rb") as fileh:
            vocab_file_contents = fileh.read()
    trf_config["return_dict"] = True
    config = AutoConfig.from_pretrained(str_path, **trf_config)
    transformer = AutoModel.from_pretrained(str_path, config=config)
    ops = get_current_ops()
    if isinstance(ops, CupyOps):
        transformer.cuda()
    return HFObjects(tokenizer, transformer, vocab_file_contents)
def huggingface_from_pretrained ( source : Union [ Path , str ] , tok_config : Dict , trf_config : Dict ) -> HFObjects : if hasattr ( source , "absolute" ) : str_path = str ( source . absolute ( ) ) else : str_path = source tokenizer = AutoTokenizer . from_pretrained ( str_path , * * tok_config ) vocab_file_contents = None if hasattr ( tokenizer , "vocab_file" ) : with open ( tokenizer . vocab_file , "rb" ) as fileh : vocab_file_contents = fileh . read ( ) trf_config [ "return_dict" ] = True config = AutoConfig . from_pretrained ( str_path , * * trf_config ) transformer = AutoModel . from_pretrained ( str_path , config = config ) ops = get_current_ops ( ) if isinstance ( ops , CupyOps ) : transformer . cuda ( ) return HFObjects ( tokenizer , transformer , vocab_file_contents )