(url: str, cache_dir: Union[str, Path] = None)
return cache_path
Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
Given a URL look for the corresponding dataset in the local cache . If it s not there download it . Then return the path to the cached file .
def get_from_cache(url: str, cache_dir: Union[str, Path] = None) -> str:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    os.makedirs(cache_dir, exist_ok=True)

    # Get eTag to add to filename, if it exists.
    if url.startswith("s3://"):
        etag = s3_etag(url)
    else:
        response = requests.head(url, allow_redirects=True)
        if response.status_code != 200:
            raise IOError("HEAD request failed for url {} with status code {}"
                          .format(url, response.status_code))
        etag = response.headers.get("ETag")

    filename = url_to_filename(url, etag)

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    if not os.path.exists(cache_path):
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:
            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)

            # GET file object
            if url.startswith("s3://"):
                s3_get(url, temp_file)
            else:
                http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
            with open(cache_path, 'wb') as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            logger.info("creating metadata file for %s", cache_path)
            meta = {'url': url, 'etag': etag}
            meta_path = cache_path + '.json'
            with open(meta_path, 'w') as meta_file:
                json.dump(meta, meta_file)

            logger.info("removing temp file %s", temp_file.name)

    return cache_path
def get_from_cache ( url : str , cache_dir : Union [ str , Path ] = None ) -> str : if cache_dir is None : cache_dir = PYTORCH_PRETRAINED_BERT_CACHE if isinstance ( cache_dir , Path ) : cache_dir = str ( cache_dir ) os . makedirs ( cache_dir , exist_ok = True ) # Get eTag to add to filename, if it exists. if url . startswith ( "s3://" ) : etag = s3_etag ( url ) else : response = requests . head ( url , allow_redirects = True ) if response . status_code != 200 : raise IOError ( "HEAD request failed for url {} with status code {}" . format ( url , response . status_code ) ) etag = response . headers . get ( "ETag" ) filename = url_to_filename ( url , etag ) # get cache path to put the file cache_path = os . path . join ( cache_dir , filename ) if not os . path . exists ( cache_path ) : # Download to temporary file, then copy to cache dir once finished. # Otherwise you get corrupt cache entries if the download gets interrupted. with tempfile . NamedTemporaryFile ( ) as temp_file : logger . info ( "%s not found in cache, downloading to %s" , url , temp_file . name ) # GET file object if url . startswith ( "s3://" ) : s3_get ( url , temp_file ) else : http_get ( url , temp_file ) # we are copying the file before closing it, so flush to avoid truncation temp_file . flush ( ) # shutil.copyfileobj() starts at the current position, so go to the start temp_file . seek ( 0 ) logger . info ( "copying %s to cache at %s" , temp_file . name , cache_path ) with open ( cache_path , 'wb' ) as cache_file : shutil . copyfileobj ( temp_file , cache_file ) logger . info ( "creating metadata file for %s" , cache_path ) meta = { 'url' : url , 'etag' : etag } meta_path = cache_path + '.json' with open ( meta_path , 'w' ) as meta_file : json . dump ( meta , meta_file ) logger . info ( "removing temp file %s" , temp_file . name ) return cache_path