(filename, length, worker_number)
Each worker claims the ``worker_number``th file in the ``filename``
    tar.gz file, and copies it to blob_db.

    Iterated file handles for blobs cannot be passed to coroutines or
    futures, because as soon as the caller iterates to the next blob,
    the previous handle is closed. We could read the data and pass that,
    but blobs can be too big to read at once.

    So threaded workers all need to read the same tar.gz file and
    iterate the files inside it, picking out the ones that belong to
    them.
Each worker claims the ``worker_number``th file in the ``filename``
    tar.gz file, and copies it to blob_db.
Each worker claims the worker_number th file in the filename tar . gz file and copies it to blob_db .
def worker(filename, length, worker_number):
    """
    Each worker claims the ``worker_number``th file in the ``filename``
    tar.gz file, and copies it to blob_db.

    Iterated file handles for blobs cannot be passed to coroutines or
    futures, because as soon as the caller iterates to the next blob,
    the previous handle is closed. We could read the data and pass that,
    but blobs can be too big to read at once.

    So threaded workers all need to read the same tar.gz file and
    iterate the files inside it, picking out the ones that belong to
    them.
    """
    # ^^^ Yeah, not great. Maybe filesystem caching helps.
    blob_db = get_blob_db()
    with tarfile.open(filename, 'r:gz') as tgzfile:
        iterable = (tarinfo for i, tarinfo in enumerate(tgzfile)
                    if i % NUM_WORKERS == worker_number)
        prefix = f"Worker {worker_number}"
        for tarinfo in with_progress_bar(iterable, length, prefix=prefix, oneline=False):
            key = tarinfo.name
            fileobj = tgzfile.extractfile(tarinfo)
            blob_db.copy_blob(fileobj, key)
def worker ( filename , length , worker_number ) : # ^^^ Yeah, not great. Maybe filesystem caching helps. blob_db = get_blob_db ( ) with tarfile . open ( filename , 'r:gz' ) as tgzfile : iterable = ( tarinfo for i , tarinfo in enumerate ( tgzfile ) if i % NUM_WORKERS == worker_number ) prefix = f"Worker {worker_number}" for tarinfo in with_progress_bar ( iterable , length , prefix = prefix , oneline = False ) : key = tarinfo . name fileobj = tgzfile . extractfile ( tarinfo ) blob_db . copy_blob ( fileobj , key )