(queue, total_docs)
Poll the queue for ProgressValue objects and log progress to logger
Poll the queue for ProgressValue objects and log progress to logger
Poll the queue for ProgressValue objects and log progress to logger
def _output_progress(queue, total_docs):
    """Poll the queue for ProgressValue objects and log progress to logger"""
    logger.debug('Starting progress reporting process')
    page_progress = {}
    progress = total_dumped = 0
    start = time.time()
    while total_dumped == 0 or progress < total_dumped:
        try:
            poll_start = time.time()
            while time.time() - poll_start < 20:
                value = queue.get(timeout=10)
                page_progress[value.page] = value
        except Empty:
            pass
        total_dumped = sum(val.total for val in page_progress.values())
        progress = sum(val.progress for val in page_progress.values())
        elapsed = time.time() - start
        docs_per_second = progress / elapsed
        docs_remaining = total_docs - progress
        try:
            time_remaining = docs_remaining / docs_per_second
            time_remaining = str(timedelta(seconds=time_remaining)).split('.')[0]
        except ArithmeticError:
            time_remaining = 'unknown'
        if total_dumped > 0:
            elapsed = str(timedelta(seconds=elapsed)).split('.')[0]
            logger.info(
                '{progress} of {total} ({percent}%) ({dumped} dumped) processed in {elapsed} '
                '(Estimated completion in {remaining}) '
                '(Avg processing rate: {rate} docs per sec)'.format(
                    progress=progress,
                    total=total_docs,
                    percent=int(progress * 100 // total_docs),
                    dumped=total_dumped,
                    elapsed=elapsed,
                    remaining=time_remaining,
                    rate=int(docs_per_second)
                ))
def _output_progress ( queue , total_docs ) : logger . debug ( 'Starting progress reporting process' ) page_progress = { } progress = total_dumped = 0 start = time . time ( ) while total_dumped == 0 or progress < total_dumped : try : poll_start = time . time ( ) while time . time ( ) - poll_start < 20 : value = queue . get ( timeout = 10 ) page_progress [ value . page ] = value except Empty : pass total_dumped = sum ( val . total for val in page_progress . values ( ) ) progress = sum ( val . progress for val in page_progress . values ( ) ) elapsed = time . time ( ) - start docs_per_second = progress / elapsed docs_remaining = total_docs - progress try : time_remaining = docs_remaining / docs_per_second time_remaining = str ( timedelta ( seconds = time_remaining ) ) . split ( '.' ) [ 0 ] except ArithmeticError : time_remaining = 'unknown' if total_dumped > 0 : elapsed = str ( timedelta ( seconds = elapsed ) ) . split ( '.' ) [ 0 ] logger . info ( '{progress} of {total} ({percent}%) ({dumped} dumped) processed in {elapsed} ' '(Estimated completion in {remaining}) ' '(Avg processing rate: {rate} docs per sec)' . format ( progress = progress , total = total_docs , percent = int ( progress * 100 // total_docs ) , dumped = total_dumped , elapsed = elapsed , remaining = time_remaining , rate = int ( docs_per_second ) ) )