()
Fetches AsyncIndicators that
        1. were not queued till now or were last queued more than 4 hours ago
        2. have failed less than ASYNC_INDICATOR_MAX_RETRIES times
        This task quits after it has run for more than
        ASYNC_INDICATOR_QUEUE_TIME - 30 seconds i.e 4 minutes 30 seconds.
        While it runs, it clubs fetched AsyncIndicators by domain and doc type and queue them for processing.
Fetches AsyncIndicators that
        1. were not queued till now or were last queued more than 4 hours ago
        2. have failed less than ASYNC_INDICATOR_MAX_RETRIES times
        This task quits after it has run for more than
        ASYNC_INDICATOR_QUEUE_TIME - 30 seconds i.e 4 minutes 30 seconds.
        While it runs, it clubs fetched AsyncIndicators by domain and doc type and queue them for processing.
Fetches AsyncIndicators that 1 . were not queued till now or were last queued more than 4 hours ago 2 . have failed less than ASYNC_INDICATOR_MAX_RETRIES times This task quits after it has run for more than ASYNC_INDICATOR_QUEUE_TIME - 30 seconds i . e 4 minutes 30 seconds . While it runs it clubs fetched AsyncIndicators by domain and doc type and queue them for processing .
def queue_async_indicators():
    """
        Fetches AsyncIndicators that
        1. were not queued till now or were last queued more than 4 hours ago
        2. have failed less than ASYNC_INDICATOR_MAX_RETRIES times
        This task quits after it has run for more than
        ASYNC_INDICATOR_QUEUE_TIME - 30 seconds i.e 4 minutes 30 seconds.
        While it runs, it clubs fetched AsyncIndicators by domain and doc type and queue them for processing.
    """
    start = datetime.utcnow()
    cutoff = start + ASYNC_INDICATOR_QUEUE_TIME - timedelta(seconds=30)
    retry_threshold = start - timedelta(hours=4)
    # don't requeue anything that has been retried more than ASYNC_INDICATOR_MAX_RETRIES times
    indicators = AsyncIndicator.objects.filter(unsuccessful_attempts__lt=ASYNC_INDICATOR_MAX_RETRIES)[:settings.ASYNC_INDICATORS_TO_QUEUE]

    indicators_by_domain_doc_type = defaultdict(list)
    # page so that envs can have arbitarily large settings.ASYNC_INDICATORS_TO_QUEUE
    for indicator in paginated_queryset(indicators, 1000):
        # only requeue things that are not in queue or were last queued earlier than the threshold
        if not indicator.date_queued or indicator.date_queued < retry_threshold:
            indicators_by_domain_doc_type[(indicator.domain, indicator.doc_type)].append(indicator)

    for k, indicators in indicators_by_domain_doc_type.items():
        _queue_indicators(indicators)
        if datetime.utcnow() > cutoff:
            break
def queue_async_indicators ( ) : start = datetime . utcnow ( ) cutoff = start + ASYNC_INDICATOR_QUEUE_TIME - timedelta ( seconds = 30 ) retry_threshold = start - timedelta ( hours = 4 ) # don't requeue anything that has been retried more than ASYNC_INDICATOR_MAX_RETRIES times indicators = AsyncIndicator . objects . filter ( unsuccessful_attempts__lt = ASYNC_INDICATOR_MAX_RETRIES ) [ : settings . ASYNC_INDICATORS_TO_QUEUE ] indicators_by_domain_doc_type = defaultdict ( list ) # page so that envs can have arbitarily large settings.ASYNC_INDICATORS_TO_QUEUE for indicator in paginated_queryset ( indicators , 1000 ) : # only requeue things that are not in queue or were last queued earlier than the threshold if not indicator . date_queued or indicator . date_queued < retry_threshold : indicators_by_domain_doc_type [ ( indicator . domain , indicator . doc_type ) ] . append ( indicator ) for k , indicators in indicators_by_domain_doc_type . items ( ) : _queue_indicators ( indicators ) if datetime . utcnow ( ) > cutoff : break