(reindex=False, overwrite=False, limit=1000)
return
Crawl through posts in batches and add them to index.
Crawl through posts in batches and add them to index.
Crawl through posts in batches and add them to index .
def crawl(reindex=False, overwrite=False, limit=1000):
    """
    Crawl through posts in batches and add them to index.
    """

    if reindex:
        logger.info(f"Setting indexed field to false on all post.")
        Post.objects.filter(indexed=True).exclude(root=None).update(indexed=False)

    # Index a limited number of posts at one time.
    posts = Post.objects.valid_posts().exclude(Q(spam=Post.SPAM) | Q(indexed=False))[:limit]

    try:
        # Add post to search index.
        index_posts(posts=posts, overwrite=overwrite)
    except Exception as exc:
        logger.error(f'Error updating index: {exc}')
        Post.objects.filter(id__in=posts.values('id')).update(indexed=False)

    # Set the indexed field to true
    Post.objects.filter(id__in=posts.values('id')).update(indexed=True)

    return
def crawl ( reindex = False , overwrite = False , limit = 1000 ) : if reindex : logger . info ( f"Setting indexed field to false on all post." ) Post . objects . filter ( indexed = True ) . exclude ( root = None ) . update ( indexed = False ) # Index a limited number of posts at one time. posts = Post . objects . valid_posts ( ) . exclude ( Q ( spam = Post . SPAM ) | Q ( indexed = False ) ) [ : limit ] try : # Add post to search index. index_posts ( posts = posts , overwrite = overwrite ) except Exception as exc : logger . error ( f'Error updating index: {exc}' ) Post . objects . filter ( id__in = posts . values ( 'id' ) ) . update ( indexed = False ) # Set the indexed field to true Post . objects . filter ( id__in = posts . values ( 'id' ) ) . update ( indexed = True ) return