from __future__ import absolute_import
from datetime import datetime

from celery.task import periodic_task
from celery.schedules import crontab

from corehq.util.datadog.gauges import datadog_counter
from corehq.blobs.models import BlobExpiration
from corehq.blobs import get_blob_db


@periodic_task(run_every=crontab(minute=0, hour='0,12'))
def delete_expired_blobs():
    blob_expirations = BlobExpiration.objects.filter(expires_on__lt=_utcnow(), deleted=False)

    db = get_blob_db()
    paths = []
    bytes_deleted = 0
    for blob_expiration in blob_expirations:
        paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
        bytes_deleted += blob_expiration.length

    db.bulk_delete(paths)
    blob_expirations.update(deleted=True)
    datadog_counter(
        'commcare.temp_blobs.bytes_deleted',
        value=bytes_deleted,
    )
    return bytes_deleted


def _utcnow():
    return datetime.utcnow()

from __future__ import absolute_import
from datetime import datetime

from celery.task import periodic_task
from celery.schedules import crontab

from corehq.util.datadog.gauges import datadog_counter
from corehq.blobs.models import BlobExpiration
from corehq.blobs import get_blob_db


@periodic_task(run_every=crontab(minute=0, hour='0,12'))
def delete_expired_blobs():
    blob_expirations = BlobExpiration.objects.filter(expires_on__lt=_utcnow(), deleted=False)

    db = get_blob_db()
    total_bytes_deleted = 0
    while blob_expirations.exists():
        paths = []
        deleted_ids = []
        bytes_deleted = 0
        for blob_expiration in blob_expirations[:1000]:
            paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
            deleted_ids.append(blob_expiration.id)
            bytes_deleted += blob_expiration.length

        db.bulk_delete(paths)
        BlobExpiration.objects.filter(id__in=deleted_ids).update(deleted=True)
        datadog_counter(
            'commcare.temp_blobs.bytes_deleted',
            value=bytes_deleted,
        )
        total_bytes_deleted += bytes_deleted

    return total_bytes_deleted


def _utcnow():
    return datetime.utcnow()

Delete expired blobs in batches
Delete expired blobs in batches
  from __future__ import absolute_import
  from datetime import datetime
  
  from celery.task import periodic_task
  from celery.schedules import crontab
  
  from corehq.util.datadog.gauges import datadog_counter
  from corehq.blobs.models import BlobExpiration
  from corehq.blobs import get_blob_db
  
  
  @periodic_task(run_every=crontab(minute=0, hour='0,12'))
  def delete_expired_blobs():
      blob_expirations = BlobExpiration.objects.filter(expires_on__lt=_utcnow(), deleted=False)
  
      db = get_blob_db()
+     total_bytes_deleted = 0
+     while blob_expirations.exists():
-     paths = []
+         paths = []
+         deleted_ids = []
-     bytes_deleted = 0
+         bytes_deleted = 0
-     for blob_expiration in blob_expirations:
+         for blob_expiration in blob_expirations[:1000]:
-         paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
+             paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
+             deleted_ids.append(blob_expiration.id)
-         bytes_deleted += blob_expiration.length
+             bytes_deleted += blob_expiration.length
  
-     db.bulk_delete(paths)
+         db.bulk_delete(paths)
-     blob_expirations.update(deleted=True)
+         BlobExpiration.objects.filter(id__in=deleted_ids).update(deleted=True)
-     datadog_counter(
+         datadog_counter(
-         'commcare.temp_blobs.bytes_deleted',
+             'commcare.temp_blobs.bytes_deleted',
-         value=bytes_deleted,
+             value=bytes_deleted,
-     )
+         )
+         total_bytes_deleted += bytes_deleted
+ 
-     return bytes_deleted
+     return total_bytes_deleted
  
  
  def _utcnow():
      return datetime.utcnow()
  
Delete expired blobs in batches
## Code Before:
from __future__ import absolute_import
from datetime import datetime

from celery.task import periodic_task
from celery.schedules import crontab

from corehq.util.datadog.gauges import datadog_counter
from corehq.blobs.models import BlobExpiration
from corehq.blobs import get_blob_db


@periodic_task(run_every=crontab(minute=0, hour='0,12'))
def delete_expired_blobs():
    blob_expirations = BlobExpiration.objects.filter(expires_on__lt=_utcnow(), deleted=False)

    db = get_blob_db()
    paths = []
    bytes_deleted = 0
    for blob_expiration in blob_expirations:
        paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
        bytes_deleted += blob_expiration.length

    db.bulk_delete(paths)
    blob_expirations.update(deleted=True)
    datadog_counter(
        'commcare.temp_blobs.bytes_deleted',
        value=bytes_deleted,
    )
    return bytes_deleted


def _utcnow():
    return datetime.utcnow()

## Instruction:
Delete expired blobs in batches
## Code After:
from __future__ import absolute_import
from datetime import datetime

from celery.task import periodic_task
from celery.schedules import crontab

from corehq.util.datadog.gauges import datadog_counter
from corehq.blobs.models import BlobExpiration
from corehq.blobs import get_blob_db


@periodic_task(run_every=crontab(minute=0, hour='0,12'))
def delete_expired_blobs():
    blob_expirations = BlobExpiration.objects.filter(expires_on__lt=_utcnow(), deleted=False)

    db = get_blob_db()
    total_bytes_deleted = 0
    while blob_expirations.exists():
        paths = []
        deleted_ids = []
        bytes_deleted = 0
        for blob_expiration in blob_expirations[:1000]:
            paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
            deleted_ids.append(blob_expiration.id)
            bytes_deleted += blob_expiration.length

        db.bulk_delete(paths)
        BlobExpiration.objects.filter(id__in=deleted_ids).update(deleted=True)
        datadog_counter(
            'commcare.temp_blobs.bytes_deleted',
            value=bytes_deleted,
        )
        total_bytes_deleted += bytes_deleted

    return total_bytes_deleted


def _utcnow():
    return datetime.utcnow()
