import api
import db
from utils import parse_timestamp

import hashlib
import json

def download_logs():
    """ Download mailgun logs and store them in the database """
    logs = []
    skip = 0
    
    # Fetch all unsaved logs and add them to a LIFO queue 
    while True:
        print("fecthing logs starting at {}".format(skip))
        for log in api.get_logs(limit=100, skip=skip)['items']:
            log_data = json.dumps(log)
            log_hash = hashlib.sha256(log_data).hexdigest()
            if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
                break
            else:
                logs[:0] = [(log_hash, log_data, parse_timestamp(log['date_created']]
        else: 
            break
        skip += 100

    # take items from LIFO queue and save to db
    for log_hash, data, timestamp in logs:
        db.MailgunLog(
            log_hash=log_hash,
            data=data,
            timestamp=timestamp
        ).save()

import api
import db
from utils import parse_timestamp
from django.db import transaction

from collections import OrderedDict

import hashlib
import json

@transaction.commit_manually
def download_logs():
    """ Download mailgun logs and store them in the database """
    # use ordered dict to protect against new logs arriving while downloading logs
    logs = OrderedDict()
    skip = 0
    
    # Fetch all unsaved logs and add them to a LIFO queue
    fetch_more = True
    while fetch_more:
        print("fecthing logs skip={}".format(skip))
        logs_tmp = api.get_logs(limit=1000, skip=skip)['items']
        if len(logs_tmp) == 0:
            break
        for log in logs_tmp:
            log_data = json.dumps(log)
            log_hash = hashlib.sha256(log_data).hexdigest()
            if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
                fetch_more = False
                break
            else:
                logs[log_hash] = (log_hash, log_data, parse_timestamp(log['created_at']))
        skip += 1000

    # take items from LIFO queue and save to db
    print("Saving {0} logs to database".format(len(logs)))
    for i, (log_hash, data, timestamp) in enumerate(logs.values()):
        db.MailgunLog(
            log_hash=log_hash,
            data=data,
            timestamp=timestamp
        ).save()
        if (i+1) % 100 == 0:
            transaction.commit()
    transaction.commit()

Handle transactions manually when saving downloaded logs
Handle transactions manually when saving downloaded logs

  import api
  import db
  from utils import parse_timestamp
+ from django.db import transaction
+ 
+ from collections import OrderedDict
  
  import hashlib
  import json
  
+ @transaction.commit_manually
  def download_logs():
      """ Download mailgun logs and store them in the database """
-     logs = []
+     # use ordered dict to protect against new logs arriving while downloading logs
+     logs = OrderedDict()
      skip = 0
      
-     # Fetch all unsaved logs and add them to a LIFO queue 
+     # Fetch all unsaved logs and add them to a LIFO queue
-     while True:
+     fetch_more = True
+     while fetch_more:
-         print("fecthing logs starting at {}".format(skip))
+         print("fecthing logs skip={}".format(skip))
-         for log in api.get_logs(limit=100, skip=skip)['items']:
+         logs_tmp = api.get_logs(limit=1000, skip=skip)['items']
+         if len(logs_tmp) == 0:
+             break
+         for log in logs_tmp:
              log_data = json.dumps(log)
              log_hash = hashlib.sha256(log_data).hexdigest()
              if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
+                 fetch_more = False
                  break
              else:
-                 logs[:0] = [(log_hash, log_data, parse_timestamp(log['date_created']]
+                 logs[log_hash] = (log_hash, log_data, parse_timestamp(log['created_at']))
-         else: 
-             break
-         skip += 100
+         skip += 1000
  
      # take items from LIFO queue and save to db
+     print("Saving {0} logs to database".format(len(logs)))
-     for log_hash, data, timestamp in logs:
+     for i, (log_hash, data, timestamp) in enumerate(logs.values()):
          db.MailgunLog(
              log_hash=log_hash,
              data=data,
              timestamp=timestamp
          ).save()
+         if (i+1) % 100 == 0:
+             transaction.commit()
+     transaction.commit()
  
Handle transactions manually when saving downloaded logs
## Code Before:
import api
import db
from utils import parse_timestamp

import hashlib
import json

def download_logs():
    """ Download mailgun logs and store them in the database """
    logs = []
    skip = 0
    
    # Fetch all unsaved logs and add them to a LIFO queue 
    while True:
        print("fecthing logs starting at {}".format(skip))
        for log in api.get_logs(limit=100, skip=skip)['items']:
            log_data = json.dumps(log)
            log_hash = hashlib.sha256(log_data).hexdigest()
            if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
                break
            else:
                logs[:0] = [(log_hash, log_data, parse_timestamp(log['date_created']]
        else: 
            break
        skip += 100

    # take items from LIFO queue and save to db
    for log_hash, data, timestamp in logs:
        db.MailgunLog(
            log_hash=log_hash,
            data=data,
            timestamp=timestamp
        ).save()

## Instruction:
Handle transactions manually when saving downloaded logs
## Code After:
import api
import db
from utils import parse_timestamp
from django.db import transaction

from collections import OrderedDict

import hashlib
import json

@transaction.commit_manually
def download_logs():
    """ Download mailgun logs and store them in the database """
    # use ordered dict to protect against new logs arriving while downloading logs
    logs = OrderedDict()
    skip = 0
    
    # Fetch all unsaved logs and add them to a LIFO queue
    fetch_more = True
    while fetch_more:
        print("fecthing logs skip={}".format(skip))
        logs_tmp = api.get_logs(limit=1000, skip=skip)['items']
        if len(logs_tmp) == 0:
            break
        for log in logs_tmp:
            log_data = json.dumps(log)
            log_hash = hashlib.sha256(log_data).hexdigest()
            if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
                fetch_more = False
                break
            else:
                logs[log_hash] = (log_hash, log_data, parse_timestamp(log['created_at']))
        skip += 1000

    # take items from LIFO queue and save to db
    print("Saving {0} logs to database".format(len(logs)))
    for i, (log_hash, data, timestamp) in enumerate(logs.values()):
        db.MailgunLog(
            log_hash=log_hash,
            data=data,
            timestamp=timestamp
        ).save()
        if (i+1) % 100 == 0:
            transaction.commit()
    transaction.commit()
