import time
import requests
import bs4

from databot.recursive import call


class DownloadErrror(Exception):
    pass


def dump_response(response):
    return {
        'headers': dict(response.headers),
        'cookies': dict(response.cookies),
        'status_code': response.status_code,
        'encoding': response.encoding,
        'content': response.content,
    }


def download(url, delay=None, update=None, **kwargs):
    update = update or {}

    def func(row):
        if delay is not None:
            time.sleep(delay)
        kw = call(kwargs, row)
        _url = url(row)
        response = requests.get(_url, **kw)
        if response.status_code == 200:
            value = dump_response(response)
            for k, fn in update.items():
                value[k] = fn(row)
            yield _url, value
        else:
            raise DownloadErrror('Error while downloading %s, returned status code was %s, response content:\n\n%s' % (
                _url, response.status_code, response.content,
            ))

    return func


def get_content(data):
    content_type = data.get('headers', {}).get('Content-Type')
    if content_type == 'text/html':
        soup = bs4.BeautifulSoup(data['content'], 'lxml')
        return data['content'].decode(soup.original_encoding)
    else:
        return data['content']

import time
import requests
import bs4
import cgi

from databot.recursive import call


class DownloadErrror(Exception):
    pass


def dump_response(response):
    return {
        'headers': dict(response.headers),
        'cookies': response.cookies.get_dict(),
        'status_code': response.status_code,
        'encoding': response.encoding,
        'content': response.content,
    }


def download(url, delay=None, update=None, **kwargs):
    update = update or {}

    def func(row):
        if delay is not None:
            time.sleep(delay)
        kw = call(kwargs, row)
        _url = url(row)
        response = requests.get(_url, **kw)
        if response.status_code == 200:
            value = dump_response(response)
            for k, fn in update.items():
                value[k] = fn(row)
            yield _url, value
        else:
            raise DownloadErrror('Error while downloading %s, returned status code was %s, response content:\n\n%s' % (
                _url, response.status_code, response.content,
            ))

    return func


def get_content(data):
    content_type_header = data.get('headers', {}).get('Content-Type', '')
    content_type, params = cgi.parse_header(content_type_header)
    if content_type == 'text/html':
        soup = bs4.BeautifulSoup(data['content'], 'lxml')
        return data['content'].decode(soup.original_encoding)
    else:
        return data['content']

Fix duplicate cookie issue and header parsing
Fix duplicate cookie issue and header parsing

  import time
  import requests
  import bs4
+ import cgi
  
  from databot.recursive import call
  
  
  class DownloadErrror(Exception):
      pass
  
  
  def dump_response(response):
      return {
          'headers': dict(response.headers),
-         'cookies': dict(response.cookies),
+         'cookies': response.cookies.get_dict(),
          'status_code': response.status_code,
          'encoding': response.encoding,
          'content': response.content,
      }
  
  
  def download(url, delay=None, update=None, **kwargs):
      update = update or {}
  
      def func(row):
          if delay is not None:
              time.sleep(delay)
          kw = call(kwargs, row)
          _url = url(row)
          response = requests.get(_url, **kw)
          if response.status_code == 200:
              value = dump_response(response)
              for k, fn in update.items():
                  value[k] = fn(row)
              yield _url, value
          else:
              raise DownloadErrror('Error while downloading %s, returned status code was %s, response content:\n\n%s' % (
                  _url, response.status_code, response.content,
              ))
  
      return func
  
  
  def get_content(data):
-     content_type = data.get('headers', {}).get('Content-Type')
+     content_type_header = data.get('headers', {}).get('Content-Type', '')
+     content_type, params = cgi.parse_header(content_type_header)
      if content_type == 'text/html':
          soup = bs4.BeautifulSoup(data['content'], 'lxml')
          return data['content'].decode(soup.original_encoding)
      else:
          return data['content']
  
Fix duplicate cookie issue and header parsing
## Code Before:
import time
import requests
import bs4

from databot.recursive import call


class DownloadErrror(Exception):
    pass


def dump_response(response):
    return {
        'headers': dict(response.headers),
        'cookies': dict(response.cookies),
        'status_code': response.status_code,
        'encoding': response.encoding,
        'content': response.content,
    }


def download(url, delay=None, update=None, **kwargs):
    update = update or {}

    def func(row):
        if delay is not None:
            time.sleep(delay)
        kw = call(kwargs, row)
        _url = url(row)
        response = requests.get(_url, **kw)
        if response.status_code == 200:
            value = dump_response(response)
            for k, fn in update.items():
                value[k] = fn(row)
            yield _url, value
        else:
            raise DownloadErrror('Error while downloading %s, returned status code was %s, response content:\n\n%s' % (
                _url, response.status_code, response.content,
            ))

    return func


def get_content(data):
    content_type = data.get('headers', {}).get('Content-Type')
    if content_type == 'text/html':
        soup = bs4.BeautifulSoup(data['content'], 'lxml')
        return data['content'].decode(soup.original_encoding)
    else:
        return data['content']

## Instruction:
Fix duplicate cookie issue and header parsing
## Code After:
import time
import requests
import bs4
import cgi

from databot.recursive import call


class DownloadErrror(Exception):
    pass


def dump_response(response):
    return {
        'headers': dict(response.headers),
        'cookies': response.cookies.get_dict(),
        'status_code': response.status_code,
        'encoding': response.encoding,
        'content': response.content,
    }


def download(url, delay=None, update=None, **kwargs):
    update = update or {}

    def func(row):
        if delay is not None:
            time.sleep(delay)
        kw = call(kwargs, row)
        _url = url(row)
        response = requests.get(_url, **kw)
        if response.status_code == 200:
            value = dump_response(response)
            for k, fn in update.items():
                value[k] = fn(row)
            yield _url, value
        else:
            raise DownloadErrror('Error while downloading %s, returned status code was %s, response content:\n\n%s' % (
                _url, response.status_code, response.content,
            ))

    return func


def get_content(data):
    content_type_header = data.get('headers', {}).get('Content-Type', '')
    content_type, params = cgi.parse_header(content_type_header)
    if content_type == 'text/html':
        soup = bs4.BeautifulSoup(data['content'], 'lxml')
        return data['content'].decode(soup.original_encoding)
    else:
        return data['content']
