import sys
import json
import requests
from string import Template

import dateutil.parser

def posts_at_url(url):
    
    current_page = 1
    max_page = sys.maxint

    while current_page <= max_page:

        resp = requests.get(url, params={'json':1,'page':current_page})
        results = json.loads(resp.content) 
        current_page += 1
        max_page = results['pages']
        for p in results['posts']:
            yield p
     


def documents(name, url, **kwargs):
    
    for post in posts_at_url(url):
        yield process_post(post)

def process_post(post):
    del post['comments']
    # del post['content']
    post['_id'] = post['slug']
    # remove fields we're not interested in
    post['category'] = [cat['title'] for cat in post['taxonomy_fj_category']]
    post['tags'] = [tag['title'] for tag in post['taxonomy_fj_tag']]
    author_template = Template("$first_name $last_name")
    post['author'] = [author['title'] for author in post['taxonomy_author']]
    dt = dateutil.parser.parse(post['date'])
    dt_string = dt.strftime('%Y-%m-%dT%H:%M:%SZ')
    post['date'] = dt_string
    return post

import sys
import json
import requests
from string import Template

import dateutil.parser

def posts_at_url(url):
    
    current_page = 1
    max_page = sys.maxint

    while current_page <= max_page:

        resp = requests.get(url, params={'json':1,'page':current_page})
        results = json.loads(resp.content) 
        current_page += 1
        max_page = results['pages']
        for p in results['posts']:
            yield p
     


def documents(name, url, **kwargs):
    
    for post in posts_at_url(url):
        yield process_post(post)

def process_post(post):
    del post['comments']
    post['_id'] = post['slug']
    # remove fields we're not interested in
    post['category'] = [cat['title'] for cat in post['taxonomy_fj_category']]
    post['tags'] = [tag['title'] for tag in post['taxonomy_fj_tag']]
    author_template = Template("$first_name $last_name")
    post['author'] = [author['title'] for author in post['taxonomy_author']]
    dt = dateutil.parser.parse(post['date'])
    dt_string = dt.strftime('%Y-%m-%dT%H:%M:%SZ')
    post['date'] = dt_string
    return post

Remove commented line we definitely will never need
Remove commented line we definitely will never need

  import sys
  import json
  import requests
  from string import Template
  
  import dateutil.parser
  
  def posts_at_url(url):
      
      current_page = 1
      max_page = sys.maxint
  
      while current_page <= max_page:
  
          resp = requests.get(url, params={'json':1,'page':current_page})
          results = json.loads(resp.content) 
          current_page += 1
          max_page = results['pages']
          for p in results['posts']:
              yield p
       
  
  
  def documents(name, url, **kwargs):
      
      for post in posts_at_url(url):
          yield process_post(post)
  
  def process_post(post):
      del post['comments']
-     # del post['content']
      post['_id'] = post['slug']
      # remove fields we're not interested in
      post['category'] = [cat['title'] for cat in post['taxonomy_fj_category']]
      post['tags'] = [tag['title'] for tag in post['taxonomy_fj_tag']]
      author_template = Template("$first_name $last_name")
      post['author'] = [author['title'] for author in post['taxonomy_author']]
      dt = dateutil.parser.parse(post['date'])
      dt_string = dt.strftime('%Y-%m-%dT%H:%M:%SZ')
      post['date'] = dt_string
      return post
  
Remove commented line we definitely will never need
## Code Before:
import sys
import json
import requests
from string import Template

import dateutil.parser

def posts_at_url(url):
    
    current_page = 1
    max_page = sys.maxint

    while current_page <= max_page:

        resp = requests.get(url, params={'json':1,'page':current_page})
        results = json.loads(resp.content) 
        current_page += 1
        max_page = results['pages']
        for p in results['posts']:
            yield p
     


def documents(name, url, **kwargs):
    
    for post in posts_at_url(url):
        yield process_post(post)

def process_post(post):
    del post['comments']
    # del post['content']
    post['_id'] = post['slug']
    # remove fields we're not interested in
    post['category'] = [cat['title'] for cat in post['taxonomy_fj_category']]
    post['tags'] = [tag['title'] for tag in post['taxonomy_fj_tag']]
    author_template = Template("$first_name $last_name")
    post['author'] = [author['title'] for author in post['taxonomy_author']]
    dt = dateutil.parser.parse(post['date'])
    dt_string = dt.strftime('%Y-%m-%dT%H:%M:%SZ')
    post['date'] = dt_string
    return post

## Instruction:
Remove commented line we definitely will never need
## Code After:
import sys
import json
import requests
from string import Template

import dateutil.parser

def posts_at_url(url):
    
    current_page = 1
    max_page = sys.maxint

    while current_page <= max_page:

        resp = requests.get(url, params={'json':1,'page':current_page})
        results = json.loads(resp.content) 
        current_page += 1
        max_page = results['pages']
        for p in results['posts']:
            yield p
     


def documents(name, url, **kwargs):
    
    for post in posts_at_url(url):
        yield process_post(post)

def process_post(post):
    del post['comments']
    post['_id'] = post['slug']
    # remove fields we're not interested in
    post['category'] = [cat['title'] for cat in post['taxonomy_fj_category']]
    post['tags'] = [tag['title'] for tag in post['taxonomy_fj_tag']]
    author_template = Template("$first_name $last_name")
    post['author'] = [author['title'] for author in post['taxonomy_author']]
    dt = dateutil.parser.parse(post['date'])
    dt_string = dt.strftime('%Y-%m-%dT%H:%M:%SZ')
    post['date'] = dt_string
    return post
