from baseparser import BaseParser
from BeautifulSoup import BeautifulSoup, Tag


class RPOParser(BaseParser):
    domains = ['www.rp-online.de']

    feeder_pat = '1\.\d*$'
    feeder_pages = ['http://www.rp-online.de/']

    def _parse(self, html):
        soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES,
                             fromEncoding='utf-8')
        self.meta = soup.findAll('meta')
        # category
        keywords = soup.find('meta', {'property': 'vr:category'})
        self.category = self.compute_category(keywords['content'] if keywords else '')
        #article headline
        elt = soup.find('meta', {'property': 'og:title'})
        if elt is None:
            self.real_article = False
            return
        self.title = elt['content']
        # byline / author
        author = soup.find('meta', {'itemprop': 'author'})
        self.byline = author['content'] if author else ''
        # article date
        created_at = soup.find('meta', {'property': 'vr:published_time'})
        self.date = created_at['content'] if created_at else ''
        #article content
        div = soup.find('div', {'class': 'main-text '})
        if div is None:
            self.real_article = False
            return
        div = self.remove_non_content(div)
        self.body = '\n' + '\n\n'.join([x.getText() for x in div.childGenerator()
                                         if isinstance(x, Tag) and x.name == 'p'])

from baseparser import BaseParser
from BeautifulSoup import BeautifulSoup, Tag


class RPOParser(BaseParser):
    domains = ['www.rp-online.de']

    feeder_pat = '(?<!(vid|bid|iid))(-1\.\d*)$'
    feeder_pages = ['http://www.rp-online.de/']

    def _parse(self, html):
        soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES,
                             fromEncoding='utf-8')
        self.meta = soup.findAll('meta')
        # category
        keywords = soup.find('meta', {'property': 'vr:category'})
        self.category = self.compute_category(keywords['content'] if keywords else '')
        #article headline
        elt = soup.find('meta', {'property': 'og:title'})
        if elt is None:
            self.real_article = False
            return
        self.title = elt['content']
        # byline / author
        author = soup.find('meta', {'itemprop': 'author'})
        self.byline = author['content'] if author else ''
        # article date
        created_at = soup.find('meta', {'property': 'vr:published_time'})
        self.date = created_at['content'] if created_at else ''
        #article content
        div = soup.find('div', {'class': 'main-text '})
        if div is None:
            self.real_article = False
            return
        div = self.remove_non_content(div)
        self.body = '\n' + '\n\n'.join([x.getText() for x in div.childGenerator()
                                         if isinstance(x, Tag) and x.name == 'p'])

Exclude Videos article and non-scrapable info-galleries and picture-galleries via URL-pattern
Exclude Videos article and non-scrapable info-galleries and picture-galleries via URL-pattern

  from baseparser import BaseParser
  from BeautifulSoup import BeautifulSoup, Tag
  
  
  class RPOParser(BaseParser):
      domains = ['www.rp-online.de']
  
-     feeder_pat = '1\.\d*$'
+     feeder_pat = '(?<!(vid|bid|iid))(-1\.\d*)$'
      feeder_pages = ['http://www.rp-online.de/']
  
      def _parse(self, html):
          soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES,
                               fromEncoding='utf-8')
          self.meta = soup.findAll('meta')
          # category
          keywords = soup.find('meta', {'property': 'vr:category'})
          self.category = self.compute_category(keywords['content'] if keywords else '')
          #article headline
          elt = soup.find('meta', {'property': 'og:title'})
          if elt is None:
              self.real_article = False
              return
          self.title = elt['content']
          # byline / author
          author = soup.find('meta', {'itemprop': 'author'})
          self.byline = author['content'] if author else ''
          # article date
          created_at = soup.find('meta', {'property': 'vr:published_time'})
          self.date = created_at['content'] if created_at else ''
          #article content
          div = soup.find('div', {'class': 'main-text '})
          if div is None:
              self.real_article = False
              return
          div = self.remove_non_content(div)
          self.body = '\n' + '\n\n'.join([x.getText() for x in div.childGenerator()
                                           if isinstance(x, Tag) and x.name == 'p'])
  
Exclude Videos article and non-scrapable info-galleries and picture-galleries via URL-pattern
## Code Before:
from baseparser import BaseParser
from BeautifulSoup import BeautifulSoup, Tag


class RPOParser(BaseParser):
    domains = ['www.rp-online.de']

    feeder_pat = '1\.\d*$'
    feeder_pages = ['http://www.rp-online.de/']

    def _parse(self, html):
        soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES,
                             fromEncoding='utf-8')
        self.meta = soup.findAll('meta')
        # category
        keywords = soup.find('meta', {'property': 'vr:category'})
        self.category = self.compute_category(keywords['content'] if keywords else '')
        #article headline
        elt = soup.find('meta', {'property': 'og:title'})
        if elt is None:
            self.real_article = False
            return
        self.title = elt['content']
        # byline / author
        author = soup.find('meta', {'itemprop': 'author'})
        self.byline = author['content'] if author else ''
        # article date
        created_at = soup.find('meta', {'property': 'vr:published_time'})
        self.date = created_at['content'] if created_at else ''
        #article content
        div = soup.find('div', {'class': 'main-text '})
        if div is None:
            self.real_article = False
            return
        div = self.remove_non_content(div)
        self.body = '\n' + '\n\n'.join([x.getText() for x in div.childGenerator()
                                         if isinstance(x, Tag) and x.name == 'p'])

## Instruction:
Exclude Videos article and non-scrapable info-galleries and picture-galleries via URL-pattern
## Code After:
from baseparser import BaseParser
from BeautifulSoup import BeautifulSoup, Tag


class RPOParser(BaseParser):
    domains = ['www.rp-online.de']

    feeder_pat = '(?<!(vid|bid|iid))(-1\.\d*)$'
    feeder_pages = ['http://www.rp-online.de/']

    def _parse(self, html):
        soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES,
                             fromEncoding='utf-8')
        self.meta = soup.findAll('meta')
        # category
        keywords = soup.find('meta', {'property': 'vr:category'})
        self.category = self.compute_category(keywords['content'] if keywords else '')
        #article headline
        elt = soup.find('meta', {'property': 'og:title'})
        if elt is None:
            self.real_article = False
            return
        self.title = elt['content']
        # byline / author
        author = soup.find('meta', {'itemprop': 'author'})
        self.byline = author['content'] if author else ''
        # article date
        created_at = soup.find('meta', {'property': 'vr:published_time'})
        self.date = created_at['content'] if created_at else ''
        #article content
        div = soup.find('div', {'class': 'main-text '})
        if div is None:
            self.real_article = False
            return
        div = self.remove_non_content(div)
        self.body = '\n' + '\n\n'.join([x.getText() for x in div.childGenerator()
                                         if isinstance(x, Tag) and x.name == 'p'])
