'''
Harvester for Dryad for the SHARE project

Example API call: http://www.datadryad.org/oai/request?verb=ListRecords&metadataPrefix=oai_dc
'''

from __future__ import unicode_literals
from scrapi.base import OAIHarvester


class DryadHarvester(OAIHarvester):
    short_name = 'dryad'
    long_name = 'Dryad Data Repository'
    url = 'http://www.datadryad.org/oai/request'

    base_url = 'http://www.datadryad.org/oai/request'
    property_list = ['rights', 'format', 'relation', 'date',
                     'identifier', 'type', 'setSpec']
    timezone_granularity = True

'''
Harvester for Dryad for the SHARE project

Example API call: http://www.datadryad.org/oai/request?verb=ListRecords&metadataPrefix=oai_dc
'''
from __future__ import unicode_literals

from lxml import etree
import logging

from scrapi.base import OAIHarvester

logger = logging.getLogger(__name__)


class DryadHarvester(OAIHarvester):
    short_name = 'dryad'
    long_name = 'Dryad Data Repository'
    url = 'http://www.datadryad.org/oai/request'

    base_url = 'http://www.datadryad.org/oai/request'
    property_list = ['rights', 'format', 'relation', 'date',
                     'identifier', 'type', 'setSpec']
    timezone_granularity = True

    def normalize(self, raw_doc):
        str_result = raw_doc.get('doc')
        result = etree.XML(str_result)

        status = (result.xpath('ns0:header/@status', namespaces=self.namespaces) or [''])[0]
        if status.lower() == 'deleted' or status.lower() == 'item is not available':
            logger.info('Not normalizing record with ID {}, status {}'.format(raw_doc['docID'], status))
            return None

        return super(OAIHarvester, self).normalize(raw_doc)

Add custom normaluze function that will not get non existant items
Add custom normaluze function that will not get non existant items

  '''
  Harvester for Dryad for the SHARE project
  
  Example API call: http://www.datadryad.org/oai/request?verb=ListRecords&metadataPrefix=oai_dc
  '''
+ from __future__ import unicode_literals
  
- from __future__ import unicode_literals
+ from lxml import etree
+ import logging
+ 
  from scrapi.base import OAIHarvester
+ 
+ logger = logging.getLogger(__name__)
  
  
  class DryadHarvester(OAIHarvester):
      short_name = 'dryad'
      long_name = 'Dryad Data Repository'
      url = 'http://www.datadryad.org/oai/request'
  
      base_url = 'http://www.datadryad.org/oai/request'
      property_list = ['rights', 'format', 'relation', 'date',
                       'identifier', 'type', 'setSpec']
      timezone_granularity = True
  
+     def normalize(self, raw_doc):
+         str_result = raw_doc.get('doc')
+         result = etree.XML(str_result)
+ 
+         status = (result.xpath('ns0:header/@status', namespaces=self.namespaces) or [''])[0]
+         if status.lower() == 'deleted' or status.lower() == 'item is not available':
+             logger.info('Not normalizing record with ID {}, status {}'.format(raw_doc['docID'], status))
+             return None
+ 
+         return super(OAIHarvester, self).normalize(raw_doc)
+ 
Add custom normaluze function that will not get non existant items
## Code Before:
'''
Harvester for Dryad for the SHARE project

Example API call: http://www.datadryad.org/oai/request?verb=ListRecords&metadataPrefix=oai_dc
'''

from __future__ import unicode_literals
from scrapi.base import OAIHarvester


class DryadHarvester(OAIHarvester):
    short_name = 'dryad'
    long_name = 'Dryad Data Repository'
    url = 'http://www.datadryad.org/oai/request'

    base_url = 'http://www.datadryad.org/oai/request'
    property_list = ['rights', 'format', 'relation', 'date',
                     'identifier', 'type', 'setSpec']
    timezone_granularity = True

## Instruction:
Add custom normaluze function that will not get non existant items
## Code After:
'''
Harvester for Dryad for the SHARE project

Example API call: http://www.datadryad.org/oai/request?verb=ListRecords&metadataPrefix=oai_dc
'''
from __future__ import unicode_literals

from lxml import etree
import logging

from scrapi.base import OAIHarvester

logger = logging.getLogger(__name__)


class DryadHarvester(OAIHarvester):
    short_name = 'dryad'
    long_name = 'Dryad Data Repository'
    url = 'http://www.datadryad.org/oai/request'

    base_url = 'http://www.datadryad.org/oai/request'
    property_list = ['rights', 'format', 'relation', 'date',
                     'identifier', 'type', 'setSpec']
    timezone_granularity = True

    def normalize(self, raw_doc):
        str_result = raw_doc.get('doc')
        result = etree.XML(str_result)

        status = (result.xpath('ns0:header/@status', namespaces=self.namespaces) or [''])[0]
        if status.lower() == 'deleted' or status.lower() == 'item is not available':
            logger.info('Not normalizing record with ID {}, status {}'.format(raw_doc['docID'], status))
            return None

        return super(OAIHarvester, self).normalize(raw_doc)
