

from __future__ import unicode_literals

from scrapi.base import OAIHarvester


tdar = OAIHarvester(
    name='tdar',
    base_url='http://core.tdar.org/oai-pmh/oai',
    property_list=['type', 'date', 'setSpec', 'type', 'publisher', 'coverage']
)

harvest = tdar.harvest
normalize = tdar.normalize



from __future__ import unicode_literals

from scrapi import util
from scrapi.base import OAIHarvester


class tdarHarvester(OAIHarvester):

    def get_ids(self, result, doc):
        """
        Gather the DOI and url from identifiers, if possible.
        Tries to save the DOI alone without a url extension.
        Tries to save a link to the original content at the source,
        instead of direct to a PDF, which is usually linked with viewcontent.cgi?
        in the url field
        """
        serviceID = doc.get('docID')

        url = 'http://core.tdar.org/document/' + serviceID.replace('oai:tdar.org:Resource:', '')
        print(url)

        return {
            'serviceID': serviceID,
            'url': util.copy_to_unicode(url),
            'doi': ''
        }


tdar = tdarHarvester(
    name='tdar',
    base_url='http://core.tdar.org/oai-pmh/oai',
    property_list=['type', 'date', 'setSpec', 'type', 'publisher', 'coverage']
)

harvest = tdar.harvest
normalize = tdar.normalize

Update tdar harvester with custom url gatheriing
Update tdar harvester with custom url gatheriing

  
  
  from __future__ import unicode_literals
  
+ from scrapi import util
  from scrapi.base import OAIHarvester
  
  
+ class tdarHarvester(OAIHarvester):
+ 
+     def get_ids(self, result, doc):
+         """
+         Gather the DOI and url from identifiers, if possible.
+         Tries to save the DOI alone without a url extension.
+         Tries to save a link to the original content at the source,
+         instead of direct to a PDF, which is usually linked with viewcontent.cgi?
+         in the url field
+         """
+         serviceID = doc.get('docID')
+ 
+         url = 'http://core.tdar.org/document/' + serviceID.replace('oai:tdar.org:Resource:', '')
+         print(url)
+ 
+         return {
+             'serviceID': serviceID,
+             'url': util.copy_to_unicode(url),
+             'doi': ''
+         }
+ 
+ 
- tdar = OAIHarvester(
+ tdar = tdarHarvester(
      name='tdar',
      base_url='http://core.tdar.org/oai-pmh/oai',
      property_list=['type', 'date', 'setSpec', 'type', 'publisher', 'coverage']
  )
  
  harvest = tdar.harvest
  normalize = tdar.normalize
  
Update tdar harvester with custom url gatheriing
## Code Before:


from __future__ import unicode_literals

from scrapi.base import OAIHarvester


tdar = OAIHarvester(
    name='tdar',
    base_url='http://core.tdar.org/oai-pmh/oai',
    property_list=['type', 'date', 'setSpec', 'type', 'publisher', 'coverage']
)

harvest = tdar.harvest
normalize = tdar.normalize

## Instruction:
Update tdar harvester with custom url gatheriing
## Code After:


from __future__ import unicode_literals

from scrapi import util
from scrapi.base import OAIHarvester


class tdarHarvester(OAIHarvester):

    def get_ids(self, result, doc):
        """
        Gather the DOI and url from identifiers, if possible.
        Tries to save the DOI alone without a url extension.
        Tries to save a link to the original content at the source,
        instead of direct to a PDF, which is usually linked with viewcontent.cgi?
        in the url field
        """
        serviceID = doc.get('docID')

        url = 'http://core.tdar.org/document/' + serviceID.replace('oai:tdar.org:Resource:', '')
        print(url)

        return {
            'serviceID': serviceID,
            'url': util.copy_to_unicode(url),
            'doi': ''
        }


tdar = tdarHarvester(
    name='tdar',
    base_url='http://core.tdar.org/oai-pmh/oai',
    property_list=['type', 'date', 'setSpec', 'type', 'publisher', 'coverage']
)

harvest = tdar.harvest
normalize = tdar.normalize
