parameters: (start_date=None)
docstring: Confirm that, for every link, there is a WARC in each storage, and that their hashes match.
  
          start_date is in the format YYYY-MM-DD
  
          Ground truth is the list of link objects: compare its list of warcs with those of each storage,
          and compare hashes when more than one such file is present.
  
          Derived from check_s3_hashes
docstring_summary: Confirm that, for every link, there is a WARC in each storage, and that their hashes match.
docstring_tokens: Confirm that for every link there is a WARC in each storage and that their hashes match .
function: def check_storage(start_date=None):
      """
          Confirm that, for every link, there is a WARC in each storage, and that their hashes match.
  
          start_date is in the format YYYY-MM-DD
  
          Ground truth is the list of link objects: compare its list of warcs with those of each storage,
          and compare hashes when more than one such file is present.
  
          Derived from check_s3_hashes
      """
      from django.core.files.storage import default_storage
      from django.db.models import Q
      from perma.models import Link, Capture
  
      from datetime import date, datetime
      from dateutil.relativedelta import relativedelta
      import pytz
      import re
  
      # check the arg
      if not start_date:
          # use first archive date
          start_datetime = Link.objects.order_by('creation_timestamp')[0].creation_timestamp
      elif re.match(r'^\d\d\d\d-\d\d-\d\d$', start_date):
          start_datetime = pytz.utc.localize(datetime.strptime(start_date, "%Y-%m-%d"))
      else:
          print("Bad argument")
          return
      end_datetime = pytz.utc.localize(datetime.now())
  
      # The abstraction of multiple storages is an artifact of the
      # transition to S3 for storage; although it's conceivable that we'd
      # want multiple storages at some point again, there's no need right now
      # to diverge from the Django norm. The abstraction remains here as a
      # point of historical interest.
      storages = {'primary': {'storage': default_storage, 'lookup': {}}}
  
      # only use cache files when all are present: link cache, and one for each storage
      link_cache = '/tmp/perma_link_cache{0}.txt'.format("" if start_date is None else start_date)
      caches = [link_cache]
      for key in storages:
          caches.append('/tmp/perma_storage_cache_{0}{1}.txt'.format(key, "" if start_date is None else start_date))
  
      if not all(os.path.exists(p) for p in caches):
          print("Building link cache ...")
          with open(link_cache, 'w') as tmp_file:
              capture_filter = (Q(role="primary") & Q(status="success")) | (Q(role="screenshot") & Q(status="success"))
              # assemble list of links by year-month, as in lockss/views.titledb:
              start_month = date(year=start_datetime.year, month=start_datetime.month, day=1)
              today = date.today()
              while start_month <= today:
                  for link in Link.objects.filter(
                          creation_timestamp__year=start_month.year,
                          creation_timestamp__month=start_month.month,
                          creation_timestamp__gte=start_datetime,
                          creation_timestamp__lt=end_datetime,
                          captures__in=Capture.objects.filter(capture_filter)
                  ).distinct():
                      tmp_file.write(f"{link.warc_storage_file()}\n")
                      # this produces strings like u'warcs/0G/GO/XR/XG/0-GGOX-RXGQ.warc.gz'; make the storage paths match
                      # by chopping off the prefix, whether storage.location, ._root_path, or .base_location
                  start_month += relativedelta(months=1)
  
          print("Building storage cache{0} ...".format("s" if len(storages) > 1 else ""))
          for key in storages:
              storage = storages[key]['storage']
              with open('/tmp/perma_storage_cache_{0}{1}.txt'.format(key, "" if start_date is None else start_date), 'w') as tmp_file:
                  if hasattr(storage, 'bucket'):
                      # S3
                      for f in storage.bucket.list('generated/warcs/'):
                          if (not start_date) or (start_datetime <= pytz.utc.localize(datetime.strptime(f.last_modified, '%Y-%m-%dT%H:%M:%S.%fZ')) < end_datetime):
                              # here we chop off the prefix aka storage.location
                              path = f.key[(len(storage.location)):]
                              # etag is a string like u'"3ea8c903d9991d466ec437d1789379a6"', so we need to
                              # knock off the extra quotation marks
                              hash = f.etag[1:-1]
                              tmp_file.write(f"{path}\t{hash}\n")
                              storages[key]['lookup'][path] = hash
                  else:
                      if hasattr(storage, '_root_path'):
                          # SFTP -- no longer in use, but leaving this here to show that different storages may have
                          # different bases
                          base = storage._root_path
                      else:
                          # local file storage -- are there other possibilities to consider?
                          base = storage.base_location
                      for f in storage.walk(os.path.join(base, 'warcs')):
                          # os.walk: "For each directory in the tree rooted at directory top (including top itself),
                          # it yields a 3-tuple (dirpath, dirnames, filenames)" -- so:
                          for filename in f[2]:
                              full_path = os.path.join(f[0], filename)
                              if (not start_date) or (start_datetime <= pytz.utc.localize(storage.modified_time(full_path)) < end_datetime):
                                  # here we chop off the prefix, whether storage._root_path or storage.base_location
                                  path = full_path[len(base):]
                                  # note that etags are not always md5sums, but should be in these cases; we can rewrite
                                  # or replace md5hash if necessary
                                  hash = md5hash(full_path, storage)
                                  tmp_file.write(f"{path}\t{hash}\n")
                                  storages[key]['lookup'][path] = hash
      else:
          print("Reading storage caches ...")
          for key in storages:
              with open('/tmp/perma_storage_cache_{0}{1}.txt'.format(key, "" if start_date is None else start_date)) as f:
                  for line in f:
                      path, hash = line[:-1].split("\t")
                      storages[key]['lookup'][path] = hash
  
      # now check ground truth against storage lookup tables
      print("Comparing link cache against storage caches ...")
      with open(link_cache) as f:
          for line in f:
              path = line[:-1]
              file_present = True
              for key in storages:
                  if path not in storages[key]['lookup']:
                      print(f"{path} not in {key}")
                      file_present = False
              if file_present and len(storages) > 1:
                  hashes = []
                  for key in storages:
                      hashes.append(storages[key]['lookup'][path])
                  # this looks funny (and is unnecessary here) but is faster than using set, per
                  # http://stackoverflow.com/a/3844948/4074877
                  if hashes.count(hashes[0]) != len(hashes):
                      print("Hash mismatch for {0}: {1}".format(path, str(zip(storages.keys(), hashes))))
function_tokens: def check_storage ( start_date = None ) : from django . core . files . storage import default_storage from django . db . models import Q from perma . models import Link , Capture from datetime import date , datetime from dateutil . relativedelta import relativedelta import pytz import re # check the arg if not start_date : # use first archive date start_datetime = Link . objects . order_by ( 'creation_timestamp' ) [ 0 ] . creation_timestamp elif re . match ( r'^\d\d\d\d-\d\d-\d\d$' , start_date ) : start_datetime = pytz . utc . localize ( datetime . strptime ( start_date , "%Y-%m-%d" ) ) else : print ( "Bad argument" ) return end_datetime = pytz . utc . localize ( datetime . now ( ) ) # The abstraction of multiple storages is an artifact of the # transition to S3 for storage; although it's conceivable that we'd # want multiple storages at some point again, there's no need right now # to diverge from the Django norm. The abstraction remains here as a # point of historical interest. storages = { 'primary' : { 'storage' : default_storage , 'lookup' : { } } } # only use cache files when all are present: link cache, and one for each storage link_cache = '/tmp/perma_link_cache{0}.txt' . format ( "" if start_date is None else start_date ) caches = [ link_cache ] for key in storages : caches . append ( '/tmp/perma_storage_cache_{0}{1}.txt' . format ( key , "" if start_date is None else start_date ) ) if not all ( os . path . exists ( p ) for p in caches ) : print ( "Building link cache ..." ) with open ( link_cache , 'w' ) as tmp_file : capture_filter = ( Q ( role = "primary" ) & Q ( status = "success" ) ) | ( Q ( role = "screenshot" ) & Q ( status = "success" ) ) # assemble list of links by year-month, as in lockss/views.titledb: start_month = date ( year = start_datetime . year , month = start_datetime . month , day = 1 ) today = date . today ( ) while start_month <= today : for link in Link . objects . filter ( creation_timestamp__year = start_month . year , creation_timestamp__month = start_month . month , creation_timestamp__gte = start_datetime , creation_timestamp__lt = end_datetime , captures__in = Capture . objects . filter ( capture_filter ) ) . distinct ( ) : tmp_file . write ( f"{link.warc_storage_file()}\n" ) # this produces strings like u'warcs/0G/GO/XR/XG/0-GGOX-RXGQ.warc.gz'; make the storage paths match # by chopping off the prefix, whether storage.location, ._root_path, or .base_location start_month += relativedelta ( months = 1 ) print ( "Building storage cache{0} ..." . format ( "s" if len ( storages ) > 1 else "" ) ) for key in storages : storage = storages [ key ] [ 'storage' ] with open ( '/tmp/perma_storage_cache_{0}{1}.txt' . format ( key , "" if start_date is None else start_date ) , 'w' ) as tmp_file : if hasattr ( storage , 'bucket' ) : # S3 for f in storage . bucket . list ( 'generated/warcs/' ) : if ( not start_date ) or ( start_datetime <= pytz . utc . localize ( datetime . strptime ( f . last_modified , '%Y-%m-%dT%H:%M:%S.%fZ' ) ) < end_datetime ) : # here we chop off the prefix aka storage.location path = f . key [ ( len ( storage . location ) ) : ] # etag is a string like u'"3ea8c903d9991d466ec437d1789379a6"', so we need to # knock off the extra quotation marks hash = f . etag [ 1 : - 1 ] tmp_file . write ( f"{path}\t{hash}\n" ) storages [ key ] [ 'lookup' ] [ path ] = hash else : if hasattr ( storage , '_root_path' ) : # SFTP -- no longer in use, but leaving this here to show that different storages may have # different bases base = storage . _root_path else : # local file storage -- are there other possibilities to consider? base = storage . base_location for f in storage . walk ( os . path . join ( base , 'warcs' ) ) : # os.walk: "For each directory in the tree rooted at directory top (including top itself), # it yields a 3-tuple (dirpath, dirnames, filenames)" -- so: for filename in f [ 2 ] : full_path = os . path . join ( f [ 0 ] , filename ) if ( not start_date ) or ( start_datetime <= pytz . utc . localize ( storage . modified_time ( full_path ) ) < end_datetime ) : # here we chop off the prefix, whether storage._root_path or storage.base_location path = full_path [ len ( base ) : ] # note that etags are not always md5sums, but should be in these cases; we can rewrite # or replace md5hash if necessary hash = md5hash ( full_path , storage ) tmp_file . write ( f"{path}\t{hash}\n" ) storages [ key ] [ 'lookup' ] [ path ] = hash else : print ( "Reading storage caches ..." ) for key in storages : with open ( '/tmp/perma_storage_cache_{0}{1}.txt' . format ( key , "" if start_date is None else start_date ) ) as f : for line in f : path , hash = line [ : - 1 ] . split ( "\t" ) storages [ key ] [ 'lookup' ] [ path ] = hash # now check ground truth against storage lookup tables print ( "Comparing link cache against storage caches ..." ) with open ( link_cache ) as f : for line in f : path = line [ : - 1 ] file_present = True for key in storages : if path not in storages [ key ] [ 'lookup' ] : print ( f"{path} not in {key}" ) file_present = False if file_present and len ( storages ) > 1 : hashes = [ ] for key in storages : hashes . append ( storages [ key ] [ 'lookup' ] [ path ] ) # this looks funny (and is unnecessary here) but is faster than using set, per # http://stackoverflow.com/a/3844948/4074877 if hashes . count ( hashes [ 0 ] ) != len ( hashes ) : print ( "Hash mismatch for {0}: {1}" . format ( path , str ( zip ( storages . keys ( ) , hashes ) ) ) )