parameters: (self, invalid_sources=None)
docstring: Provides access to the packages in this repository
  
          :param invalid_sources:
              A list of URLs that are permissible to fetch data from
  
          :raises:
              ProviderException: when an error occurs trying to open a file
              DownloaderException: when there is an issue download package info
              ClientException: when there is an issue parsing package info
  
          :return:
              A generator of
              (
                  'Package Name',
                  {
                      'name': name,
                      'description': description,
                      'author': author,
                      'homepage': homepage,
                      'last_modified': last modified date,
                      'download': {
                          'url': url,
                          'date': date,
                          'version': version
                      },
                      'previous_names': [old_name, ...],
                      'labels': [label, ...],
                      'sources': [url, ...],
                      'readme': url,
                      'issues': url,
                      'donate': url,
                      'buy': url
                  }
              )
              tuples
docstring_summary: Provides access to the packages in this repository
docstring_tokens: Provides access to the packages in this repository
function: def get_packages(self, invalid_sources=None):
          """
          Provides access to the packages in this repository
  
          :param invalid_sources:
              A list of URLs that are permissible to fetch data from
  
          :raises:
              ProviderException: when an error occurs trying to open a file
              DownloaderException: when there is an issue download package info
              ClientException: when there is an issue parsing package info
  
          :return:
              A generator of
              (
                  'Package Name',
                  {
                      'name': name,
                      'description': description,
                      'author': author,
                      'homepage': homepage,
                      'last_modified': last modified date,
                      'download': {
                          'url': url,
                          'date': date,
                          'version': version
                      },
                      'previous_names': [old_name, ...],
                      'labels': [label, ...],
                      'sources': [url, ...],
                      'readme': url,
                      'issues': url,
                      'donate': url,
                      'buy': url
                  }
              )
              tuples
          """
  
          if 'get_packages' in self.cache:
              for key, value in self.cache['get_packages'].items():
                  yield (key, value)
              return
  
          if invalid_sources != None and self.repo in invalid_sources:
              raise StopIteration()
  
          self.fetch()
  
          def fail(message):
              exception = ProviderException(message)
              self.failed_sources[self.repo] = exception
              self.cache['get_packages'] = {}
              return
          schema_error = u'Repository %s does not appear to be a valid repository file because ' % self.repo
  
          if 'schema_version' not in self.repo_info:
              error_string = u'%s the "schema_version" JSON key is missing.' % schema_error
              fail(error_string)
              return
  
          try:
              self.schema_version = float(self.repo_info.get('schema_version'))
          except (ValueError):
              error_string = u'%s the "schema_version" is not a valid number.' % schema_error
              fail(error_string)
              return
  
          if self.schema_version not in [1.0, 1.1, 1.2, 2.0]:
              error_string = u'%s the "schema_version" is not recognized. Must be one of: 1.0, 1.1, 1.2 or 2.0.' % schema_error
              fail(error_string)
              return
  
          if 'packages' not in self.repo_info:
              error_string = u'%s the "packages" JSON key is missing.' % schema_error
              fail(error_string)
              return
  
          github_client = GitHubClient(self.settings)
          bitbucket_client = BitBucketClient(self.settings)
  
          # Backfill the "previous_names" keys for old schemas
          previous_names = {}
          if self.schema_version < 2.0:
              renamed = self.get_renamed_packages()
              for old_name in renamed:
                  new_name = renamed[old_name]
                  if new_name not in previous_names:
                      previous_names[new_name] = []
                  previous_names[new_name].append(old_name)
  
          output = {}
          for package in self.repo_info['packages']:
              info = {
                  'sources': [self.repo]
              }
  
              for field in ['name', 'description', 'author', 'last_modified', 'previous_names',
                      'labels', 'homepage', 'readme', 'issues', 'donate', 'buy']:
                  if package.get(field):
                      info[field] = package.get(field)
  
              # Schema version 2.0 allows for grabbing details about a pacakge, or its
              # download from "details" urls. See the GitHubClient and BitBucketClient
              # classes for valid URLs.
              if self.schema_version >= 2.0:
                  details = package.get('details')
                  releases = package.get('releases')
  
                  # Try to grab package-level details from GitHub or BitBucket
                  if details:
                      if invalid_sources != None and details in invalid_sources:
                          continue
  
                      info['sources'].append(details)
  
                      try:
                          github_repo_info = github_client.repo_info(details)
                          bitbucket_repo_info = bitbucket_client.repo_info(details)
  
                          # When grabbing details, prefer explicit field values over the values
                          # from the GitHub or BitBucket API
                          if github_repo_info:
                              info = dict(chain(github_repo_info.items(), info.items()))
                          elif bitbucket_repo_info:
                              info = dict(chain(bitbucket_repo_info.items(), info.items()))
                          else:
                              raise ProviderException(u'Invalid "details" value "%s" for one of the packages in the repository %s.' % (details, self.repo))
  
                      except (DownloaderException, ClientException, ProviderException) as e:
                          if 'name' in info:
                              self.broken_packages[info['name']] = e
                          self.failed_sources[details] = e
                          continue
  
                  # If no releases info was specified, also grab the download info from GH or BB
                  if not releases and details:
                      releases = [{'details': details}]
  
                  # This allows developers to specify a GH or BB location to get releases from,
                  # especially tags URLs (https://github.com/user/repo/tags or
                  # https://bitbucket.org/user/repo#tags)
                  info['releases'] = []
                  for release in releases:
                      download_details = None
                      download_info = {}
  
                      # Make sure that explicit fields are copied over
                      for field in ['platforms', 'sublime_text', 'version', 'url', 'date']:
                          if field in release:
                              download_info[field] = release[field]
  
                      if 'details' in release:
                          download_details = release['details']
  
                          try:
                              github_download = github_client.download_info(download_details)
                              bitbucket_download = bitbucket_client.download_info(download_details)
  
                              # Overlay the explicit field values over values fetched from the APIs
                              if github_download:
                                  download_info = dict(chain(github_download.items(), download_info.items()))
                              # No matching tags
                              elif github_download == False:
                                  download_info = {}
                              elif bitbucket_download:
                                  download_info = dict(chain(bitbucket_download.items(), download_info.items()))
                              # No matching tags
                              elif bitbucket_download == False:
                                  download_info = {}
                              else:
                                  raise ProviderException(u'Invalid "details" value "%s" under the "releases" key for the package "%s" in the repository %s.' % (download_details, info['name'], self.repo))
  
                          except (DownloaderException, ClientException, ProviderException) as e:
                              if 'name' in info:
                                  self.broken_packages[info['name']] = e
                              self.failed_sources[download_details] = e
                              continue
  
                      if download_info:
                          info['releases'].append(download_info)
  
                  info = self.select_release(info)
  
              # Schema version 1.0, 1.1 and 1.2 just require that all values be
              # explicitly specified in the package JSON
              else:
                  info['platforms'] = package.get('platforms')
                  info = self.select_platform(info)
  
              if not info:
                  self.unavailable_packages.append(package['name'])
                  continue
  
              if 'download' not in info and 'releases' not in info:
                  self.broken_packages[info['name']] = ProviderException(u'No "releases" key for the package "%s" in the repository %s.' % (info['name'], self.repo))
                  continue
  
              for field in ['previous_names', 'labels']:
                  if field not in info:
                      info[field] = []
  
              for field in ['readme', 'issues', 'donate', 'buy']:
                  if field not in info:
                      info[field] = None
  
              if 'homepage' not in info:
                  info['homepage'] = self.repo
  
              if 'download' in info:
                  # Rewrites the legacy "zipball" URLs to the new "zip" format
                  info['download']['url'] = re.sub(
                      '^(https://nodeload.github.com/[^/]+/[^/]+/)zipball(/.*)$',
                      '\\1zip\\2', info['download']['url'])
  
                  # Rewrites the legacy "nodeload" URLs to the new "codeload" subdomain
                  info['download']['url'] = info['download']['url'].replace(
                      'nodeload.github.com', 'codeload.github.com')
  
                  # Extract the date from the download
                  if 'last_modified' not in info:
                      info['last_modified'] = info['download']['date']
  
              elif 'releases' in info and 'last_modified' not in info:
                  # Extract a date from the newest download
                  date = '1970-01-01 00:00:00'
                  for release in info['releases']:
                      if 'date' in release and release['date'] > date:
                          date = release['date']
                  info['last_modified'] = date
  
              if info['name'] in previous_names:
                  info['previous_names'].extend(previous_names[info['name']])
  
              output[info['name']] = info
              yield (info['name'], info)
  
          self.cache['get_packages'] = output
function_tokens: def get_packages ( self , invalid_sources = None ) : if 'get_packages' in self . cache : for key , value in self . cache [ 'get_packages' ] . items ( ) : yield ( key , value ) return if invalid_sources != None and self . repo in invalid_sources : raise StopIteration ( ) self . fetch ( ) def fail ( message ) : exception = ProviderException ( message ) self . failed_sources [ self . repo ] = exception self . cache [ 'get_packages' ] = { } return schema_error = u'Repository %s does not appear to be a valid repository file because ' % self . repo if 'schema_version' not in self . repo_info : error_string = u'%s the "schema_version" JSON key is missing.' % schema_error fail ( error_string ) return try : self . schema_version = float ( self . repo_info . get ( 'schema_version' ) ) except ( ValueError ) : error_string = u'%s the "schema_version" is not a valid number.' % schema_error fail ( error_string ) return if self . schema_version not in [ 1.0 , 1.1 , 1.2 , 2.0 ] : error_string = u'%s the "schema_version" is not recognized. Must be one of: 1.0, 1.1, 1.2 or 2.0.' % schema_error fail ( error_string ) return if 'packages' not in self . repo_info : error_string = u'%s the "packages" JSON key is missing.' % schema_error fail ( error_string ) return github_client = GitHubClient ( self . settings ) bitbucket_client = BitBucketClient ( self . settings ) # Backfill the "previous_names" keys for old schemas previous_names = { } if self . schema_version < 2.0 : renamed = self . get_renamed_packages ( ) for old_name in renamed : new_name = renamed [ old_name ] if new_name not in previous_names : previous_names [ new_name ] = [ ] previous_names [ new_name ] . append ( old_name ) output = { } for package in self . repo_info [ 'packages' ] : info = { 'sources' : [ self . repo ] } for field in [ 'name' , 'description' , 'author' , 'last_modified' , 'previous_names' , 'labels' , 'homepage' , 'readme' , 'issues' , 'donate' , 'buy' ] : if package . get ( field ) : info [ field ] = package . get ( field ) # Schema version 2.0 allows for grabbing details about a pacakge, or its # download from "details" urls. See the GitHubClient and BitBucketClient # classes for valid URLs. if self . schema_version >= 2.0 : details = package . get ( 'details' ) releases = package . get ( 'releases' ) # Try to grab package-level details from GitHub or BitBucket if details : if invalid_sources != None and details in invalid_sources : continue info [ 'sources' ] . append ( details ) try : github_repo_info = github_client . repo_info ( details ) bitbucket_repo_info = bitbucket_client . repo_info ( details ) # When grabbing details, prefer explicit field values over the values # from the GitHub or BitBucket API if github_repo_info : info = dict ( chain ( github_repo_info . items ( ) , info . items ( ) ) ) elif bitbucket_repo_info : info = dict ( chain ( bitbucket_repo_info . items ( ) , info . items ( ) ) ) else : raise ProviderException ( u'Invalid "details" value "%s" for one of the packages in the repository %s.' % ( details , self . repo ) ) except ( DownloaderException , ClientException , ProviderException ) as e : if 'name' in info : self . broken_packages [ info [ 'name' ] ] = e self . failed_sources [ details ] = e continue # If no releases info was specified, also grab the download info from GH or BB if not releases and details : releases = [ { 'details' : details } ] # This allows developers to specify a GH or BB location to get releases from, # especially tags URLs (https://github.com/user/repo/tags or # https://bitbucket.org/user/repo#tags) info [ 'releases' ] = [ ] for release in releases : download_details = None download_info = { } # Make sure that explicit fields are copied over for field in [ 'platforms' , 'sublime_text' , 'version' , 'url' , 'date' ] : if field in release : download_info [ field ] = release [ field ] if 'details' in release : download_details = release [ 'details' ] try : github_download = github_client . download_info ( download_details ) bitbucket_download = bitbucket_client . download_info ( download_details ) # Overlay the explicit field values over values fetched from the APIs if github_download : download_info = dict ( chain ( github_download . items ( ) , download_info . items ( ) ) ) # No matching tags elif github_download == False : download_info = { } elif bitbucket_download : download_info = dict ( chain ( bitbucket_download . items ( ) , download_info . items ( ) ) ) # No matching tags elif bitbucket_download == False : download_info = { } else : raise ProviderException ( u'Invalid "details" value "%s" under the "releases" key for the package "%s" in the repository %s.' % ( download_details , info [ 'name' ] , self . repo ) ) except ( DownloaderException , ClientException , ProviderException ) as e : if 'name' in info : self . broken_packages [ info [ 'name' ] ] = e self . failed_sources [ download_details ] = e continue if download_info : info [ 'releases' ] . append ( download_info ) info = self . select_release ( info ) # Schema version 1.0, 1.1 and 1.2 just require that all values be # explicitly specified in the package JSON else : info [ 'platforms' ] = package . get ( 'platforms' ) info = self . select_platform ( info ) if not info : self . unavailable_packages . append ( package [ 'name' ] ) continue if 'download' not in info and 'releases' not in info : self . broken_packages [ info [ 'name' ] ] = ProviderException ( u'No "releases" key for the package "%s" in the repository %s.' % ( info [ 'name' ] , self . repo ) ) continue for field in [ 'previous_names' , 'labels' ] : if field not in info : info [ field ] = [ ] for field in [ 'readme' , 'issues' , 'donate' , 'buy' ] : if field not in info : info [ field ] = None if 'homepage' not in info : info [ 'homepage' ] = self . repo if 'download' in info : # Rewrites the legacy "zipball" URLs to the new "zip" format info [ 'download' ] [ 'url' ] = re . sub ( '^(https://nodeload.github.com/[^/]+/[^/]+/)zipball(/.*)$' , '\\1zip\\2' , info [ 'download' ] [ 'url' ] ) # Rewrites the legacy "nodeload" URLs to the new "codeload" subdomain info [ 'download' ] [ 'url' ] = info [ 'download' ] [ 'url' ] . replace ( 'nodeload.github.com' , 'codeload.github.com' ) # Extract the date from the download if 'last_modified' not in info : info [ 'last_modified' ] = info [ 'download' ] [ 'date' ] elif 'releases' in info and 'last_modified' not in info : # Extract a date from the newest download date = '1970-01-01 00:00:00' for release in info [ 'releases' ] : if 'date' in release and release [ 'date' ] > date : date = release [ 'date' ] info [ 'last_modified' ] = date if info [ 'name' ] in previous_names : info [ 'previous_names' ] . extend ( previous_names [ info [ 'name' ] ] ) output [ info [ 'name' ] ] = info yield ( info [ 'name' ] , info ) self . cache [ 'get_packages' ] = output