(self, url, page)
Process the contents of a PyPI page
Process the contents of a PyPI page
Process the contents of a PyPI page
def process_index(self, url, page):
        """Process the contents of a PyPI page"""

        def scan(link):
            # Process a URL to see if it's for a package page
            if link.startswith(self.index_url):
                parts = list(map(
                    urllib.parse.unquote, link[len(self.index_url):].split('/')
                ))
                if len(parts) == 2 and '#' not in parts[1]:
                    # it's a package page, sanitize and index it
                    pkg = safe_name(parts[0])
                    ver = safe_version(parts[1])
                    self.package_pages.setdefault(pkg.lower(), {})[link] = True
                    return to_filename(pkg), to_filename(ver)
            return None, None

        # process an index page into the package-page index
        for match in HREF.finditer(page):
            try:
                scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))
            except ValueError:
                pass

        pkg, ver = scan(url)  # ensure this page is in the page index
        if pkg:
            # process individual package page
            for new_url in find_external_links(url, page):
                # Process the found URL
                base, frag = egg_info_for_url(new_url)
                if base.endswith('.py') and not frag:
                    if ver:
                        new_url += '#egg=%s-%s' % (pkg, ver)
                    else:
                        self.need_version_info(url)
                self.scan_url(new_url)

            return PYPI_MD5.sub(
                lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page
            )
        else:
            return ""
def process_index ( self , url , page ) : def scan ( link ) : # Process a URL to see if it's for a package page if link . startswith ( self . index_url ) : parts = list ( map ( urllib . parse . unquote , link [ len ( self . index_url ) : ] . split ( '/' ) ) ) if len ( parts ) == 2 and '#' not in parts [ 1 ] : # it's a package page, sanitize and index it pkg = safe_name ( parts [ 0 ] ) ver = safe_version ( parts [ 1 ] ) self . package_pages . setdefault ( pkg . lower ( ) , { } ) [ link ] = True return to_filename ( pkg ) , to_filename ( ver ) return None , None # process an index page into the package-page index for match in HREF . finditer ( page ) : try : scan ( urllib . parse . urljoin ( url , htmldecode ( match . group ( 1 ) ) ) ) except ValueError : pass pkg , ver = scan ( url ) # ensure this page is in the page index if pkg : # process individual package page for new_url in find_external_links ( url , page ) : # Process the found URL base , frag = egg_info_for_url ( new_url ) if base . endswith ( '.py' ) and not frag : if ver : new_url += '#egg=%s-%s' % ( pkg , ver ) else : self . need_version_info ( url ) self . scan_url ( new_url ) return PYPI_MD5 . sub ( lambda m : '<a href="%s#md5=%s">%s</a>' % m . group ( 1 , 3 , 2 ) , page ) else : return ""