(self, url)
return result
Get the HTML for an URL, possibly from an in-memory cache.

        XXX TODO Note: this cache is never actually cleared. It's assumed that
        the data won't get stale over the lifetime of a locator instance (not
        necessarily true for the default_locator).
Get the HTML for an URL, possibly from an in-memory cache.
Get the HTML for an URL possibly from an in - memory cache .
def get_page(self, url):
        """
        Get the HTML for an URL, possibly from an in-memory cache.

        XXX TODO Note: this cache is never actually cleared. It's assumed that
        the data won't get stale over the lifetime of a locator instance (not
        necessarily true for the default_locator).
        """
        # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
        scheme, netloc, path, _, _, _ = urlparse(url)
        if scheme == 'file' and os.path.isdir(url2pathname(path)):
            url = urljoin(ensure_slash(url), 'index.html')

        if url in self._page_cache:
            result = self._page_cache[url]
            logger.debug('Returning %s from cache: %s', url, result)
        else:
            host = netloc.split(':', 1)[0]
            result = None
            if host in self._bad_hosts:
                logger.debug('Skipping %s due to bad host %s', url, host)
            else:
                req = Request(url, headers={'Accept-encoding': 'identity'})
                try:
                    logger.debug('Fetching %s', url)
                    resp = self.opener.open(req, timeout=self.timeout)
                    logger.debug('Fetched %s', url)
                    headers = resp.info()
                    content_type = headers.get('Content-Type', '')
                    if HTML_CONTENT_TYPE.match(content_type):
                        final_url = resp.geturl()
                        data = resp.read()
                        encoding = headers.get('Content-Encoding')
                        if encoding:
                            decoder = self.decoders[encoding]   # fail if not found
                            data = decoder(data)
                        encoding = 'utf-8'
                        m = CHARSET.search(content_type)
                        if m:
                            encoding = m.group(1)
                        try:
                            data = data.decode(encoding)
                        except UnicodeError:  # pragma: no cover
                            data = data.decode('latin-1')    # fallback
                        result = Page(data, final_url)
                        self._page_cache[final_url] = result
                except HTTPError as e:
                    if e.code != 404:
                        logger.exception('Fetch failed: %s: %s', url, e)
                except URLError as e:  # pragma: no cover
                    logger.exception('Fetch failed: %s: %s', url, e)
                    with self._lock:
                        self._bad_hosts.add(host)
                except Exception as e:  # pragma: no cover
                    logger.exception('Fetch failed: %s: %s', url, e)
                finally:
                    self._page_cache[url] = result   # even if None (failure)
        return result
def get_page ( self , url ) : # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api scheme , netloc , path , _ , _ , _ = urlparse ( url ) if scheme == 'file' and os . path . isdir ( url2pathname ( path ) ) : url = urljoin ( ensure_slash ( url ) , 'index.html' ) if url in self . _page_cache : result = self . _page_cache [ url ] logger . debug ( 'Returning %s from cache: %s' , url , result ) else : host = netloc . split ( ':' , 1 ) [ 0 ] result = None if host in self . _bad_hosts : logger . debug ( 'Skipping %s due to bad host %s' , url , host ) else : req = Request ( url , headers = { 'Accept-encoding' : 'identity' } ) try : logger . debug ( 'Fetching %s' , url ) resp = self . opener . open ( req , timeout = self . timeout ) logger . debug ( 'Fetched %s' , url ) headers = resp . info ( ) content_type = headers . get ( 'Content-Type' , '' ) if HTML_CONTENT_TYPE . match ( content_type ) : final_url = resp . geturl ( ) data = resp . read ( ) encoding = headers . get ( 'Content-Encoding' ) if encoding : decoder = self . decoders [ encoding ] # fail if not found data = decoder ( data ) encoding = 'utf-8' m = CHARSET . search ( content_type ) if m : encoding = m . group ( 1 ) try : data = data . decode ( encoding ) except UnicodeError : # pragma: no cover data = data . decode ( 'latin-1' ) # fallback result = Page ( data , final_url ) self . _page_cache [ final_url ] = result except HTTPError as e : if e . code != 404 : logger . exception ( 'Fetch failed: %s: %s' , url , e ) except URLError as e : # pragma: no cover logger . exception ( 'Fetch failed: %s: %s' , url , e ) with self . _lock : self . _bad_hosts . add ( host ) except Exception as e : # pragma: no cover logger . exception ( 'Fetch failed: %s: %s' , url , e ) finally : self . _page_cache [ url ] = result # even if None (failure) return result