(page, contentEncoding, contentType)
return page
Decode compressed/charset HTTP response
Decode compressed/charset HTTP response
Decode compressed / charset HTTP response
def decodePage(page, contentEncoding, contentType):
    """
    Decode compressed/charset HTTP response
    """

    if not page or (conf.nullConnection and len(page) < 2):
        return getUnicode(page)

    if isinstance(contentEncoding, basestring) and contentEncoding.lower() in ("gzip", "x-gzip", "deflate"):
        if not kb.pageCompress:
            return None

        try:
            if contentEncoding.lower() == "deflate":
                data = StringIO.StringIO(zlib.decompress(page, -15))  # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
            else:
                data = gzip.GzipFile("", "rb", 9, StringIO.StringIO(page))
                size = struct.unpack("<l", page[-4:])[0]  # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py
                if size > MAX_CONNECTION_TOTAL_SIZE:
                    raise Exception("size too large")

            page = data.read()
        except Exception, msg:
            errMsg = "detected invalid data for declared content "
            errMsg += "encoding '%s' ('%s')" % (contentEncoding, msg)
            singleTimeLogMessage(errMsg, logging.ERROR)

            warnMsg = "turning off page compression"
            singleTimeWarnMessage(warnMsg)

            kb.pageCompress = False
            raise SqlmapCompressionException

    if not conf.charset:
        httpCharset, metaCharset = None, None

        # Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode
        if contentType and (contentType.find("charset=") != -1):
            httpCharset = checkCharEncoding(contentType.split("charset=")[-1])

        metaCharset = checkCharEncoding(extractRegexResult(META_CHARSET_REGEX, page))

        if (any((httpCharset, metaCharset)) and not all((httpCharset, metaCharset)))\
            or (httpCharset == metaCharset and all((httpCharset, metaCharset))):
            kb.pageEncoding = httpCharset or metaCharset  # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence
            debugMsg = "declared web page charset '%s'" % kb.pageEncoding
            singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg)
        else:
            kb.pageEncoding = None
    else:
        kb.pageEncoding = conf.charset

    # can't do for all responses because we need to support binary files too
    if contentType and not isinstance(page, unicode) and "text/" in contentType.lower():
        if kb.heuristicMode:
            kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
            page = getUnicode(page, kb.pageEncoding)
        else:
            # e.g. &#195;&#235;&#224;&#226;&#224;
            if "&#" in page:
                page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)

            # e.g. %20%28%29
            if "%" in page:
                page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page)

            # e.g. &amp;
            page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)

            kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
            page = getUnicode(page, kb.pageEncoding)

            # e.g. &#8217;&#8230;&#8482;
            if "&#" in page:
                def _(match):
                    retVal = match.group(0)
                    try:
                        retVal = unichr(int(match.group(1)))
                    except ValueError:
                        pass
                    return retVal
                page = re.sub(r"&#(\d+);", _, page)

            # e.g. &zeta;
            page = re.sub(r"&([^;]+);", lambda _: unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)

    return page
def decodePage ( page , contentEncoding , contentType ) : if not page or ( conf . nullConnection and len ( page ) < 2 ) : return getUnicode ( page ) if isinstance ( contentEncoding , basestring ) and contentEncoding . lower ( ) in ( "gzip" , "x-gzip" , "deflate" ) : if not kb . pageCompress : return None try : if contentEncoding . lower ( ) == "deflate" : data = StringIO . StringIO ( zlib . decompress ( page , - 15 ) ) # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations else : data = gzip . GzipFile ( "" , "rb" , 9 , StringIO . StringIO ( page ) ) size = struct . unpack ( "<l" , page [ - 4 : ] ) [ 0 ] # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py if size > MAX_CONNECTION_TOTAL_SIZE : raise Exception ( "size too large" ) page = data . read ( ) except Exception , msg : errMsg = "detected invalid data for declared content " errMsg += "encoding '%s' ('%s')" % ( contentEncoding , msg ) singleTimeLogMessage ( errMsg , logging . ERROR ) warnMsg = "turning off page compression" singleTimeWarnMessage ( warnMsg ) kb . pageCompress = False raise SqlmapCompressionException if not conf . charset : httpCharset , metaCharset = None , None # Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode if contentType and ( contentType . find ( "charset=" ) != - 1 ) : httpCharset = checkCharEncoding ( contentType . split ( "charset=" ) [ - 1 ] ) metaCharset = checkCharEncoding ( extractRegexResult ( META_CHARSET_REGEX , page ) ) if ( any ( ( httpCharset , metaCharset ) ) and not all ( ( httpCharset , metaCharset ) ) ) or ( httpCharset == metaCharset and all ( ( httpCharset , metaCharset ) ) ) : kb . pageEncoding = httpCharset or metaCharset # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence debugMsg = "declared web page charset '%s'" % kb . pageEncoding singleTimeLogMessage ( debugMsg , logging . DEBUG , debugMsg ) else : kb . pageEncoding = None else : kb . pageEncoding = conf . charset # can't do for all responses because we need to support binary files too if contentType and not isinstance ( page , unicode ) and "text/" in contentType . lower ( ) : if kb . heuristicMode : kb . pageEncoding = kb . pageEncoding or checkCharEncoding ( getHeuristicCharEncoding ( page ) ) page = getUnicode ( page , kb . pageEncoding ) else : # e.g. &#195;&#235;&#224;&#226;&#224; if "&#" in page : page = re . sub ( r"&#(\d{1,3});" , lambda _ : chr ( int ( _ . group ( 1 ) ) ) if int ( _ . group ( 1 ) ) < 256 else _ . group ( 0 ) , page ) # e.g. %20%28%29 if "%" in page : page = re . sub ( r"%([0-9a-fA-F]{2})" , lambda _ : _ . group ( 1 ) . decode ( "hex" ) , page ) # e.g. &amp; page = re . sub ( r"&([^;]+);" , lambda _ : chr ( htmlEntities [ _ . group ( 1 ) ] ) if htmlEntities . get ( _ . group ( 1 ) , 256 ) < 256 else _ . group ( 0 ) , page ) kb . pageEncoding = kb . pageEncoding or checkCharEncoding ( getHeuristicCharEncoding ( page ) ) page = getUnicode ( page , kb . pageEncoding ) # e.g. &#8217;&#8230;&#8482; if "&#" in page : def _ ( match ) : retVal = match . group ( 0 ) try : retVal = unichr ( int ( match . group ( 1 ) ) ) except ValueError : pass return retVal page = re . sub ( r"&#(\d+);" , _ , page ) # e.g. &zeta; page = re . sub ( r"&([^;]+);" , lambda _ : unichr ( htmlEntities [ _ . group ( 1 ) ] ) if htmlEntities . get ( _ . group ( 1 ) , 0 ) > 255 else _ . group ( 0 ) , page ) return page