(**kwargs)
return page, responseHeaders, code
This method connects to the target URL or proxy and returns
        the target URL page content
This method connects to the target URL or proxy and returns
        the target URL page content
This method connects to the target URL or proxy and returns the target URL page content
def getPage(**kwargs):
        """
        This method connects to the target URL or proxy and returns
        the target URL page content
        """

        if isinstance(conf.delay, (int, float)) and conf.delay > 0:
            time.sleep(conf.delay)
        elif conf.cpuThrottle:
            cpuThrottle(conf.cpuThrottle)

        if conf.offline:
            return None, None, None
        elif conf.dummy:
            return getUnicode(randomStr(int(randomInt()), alphabet=[chr(_) for _ in xrange(256)]), {}, int(randomInt())), None, None

        threadData = getCurrentThreadData()
        with kb.locks.request:
            kb.requestCounter += 1
            threadData.lastRequestUID = kb.requestCounter

        url = kwargs.get("url",                     None) or conf.url
        get = kwargs.get("get",                     None)
        post = kwargs.get("post",                   None)
        method = kwargs.get("method",               None)
        cookie = kwargs.get("cookie",               None)
        ua = kwargs.get("ua",                       None) or conf.agent
        referer = kwargs.get("referer",             None) or conf.referer
        host = kwargs.get("host",                   None) or conf.host
        direct_ = kwargs.get("direct",              False)
        multipart = kwargs.get("multipart",         False)
        silent = kwargs.get("silent",               False)
        raise404 = kwargs.get("raise404",           True)
        timeout = kwargs.get("timeout",             None) or conf.timeout
        auxHeaders = kwargs.get("auxHeaders",       None)
        response = kwargs.get("response",           False)
        ignoreTimeout = kwargs.get("ignoreTimeout", False) or kb.ignoreTimeout
        refreshing = kwargs.get("refreshing",       False)
        retrying = kwargs.get("retrying",           False)
        crawling = kwargs.get("crawling",           False)
        skipRead = kwargs.get("skipRead",           False)

        websocket_ = url.lower().startswith("ws")

        if not urlparse.urlsplit(url).netloc:
            url = urlparse.urljoin(conf.url, url)

        # flag to know if we are dealing with the same target host
        target = reduce(lambda x, y: x == y, map(lambda x: urlparse.urlparse(x).netloc.split(':')[0], [url, conf.url or ""]))

        if not retrying:
            # Reset the number of connection retries
            threadData.retriesCount = 0

        # fix for known issue when urllib2 just skips the other part of provided
        # url splitted with space char while urlencoding it in the later phase
        url = url.replace(" ", "%20")

        conn = None
        code = None
        page = None

        _ = urlparse.urlsplit(url)
        requestMsg = u"HTTP request [#%d]:\n%s " % (threadData.lastRequestUID, method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET))
        requestMsg += ("%s%s" % (_.path or "/", ("?%s" % _.query) if _.query else "")) if not any((refreshing, crawling)) else url
        responseMsg = u"HTTP response "
        requestHeaders = u""
        responseHeaders = None
        logHeaders = u""
        skipLogTraffic = False

        raise404 = raise404 and not kb.ignoreNotFound

        # support for non-latin (e.g. cyrillic) URLs as urllib/urllib2 doesn't
        # support those by default
        url = asciifyUrl(url)

        try:
            socket.setdefaulttimeout(timeout)

            if direct_:
                if '?' in url:
                    url, params = url.split('?', 1)
                    params = urlencode(params)
                    url = "%s?%s" % (url, params)

            elif multipart:
                # Needed in this form because of potential circle dependency
                # problem (option -> update -> connect -> option)
                from lib.core.option import proxyHandler

                multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler)
                conn = multipartOpener.open(unicodeencode(url), multipart)
                page = Connect._connReadProxy(conn) if not skipRead else None
                responseHeaders = conn.info()
                responseHeaders[URI_HTTP_HEADER] = conn.geturl()
                page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))

                return page

            elif any((refreshing, crawling)):
                pass

            elif target:
                if conf.forceSSL and urlparse.urlparse(url).scheme != "https":
                    url = re.sub("\Ahttp:", "https:", url, re.I)
                    url = re.sub(":80/", ":443/", url, re.I)

                if PLACE.GET in conf.parameters and not get:
                    get = conf.parameters[PLACE.GET]

                    if not conf.skipUrlEncode:
                        get = urlencode(get, limit=True)

                if get:
                    if '?' in url:
                        url = "%s%s%s" % (url, DEFAULT_GET_POST_DELIMITER, get)
                        requestMsg += "%s%s" % (DEFAULT_GET_POST_DELIMITER, get)
                    else:
                        url = "%s?%s" % (url, get)
                        requestMsg += "?%s" % get

                if PLACE.POST in conf.parameters and not post and method != HTTPMETHOD.GET:
                    post = conf.parameters[PLACE.POST]

            elif get:
                url = "%s?%s" % (url, get)
                requestMsg += "?%s" % get

            requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str

            # Prepare HTTP headers
            headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie, HTTP_HEADER.USER_AGENT: ua, HTTP_HEADER.REFERER: referer, HTTP_HEADER.HOST: host})

            if kb.authHeader:
                headers[HTTP_HEADER.AUTHORIZATION] = kb.authHeader

            if kb.proxyAuthHeader:
                headers[HTTP_HEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader

            if not getHeader(headers, HTTP_HEADER.ACCEPT):
                headers[HTTP_HEADER.ACCEPT] = HTTP_ACCEPT_HEADER_VALUE

            if not getHeader(headers, HTTP_HEADER.HOST) or not target:
                headers[HTTP_HEADER.HOST] = getHostHeader(url)

            if not getHeader(headers, HTTP_HEADER.ACCEPT_ENCODING):
                headers[HTTP_HEADER.ACCEPT_ENCODING] = HTTP_ACCEPT_ENCODING_HEADER_VALUE if kb.pageCompress else "identity"

            if post is not None and not getHeader(headers, HTTP_HEADER.CONTENT_TYPE):
                headers[HTTP_HEADER.CONTENT_TYPE] = POST_HINT_CONTENT_TYPES.get(kb.postHint, DEFAULT_CONTENT_TYPE)

            if headers.get(HTTP_HEADER.CONTENT_TYPE) == POST_HINT_CONTENT_TYPES[POST_HINT.MULTIPART]:
                warnMsg = "missing 'boundary parameter' in '%s' header. " % HTTP_HEADER.CONTENT_TYPE
                warnMsg += "Will try to reconstruct"
                singleTimeWarnMessage(warnMsg)

                boundary = findMultipartPostBoundary(conf.data)
                if boundary:
                    headers[HTTP_HEADER.CONTENT_TYPE] = "%s; boundary=%s" % (headers[HTTP_HEADER.CONTENT_TYPE], boundary)

            if auxHeaders:
                for key, item in auxHeaders.items():
                    for _ in headers.keys():
                        if _.upper() == key.upper():
                            del headers[_]
                    headers[key] = item

            for key, item in headers.items():
                del headers[key]
                headers[unicodeencode(key, kb.pageEncoding)] = unicodeencode(item, kb.pageEncoding)

            url = unicodeencode(url)
            post = unicodeencode(post)

            if websocket_:
                ws = websocket.WebSocket()
                ws.connect(url, header=("%s: %s" % _ for _ in headers.items() if _[0] not in ("Host",)), cookie=cookie)  # WebSocket will add Host field of headers automatically
                ws.send(urldecode(post or ""))
                page = ws.recv()
                ws.close()
                code = ws.status
                status = httplib.responses[code]
                class _(dict):
                    pass
                responseHeaders = _(ws.getheaders())
                responseHeaders.headers = ["%s: %s\r\n" % (_[0].capitalize(), _[1]) for _ in responseHeaders.items()]

                requestHeaders += "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items())
                requestMsg += "\n%s" % requestHeaders

                if post is not None:
                    requestMsg += "\n\n%s" % getUnicode(post)

                requestMsg += "\n"

                threadData.lastRequestMsg = requestMsg

                logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
            else:
                if method and method not in (HTTPMETHOD.GET, HTTPMETHOD.POST):
                    method = unicodeencode(method)
                    req = MethodRequest(url, post, headers)
                    req.set_method(method)
                else:
                    req = urllib2.Request(url, post, headers)

                requestHeaders += "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in req.header_items())

                if not getRequestHeader(req, HTTP_HEADER.COOKIE) and conf.cj:
                    conf.cj._policy._now = conf.cj._now = int(time.time())
                    cookies = conf.cj._cookies_for_request(req)
                    requestHeaders += "\n%s" % ("Cookie: %s" % ";".join("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value)) for cookie in cookies))

                if post is not None:
                    if not getRequestHeader(req, HTTP_HEADER.CONTENT_LENGTH):
                        requestHeaders += "\n%s: %d" % (string.capwords(HTTP_HEADER.CONTENT_LENGTH), len(post))

                if not getRequestHeader(req, HTTP_HEADER.CONNECTION):
                    requestHeaders += "\n%s: close" % HTTP_HEADER.CONNECTION

                requestMsg += "\n%s" % requestHeaders

                if post is not None:
                    requestMsg += "\n\n%s" % getUnicode(post)

                requestMsg += "\n"

                threadData.lastRequestMsg = requestMsg

                logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)

                if conf.cj:
                    for cookie in conf.cj:
                        if cookie.value is None:
                            cookie.value = ""

                conn = urllib2.urlopen(req)

                if not kb.authHeader and getRequestHeader(req, HTTP_HEADER.AUTHORIZATION) and (conf.authType or "").lower() == AUTH_TYPE.BASIC.lower():
                    kb.authHeader = getRequestHeader(req, HTTP_HEADER.AUTHORIZATION)

                if not kb.proxyAuthHeader and getRequestHeader(req, HTTP_HEADER.PROXY_AUTHORIZATION):
                    kb.proxyAuthHeader = getRequestHeader(req, HTTP_HEADER.PROXY_AUTHORIZATION)

                # Return response object
                if response:
                    return conn, None, None

                # Get HTTP response
                if hasattr(conn, 'redurl'):
                    page = (threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO\
                    else Connect._connReadProxy(conn)) if not skipRead else None
                    skipLogTraffic = kb.redirectChoice == REDIRECTION.NO
                    code = conn.redcode
                else:
                    page = Connect._connReadProxy(conn) if not skipRead else None

                code = code or conn.code
                responseHeaders = conn.info()
                responseHeaders[URI_HTTP_HEADER] = conn.geturl()
                page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
                status = getUnicode(conn.msg)

            if extractRegexResult(META_REFRESH_REGEX, page) and not refreshing:
                url = extractRegexResult(META_REFRESH_REGEX, page)

                debugMsg = "got HTML meta refresh header"
                logger.debug(debugMsg)

                if kb.alwaysRefresh is None:
                    msg = "sqlmap got a refresh request "
                    msg += "(redirect like response common to login pages). "
                    msg += "Do you want to apply the refresh "
                    msg += "from now on (or stay on the original page)? [Y/n]"
                    choice = readInput(msg, default="Y")

                    kb.alwaysRefresh = choice not in ("n", "N")

                if kb.alwaysRefresh:
                    if url.lower().startswith('http://'):
                        kwargs['url'] = url
                    else:
                        kwargs['url'] = conf.url[:conf.url.rfind('/') + 1] + url

                    threadData.lastRedirectMsg = (threadData.lastRequestUID, page)
                    kwargs['refreshing'] = True
                    kwargs['get'] = None
                    kwargs['post'] = None

                    try:
                        return Connect._getPageProxy(**kwargs)
                    except SqlmapSyntaxException:
                        pass

            # Explicit closing of connection object
            if conn and not conf.keepAlive:
                try:
                    if hasattr(conn.fp, '_sock'):
                        conn.fp._sock.close()
                    conn.close()
                except Exception, ex:
                    warnMsg = "problem occurred during connection closing ('%s')" % getSafeExString(ex)
                    logger.warn(warnMsg)

        except urllib2.HTTPError, ex:
            page = None
            responseHeaders = None

            try:
                page = ex.read() if not skipRead else None
                responseHeaders = ex.info()
                responseHeaders[URI_HTTP_HEADER] = ex.geturl()
                page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
            except socket.timeout:
                warnMsg = "connection timed out while trying "
                warnMsg += "to get error page information (%d)" % ex.code
                logger.warn(warnMsg)
                return None, None, None
            except KeyboardInterrupt:
                raise
            except:
                pass
            finally:
                page = page if isinstance(page, unicode) else getUnicode(page)

            code = ex.code

            kb.originalCode = kb.originalCode or code
            threadData.lastHTTPError = (threadData.lastRequestUID, code)
            kb.httpErrorCodes[code] = kb.httpErrorCodes.get(code, 0) + 1

            status = getUnicode(ex.msg)
            responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)

            if responseHeaders:
                logHeaders = "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items())

            logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]))

            skipLogTraffic = True

            if conf.verbose <= 5:
                responseMsg += getUnicode(logHeaders)
            elif conf.verbose > 5:
                responseMsg += "%s\n\n%s" % (logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE])

            logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)

            if ex.code == httplib.UNAUTHORIZED and not conf.ignore401:
                errMsg = "not authorized, try to provide right HTTP "
                errMsg += "authentication type and valid credentials (%d)" % code
                raise SqlmapConnectionException(errMsg)
            elif ex.code == httplib.NOT_FOUND:
                if raise404:
                    errMsg = "page not found (%d)" % code
                    raise SqlmapConnectionException(errMsg)
                else:
                    debugMsg = "page not found (%d)" % code
                    singleTimeLogMessage(debugMsg, logging.DEBUG)
                    processResponse(page, responseHeaders)
            elif ex.code == httplib.GATEWAY_TIMEOUT:
                if ignoreTimeout:
                    return None, None, None
                else:
                    warnMsg = "unable to connect to the target URL (%d - %s)" % (ex.code, httplib.responses[ex.code])
                    if threadData.retriesCount < conf.retries and not kb.threadException:
                        warnMsg += ". sqlmap is going to retry the request"
                        logger.critical(warnMsg)
                        return Connect._retryProxy(**kwargs)
                    elif kb.testMode:
                        logger.critical(warnMsg)
                        return None, None, None
                    else:
                        raise SqlmapConnectionException(warnMsg)
            else:
                debugMsg = "got HTTP error code: %d (%s)" % (code, status)
                logger.debug(debugMsg)

        except (urllib2.URLError, socket.error, socket.timeout, httplib.HTTPException, struct.error, binascii.Error, ProxyError, SqlmapCompressionException, WebSocketException):
            tbMsg = traceback.format_exc()

            if "no host given" in tbMsg:
                warnMsg = "invalid URL address used (%s)" % repr(url)
                raise SqlmapSyntaxException(warnMsg)
            elif "forcibly closed" in tbMsg or "Connection is already closed" in tbMsg:
                warnMsg = "connection was forcibly closed by the target URL"
            elif "timed out" in tbMsg:
                if kb.testMode and kb.testType not in (None, PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED):
                    singleTimeWarnMessage("there is a possibility that the target (or WAF) is dropping 'suspicious' requests")
                warnMsg = "connection timed out to the target URL"
            elif "URLError" in tbMsg or "error" in tbMsg:
                warnMsg = "unable to connect to the target URL"
            elif "NTLM" in tbMsg:
                warnMsg = "there has been a problem with NTLM authentication"
            elif "BadStatusLine" in tbMsg:
                warnMsg = "connection dropped or unknown HTTP "
                warnMsg += "status code received"
                if not conf.agent and not conf.randomAgent:
                    warnMsg += ". Try to force the HTTP User-Agent "
                    warnMsg += "header with option '--user-agent' or switch '--random-agent'"
            elif "IncompleteRead" in tbMsg:
                warnMsg = "there was an incomplete read error while retrieving data "
                warnMsg += "from the target URL"
            elif "Handshake status" in tbMsg:
                status = re.search("Handshake status ([\d]{3})", tbMsg)
                errMsg = "websocket handshake status %s" % status.group(1) if status else "unknown"
                raise SqlmapConnectionException(errMsg)
            else:
                warnMsg = "unable to connect to the target URL"

            if "BadStatusLine" not in tbMsg:
                warnMsg += " or proxy"

            if silent:
                return None, None, None
            elif "forcibly closed" in tbMsg:
                logger.critical(warnMsg)
                return None, None, None
            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead")):
                return None, None, None
            elif threadData.retriesCount < conf.retries and not kb.threadException:
                warnMsg += ". sqlmap is going to retry the request"
                if not retrying:
                    warnMsg += "(s)"
                    logger.critical(warnMsg)
                else:
                    logger.debug(warnMsg)
                return Connect._retryProxy(**kwargs)
            elif kb.testMode:
                logger.critical(warnMsg)
                return None, None, None
            else:
                raise SqlmapConnectionException(warnMsg)

        finally:
            if isinstance(page, basestring) and not isinstance(page, unicode):
                if HTTP_HEADER.CONTENT_TYPE in (responseHeaders or {}) and not re.search(TEXT_CONTENT_TYPE_REGEX, responseHeaders[HTTP_HEADER.CONTENT_TYPE]):
                    page = unicode(page, errors="ignore")
                else:
                    page = getUnicode(page)
            socket.setdefaulttimeout(conf.timeout)

        processResponse(page, responseHeaders)

        if conn and getattr(conn, "redurl", None):
            _ = urlparse.urlsplit(conn.redurl)
            _ = ("%s%s" % (_.path or "/", ("?%s" % _.query) if _.query else ""))
            requestMsg = re.sub("(\n[A-Z]+ ).+?( HTTP/\d)", "\g<1>%s\g<2>" % re.escape(getUnicode(_)), requestMsg, 1)

            if kb.resendPostOnRedirect is False:
                requestMsg = re.sub("(\[#\d+\]:\n)POST ", "\g<1>GET ", requestMsg)
                requestMsg = re.sub("(?i)Content-length: \d+\n", "", requestMsg)
                requestMsg = re.sub("(?s)\n\n.+", "\n", requestMsg)

            responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, conn.code, status)
        else:
            responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)

        if responseHeaders:
            logHeaders = "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items())

        if not skipLogTraffic:
            logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]))

        if conf.verbose <= 5:
            responseMsg += getUnicode(logHeaders)
        elif conf.verbose > 5:
            responseMsg += "%s\n\n%s" % (logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE])

        logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)

        return page, responseHeaders, code
def getPage ( * * kwargs ) : if isinstance ( conf . delay , ( int , float ) ) and conf . delay > 0 : time . sleep ( conf . delay ) elif conf . cpuThrottle : cpuThrottle ( conf . cpuThrottle ) if conf . offline : return None , None , None elif conf . dummy : return getUnicode ( randomStr ( int ( randomInt ( ) ) , alphabet = [ chr ( _ ) for _ in xrange ( 256 ) ] ) , { } , int ( randomInt ( ) ) ) , None , None threadData = getCurrentThreadData ( ) with kb . locks . request : kb . requestCounter += 1 threadData . lastRequestUID = kb . requestCounter url = kwargs . get ( "url" , None ) or conf . url get = kwargs . get ( "get" , None ) post = kwargs . get ( "post" , None ) method = kwargs . get ( "method" , None ) cookie = kwargs . get ( "cookie" , None ) ua = kwargs . get ( "ua" , None ) or conf . agent referer = kwargs . get ( "referer" , None ) or conf . referer host = kwargs . get ( "host" , None ) or conf . host direct_ = kwargs . get ( "direct" , False ) multipart = kwargs . get ( "multipart" , False ) silent = kwargs . get ( "silent" , False ) raise404 = kwargs . get ( "raise404" , True ) timeout = kwargs . get ( "timeout" , None ) or conf . timeout auxHeaders = kwargs . get ( "auxHeaders" , None ) response = kwargs . get ( "response" , False ) ignoreTimeout = kwargs . get ( "ignoreTimeout" , False ) or kb . ignoreTimeout refreshing = kwargs . get ( "refreshing" , False ) retrying = kwargs . get ( "retrying" , False ) crawling = kwargs . get ( "crawling" , False ) skipRead = kwargs . get ( "skipRead" , False ) websocket_ = url . lower ( ) . startswith ( "ws" ) if not urlparse . urlsplit ( url ) . netloc : url = urlparse . urljoin ( conf . url , url ) # flag to know if we are dealing with the same target host target = reduce ( lambda x , y : x == y , map ( lambda x : urlparse . urlparse ( x ) . netloc . split ( ':' ) [ 0 ] , [ url , conf . url or "" ] ) ) if not retrying : # Reset the number of connection retries threadData . retriesCount = 0 # fix for known issue when urllib2 just skips the other part of provided # url splitted with space char while urlencoding it in the later phase url = url . replace ( " " , "%20" ) conn = None code = None page = None _ = urlparse . urlsplit ( url ) requestMsg = u"HTTP request [#%d]:\n%s " % ( threadData . lastRequestUID , method or ( HTTPMETHOD . POST if post is not None else HTTPMETHOD . GET ) ) requestMsg += ( "%s%s" % ( _ . path or "/" , ( "?%s" % _ . query ) if _ . query else "" ) ) if not any ( ( refreshing , crawling ) ) else url responseMsg = u"HTTP response " requestHeaders = u"" responseHeaders = None logHeaders = u"" skipLogTraffic = False raise404 = raise404 and not kb . ignoreNotFound # support for non-latin (e.g. cyrillic) URLs as urllib/urllib2 doesn't # support those by default url = asciifyUrl ( url ) try : socket . setdefaulttimeout ( timeout ) if direct_ : if '?' in url : url , params = url . split ( '?' , 1 ) params = urlencode ( params ) url = "%s?%s" % ( url , params ) elif multipart : # Needed in this form because of potential circle dependency # problem (option -> update -> connect -> option) from lib . core . option import proxyHandler multipartOpener = urllib2 . build_opener ( proxyHandler , multipartpost . MultipartPostHandler ) conn = multipartOpener . open ( unicodeencode ( url ) , multipart ) page = Connect . _connReadProxy ( conn ) if not skipRead else None responseHeaders = conn . info ( ) responseHeaders [ URI_HTTP_HEADER ] = conn . geturl ( ) page = decodePage ( page , responseHeaders . get ( HTTP_HEADER . CONTENT_ENCODING ) , responseHeaders . get ( HTTP_HEADER . CONTENT_TYPE ) ) return page elif any ( ( refreshing , crawling ) ) : pass elif target : if conf . forceSSL and urlparse . urlparse ( url ) . scheme != "https" : url = re . sub ( "\Ahttp:" , "https:" , url , re . I ) url = re . sub ( ":80/" , ":443/" , url , re . I ) if PLACE . GET in conf . parameters and not get : get = conf . parameters [ PLACE . GET ] if not conf . skipUrlEncode : get = urlencode ( get , limit = True ) if get : if '?' in url : url = "%s%s%s" % ( url , DEFAULT_GET_POST_DELIMITER , get ) requestMsg += "%s%s" % ( DEFAULT_GET_POST_DELIMITER , get ) else : url = "%s?%s" % ( url , get ) requestMsg += "?%s" % get if PLACE . POST in conf . parameters and not post and method != HTTPMETHOD . GET : post = conf . parameters [ PLACE . POST ] elif get : url = "%s?%s" % ( url , get ) requestMsg += "?%s" % get requestMsg += " %s" % httplib . HTTPConnection . _http_vsn_str # Prepare HTTP headers headers = forgeHeaders ( { HTTP_HEADER . COOKIE : cookie , HTTP_HEADER . USER_AGENT : ua , HTTP_HEADER . REFERER : referer , HTTP_HEADER . HOST : host } ) if kb . authHeader : headers [ HTTP_HEADER . AUTHORIZATION ] = kb . authHeader if kb . proxyAuthHeader : headers [ HTTP_HEADER . PROXY_AUTHORIZATION ] = kb . proxyAuthHeader if not getHeader ( headers , HTTP_HEADER . ACCEPT ) : headers [ HTTP_HEADER . ACCEPT ] = HTTP_ACCEPT_HEADER_VALUE if not getHeader ( headers , HTTP_HEADER . HOST ) or not target : headers [ HTTP_HEADER . HOST ] = getHostHeader ( url ) if not getHeader ( headers , HTTP_HEADER . ACCEPT_ENCODING ) : headers [ HTTP_HEADER . ACCEPT_ENCODING ] = HTTP_ACCEPT_ENCODING_HEADER_VALUE if kb . pageCompress else "identity" if post is not None and not getHeader ( headers , HTTP_HEADER . CONTENT_TYPE ) : headers [ HTTP_HEADER . CONTENT_TYPE ] = POST_HINT_CONTENT_TYPES . get ( kb . postHint , DEFAULT_CONTENT_TYPE ) if headers . get ( HTTP_HEADER . CONTENT_TYPE ) == POST_HINT_CONTENT_TYPES [ POST_HINT . MULTIPART ] : warnMsg = "missing 'boundary parameter' in '%s' header. " % HTTP_HEADER . CONTENT_TYPE warnMsg += "Will try to reconstruct" singleTimeWarnMessage ( warnMsg ) boundary = findMultipartPostBoundary ( conf . data ) if boundary : headers [ HTTP_HEADER . CONTENT_TYPE ] = "%s; boundary=%s" % ( headers [ HTTP_HEADER . CONTENT_TYPE ] , boundary ) if auxHeaders : for key , item in auxHeaders . items ( ) : for _ in headers . keys ( ) : if _ . upper ( ) == key . upper ( ) : del headers [ _ ] headers [ key ] = item for key , item in headers . items ( ) : del headers [ key ] headers [ unicodeencode ( key , kb . pageEncoding ) ] = unicodeencode ( item , kb . pageEncoding ) url = unicodeencode ( url ) post = unicodeencode ( post ) if websocket_ : ws = websocket . WebSocket ( ) ws . connect ( url , header = ( "%s: %s" % _ for _ in headers . items ( ) if _ [ 0 ] not in ( "Host" , ) ) , cookie = cookie ) # WebSocket will add Host field of headers automatically ws . send ( urldecode ( post or "" ) ) page = ws . recv ( ) ws . close ( ) code = ws . status status = httplib . responses [ code ] class _ ( dict ) : pass responseHeaders = _ ( ws . getheaders ( ) ) responseHeaders . headers = [ "%s: %s\r\n" % ( _ [ 0 ] . capitalize ( ) , _ [ 1 ] ) for _ in responseHeaders . items ( ) ] requestHeaders += "\n" . join ( "%s: %s" % ( getUnicode ( key . capitalize ( ) if isinstance ( key , basestring ) else key ) , getUnicode ( value ) ) for ( key , value ) in responseHeaders . items ( ) ) requestMsg += "\n%s" % requestHeaders if post is not None : requestMsg += "\n\n%s" % getUnicode ( post ) requestMsg += "\n" threadData . lastRequestMsg = requestMsg logger . log ( CUSTOM_LOGGING . TRAFFIC_OUT , requestMsg ) else : if method and method not in ( HTTPMETHOD . GET , HTTPMETHOD . POST ) : method = unicodeencode ( method ) req = MethodRequest ( url , post , headers ) req . set_method ( method ) else : req = urllib2 . Request ( url , post , headers ) requestHeaders += "\n" . join ( "%s: %s" % ( getUnicode ( key . capitalize ( ) if isinstance ( key , basestring ) else key ) , getUnicode ( value ) ) for ( key , value ) in req . header_items ( ) ) if not getRequestHeader ( req , HTTP_HEADER . COOKIE ) and conf . cj : conf . cj . _policy . _now = conf . cj . _now = int ( time . time ( ) ) cookies = conf . cj . _cookies_for_request ( req ) requestHeaders += "\n%s" % ( "Cookie: %s" % ";" . join ( "%s=%s" % ( getUnicode ( cookie . name ) , getUnicode ( cookie . value ) ) for cookie in cookies ) ) if post is not None : if not getRequestHeader ( req , HTTP_HEADER . CONTENT_LENGTH ) : requestHeaders += "\n%s: %d" % ( string . capwords ( HTTP_HEADER . CONTENT_LENGTH ) , len ( post ) ) if not getRequestHeader ( req , HTTP_HEADER . CONNECTION ) : requestHeaders += "\n%s: close" % HTTP_HEADER . CONNECTION requestMsg += "\n%s" % requestHeaders if post is not None : requestMsg += "\n\n%s" % getUnicode ( post ) requestMsg += "\n" threadData . lastRequestMsg = requestMsg logger . log ( CUSTOM_LOGGING . TRAFFIC_OUT , requestMsg ) if conf . cj : for cookie in conf . cj : if cookie . value is None : cookie . value = "" conn = urllib2 . urlopen ( req ) if not kb . authHeader and getRequestHeader ( req , HTTP_HEADER . AUTHORIZATION ) and ( conf . authType or "" ) . lower ( ) == AUTH_TYPE . BASIC . lower ( ) : kb . authHeader = getRequestHeader ( req , HTTP_HEADER . AUTHORIZATION ) if not kb . proxyAuthHeader and getRequestHeader ( req , HTTP_HEADER . PROXY_AUTHORIZATION ) : kb . proxyAuthHeader = getRequestHeader ( req , HTTP_HEADER . PROXY_AUTHORIZATION ) # Return response object if response : return conn , None , None # Get HTTP response if hasattr ( conn , 'redurl' ) : page = ( threadData . lastRedirectMsg [ 1 ] if kb . redirectChoice == REDIRECTION . NO else Connect . _connReadProxy ( conn ) ) if not skipRead else None skipLogTraffic = kb . redirectChoice == REDIRECTION . NO code = conn . redcode else : page = Connect . _connReadProxy ( conn ) if not skipRead else None code = code or conn . code responseHeaders = conn . info ( ) responseHeaders [ URI_HTTP_HEADER ] = conn . geturl ( ) page = decodePage ( page , responseHeaders . get ( HTTP_HEADER . CONTENT_ENCODING ) , responseHeaders . get ( HTTP_HEADER . CONTENT_TYPE ) ) status = getUnicode ( conn . msg ) if extractRegexResult ( META_REFRESH_REGEX , page ) and not refreshing : url = extractRegexResult ( META_REFRESH_REGEX , page ) debugMsg = "got HTML meta refresh header" logger . debug ( debugMsg ) if kb . alwaysRefresh is None : msg = "sqlmap got a refresh request " msg += "(redirect like response common to login pages). " msg += "Do you want to apply the refresh " msg += "from now on (or stay on the original page)? [Y/n]" choice = readInput ( msg , default = "Y" ) kb . alwaysRefresh = choice not in ( "n" , "N" ) if kb . alwaysRefresh : if url . lower ( ) . startswith ( 'http://' ) : kwargs [ 'url' ] = url else : kwargs [ 'url' ] = conf . url [ : conf . url . rfind ( '/' ) + 1 ] + url threadData . lastRedirectMsg = ( threadData . lastRequestUID , page ) kwargs [ 'refreshing' ] = True kwargs [ 'get' ] = None kwargs [ 'post' ] = None try : return Connect . _getPageProxy ( * * kwargs ) except SqlmapSyntaxException : pass # Explicit closing of connection object if conn and not conf . keepAlive : try : if hasattr ( conn . fp , '_sock' ) : conn . fp . _sock . close ( ) conn . close ( ) except Exception , ex : warnMsg = "problem occurred during connection closing ('%s')" % getSafeExString ( ex ) logger . warn ( warnMsg ) except urllib2 . HTTPError , ex : page = None responseHeaders = None try : page = ex . read ( ) if not skipRead else None responseHeaders = ex . info ( ) responseHeaders [ URI_HTTP_HEADER ] = ex . geturl ( ) page = decodePage ( page , responseHeaders . get ( HTTP_HEADER . CONTENT_ENCODING ) , responseHeaders . get ( HTTP_HEADER . CONTENT_TYPE ) ) except socket . timeout : warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % ex . code logger . warn ( warnMsg ) return None , None , None except KeyboardInterrupt : raise except : pass finally : page = page if isinstance ( page , unicode ) else getUnicode ( page ) code = ex . code kb . originalCode = kb . originalCode or code threadData . lastHTTPError = ( threadData . lastRequestUID , code ) kb . httpErrorCodes [ code ] = kb . httpErrorCodes . get ( code , 0 ) + 1 status = getUnicode ( ex . msg ) responseMsg += "[#%d] (%d %s):\n" % ( threadData . lastRequestUID , code , status ) if responseHeaders : logHeaders = "\n" . join ( "%s: %s" % ( getUnicode ( key . capitalize ( ) if isinstance ( key , basestring ) else key ) , getUnicode ( value ) ) for ( key , value ) in responseHeaders . items ( ) ) logHTTPTraffic ( requestMsg , "%s%s\n\n%s" % ( responseMsg , logHeaders , ( page or "" ) [ : MAX_CONNECTION_CHUNK_SIZE ] ) ) skipLogTraffic = True if conf . verbose <= 5 : responseMsg += getUnicode ( logHeaders ) elif conf . verbose > 5 : responseMsg += "%s\n\n%s" % ( logHeaders , ( page or "" ) [ : MAX_CONNECTION_CHUNK_SIZE ] ) logger . log ( CUSTOM_LOGGING . TRAFFIC_IN , responseMsg ) if ex . code == httplib . UNAUTHORIZED and not conf . ignore401 : errMsg = "not authorized, try to provide right HTTP " errMsg += "authentication type and valid credentials (%d)" % code raise SqlmapConnectionException ( errMsg ) elif ex . code == httplib . NOT_FOUND : if raise404 : errMsg = "page not found (%d)" % code raise SqlmapConnectionException ( errMsg ) else : debugMsg = "page not found (%d)" % code singleTimeLogMessage ( debugMsg , logging . DEBUG ) processResponse ( page , responseHeaders ) elif ex . code == httplib . GATEWAY_TIMEOUT : if ignoreTimeout : return None , None , None else : warnMsg = "unable to connect to the target URL (%d - %s)" % ( ex . code , httplib . responses [ ex . code ] ) if threadData . retriesCount < conf . retries and not kb . threadException : warnMsg += ". sqlmap is going to retry the request" logger . critical ( warnMsg ) return Connect . _retryProxy ( * * kwargs ) elif kb . testMode : logger . critical ( warnMsg ) return None , None , None else : raise SqlmapConnectionException ( warnMsg ) else : debugMsg = "got HTTP error code: %d (%s)" % ( code , status ) logger . debug ( debugMsg ) except ( urllib2 . URLError , socket . error , socket . timeout , httplib . HTTPException , struct . error , binascii . Error , ProxyError , SqlmapCompressionException , WebSocketException ) : tbMsg = traceback . format_exc ( ) if "no host given" in tbMsg : warnMsg = "invalid URL address used (%s)" % repr ( url ) raise SqlmapSyntaxException ( warnMsg ) elif "forcibly closed" in tbMsg or "Connection is already closed" in tbMsg : warnMsg = "connection was forcibly closed by the target URL" elif "timed out" in tbMsg : if kb . testMode and kb . testType not in ( None , PAYLOAD . TECHNIQUE . TIME , PAYLOAD . TECHNIQUE . STACKED ) : singleTimeWarnMessage ( "there is a possibility that the target (or WAF) is dropping 'suspicious' requests" ) warnMsg = "connection timed out to the target URL" elif "URLError" in tbMsg or "error" in tbMsg : warnMsg = "unable to connect to the target URL" elif "NTLM" in tbMsg : warnMsg = "there has been a problem with NTLM authentication" elif "BadStatusLine" in tbMsg : warnMsg = "connection dropped or unknown HTTP " warnMsg += "status code received" if not conf . agent and not conf . randomAgent : warnMsg += ". Try to force the HTTP User-Agent " warnMsg += "header with option '--user-agent' or switch '--random-agent'" elif "IncompleteRead" in tbMsg : warnMsg = "there was an incomplete read error while retrieving data " warnMsg += "from the target URL" elif "Handshake status" in tbMsg : status = re . search ( "Handshake status ([\d]{3})" , tbMsg ) errMsg = "websocket handshake status %s" % status . group ( 1 ) if status else "unknown" raise SqlmapConnectionException ( errMsg ) else : warnMsg = "unable to connect to the target URL" if "BadStatusLine" not in tbMsg : warnMsg += " or proxy" if silent : return None , None , None elif "forcibly closed" in tbMsg : logger . critical ( warnMsg ) return None , None , None elif ignoreTimeout and any ( _ in tbMsg for _ in ( "timed out" , "IncompleteRead" ) ) : return None , None , None elif threadData . retriesCount < conf . retries and not kb . threadException : warnMsg += ". sqlmap is going to retry the request" if not retrying : warnMsg += "(s)" logger . critical ( warnMsg ) else : logger . debug ( warnMsg ) return Connect . _retryProxy ( * * kwargs ) elif kb . testMode : logger . critical ( warnMsg ) return None , None , None else : raise SqlmapConnectionException ( warnMsg ) finally : if isinstance ( page , basestring ) and not isinstance ( page , unicode ) : if HTTP_HEADER . CONTENT_TYPE in ( responseHeaders or { } ) and not re . search ( TEXT_CONTENT_TYPE_REGEX , responseHeaders [ HTTP_HEADER . CONTENT_TYPE ] ) : page = unicode ( page , errors = "ignore" ) else : page = getUnicode ( page ) socket . setdefaulttimeout ( conf . timeout ) processResponse ( page , responseHeaders ) if conn and getattr ( conn , "redurl" , None ) : _ = urlparse . urlsplit ( conn . redurl ) _ = ( "%s%s" % ( _ . path or "/" , ( "?%s" % _ . query ) if _ . query else "" ) ) requestMsg = re . sub ( "(\n[A-Z]+ ).+?( HTTP/\d)" , "\g<1>%s\g<2>" % re . escape ( getUnicode ( _ ) ) , requestMsg , 1 ) if kb . resendPostOnRedirect is False : requestMsg = re . sub ( "(\[#\d+\]:\n)POST " , "\g<1>GET " , requestMsg ) requestMsg = re . sub ( "(?i)Content-length: \d+\n" , "" , requestMsg ) requestMsg = re . sub ( "(?s)\n\n.+" , "\n" , requestMsg ) responseMsg += "[#%d] (%d %s):\n" % ( threadData . lastRequestUID , conn . code , status ) else : responseMsg += "[#%d] (%d %s):\n" % ( threadData . lastRequestUID , code , status ) if responseHeaders : logHeaders = "\n" . join ( "%s: %s" % ( getUnicode ( key . capitalize ( ) if isinstance ( key , basestring ) else key ) , getUnicode ( value ) ) for ( key , value ) in responseHeaders . items ( ) ) if not skipLogTraffic : logHTTPTraffic ( requestMsg , "%s%s\n\n%s" % ( responseMsg , logHeaders , ( page or "" ) [ : MAX_CONNECTION_CHUNK_SIZE ] ) ) if conf . verbose <= 5 : responseMsg += getUnicode ( logHeaders ) elif conf . verbose > 5 : responseMsg += "%s\n\n%s" % ( logHeaders , ( page or "" ) [ : MAX_CONNECTION_CHUNK_SIZE ] ) logger . log ( CUSTOM_LOGGING . TRAFFIC_IN , responseMsg ) return page , responseHeaders , code