(reqFile, addedTargetUrls)
Parses web scarab and burp logs and adds results to the target URL list
Parses web scarab and burp logs and adds results to the target URL list
Parses web scarab and burp logs and adds results to the target URL list
def _feedTargetsDict(reqFile, addedTargetUrls):
    """
    Parses web scarab and burp logs and adds results to the target URL list
    """

    def _parseWebScarabLog(content):
        """
        Parses web scarab logs (POST method not supported)
        """

        reqResList = content.split(WEBSCARAB_SPLITTER)

        for request in reqResList:
            url = extractRegexResult(r"URL: (?P<result>.+?)\n", request, re.I)
            method = extractRegexResult(r"METHOD: (?P<result>.+?)\n", request, re.I)
            cookie = extractRegexResult(r"COOKIE: (?P<result>.+?)\n", request, re.I)

            if not method or not url:
                logger.debug("not a valid WebScarab log data")
                continue

            if method.upper() == HTTPMETHOD.POST:
                warnMsg = "POST requests from WebScarab logs aren't supported "
                warnMsg += "as their body content is stored in separate files. "
                warnMsg += "Nevertheless you can use -r to load them individually."
                logger.warning(warnMsg)
                continue

            if not(conf.scope and not re.search(conf.scope, url, re.I)):
                if not kb.targets or url not in addedTargetUrls:
                    kb.targets.add((url, method, None, cookie, None))
                    addedTargetUrls.add(url)

    def _parseBurpLog(content):
        """
        Parses burp logs
        """

        if not re.search(BURP_REQUEST_REGEX, content, re.I | re.S):
            if re.search(BURP_XML_HISTORY_REGEX, content, re.I | re.S):
                reqResList = []
                for match in re.finditer(BURP_XML_HISTORY_REGEX, content, re.I | re.S):
                    port, request = match.groups()
                    request = request.decode("base64")
                    _ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
                    if _:
                        host = _.group(0).strip()
                        if not re.search(r":\d+\Z", host):
                            request = request.replace(host, "%s:%d" % (host, int(port)))
                    reqResList.append(request)
            else:
                reqResList = [content]
        else:
            reqResList = re.finditer(BURP_REQUEST_REGEX, content, re.I | re.S)

        for match in reqResList:
            request = match if isinstance(match, basestring) else match.group(0)
            request = re.sub(r"\A[^\w]+", "", request)

            schemePort = re.search(r"(http[\w]*)\:\/\/.*?\:([\d]+).+?={10,}", request, re.I | re.S)

            if schemePort:
                scheme = schemePort.group(1)
                port = schemePort.group(2)
            else:
                scheme, port = None, None

            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
                continue

            if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
                continue

            getPostReq = False
            url = None
            host = None
            method = None
            data = None
            cookie = None
            params = False
            newline = None
            lines = request.split('\n')
            headers = []

            for index in xrange(len(lines)):
                line = lines[index]

                if not line.strip() and index == len(lines) - 1:
                    break

                newline = "\r\n" if line.endswith('\r') else '\n'
                line = line.strip('\r')
                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None

                if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                    data = ""
                    params = True

                elif match:
                    method = match.group(1)
                    url = match.group(2)

                    if any(_ in line for _ in ('?', '=', CUSTOM_INJECTION_MARK_CHAR)):
                        params = True

                    getPostReq = True

                # POST parameters
                elif data is not None and params:
                    data += "%s%s" % (line, newline)

                # GET parameters
                elif "?" in line and "=" in line and ": " not in line:
                    params = True

                # Headers
                elif re.search(r"\A\S+:", line):
                    key, value = line.split(":", 1)
                    value = value.strip().replace("\r", "").replace("\n", "")

                    # Cookie and Host headers
                    if key.upper() == HTTP_HEADER.COOKIE.upper():
                        cookie = value
                    elif key.upper() == HTTP_HEADER.HOST.upper():
                        if '://' in value:
                            scheme, value = value.split('://')[:2]
                        splitValue = value.split(":")
                        host = splitValue[0]

                        if len(splitValue) > 1:
                            port = filterStringValue(splitValue[1], "[0-9]")

                    # Avoid to add a static content length header to
                    # headers and consider the following lines as
                    # POSTed data
                    if key.upper() == HTTP_HEADER.CONTENT_LENGTH.upper():
                        params = True

                    # Avoid proxy and connection type related headers
                    elif key not in (HTTP_HEADER.PROXY_CONNECTION, HTTP_HEADER.CONNECTION):
                        headers.append((getUnicode(key), getUnicode(value)))

                    if CUSTOM_INJECTION_MARK_CHAR in re.sub(PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "", value or ""):
                        params = True

            data = data.rstrip("\r\n") if data else data

            if getPostReq and (params or cookie):
                if not port and isinstance(scheme, basestring) and scheme.lower() == "https":
                    port = "443"
                elif not scheme and port == "443":
                    scheme = "https"

                if conf.forceSSL:
                    scheme = "https"
                    port = port or "443"

                if not host:
                    errMsg = "invalid format of a request file"
                    raise SqlmapSyntaxException, errMsg

                if not url.startswith("http"):
                    url = "%s://%s:%s%s" % (scheme or "http", host, port or "80", url)
                    scheme = None
                    port = None

                if not(conf.scope and not re.search(conf.scope, url, re.I)):
                    if not kb.targets or url not in addedTargetUrls:
                        kb.targets.add((url, method, data, cookie, tuple(headers)))
                        addedTargetUrls.add(url)

    checkFile(reqFile)
    try:
        with openFile(reqFile, "rb") as f:
            content = f.read()
    except (IOError, OSError, MemoryError), ex:
        errMsg = "something went wrong while trying "
        errMsg += "to read the content of file '%s' ('%s')" % (reqFile, ex)
        raise SqlmapSystemException(errMsg)

    if conf.scope:
        logger.info("using regular expression '%s' for filtering targets" % conf.scope)

    _parseBurpLog(content)
    _parseWebScarabLog(content)

    if not addedTargetUrls:
        errMsg = "unable to find usable request(s) "
        errMsg += "in provided file ('%s')" % reqFile
        raise SqlmapGenericException(errMsg)
def _feedTargetsDict ( reqFile , addedTargetUrls ) : def _parseWebScarabLog ( content ) : """
        Parses web scarab logs (POST method not supported)
        """ reqResList = content . split ( WEBSCARAB_SPLITTER ) for request in reqResList : url = extractRegexResult ( r"URL: (?P<result>.+?)\n" , request , re . I ) method = extractRegexResult ( r"METHOD: (?P<result>.+?)\n" , request , re . I ) cookie = extractRegexResult ( r"COOKIE: (?P<result>.+?)\n" , request , re . I ) if not method or not url : logger . debug ( "not a valid WebScarab log data" ) continue if method . upper ( ) == HTTPMETHOD . POST : warnMsg = "POST requests from WebScarab logs aren't supported " warnMsg += "as their body content is stored in separate files. " warnMsg += "Nevertheless you can use -r to load them individually." logger . warning ( warnMsg ) continue if not ( conf . scope and not re . search ( conf . scope , url , re . I ) ) : if not kb . targets or url not in addedTargetUrls : kb . targets . add ( ( url , method , None , cookie , None ) ) addedTargetUrls . add ( url ) def _parseBurpLog ( content ) : """
        Parses burp logs
        """ if not re . search ( BURP_REQUEST_REGEX , content , re . I | re . S ) : if re . search ( BURP_XML_HISTORY_REGEX , content , re . I | re . S ) : reqResList = [ ] for match in re . finditer ( BURP_XML_HISTORY_REGEX , content , re . I | re . S ) : port , request = match . groups ( ) request = request . decode ( "base64" ) _ = re . search ( r"%s:.+" % re . escape ( HTTP_HEADER . HOST ) , request ) if _ : host = _ . group ( 0 ) . strip ( ) if not re . search ( r":\d+\Z" , host ) : request = request . replace ( host , "%s:%d" % ( host , int ( port ) ) ) reqResList . append ( request ) else : reqResList = [ content ] else : reqResList = re . finditer ( BURP_REQUEST_REGEX , content , re . I | re . S ) for match in reqResList : request = match if isinstance ( match , basestring ) else match . group ( 0 ) request = re . sub ( r"\A[^\w]+" , "" , request ) schemePort = re . search ( r"(http[\w]*)\:\/\/.*?\:([\d]+).+?={10,}" , request , re . I | re . S ) if schemePort : scheme = schemePort . group ( 1 ) port = schemePort . group ( 2 ) else : scheme , port = None , None if not re . search ( r"^[\n]*(%s).*?\sHTTP\/" % "|" . join ( getPublicTypeMembers ( HTTPMETHOD , True ) ) , request , re . I | re . M ) : continue if re . search ( r"^[\n]*%s.*?\.(%s)\sHTTP\/" % ( HTTPMETHOD . GET , "|" . join ( CRAWL_EXCLUDE_EXTENSIONS ) ) , request , re . I | re . M ) : continue getPostReq = False url = None host = None method = None data = None cookie = None params = False newline = None lines = request . split ( '\n' ) headers = [ ] for index in xrange ( len ( lines ) ) : line = lines [ index ] if not line . strip ( ) and index == len ( lines ) - 1 : break newline = "\r\n" if line . endswith ( '\r' ) else '\n' line = line . strip ( '\r' ) match = re . search ( r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|" . join ( getPublicTypeMembers ( HTTPMETHOD , True ) ) , line ) if not method else None if len ( line . strip ( ) ) == 0 and method and method != HTTPMETHOD . GET and data is None : data = "" params = True elif match : method = match . group ( 1 ) url = match . group ( 2 ) if any ( _ in line for _ in ( '?' , '=' , CUSTOM_INJECTION_MARK_CHAR ) ) : params = True getPostReq = True # POST parameters elif data is not None and params : data += "%s%s" % ( line , newline ) # GET parameters elif "?" in line and "=" in line and ": " not in line : params = True # Headers elif re . search ( r"\A\S+:" , line ) : key , value = line . split ( ":" , 1 ) value = value . strip ( ) . replace ( "\r" , "" ) . replace ( "\n" , "" ) # Cookie and Host headers if key . upper ( ) == HTTP_HEADER . COOKIE . upper ( ) : cookie = value elif key . upper ( ) == HTTP_HEADER . HOST . upper ( ) : if '://' in value : scheme , value = value . split ( '://' ) [ : 2 ] splitValue = value . split ( ":" ) host = splitValue [ 0 ] if len ( splitValue ) > 1 : port = filterStringValue ( splitValue [ 1 ] , "[0-9]" ) # Avoid to add a static content length header to # headers and consider the following lines as # POSTed data if key . upper ( ) == HTTP_HEADER . CONTENT_LENGTH . upper ( ) : params = True # Avoid proxy and connection type related headers elif key not in ( HTTP_HEADER . PROXY_CONNECTION , HTTP_HEADER . CONNECTION ) : headers . append ( ( getUnicode ( key ) , getUnicode ( value ) ) ) if CUSTOM_INJECTION_MARK_CHAR in re . sub ( PROBLEMATIC_CUSTOM_INJECTION_PATTERNS , "" , value or "" ) : params = True data = data . rstrip ( "\r\n" ) if data else data if getPostReq and ( params or cookie ) : if not port and isinstance ( scheme , basestring ) and scheme . lower ( ) == "https" : port = "443" elif not scheme and port == "443" : scheme = "https" if conf . forceSSL : scheme = "https" port = port or "443" if not host : errMsg = "invalid format of a request file" raise SqlmapSyntaxException , errMsg if not url . startswith ( "http" ) : url = "%s://%s:%s%s" % ( scheme or "http" , host , port or "80" , url ) scheme = None port = None if not ( conf . scope and not re . search ( conf . scope , url , re . I ) ) : if not kb . targets or url not in addedTargetUrls : kb . targets . add ( ( url , method , data , cookie , tuple ( headers ) ) ) addedTargetUrls . add ( url ) checkFile ( reqFile ) try : with openFile ( reqFile , "rb" ) as f : content = f . read ( ) except ( IOError , OSError , MemoryError ) , ex : errMsg = "something went wrong while trying " errMsg += "to read the content of file '%s' ('%s')" % ( reqFile , ex ) raise SqlmapSystemException ( errMsg ) if conf . scope : logger . info ( "using regular expression '%s' for filtering targets" % conf . scope ) _parseBurpLog ( content ) _parseWebScarabLog ( content ) if not addedTargetUrls : errMsg = "unable to find usable request(s) " errMsg += "in provided file ('%s')" % reqFile raise SqlmapGenericException ( errMsg )