(page)
return data
Returns resulting items from UNION query inside provided page content
Returns resulting items from UNION query inside provided page content
Returns resulting items from UNION query inside provided page content
def parseUnionPage(page):
    """
    Returns resulting items from UNION query inside provided page content
    """

    if page is None:
        return None

    if re.search("(?si)\A%s.*%s\Z" % (kb.chars.start, kb.chars.stop), page):
        if len(page) > LARGE_OUTPUT_THRESHOLD:
            warnMsg = "large output detected. This might take a while"
            logger.warn(warnMsg)

        data = BigArray()
        keys = set()

        for match in re.finditer("%s(.*?)%s" % (kb.chars.start, kb.chars.stop), page, re.DOTALL | re.IGNORECASE):
            entry = match.group(1)

            if kb.chars.start in entry:
                entry = entry.split(kb.chars.start)[-1]

            if kb.unionDuplicates:
                key = entry.lower()
                if key not in keys:
                    keys.add(key)
                else:
                    continue

            entry = entry.split(kb.chars.delimiter)

            if conf.hexConvert:
                entry = applyFunctionRecursively(entry, decodeHexValue)

            if kb.safeCharEncode:
                entry = applyFunctionRecursively(entry, safecharencode)

            data.append(entry[0] if len(entry) == 1 else entry)
    else:
        data = page

    if len(data) == 1 and isinstance(data[0], basestring):
        data = data[0]

    return data
def parseUnionPage ( page ) : if page is None : return None if re . search ( "(?si)\A%s.*%s\Z" % ( kb . chars . start , kb . chars . stop ) , page ) : if len ( page ) > LARGE_OUTPUT_THRESHOLD : warnMsg = "large output detected. This might take a while" logger . warn ( warnMsg ) data = BigArray ( ) keys = set ( ) for match in re . finditer ( "%s(.*?)%s" % ( kb . chars . start , kb . chars . stop ) , page , re . DOTALL | re . IGNORECASE ) : entry = match . group ( 1 ) if kb . chars . start in entry : entry = entry . split ( kb . chars . start ) [ - 1 ] if kb . unionDuplicates : key = entry . lower ( ) if key not in keys : keys . add ( key ) else : continue entry = entry . split ( kb . chars . delimiter ) if conf . hexConvert : entry = applyFunctionRecursively ( entry , decodeHexValue ) if kb . safeCharEncode : entry = applyFunctionRecursively ( entry , safecharencode ) data . append ( entry [ 0 ] if len ( entry ) == 1 else entry ) else : data = page if len ( data ) == 1 and isinstance ( data [ 0 ] , basestring ) : data = data [ 0 ] return data