(SpiderGlobalVariable,UrlRule)
SpiderGlobalVariable
SpiderGlobalVariable
SpiderGlobalVariable
def spider_scheduling(SpiderGlobalVariable,UrlRule):
    '''
    SpiderGlobalVariable
    '''
    for i in init_urlnode(SpiderGlobalVariable.start_url,UrlRule):
        SpiderGlobalVariable.global_urlnode_queue.put((0,i))

    while exit_condition(SpiderGlobalVariable):
        if SpiderGlobalVariable.htmlnode_queue.qsize() > 0:
            html_node = SpiderGlobalVariable.htmlnode_queue.get()
            linklist = crawl(html_node.url, html_node.html)
            for i in linklist:
                url = i[1]
                method = i[0]
                data = i[2]
                depth = html_node.depth
                referer = html_node.url
                i = UrlNode(url, referer, depth, method, data)

                if i.depth <= SpiderGlobalVariable.depth and UrlRule.check_url(i.check_url):
                    if is_netloc(i.url):
                        SpiderGlobalVariable.global_urlnode_queue.put((0,i))
                    else:
                        SpiderGlobalVariable.global_urlnode_queue.put((random.randint(1,5),i))

                else:
                    SpiderGlobalVariable.refuse_count += 1
def spider_scheduling ( SpiderGlobalVariable , UrlRule ) : for i in init_urlnode ( SpiderGlobalVariable . start_url , UrlRule ) : SpiderGlobalVariable . global_urlnode_queue . put ( ( 0 , i ) ) while exit_condition ( SpiderGlobalVariable ) : if SpiderGlobalVariable . htmlnode_queue . qsize ( ) > 0 : html_node = SpiderGlobalVariable . htmlnode_queue . get ( ) linklist = crawl ( html_node . url , html_node . html ) for i in linklist : url = i [ 1 ] method = i [ 0 ] data = i [ 2 ] depth = html_node . depth referer = html_node . url i = UrlNode ( url , referer , depth , method , data ) if i . depth <= SpiderGlobalVariable . depth and UrlRule . check_url ( i . check_url ) : if is_netloc ( i . url ) : SpiderGlobalVariable . global_urlnode_queue . put ( ( 0 , i ) ) else : SpiderGlobalVariable . global_urlnode_queue . put ( ( random . randint ( 1 , 5 ) , i ) ) else : SpiderGlobalVariable . refuse_count += 1