Python 抓取百度百科数据

正文

请到「今天看啥」查看全文

== 10 :

break

count = count + 1

except :

print ( 'craw failed' )

self . outputer . output_html ()

if __name__ == '__main__' :

root_url = 'http://baike.baidu.com/view/21087.htm'

obj_spider = SpiderMain ()

obj_spider . craw ( root_url )

url_manger.py

# coding:utf-8

class UrlManager ( object ) :

def __init__ ( self ) :

self . new_urls = set ()

self . old_urls = set ()

def add_new_url ( self , url ) :

if url is None :

return

if url not in self . new_urls and url not in self . old_urls :

self . new_urls . add ( url )

def add_new_urls ( self , urls ) :

if urls is None or len ( urls ) == 0 :

return

for url in urls :

self . add_new_url ( url )

def has_new_url ( self ) :

return len ( self . new_urls ) != 0

def

请到「今天看啥」查看全文