13518219792
Python爬虫实现爬取百度百科词条功能？相信很多没有经验的人对此束手无策，为此本文总结了问题出现的原因和解决方法，通过这篇文章希望你能解决这个问题。
爬虫主程序入口
from crawler_test.html_downloader import UrlDownLoader
from crawler_test.html_outer import HtmlOuter
from crawler_test.html_parser import HtmlParser
from crawler_test.url_manager import UrlManager
# 爬虫主程序入口
class MainCrawler():
def __init__(self):
# 初始值，实例化四大处理器：url管理器，下载器，解析器，输出器
self.urls = UrlManager()
self.downloader = UrlDownLoader()
self.parser = HtmlParser()
self.outer = HtmlOuter()
# 开始爬虫方法
def start_craw(self, main_url):
print('爬虫开始...')
count = 1
self.urls.add_new_url(main_url)
while self.urls.has_new_url():
try:
new_url = self.urls.get_new_url()
print('爬虫%d,%s' % (count, new_url))
html_cont = self.downloader.down_load(new_url)
new_urls, new_data = self.parser.parse(new_url, html_cont)
# 将解析出的url放入url管理器，解析出的数据放入输出器中
self.urls.add_new_urls(new_urls)
self.outer.conllect_data(new_data)
if count >= 10:# 控制爬取的数量
break
count += 1
except:
print('爬虫失败一条')
self.outer.output()
print('爬虫结束。')
if __name__ == '__main__':
main_url = 'https://baike.baidu.com/item/Python/407313'
mc = MainCrawler()
mc.start_craw(main_url)URL管理器
# URL管理器 class UrlManager(): def __init__(self): self.new_urls = set() # 待爬取 self.old_urls = set() # 已爬取 # 添加一个新的url def add_new_url(self, url): if url is None: return elif url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) # 批量添加url def add_new_urls(self, urls): if urls is None or len(urls) == 0: return else: for url in urls: self.add_new_url(url) # 判断是否有url def has_new_url(self): return len(self.new_urls) != 0 # 从待爬取的集合中获取一个url def get_new_url(self): new_url = self.new_urls.pop() self.old_urls.add(new_url) return new_url
网页下载器
from urllib import request # 网页下载器 class UrlDownLoader(): def down_load(self, url): if url is None: return None else: rt = request.Request(url=url, method='GET') # 发GET请求 with request.urlopen(rt) as rp: # 打开网页 if rp.status != 200: return None else: return rp.read() # 读取网页内容
网页解析器
