python爬取百度貼吧文字內容
阿新 • • 發佈:2022-03-20
爬取百度貼吧文字內容
方法1:
點選檢視程式碼
**匯入urllib庫** from urllib import request **匯入re正則模組庫** import re #指定爬取頁數 url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8" try: #指定請求體瀏覽器UA headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"} req = request.Request(url, headers=headers) resp = request.urlopen(req) #把內容進行解碼 content = resp.read().decode('utf-8') print(content) #定義正則規則進行內容爬取 pattern = re.compile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</a>') items = re.findall(pattern, content) #遍歷輸出內容 for i in items: print(i[0] + "\t" + i[1]) except request.URLError as e: if hasattr(e, 'code'): print(e.code) if hasattr(e, 'reason'): print(e.reason)
方法2(封裝成方法形式):
點選檢視程式碼
from urllib import request import re class TieBa: # 初始化方法 def __init__(self): # 連結 self.url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn=" # 請求頭 self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"} # 列表 ,儲存解析後的結果 self.stories = [] # 下載頁面 def getPage(self, page_number): try: # 定義請求 req = request.Request(self.url + str(page_number), headers=self.headers) # 下載頁面 resp = request.urlopen(req) # 解碼 content = resp.read().decode('utf-8') # 返回結果 return content except request.URLError as e: # 列印異常的響應碼 if hasattr(e, 'code'): print(e.code) if hasattr(e, 'reason'): print(e.reason) # 解析頁面 # global items def rexgPage(self, content): pattern = re.compile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</ a>') items = re.findall(pattern, content) for i in items: self.stories.append(i[0] + "\t" + i[1]) # 顯示解析的內容 def getContent(self): for i in self.stories: print(i) # 呼叫方法下載然後解析頁面 - 顯示解析的內容 s1 = TieBa() print(s1.stories) s1.rexgPage(s1.getPage(50)) s1.getContent()