1. 程式人生 > 其它 >python爬取百度貼吧文字內容

python爬取百度貼吧文字內容

爬取百度貼吧文字內容

方法1:

點選檢視程式碼
**匯入urllib庫**
from urllib import request
**匯入re正則模組庫**
import re
	#指定爬取頁數
url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8"
try:
	#指定請求體瀏覽器UA
    headers = {"User-Agent":
                   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"}
    req = request.Request(url, headers=headers)
    resp = request.urlopen(req)
	#把內容進行解碼
    content = resp.read().decode('utf-8')
    print(content)
	#定義正則規則進行內容爬取
    pattern = re.compile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</a>')
    items = re.findall(pattern, content)
	#遍歷輸出內容
    for i in items:
        print(i[0] + "\t" + i[1])
except request.URLError as e:
    if hasattr(e, 'code'):
        print(e.code)
    if hasattr(e, 'reason'):
        print(e.reason)

方法2(封裝成方法形式):

點選檢視程式碼
from urllib import request
import re


class TieBa:
    # 初始化方法
    def __init__(self):
        # 連結
        self.url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn="
        # 請求頭
        self.headers = {"User-Agent":
                            "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"}
        # 列表 ,儲存解析後的結果
        self.stories = []

    # 下載頁面
    def getPage(self, page_number):
        try:
            # 定義請求
            req = request.Request(self.url + str(page_number), headers=self.headers)
            # 下載頁面
            resp = request.urlopen(req)
            # 解碼
            content = resp.read().decode('utf-8')
            # 返回結果
            return content
        except request.URLError as e:
            # 列印異常的響應碼
            if hasattr(e, 'code'):
                print(e.code)
            if hasattr(e, 'reason'):
                print(e.reason)

    # 解析頁面
    # global items

    def rexgPage(self, content):
        pattern = re.compile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</ a>')
        items = re.findall(pattern, content)
        for i in items:
            self.stories.append(i[0] + "\t" + i[1])

    # 顯示解析的內容
    def getContent(self):
        for i in self.stories:
            print(i)


	# 呼叫方法下載然後解析頁面 - 顯示解析的內容
s1 = TieBa()

print(s1.stories)
s1.rexgPage(s1.getPage(50))
s1.getContent()