pip3 install requests

以最近比較火的小說“魔道祖師”為例。

下面是整個腳本

import requests,re

def get_content(url,timeout=10):
    req = requests.get(url=url,timeout=timeout)
    return req.text

def get_title(html,re_title):
    ret = re_title.search(html)
    if ret:
        ret = ret.group()
        tmp = ret.split('_')[0]
        tmp = tmp.replace('<title>','')
        tmp = tmp.strip()
        return tmp

def get_body(html,ret_body):
    ret_body = re_body.search(html)
    if ret_body:
        ret = ret_body.group()
        tmp = re_clear_header.sub(r'\2',ret)
        tmp = tmp.replace(r'&nbsp;',' ').replace(r'<br /><br />','\n').replace(r'<br />','\n')
        tmp = tmp.replace(r'2k小說閱讀網</p>','\n\n')
        return tmp

if __name__ == '__main__':
    mdzs = open('mdzs.txt','w')
    re_title = re.compile(r'<title>(.*?)</title>')
    re_body = re.compile(r'<p class="Text">(.*?)</p>',re.S)
    re_clear_header = re.compile(r'(.*</script>)(.*)',re.S)
    first_page = 19613532
    for i in range(116):
        page = first_page + i
        url = r'https://www.2kxs.com/xiaoshuo/96/96717/{}.html'.format(page)
        try:
            html = get_content(url)
            title = get_title(html,re_title)
            mdzs.write(title + '\n\n')
            body = get_body(html,re_body)
            mdzs.write(body)
            print('{} is success'.format(url))
        except Exception as e:
            print('url :{} , error: {}'.format(url,e))

該網站是小說網站，排版和網頁的url比較有規律性，所以實現起來比較簡單

用python“爬”一篇小說