技術標籤：nodejs

用nodejs寫了個爬表情包的功能

用到的包：axios cheerio

首先安裝node,再安裝包：

npm i axios
npm i cheerio

下面是程式碼：

//表情包爬取

const axios = require('axios')
const cheerio = require('cheerio')
const { fstat } = require('fs')
const path = require('path')
const fs = require('fs')
const { time, assert, error } = require
 
('console')
const { func } = require('assert-plus')
const { dirname } = require('path')

//表情包網站
var baseUrl = "https://fabiaoqing.com/"

//爬取五頁表情
main(5)

//主程式
function main(downloadPage){
    new Promise(()=>{
        for (let i = 0; i < downloadPage; i++) {
            var httpUrl = "https://fabiaoqing.com/bqb/lists/type/hot/page/"
 
+i+".html"
            parsePageOne(httpUrl)     
        }
    })
    // .finally(()=>{
    //     console.log("所有圖片寫入完成");
    // })
}

//解析主頁面，獲取各個表情包地址
async function parsePageOne(httpUrl){
    let res = await axios.get(httpUrl)
    let $ = cheerio.load(res.data)

    $('#bqblist a.bqba'
 
).each((index, element)=> {
        let url = $(element).attr('href');

        let imgsUrl = new URL(url, baseUrl).href 
         
        parsePageTwo(imgsUrl).catch(error => {
            if (error) {
                //處理請求失敗的url
                let failUrl = imgsUrl
                let reqNum = 0//計數器
                dealFaileUrl(failUrl, reqNum)
           }       
       }) 
    })
}

//遞迴暴力請求，失敗了就繼續請求，成功就終止
function dealFaileUrl(failUrl, reqNum){
	//失敗後0.8秒後再次請求
    setTimeout(() => {
        parsePageTwo(failUrl).catch(error => {
            throw error
        }).then(()=>{//處理成功
            console.log(failUrl+"請求完成！");
            return
        }, ()=>{//處理失敗
            reqNum++
            dealFaileUrl(failUrl,reqNum)
            console.log(failUrl+"請求了"+reqNum+"次");
        })
    }, 800);
    
}

//parsePageTwo('https://fabiaoqing.com/bqb/detail/id/54322.html')

//解析各個表情包的地址
async function parsePageTwo(imgsUrl){
    let res = await axios.get(imgsUrl)
    let $ = cheerio.load(res.data)

    let dirName = $('#bqb h1').html()
    
    //去除檔名裡的斜槓
    //檔名有特殊字元會報錯
    if (dirName) {
        dirName = dirName.replace(/\//g, '')
        dirName = dirName.replace(/\|/g, '')
    }else{
        throw new Error('無法獲取表情包名！')
    }

    //建立資料夾
    fs.mkdir(`./image/${dirName}`, { recursive: true }, (error) => {
        if (error) {
            throw error
        }
        console.log("目錄建立完成:"+dirName);

        //下載檔案
        $('.swiper-wrapper img').each((index, element)=>{
            let imgUrl = $(element).attr('data-original')
    
            //下載
            download(imgUrl, dirName).catch(error => {
                if (error) {
                    console.log("下載失敗:"+error);
                }
            })   
        })  
    })
    
}

//download("https://fabiaoqing.com/bqb/detail/id/54329.html")

//下載表情
async function download(imgUrl, dirName){   
        //檔名
        let fileName = path.parse(imgUrl).base
        
        let ws = fs.createWriteStream(`./image/${dirName}/${fileName}`)
        axios.get(imgUrl, { responseType:'stream'}).then(res => {
            res.data.pipe(ws)            
        })

        ws.on('close', ()=>{
            console.log(fileName+"下載完成!");
        })
}

Q: 檔案下載過多後，控制檯不會自動關閉，而是處於堵塞狀態？這個問題一直沒有解決，有解決了的可以交流交流...

下載的檔案會放在當前目錄的image資料夾的各個表情包目錄下：