爬取目標：javlib，使用框架Scrapy

首先使用在命令列裡

scrapy startproject projectname

和

scrapy genspider spidername

首先定義items.py

import scrapy


class AvmoItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()
    pic = scrapy.Field()
    url = scrapy.Field()
    id_ = scrapy.Field()
 

這是spiders資料夾中的爬蟲檔案

spidername.py

# -*- coding: utf-8 -*-
import scrapy
import os
import sys
import re
import urllib.parse
sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
import items


class JavbusSpider(scrapy.Spider):
    name = 'javlibrary'
    allowed_domains = ['www.ja14b.com']
    start_urls = ['http://www.ja14b.com/cn/vl_genre.php?g=cu']

    # 爬取目錄頁
    def parse(self, response):
        # 通過xpath指令找到需要爬取的每部電影的url。
        for av in response.xpath("//div[@class=\"video\"]"):
            url = urllib.parse.urljoin('http://www.ja14b.com/cn/', av.xpath("a/@href").extract()[0])
            # 將每部電影的URL傳送到parse_movie中進行下一步處理。
            yield scrapy.Request(url, callback=self.parse_movie)
        # 檢查是否有下一頁，如果有下一頁，通過parse遞迴爬取下一頁的內容
        next_page = response.xpath("//*[@class=\"page next\"]")
        if next_page:
            next_page_url = "http://www.ja14b.com" + response.xpath("//*[@id=\"rightcolumn\"]/div[3]/a[9]/@href").extract()[0]
            yield scrapy.Request(next_page_url, callback=self.parse)

    # 對每一部電影進行分析
    def parse_movie(self, response):
        title = response.xpath("//*[@id=\"video_title\"]/h3/a/text()").extract()[0]
        # 定義新的ITEM
        item = items.AvmoItem()
        # 通過regex獲取番號。
        id_ = re.search("[a-zA-Z]{2,5}-?\d{2,5}", title).group(0)
        name = title[len(id_)+1:]
        pic = "http:" + response.xpath("//*[@id=\"video_jacket_img\"]/@src").extract()[0]
        # 將每部電影的屬性新增到item中。
        item["url"] = response.url
        item["id_"] = id_
        item["name"] = name
        item["pic"] = pic
        yield item
 

然後修改pipelines.py

# 通過pymysql連線電腦上的MySQL
import pymysql


class AvmoPipeline(object):
    def __init__(self):
        # 連線database
        self.db = pymysql.connect("localhost", "root", "6966xx511", "testDB", charset='utf8mb4', )
        # 通過cursor傳達sql指令
        self.cursor = self.db.cursor()
        # 如果table存在，則drop，防止引發衝突。
        self.cursor.execute("DROP TABLE IF EXISTS AV")
        # 建立table av，以番號（id）為primary key，防止重複。
        sql = """CREATE TABLE AV (
                         id  VARCHAR(255) NOT NULL,
                         name  VARCHAR(255),
                         pic  VARCHAR(255),
                         url VARCHAR(255),
                         magnet VARCHAR(255),
                         PRIMARY KEY (id)
                         );"""
        self.cursor.execute(sql)
        self.db.commit()

    def process_item(self, item, spider):
        # 每次收到一個item即將其加入table
        self.cursor.execute("ALTER TABLE AV CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;")
        sql = """INSERT INTO AV(id,name,pic,url)
                VALUES ('{}','{}','{}','{}');""".format(item['id_'], item['name'], item['pic'], item['url'])
        self.cursor.execute(sql)
        self.db.commit()
        return item

    def close_spider(self, spider):
        self.db.close()