crawl將自動對所有連結進行分析，將符合的連結資料爬取。官方文件

，其中價格，好評率需要用瀏覽器抓包分析真實地址，本文所用的基礎技術包括：sql語句，re表示式,xpath表示式，基本的網路知識和python基礎

jd.py

# -*- coding: utf-8 -*-
import scrapy
import urllib.request
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import re
from jingdong.items import
 
 JingdongItem


class JdSpider(CrawlSpider):
    name = 'jd'
    allowed_domains = ['jd.com']
    start_urls = ['http://jd.com/']

    rules = (
        Rule(LinkExtractor(allow=''), callback='parse_item', follow=True),#爬取所有連結，均不加限制
    )

    def parse_item(self, response):
        i = JingdongItem()
        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
 

        #i['name'] = response.xpath('//div[@id="name"]').extract()
        #i['description'] = response.xpath('//div[@id="description"]').extract()
        thisurl = response.url#獲取當前所在的地址，下一步判斷是否是商品地址
        pat = 'item.jd.com/(.*?).html'
        url = re.search(pat,thisurl)
        if(url):#如果url是商品頁
 

            product_id = re.findall(pat,thisurl)[0]
            price_link = 'http://p.3.cn/prices/mgets?callback=jQuery6325563&type=1&area=1_72_2799_0&pdtk=&pduid=1509845912914927705768&pdpin=&pin=null&pdbp=0&skuIds=J_'+str(product_id)+'&ext=11000000&source=item-pc'#當前商品價格連結地址
            pat_price = 'p":"(.*?)"'#獲取價格的正則
            price_str=urllib.request.urlopen(price_link).read().decode('utf-8','ignore')#價格所在url
            i['price'] = re.compile(pat_price).findall(price_str)[0]#獲取價格
            goodRate_link = 'http://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv244&productId='+str(product_id)+'&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'#當前商品好評率連結地址
            pat_goodrate = '{"goodRateShow":(.*?),'#正則表示式匹配價格
            goodRate_str = urllib.request.urlopen(goodRate_link).read().decode('utf-8','ignore')#好評地址
            i['goodRate'] = re.compile(pat_goodrate).findall(goodRate_str)[0]#好評率
            i['title'] = response.xpath('//title/text()').extract()#商品標題
            i['store'] = response.xpath('//div[@class="name"]/a/text()').extract()#店鋪名字
            i['link'] = response.xpath('//link[@rel="canonical"]/@href').extract()#商品連結
            print(i['price'],i['goodRate'],i['title'],i['store'],i['link'])
        else:#不是商品葉
            pass
            #print("不是商品")
        return i



#價格地址
#http://p.3.cn/prices/mgets?callback=jQuery6325563&type=1&area=1_72_2799_0&pdtk=&pduid=1509845912914927705768&pdpin=&pin=null&pdbp=0&skuIds=J_5560552&ext=11000000&source=item-pc
#好評率地址
#http://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv244&productId=5560552&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1
#簡介xpath
#/html/body/div[7]/div/div[2]/div[1]/text()
#店鋪xpath
#//div[@class="name"]/a/text()
#商品連結
#//link[@rel="canonical"]/@href

嘗試執行

pipelines.py

# -*- coding: utf-8 -*-
import mysql.connector
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html


class JingdongPipeline(object):
    def process_item(self, item, spider):
        db = mysql.connector.connect(host='localhost',
                                     user='root',
                                     passwd='123456',
                                     db='python')  # 連結資料庫
        cur = db.cursor()  # 獲取資料庫遊標
        title = item['title']
        link = item['link']
        price = item['price']
        goodRate = item['goodRate']
        store = item['store']
        print(title,link,price,goodRate,store)
        cur.execute("insert into jingdong VALUES ('"+title+"','"+link+"','"+price+"','"+goodRate+"','"+store+"')")  # 執行語句
        db.commit()  # 提交事務，沒有此句在資料庫中不能查詢到資料
        cur.close()  # 關閉遊標
        db.close()  # 關閉資料庫