使用scrapy爬取suning

# -*- coding: utf-8 -*-
import scrapy
from copy import deepcopy


class SuSpider(scrapy.Spider):
    name = 'su'
    allowed_domains = ['suning.com']
    start_urls = ['http://list.suning.com/?safp=d488778a.error1.0.4786e76351']

    def parse(self, response):
        # 获取大分类列表
        bcate_list = response.xpath("//div[@class='allsortLeft']/ul/li")
        for bcate in bcate_list:
            item = {}
            # 获取大分类class的值
            class_name = bcate.xpath("./@class").extract_first()
            # 获取所有大分类的名称
            item["BCate"] = bcate.xpath("./a/span/text()").extract_first()
            # print(item["BCate"])
            # 根据大分类的class定位每个大分类下的所有小分类
            scate_list = response.xpath("//div[@class='{}']/div".format(class_name))
            for scate in scate_list:
                # 小分类的名称
                item["SCate"] = scate.xpath("./div[1]/a/@title").extract_first()
                # 获取每个小分类下的所有标签
                tag_list = scate.xpath("./div[2]/a")
                for tag in tag_list:
                    # 每个标签的链接和名称
                    item["tag"] = tag.xpath("./text()").extract_first()
                    item["tag_link"] = "http:" + tag.xpath("./@href").extract_first()
                    # 进入列表页
                    yield scrapy.Request(
                        item["tag_link"],
                        callback=self.good_list,
                        meta={"item": deepcopy(item)}
                    )

    def good_list(self, response):
        item = deepcopy(response.meta["item"])
        # 获取当前页的所有商品列表
        li_list = response.xpath("//div[@id='product-wrap']/div/ul/li")
        for li in li_list:
            # 获取商品的图片地址，名称，价格，商品详情页的链接
            item["good_img"] = "http:"+li.xpath(".//div[@class='res-img']/div/a/img/@src").extract_first()
            item["good_name"] = li.xpath(".//div[@class='res-info']/div/a/text()").extract_first()
            item["good_price"] = li.xpath(".//div[@class='res-info']/div/span/text()").extract_first()
            item["good_href"] = li.xpath(".//div[@class='res-info']/div/a/@href").extract_first()
            # 进入商品详情页
            if item["good_href"] != "javascript:void(0);":
                yield scrapy.Request(
                    "http:"+item["good_href"],
                    callback=self.good_detail,
                    meta={"item": deepcopy(item)}
                )
        # 翻页
        next_url = response.xpath("//a[@id='nextPage']/@href").extract_first()
        if next_url:
            yield scrapy.Request(
                next_url,
                callback=self.good_list,
                meta={"item": response.meta["item"]}
            )

    def good_detail(self, response):
        item = response.meta["item"]
        # 获取当前商品的属性规格：颜色、版本、
        size_list = response.xpath("//div[@id='J-TZM']/dl")
        for size in size_list:
            size_name = size.xpath("./dt/span/text()").extract_first()
            size_value = size.xpath("./dd/ul/li/@title").extract()
            item[size_name] = size_value
        print(item)