Scrapy框架爬取妹子图保存到不同目录下

本文来源吾爱破解论坛

进行设置settings
[Asm] 纯文本查看 复制代码

#启动图片管道
ITEM_PIPELINES = {
   'mztu.pipelines.ImagesPipelinse': 300,
}
#设置默认目录地址  注意下载图片的话默认地址必须设置！！！
IMAGES_STORE = "E:\study\Python\scrapy\mztu\imges"
#设置图片通道失效时间
IMAGES_EXPIRES =90
#缩略图生成
#IMAGES_THUMBS = {
 #   'small': (50, 50),
#    'big': (270, 270),
#}

# -*- coding: utf-8 -*-
import scrapy
from mztu.items import MztuItem

class ZimdgSpider(scrapy.Spider):
    name = 'zimdg'
    allowed_domains = ['mzitu.com']
    #生成链接列表
    start_urls = ['http://www.mzitu.com/xinggan/page/{}/'.format(str(x)) for x in range(118)]
    def parse(self, response):
        #解析出链接
        set_li = response.xpath("//div[@class='postlist']/ul/li")
        for ecth in set_li:
            ed = ecth.xpath('./a/@href').extract()
            #进行二次分类解析
            yield scrapy.Request(ed[0],callback=self.parse_item)


    def parse_item(self,response):
        itme = MztuItem()
        # 获取页数链接进行访问
        offset = int(response.xpath('//div[@class="pagenavi"]/a/span/text()')[4].extract())
        #生成链接访问
        #遍历链接访问
        for i in [response.url+"/{}".format(str(x))  for x in range(1,offset+1)]:
            itme['Referer']=i
            #将meta传入链接
            yield scrapy.Request(itme['Referer'],meta={'meta_1':itme}, callback=self.parse_ponse)
        # for i in url:

    def parse_ponse(self,response):
        #获取itme资源
        itme = response.meta['meta_1']
        #获取图片地址
        imgs = response.xpath('//div[@class="main-image"]/p/a/img/@src')[0].extract()
        #获取图片目录
        title = response.xpath('//div[@class="main-image"]/p/a/img/@alt')[0].extract()
        itme["title"]= title
        itme["imge_url"]= imgs
        #itme["nickname"] = itme["Referer"][itme["Referer"].rfind("/"):]+itme["imge_url"][itme["imge_url"].rfind('/')+1:itme["imge_url"].rfind('.')]
        #itme["nickname"] = itme["imge_url"][itme["imge_url"].rfind('/')+1:itme["imge_url"].rfind('.')]
        yield itme

import scrapy


class MztuItem(scrapy.Item):
    #目录
    title = scrapy.Field()
    #图片地址
    imge_url = scrapy.Field()
    #请求头
    Referer = scrapy.Field()

    image_Path = scrapy.Field()
    #图片名称
   # nickname = scrapy.Field()

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# 导入这个包为了移动文件
import shutil
#此包不解释
import scrapy
# 导入项目设置
from scrapy.utils.project import get_project_settings
# 导入scrapy框架的图片下载类
from scrapy.pipelines.images import ImagesPipeline
#此包不解释
import os

class ImagesPipelinse(ImagesPipeline):
    #def process_item(self, item, spider):
    #    return item
    # 获取settings文件里设置的变量值
    IMAGES_STORE = get_project_settings().get("IMAGES_STORE")
    # 重写ImagesPipeline类的此方法
    # 发送图片下载请求
    def get_media_requests(self, item, info):
        image_url = item["imge_url"]
        #headers是请求头主要是防反爬虫
        yield scrapy.Request(image_url,headers={'Referer':item['Referer']})

    def item_completed(self, result, item, info):
        image_path = [x["path"] for ok, x in result if ok]
        # 定义分类保存的路径
        img_path = "%s\%s" % (self.IMAGES_STORE, item['title'])
        # 目录不存在则创建目录
        if os.path.exists(img_path) == False:
            os.mkdir(img_path)
        # 将文件从默认下路路径移动到指定路径下
        shutil.move(self.IMAGES_STORE + "\\" +image_path[0], img_path + "\\" +image_path[0][image_path[0].find("full\\")+6:])
        item['image_Path'] = img_path + "\\" + image_path[0][image_path[0].find("full\\")+6:]
        return item