1. items.py

class DouyuspiderItem(scrapy.Item):name = scrapy.Field()# 存儲照片的名字imagesUrls = scrapy.Field()# 照片的url路徑imagesPath = scrapy.Field()# 照片保存在本地的路徑

2. spiders/douyu.py

import scrapy
import json
from douyuSpider.items import DouyuspiderItemclass DouyuSpider(scrapy.Spider):name = "douyu"allowd_domains = ["http://capi.douyucdn.cn"]offset = 0url = "http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset="start_urls = [url + str(offset)]def parse(self, response):# 返回從json里獲取 data段數據集合data = json.loads(response.text)["data"]for each in data:item = DouyuspiderItem()item["name"] = each["nickname"]item["imagesUrls"] = each["vertical_src"]yield itemself.offset += 20yield scrapy.Request(self.url + str(self.offset), callback = self.parse)

3. 設置setting.py

ITEM_PIPELINES = {'douyuSpider.pipelines.ImagesPipeline': 1}# Images 的存放位置，之后會在pipelines.py里調用
IMAGES_STORE = "/Users/Power/lesson_python/douyuSpider/Images"# user-agent
USER_AGENT = 'DYZB/2.290 (iPhone; iOS 9.3.4; Scale/2.00)'

4. pipelines.py

import scrapy
import os
from scrapy.pipelines.images import ImagesPipeline
from scrapy.utils.project import get_project_settingsclass ImagesPipeline(ImagesPipeline):IMAGES_STORE = get_project_settings().get("IMAGES_STORE")def get_media_requests(self, item, info):image_url = item["imagesUrls"]yield scrapy.Request(image_url)def item_completed(self, results, item, info):# 固定寫法，獲取圖片路徑，同時判斷這個路徑是否正確，如果正確，就放到 image_path里，ImagesPipeline源碼剖析可見image_path = [x["path"] for ok, x in results if ok]os.rename(self.IMAGES_STORE + "/" + image_path[0], self.IMAGES_STORE + "/" + item["name"] + ".jpg")item["imagesPath"] = self.IMAGES_STORE + "/" + item["name"]return item#get_media_requests的作用就是為每一個圖片鏈接生成一個Request對象，這個方法的輸出將作為item_completed的輸入中的results，results是一個元組，每個元組包括(success, imageinfoorfailure)。如果success=true，imageinfoor_failure是一個字典，包括url/path/checksum三個key。

在項目根目錄下新建main.py文件,用于調試

from scrapy import cmdline
cmdline.execute('scrapy crawl douyu'.split())

執行程序

py2 main.py

總結

以上是生活随笔為你收集整理的Python:Scrapy实战项目手机App抓包爬虫的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。