1.隨機請求頭

# -*- coding: utf-8 -*- """ 所有請求頭的USER_AGENTS網址 http://www.useragentstring.com/pages/useragentstring.php?name=All """ import json import random import requestsUSER_AGENTS = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36','Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36','Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36 Chrome 41.0.2227.0','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36','Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36','Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36', ]#從列表中隨機選擇一個 **User_Agent = random.choice(USER_AGENTS)**headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en','User-Agent': User_Agent }def testUserAgent():resp = requests.get("http://httpbin.org/user-agent", headers=headers)user_agent = json.loads(resp.text)['user-agent']#將返回的字符串loads為字典print(user_agent)def testProxy():proxies = {"https": "223.199.25.183:9999"}resp = requests.get("http://httpbin.org/ip", proxies=proxies)origin = json.loads(resp.text)['origin']print(origin)# https://www.xicidaili.com/nn/ 西刺代理 if __name__ == "__main__":testUserAgent()testProxy()

在middleware.py文件進行修改，

#將多個請求頭放在一個列表中 User_Agent = ['請求頭1'，'請求頭2'，'請求頭3'，...] #需要使用的時候從列表中隨機抽取一個請求頭，代碼如下 ```python class user_agentdownloadmiddleware(Object):def process_request(self,request,spider):User_Agent = random.chioce(self.User_Agent)request.headers['User_Agent'] = User_Agent

在setting.py中進行修改

將配置文件中的downloadmiddleware取消注銷即可

取消鏈接去重

def parse():...#此處的dont_filter即為去重的函數，將其設置為True則表示不要去重復的鏈接進行去重，即可以對同一個鏈接進行反復地請求并且獲得數據yield scrapy.Request(self.start_urls[0],dont_filter = True)...

總結

以上是生活随笔為你收集整理的NO.2_python_scrapy_反爬虫（随机请求头IP代理）取消链接去重的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。