多線程

1. threading

import threading
thread = threading.Thread(target=target, args=[i])

def targets(second):
print(f’Threading {threading.current_thread().name} is running’)
print(f’Threading {threading.current_thread().name} sleep {second}s’)
time.sleep(second)
print(f’Threading {threading.current_thread().name} is ended’)

print(f’Threading {threading.current_thread().name} is running’)

t = []
for i in [1, 5]:
thread = threading.Thread(target=targets, args=[i])
t.append(thread)
thread.start()
for i in t:
i.join()
print(f’Threading {threading.current_thread().name} is ended’)

2. 線程池

1名員工 10雙鞋
10名員工 100雙

10名員工 半天 50雙
5名員工 一天 50雙

給線程一個數量 只有這么多人

from concurrent.futures import ThreadPoolExecutordef crawl(url):print(url)if __name__ == '__main__':base_url = 'https://jobs.51job.com/pachongkaifa/p{}/'with ThreadPoolExecutor(10) as f:for i in range(1,15):f.submit(crawl,url=base_url.format(i))

多線程采集案例

1、 先提取json文件 獲取英雄的Id
2、根據iD 找到英雄的詳情頁地址
3、從詳情頁里面提取頭像地址 構造 皮膚地址

import requests import os import json import threading from lxml import etree import time h=[] s=time.time()#1.先提取json文件 獲取英雄的Id def duo():'''處理多任務:return:'''response=requests.get('https://pvp.qq.com/web201605/js/herolist.json')data=json.loads(response.text)#print(data)for j in data:t=threading.Thread(target=pa,args=(j,))t.start()h.append(t)for k in h:k.join()#2.根據iD 找到英雄的詳情頁地址 def pa(j):num = j['ename'] #從data中獲取ename的值name = j['cname']res2 = requests.get("https://pvp.qq.com/web201605/herodetail/{}.shtml".format(num))res2_decode = res2.content.decode('gbk') # 返回相應的html頁面，字符串格式，解碼為utf-8_element = etree.HTML(res2_decode) # 將html轉換為_Element對象，可以方便的使用getparent()、remove()、xpath()等方法element_img = _element.xpath('//div[@class="pic-pf"]/ul/@data-imgname')#print(element_img)name_img = element_img[0].split('|') # 去掉字符串中的|字符,并分割#print(name_img)for i in range(0,10):res1=requests.get("https://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/{0}/{0}-bigskin-{1}.jpg".format(num,i+1)) #返回響應包if res1.status_code == 200:aa=name_img[i].find('&')#print(aa)bb=name_img[i][:aa]res_img=res1.content #把相應包內容轉換為2進制a = './王者榮耀/' + str(name)b='./王者榮耀/'+str(name)+'/'+bb+'.jpg'if not os.path.exists('./王者榮耀/'):os.mkdir('./王者榮耀/')if not os.path.exists(a):os.mkdir(a)#3.從詳情頁里面提取頭像地址 構造 皮膚地址with open(b,"wb") as f: #創建一個名為1.jpg的圖片f.write(res_img) #把響應包2進制內容寫入到1.jpg中print(name, bb)else:breakif __name__=='__main__':duo()g=time.time()print("用時：",g-s,"秒")

多進程

multiprocessing
from multiprocessing import Pool #進程池

import multiprocessingfrom multiprocessing import Pool import requestsdef process(index):print(f'Proess:{index}')def scrape(url):try:requests.get(url)print(f'URL {url} Scraped')except requests.ConnectionError:print(f'URL {url} not Scraped')if __name__ == '__main__':# for i in range(5):# p = multiprocessing.Process(target=process,args=(i,))# p.start()pool = Pool(processes=3)urls = ['https://www.baidu.com','http://www.meituan.com/','http://blog.csdn.net/','http://xxxyxxx.net']pool.map(scrape, urls)pool.close()

異步攜程

#異步函數聲明 async

實例

import asyncio import time import httpx async def req(client, i):res = await client.get('https://www.example.com')print(f'第{i + 1}次請求，status_code = {res.status_code}')return resasync def main():async with httpx.AsyncClient() as client:task_list = [] # 任務列表for i in range(50):res = req(client, i)task = asyncio.create_task(res) # 創建任務task_list.append(task)#await 耗時任務給他掛起await asyncio.gather(*task_list) # 收集任務if __name__ == '__main__':start = time.time()asyncio.run(main())end = time.time()print(f'異步發送50次請求，耗時：{end - start}')

總結

以上是生活随笔為你收集整理的2-5.多进程、多线程、异步携程的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。