一．多進程

當計算機運行程序時，就會創建包含代碼和狀態的進程。這些進程會通過計算機的一個或多個CPU執行。不過，同一時刻每個CPU只會執行一個進程，然后不同進程間快速切換，給我們一種錯覺，感覺好像多個程序在同時進行。例如：有一個大型工廠，該工廠負責生產電腦，工廠有很多的車間用來生產不同的電腦部件。每個車間又有很多工人互相合作共享資源來生產某個電腦部件。這里的工廠相當于一個爬蟲工程，每個車間相當于一個進程，每個工人就相當于線程。線程是CPU調度的基本單元。

也就是進程間是獨立的,這表現在內存空間,上下文環境;而線程運行在進程空間內.也就是同一進程產生的線程共享同一內存空間.

需要注意的是單核CPU系統中，真正的并發是不可能的．

１．順序執行?

２．多進程并發 注意除了時間的加速意外也要看看函數返回值的寫法，帶有多進程的map,是返回一個列表

import requests import re import time from multiprocessing import Pool from multiprocessing.dummy import Pool as ThreadPool def spyder(url):# res = []res = {'init:':'hello'}print('hahah:{}'.format(url))time.sleep(1)# res.append(url)res.update({'entr:'+url:url})return resdef use_process():urls = ["https://www.qiushibaike.com/text/page/{}/".format(str(i)) for i in range(0, 4)]start_1 = time.time()#獲取函數返回結果res1 = []for url in urls:res_ = spyder(url)res1.append(res_)end_1 = time.time()print("單進程：", end_1 - start_1)print('res1:', res1)# 獲取函數返回結果# 　進程池start_2 = time.time()pool = Pool(processes=2)res2 = pool.map(spyder, urls)pool.close()pool.join()print('res2:', res2)end_2 = time.time()print("2進程：", end_2 - start_2)# 獲取函數返回結果# 進程池start_3 = time.time()pool = Pool(processes=4)res3 = pool.map(spyder, urls)pool.close()pool.join()print('res2:', res3)end_3 = time.time()print("4進程：", end_3 - start_3) if __name__ == "__main__":use_process()

二.多線程

實際上由于GIL（全局解釋器鎖）的限制，哪個線程想要執行代碼就需要去申請鎖，否則只能等著，所以這個鎖阻礙了真正的多線程并發，這是解釋器cpython的鍋，一般不推薦用多線程，而是用多進程multiprocess來繞過GIL.

2.1 thread多線程

import time import _thread from threading import Thread # 使用線程鎖，防止線程死鎖 mutex = _thread.allocate_lock() def test(d_num):d_num.append(89)print("test: %s"% str(d_num)) def test1(d_num):print("test1: %s"% str(d_num)) def main():d_num = [100, 58]t1 = Thread(target=test, args=(d_num,))t2 = Thread(target=test1, args=(d_num,))t1.start()time.sleep(1)t2.start()time.sleep(1)if __name__ == '__main__':main()

2.2 多線程隊列版

import time import _thread from threading import Thread import queue # 使用線程鎖，防止線程死鎖 mutex = _thread.allocate_lock() frame_queue = queue.Queue() def test(d_num):print("test: %s" % str(d_num))for i in range(d_num):frame_queue.put(i)def test1():while 1:if frame_queue.empty() != True:# 從隊列中取出圖片value = frame_queue.get()print('==value:', value)time.sleep(1)else:break def main():d_num = 10t1 = Thread(target=test, args=(d_num,))t1.start()t2 = Thread(target=test1)t2.start()if __name__ == '__main__':main()

2.3 注意傳參與多進程的區別,線程池

from functools import partial from itertools import repeat from multiprocessing import Pool, freeze_supportdef func(a, b):return a + bdef main():a_args = [1, 2, 3]second_arg = 1with Pool() as pool:L = pool.starmap(func, [(1, 1), (2, 1), (3, 1)])print('L:', L)M = pool.starmap(func, zip(a_args, repeat(second_arg)))print('M:', M)N = pool.map(partial(func, b=second_arg), a_args)print('N:', N) main()

import requests import re import time from multiprocessing import Pool from multiprocessing.dummy import Pool as ThreadPool def spyder(url):# res = []res = {'init:':'hello'}print('hahah:{}'.format(url))time.sleep(1)# res.append(url)res.update({'entr:'+url:url})return resdef use_process():urls = ["https://www.qiushibaike.com/text/page/{}/".format(str(i)) for i in range(0, 4)]start_1 = time.time()#獲取函數返回結果res1 = []for url in urls:res_ = spyder(url)res1.append(res_)end_1 = time.time()print("單進程：", end_1 - start_1)print('res1:', res1)# 獲取函數返回結果# 　進程池start_2 = time.time()pool = Pool(processes=2)res2 = pool.map(spyder, urls)pool.close()pool.join()print('res2:', res2)end_2 = time.time()print("2進程：", end_2 - start_2)# 獲取函數返回結果# 進程池start_3 = time.time()pool = Pool(processes=4)res3 = pool.map(spyder, urls)pool.close()pool.join()print('res2:', res3)end_3 = time.time()print("4進程：", end_3 - start_3)def use_threadpool():urls = [["https://www.qiushibaike.com/text/page/{}/".format(str(i))] for i in range(0, 4)]print('urls:', urls)# 線程池start = time.time()pool = ThreadPool(processes=4)res = pool.starmap(spyder, urls)pool.close()pool.join()end = time.time()print('res:', res)print("4線程：", end - start) if __name__ == "__main__":# use_process()use_threadpool()

實際應用將圖片路徑和名字傳入，用zip方式打包傳參

import osimport cv2 import time import itertools from multiprocessing.dummy import Pool as ThreadPoolSIZE = (75,75) SAVE_DIRECTORY='thumbs' def save_img(filename,save_path):save_path+= filename.split('/')[-1]im = cv2.imread(filename)im=cv2.resize(im,SIZE)cv2.imwrite(save_path,im)if __name__ == '__main__':path='./data/testlabel'print(path)output_path='./data/thumbs/'if not os.path.exists(output_path):os.mkdir(output_path)print(output_path)imgs_list_path=[os.path.join(path,i) for i in os.listdir(path)]print(len(imgs_list_path))start_time=time.time()pool = ThreadPool(processes=8)print(list(zip(imgs_list_path,[output_path]*len(imgs_list_path))))pool.starmap(save_img,zip(imgs_list_path,[output_path]*len(imgs_list_path)))pool.close()pool.join()end_time=time.time()print('use time=',end_time-start_time)

?

總結

以上是生活随笔為你收集整理的python多进程并发与pool多线程的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。