菜鸟追梦旅行

这个屌丝很懒，什么也没留下！

热门标签

Python高性能编程

作者：菜鸟追梦旅行 | 2024-02-10 18:42:18

踩

python高性能编程

一、进程池和线程池

1.串行

import time

import requests

url_lists = [

'http://www.baidu.com',

'http://fanyi.baidu.com',

'http://map.baidu.com',

'http://music.baidu.com/',

'http://tieba.baidu.com',

'http://v.baidu.com',

'http://image.baidu.com',

'http://zhidao.baidu.com',

'http://news.baidu.com',

'http://xueshu.baidu.com']

start_time = time.time()

for url in url_lists:

response = requests.get(url)

print(response.text)

print("Runtime: {}".format(time.time()-start_time))

# Runtime: 1.95

2.多进程

import time

import requests

from multiprocessing import Process

url_lists = [

'http://www.baidu.com',

'http://fanyi.baidu.com',

'http://map.baidu.com',

'http://music.baidu.com/',

'http://tieba.baidu.com',

'http://v.baidu.com',

'http://image.baidu.com',

'http://zhidao.baidu.com',

'http://news.baidu.com',

'http://xueshu.baidu.com']

def task(url):

response = requests.get(url)

print(response.text)

if __name__ == '__main__':

p_list = []

start_time = time.time()

for url in url_lists:

p = Process(target=task, args=(url,))

p_list.append(p)

p.start()

for p in p_list:

p.join()

print("Runtime: {}".format(time.time() - start_time))

# Runtime: 1.91

3.进程池（1）

import time

import requests

from concurrent.futures import ProcessPoolExecutor

"""

Py2里没有线程池但是有进程池

Py3里有线程池有进程池

"""

url_lists = [

'http://www.baidu.com',

'http://fanyi.baidu.com',

'http://map.baidu.com',

'http://music.baidu.com/',

'http://tieba.baidu.com',

'http://v.baidu.com',

'http://image.baidu.com',

'http://zhidao.baidu.com',

'http://news.baidu.com',

'http://xueshu.baidu.com']

def task(url):

response = requests.get(url)

print(response.content)

if __name__ == '__main__':

start_time = time.time()

pool = ProcessPoolExecutor(10)

for url in url_lists:

pool.submit(task,url)

pool.shutdown(wait=True)

print("Runtime: {}".format(time.time() - start_time))

# Runtime: 2.00

# 进程池 + 回调函数

import time

import requests

from concurrent.futures import ProcessPoolExecutor

"""

Py2里没有线程池但是有进程池

Py3里有线程池有进程池

"""

url_lists = [

'http://www.baidu.com',

'http://fanyi.baidu.com',

'http://map.baidu.com',

'http://music.baidu.com/',

'http://tieba.baidu.com',

'http://v.baidu.com',

'http://image.baidu.com',

'http://zhidao.baidu.com',

'http://news.baidu.com',

'http://xueshu.baidu.com']

def task(url):

response = requests.get(url)

return response.content

def callback(future):

print(future.result())

if __name__ == '__main__':

start_time = time.time()

pool = ProcessPoolExecutor(10)

for url in url_lists:

v = pool.submit(task,url)

v.add_done_callback(callback)

pool.shutdown(wait=True)

print("Runtime: {}".format(time.time() - start_time))

3.进程池（2）

import time

import requests

from multiprocessing import Pool

url_lists = [

'http://www.baidu.com',

'http://fanyi.baidu.com',

'http://map.baidu.com',

'http://music.baidu.com/',

'http://tieba.baidu.com',

'http://v.baidu.com',

'http://image.baidu.com',

'http://zhidao.baidu.com',

'http://news.baidu.com',

'http://xueshu.baidu.com']

def task(url):

response = requests.get(url)

return response.content

def callBackFunc(content):

print(content)

if __name__ == '__main__':

start_time = time.time()

pool = Pool(10)

for url in url_lists:

pool.apply_async(func=task,args=(url,),callback=callBackFunc)

pool.close()

pool.join()

print("Runtime: {}".format(time.time() - start_time))

# Runtime: 1.96

2019-03-06 補充

# Parallel 底层調用的是 multiprocessing

import time

from joblib import Parallel, delayed

def func(idx):

print(idx)

time.sleep(1)

return {'idx':idx}

start_ts = time.time()

results = Parallel(-1)(

delayed(func)(x) for x in range(4)

)

print(results)

print('Runtime : {}'.format(time.time()-start_ts

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/菜鸟追梦旅行/article/detail/75012?site