1import concurrent.futures
2import requests
3import threading
4import time
5
6
7thread_local = threading.local()
8
9
10def get_session():
11 if not hasattr(thread_local, "session"):
12 thread_local.session = requests.Session()
13 return thread_local.session
14
15
16def download_site(url):
17 session = get_session()
18 with session.get(url) as response:
19 print(f"Read {len(response.content)} from {url}")
20
21
22def download_all_sites(sites):
23 with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
24 executor.map(download_site, sites)
25
26
27if __name__ == "__main__":
28 sites = [
29 "https://www.jython.org",
30 "http://olympus.realpython.org/dice",
31 ] * 80
32 start_time = time.time()
33 download_all_sites(sites)
34 duration = time.time() - start_time
35 print(f"Downloaded {len(sites)} in {duration} seconds")
36