requests with caching function

1import requests
2import json
3
4PERMANENT_CACHE_FNAME = "permanent_cache.txt"
5TEMP_CACHE_FNAME = "this_page_cache.txt"
6
7def _write_to_file(cache, fname):
8    with open(fname, 'w') as outfile:
9        outfile.write(json.dumps(cache, indent=2))
10
11def _read_from_file(fname):
12    try:
13        with open(fname, 'r') as infile:
14            res = infile.read()
15            return json.loads(res)
16    except:
17        return {}
18
19def add_to_cache(cache_file, cache_key, cache_value):
20    temp_cache = _read_from_file(cache_file)
21    temp_cache[cache_key] = cache_value
22    _write_to_file(temp_cache, cache_file)
23
24def clear_cache(cache_file=TEMP_CACHE_FNAME):
25    _write_to_file({}, cache_file)
26
27def make_cache_key(baseurl, params_d, private_keys=["api_key"]):
28    """Makes a long string representing the query.
29    Alphabetize the keys from the params dictionary so we get the same order each time.
30    Omit keys with private info."""
31    alphabetized_keys = sorted(params_d.keys())
32    res = []
33    for k in alphabetized_keys:
34        if k not in private_keys:
35            res.append("{}-{}".format(k, params_d[k]))
36    return baseurl + "_".join(res)
37
38def get(baseurl, params={}, private_keys_to_ignore=["api_key"], permanent_cache_file=PERMANENT_CACHE_FNAME, temp_cache_file=TEMP_CACHE_FNAME):
39    full_url = requests.requestURL(baseurl, params)
40    cache_key = make_cache_key(baseurl, params, private_keys_to_ignore)
41    # Load the permanent and page-specific caches from files
42    permanent_cache = _read_from_file(permanent_cache_file)
43    temp_cache = _read_from_file(temp_cache_file)
44    if cache_key in temp_cache:
45        print("found in temp_cache")
46        # make a Response object containing text from the change, and the full_url that would have been fetched
47        return requests.Response(temp_cache[cache_key], full_url)
48    elif cache_key in permanent_cache:
49        print("found in permanent_cache")
50        # make a Response object containing text from the change, and the full_url that would have been fetched
51        return requests.Response(permanent_cache[cache_key], full_url)
52    else:
53        print("new; adding to cache")
54        # actually request it
55        resp = requests.get(baseurl, params)
56        # save it
57        add_to_cache(temp_cache_file, cache_key, resp.text)
58        return resp