#!/usr/bin/env python3 import hashlib import math import subprocess import time import uuid from concurrent.futures import ThreadPoolExecutor from client import Client, Failure, arguments, check, run def docker(*command): return subprocess.run( ["docker", "compose", *command], check=True, capture_output=True, text=True, timeout=15, ) def fill_events(request_ids): logs = docker("logs", "--no-color", "--since", "5m", "api").stdout return { request_id for request_id in request_ids if "profile_cache_fill request_id=" + request_id in logs } def ready(client): for _ in range(60): try: client.call("GET", "/health/ready", schema="Health") return except Failure: time.sleep(1) raise Failure("API did not become ready after Redis restart") def p95_milliseconds(results): elapsed = sorted(result["elapsed"] for result in results) return elapsed[math.ceil(0.95 * len(elapsed)) - 1] * 1000 def main(): args = arguments("Redis profile-cache single-flight consistency check") client = Client(args.base_url, args.ca_file, timeout=10) client.call("GET", "/health/ready", schema="Health") actor = client.user("cache") cache_key = "session:" + hashlib.sha256(actor["token"].encode()).hexdigest() docker("exec", "-T", "redis", "redis-cli", "DEL", cache_key) def get_profile(request_id): return Client(args.base_url, args.ca_file, timeout=10).call( "GET", "/api/v1/users/me", actor=actor, schema="User", headers={"X-Request-Id": request_id}, ) cold_ids = ["cache-cold-" + uuid.uuid4().hex for _ in range(16)] with ThreadPoolExecutor(max_workers=len(cold_ids)) as executor: cold_results = list(executor.map(get_profile, cold_ids)) check( all(result["json"] == actor["user"] for result in cold_results), "concurrent cold-cache requests returned inconsistent profiles", ) cold_fills = fill_events(cold_ids) check( len(cold_fills) == 1, "cold-cache request burst filled the profile cache " + str(len(cold_fills)) + " times instead of once", ) warm_ids = ["cache-warm-" + uuid.uuid4().hex for _ in range(16)] with ThreadPoolExecutor(max_workers=len(warm_ids)) as executor: warm_results = list(executor.map(get_profile, warm_ids)) check( all(result["json"] == actor["user"] for result in warm_results), "concurrent warm-cache requests returned inconsistent profiles", ) check(not fill_events(warm_ids), "warm-cache requests unexpectedly queried profiles") docker("restart", "redis") ready(client) docker("exec", "-T", "redis", "redis-cli", "DEL", cache_key) restart_id = "cache-restart-" + uuid.uuid4().hex restarted_result = get_profile(restart_id) check( restarted_result["json"] == actor["user"], "profile read-after-Redis-restart returned inconsistent data", ) check( fill_events([restart_id]) == {restart_id}, "profile was not reloaded from PostgreSQL after Redis restart", ) print( "PASS: Redis single-flight; " + str(len(cold_ids)) + " concurrent cold reads caused 1 profile SQL read " + "(vs up to " + str(len(cold_ids)) + " without single-flight), " + str(len(warm_ids)) + " warm reads caused 0; " + "cold p95=" + format(p95_milliseconds(cold_results), ".1f") + "ms, warm p95=" + format(p95_milliseconds(warm_results), ".1f") + "ms; cache restart recovered from PostgreSQL" ) if __name__ == "__main__": run(main)