120 lines
3.7 KiB
Python
120 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
import hashlib
|
|
import math
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def docker(*command):
|
|
return subprocess.run(
|
|
["docker", "compose", *command],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
)
|
|
|
|
|
|
def fill_events(request_ids):
|
|
logs = docker("logs", "--no-color", "--since", "5m", "api").stdout
|
|
return {
|
|
request_id
|
|
for request_id in request_ids
|
|
if "profile_cache_fill request_id=" + request_id in logs
|
|
}
|
|
|
|
|
|
def ready(client):
|
|
for _ in range(60):
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure:
|
|
time.sleep(1)
|
|
raise Failure("API did not become ready after Redis restart")
|
|
|
|
|
|
def p95_milliseconds(results):
|
|
elapsed = sorted(result["elapsed"] for result in results)
|
|
return elapsed[math.ceil(0.95 * len(elapsed)) - 1] * 1000
|
|
|
|
|
|
def main():
|
|
args = arguments("Redis profile-cache single-flight consistency check")
|
|
client = Client(args.base_url, args.ca_file, timeout=10)
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
actor = client.user("cache")
|
|
cache_key = "session:" + hashlib.sha256(actor["token"].encode()).hexdigest()
|
|
docker("exec", "-T", "redis", "redis-cli", "DEL", cache_key)
|
|
|
|
def get_profile(request_id):
|
|
return Client(args.base_url, args.ca_file, timeout=10).call(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
actor=actor,
|
|
schema="User",
|
|
headers={"X-Request-Id": request_id},
|
|
)
|
|
|
|
cold_ids = ["cache-cold-" + uuid.uuid4().hex for _ in range(16)]
|
|
with ThreadPoolExecutor(max_workers=len(cold_ids)) as executor:
|
|
cold_results = list(executor.map(get_profile, cold_ids))
|
|
check(
|
|
all(result["json"] == actor["user"] for result in cold_results),
|
|
"concurrent cold-cache requests returned inconsistent profiles",
|
|
)
|
|
cold_fills = fill_events(cold_ids)
|
|
check(
|
|
len(cold_fills) == 1,
|
|
"cold-cache request burst filled the profile cache "
|
|
+ str(len(cold_fills))
|
|
+ " times instead of once",
|
|
)
|
|
|
|
warm_ids = ["cache-warm-" + uuid.uuid4().hex for _ in range(16)]
|
|
with ThreadPoolExecutor(max_workers=len(warm_ids)) as executor:
|
|
warm_results = list(executor.map(get_profile, warm_ids))
|
|
check(
|
|
all(result["json"] == actor["user"] for result in warm_results),
|
|
"concurrent warm-cache requests returned inconsistent profiles",
|
|
)
|
|
check(not fill_events(warm_ids), "warm-cache requests unexpectedly queried profiles")
|
|
|
|
docker("restart", "redis")
|
|
ready(client)
|
|
docker("exec", "-T", "redis", "redis-cli", "DEL", cache_key)
|
|
restart_id = "cache-restart-" + uuid.uuid4().hex
|
|
restarted_result = get_profile(restart_id)
|
|
check(
|
|
restarted_result["json"] == actor["user"],
|
|
"profile read-after-Redis-restart returned inconsistent data",
|
|
)
|
|
check(
|
|
fill_events([restart_id]) == {restart_id},
|
|
"profile was not reloaded from PostgreSQL after Redis restart",
|
|
)
|
|
|
|
print(
|
|
"PASS: Redis single-flight; "
|
|
+ str(len(cold_ids))
|
|
+ " concurrent cold reads caused 1 profile SQL read "
|
|
+ "(vs up to "
|
|
+ str(len(cold_ids))
|
|
+ " without single-flight), "
|
|
+ str(len(warm_ids))
|
|
+ " warm reads caused 0; "
|
|
+ "cold p95="
|
|
+ format(p95_milliseconds(cold_results), ".1f")
|
|
+ "ms, warm p95="
|
|
+ format(p95_milliseconds(warm_results), ".1f")
|
|
+ "ms; cache restart recovered from PostgreSQL"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|