229 lines
9.3 KiB
Python
229 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Трек 5: согласованность кэша истории. Фазы (переключаются bat-ом через CACHE_MODE):
|
|
off — baseline: кэша нет. 32 конкурентных холодных чтения = ~32 SQL
|
|
(демонстрация cache stampede «до»), 100 последовательных чтений = 100 SQL,
|
|
latency p50/p95/p99 «до».
|
|
naive — кэш БЕЗ инвалидации версией: ожидаем НАБЛЮДАТЬ stale read
|
|
(запись не видна из тёплого кэша). Это демонстрация проблемы согласованности.
|
|
versioned — защищённый кэш: read-after-write OK; 32 конкурентных холодных чтения
|
|
схлопываются single-flight'ом в 1 SQL (защита от stampede «после»);
|
|
100 тёплых чтений обслуживает кэш (sql<=2), latency «после».
|
|
recovery — restart redis: сессия и read-after-write корректны, кэш перестраивается.
|
|
Запуск: scripts/cache_demo.bat (сам переключает CACHE_MODE и перезапускает api).
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
FAILURES = []
|
|
BASE = "http://localhost:8080"
|
|
PASSWORD = "Example-only-not-a-real-secret-123"
|
|
READS = 100 # последовательных чтений для SQL-счётчика и latency
|
|
STAMPEDE = 32 # конкурентных холодных чтений
|
|
|
|
|
|
def check(name, cond, detail=""):
|
|
if not cond:
|
|
FAILURES.append(name)
|
|
print(f"[{'PASS' if cond else 'FAIL'}] {name}" + (f" -- {detail}" if not cond and detail else ""))
|
|
|
|
|
|
def sh(cmd):
|
|
subprocess.run(cmd, shell=True, check=True, capture_output=True)
|
|
|
|
|
|
def req(method, path, body=None, token=None, basic=None):
|
|
headers, data = {}, None
|
|
if body is not None:
|
|
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
if token:
|
|
headers["Authorization"] = "Bearer " + token
|
|
if basic:
|
|
headers["Authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
|
|
r = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(r, timeout=30) as resp:
|
|
payload = resp.read().decode()
|
|
return resp.status, (json.loads(payload) if payload else None)
|
|
except urllib.error.HTTPError as e:
|
|
payload = e.read().decode()
|
|
try:
|
|
return e.code, json.loads(payload)
|
|
except ValueError:
|
|
return e.code, None
|
|
|
|
|
|
def wait_ready(timeout=90, want=200):
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
st, _ = req("GET", "/health/ready")
|
|
if st == want:
|
|
return True
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def register_login():
|
|
uname = "t5" + uuid.uuid4().hex[:10]
|
|
st, _ = req("POST", "/api/v1/users", {"username": uname, "display_name": "T5", "password": PASSWORD})
|
|
assert st == 201, st
|
|
st, sess = req("POST", "/api/v1/auth/sessions", basic=f"{uname}:{PASSWORD}")
|
|
assert st == 201, st
|
|
return sess["token"], sess["user"]["id"]
|
|
|
|
|
|
def setup_chat(token, uid, n):
|
|
st, chat = req("POST", "/api/v1/chats",
|
|
{"title": "track5-" + uuid.uuid4().hex[:8], "member_ids": [uid]}, token=token)
|
|
assert st == 201, st
|
|
for i in range(n):
|
|
st, _ = req("POST", f"/api/v1/chats/{chat['id']}/messages",
|
|
{"text": f"t5-{i:03d}"}, token=token)
|
|
assert st == 201, st
|
|
return chat
|
|
|
|
|
|
def read_page(token, chat_id):
|
|
st, body = req("GET", f"/api/v1/chats/{chat_id}/messages?limit=50", token=token)
|
|
assert st == 200, (st, body)
|
|
return body
|
|
|
|
|
|
def sql_count(since_unix, chat_id):
|
|
out = subprocess.run(f"docker compose logs api --since {since_unix}",
|
|
shell=True, capture_output=True, text=True).stdout
|
|
return sum(1 for line in out.splitlines()
|
|
if "sql_list_messages" in line and chat_id in line)
|
|
|
|
|
|
def concurrent_burst(token, chat_id):
|
|
"""32 конкурентных холодных чтения через barrier; возвращает число SQL-запросов."""
|
|
since = str(int(time.time()))
|
|
barrier = threading.Barrier(STAMPEDE)
|
|
|
|
def reader():
|
|
barrier.wait()
|
|
read_page(token, chat_id)
|
|
|
|
ts = [threading.Thread(target=reader) for _ in range(STAMPEDE)]
|
|
for t in ts:
|
|
t.start()
|
|
for t in ts:
|
|
t.join()
|
|
return sql_count(since, chat_id)
|
|
|
|
|
|
def pctl(xs):
|
|
xs = sorted(xs)
|
|
n = len(xs)
|
|
|
|
def p(q):
|
|
return xs[min(n - 1, int(q * n // 100))]
|
|
|
|
return round(p(50), 2), round(p(95), 2), round(p(99), 2)
|
|
|
|
|
|
def phase_off():
|
|
token, uid = register_login()
|
|
chat = setup_chat(token, uid, 5)
|
|
# Stampede БЕЗ кэша: каждый из 32 конкурентных читателей идёт в SQL.
|
|
n_burst = concurrent_burst(token, chat["id"])
|
|
check("off: concurrent cold reads -> each hits SQL (stampede without cache)",
|
|
n_burst >= STAMPEDE // 2, f"sql={n_burst} clients={STAMPEDE}")
|
|
# Baseline: каждое последовательное чтение = SQL.
|
|
since = str(int(time.time()))
|
|
lats = []
|
|
for _ in range(READS):
|
|
t0 = time.perf_counter()
|
|
read_page(token, chat["id"])
|
|
lats.append((time.perf_counter() - t0) * 1000)
|
|
n = sql_count(since, chat["id"])
|
|
check("off: every read hits SQL", n >= READS * 0.9, f"sql={n} reads={READS}")
|
|
p50, p95, p99 = pctl(lats)
|
|
print(f"RESULT mode=off reads={READS} sql={n} stampede_sql={n_burst} p50={p50} p95={p95} p99={p99}")
|
|
|
|
|
|
def phase_naive():
|
|
token, uid = register_login()
|
|
# Stale read: тёплый кэш + запись + повторное чтение.
|
|
chat = setup_chat(token, uid, 2)
|
|
read_page(token, chat["id"]) # прогрев кэша
|
|
req("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "stale-probe"}, token=token)
|
|
page = read_page(token, chat["id"])
|
|
visible = any(m["text"] == "stale-probe" for m in page["items"])
|
|
check("naive: stale read OBSERVED (cache returned old page after write)", not visible,
|
|
f"visible={visible}")
|
|
print(f"RESULT mode=naive stale_observed={not visible}")
|
|
|
|
|
|
def phase_versioned():
|
|
token, uid = register_login()
|
|
# Read-after-write: запись видна сразу даже при тёплом кэше.
|
|
chat = setup_chat(token, uid, 2)
|
|
read_page(token, chat["id"])
|
|
req("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "stale-probe"}, token=token)
|
|
page = read_page(token, chat["id"])
|
|
visible = any(m["text"] == "stale-probe" for m in page["items"])
|
|
check("versioned: read-after-write OK (no stale)", visible)
|
|
# Stampede ПОД защитой: single-flight схлопывает 32 холодных чтения в 1 SQL.
|
|
chat2 = setup_chat(token, uid, 5)
|
|
n_burst = concurrent_burst(token, chat2["id"])
|
|
check("versioned: single-flight -> exactly 1 SQL on cold key", n_burst == 1,
|
|
f"sql={n_burst} clients={STAMPEDE}")
|
|
# Тёплые чтения обслуживает кэш.
|
|
since = str(int(time.time()))
|
|
lats = []
|
|
for _ in range(READS):
|
|
t0 = time.perf_counter()
|
|
read_page(token, chat["id"])
|
|
lats.append((time.perf_counter() - t0) * 1000)
|
|
n = sql_count(since, chat["id"])
|
|
check("versioned: warm reads served from cache", n <= 2, f"sql={n} reads={READS}")
|
|
p50, p95, p99 = pctl(lats)
|
|
print(f"RESULT mode=versioned reads={READS} sql={n} stampede_sql={n_burst} p50={p50} p95={p95} p99={p99}")
|
|
|
|
|
|
def phase_recovery():
|
|
token, uid = register_login()
|
|
chat = setup_chat(token, uid, 3)
|
|
read_page(token, chat["id"])
|
|
sh("docker compose restart redis")
|
|
check("recovery: api ready after redis restart", wait_ready(90, 200))
|
|
st, _ = req("GET", "/api/v1/users/me", token=token)
|
|
check("recovery: session still valid (source of truth = postgres)", st == 200, str(st))
|
|
since = str(int(time.time()))
|
|
page = read_page(token, chat["id"])
|
|
check("recovery: history readable after cache loss", len(page["items"]) == 3)
|
|
req("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "after-restart"}, token=token)
|
|
page = read_page(token, chat["id"])
|
|
check("recovery: read-after-write OK with rebuilt cache",
|
|
any(m["text"] == "after-restart" for m in page["items"]))
|
|
n = sql_count(since, chat["id"])
|
|
check("recovery: cache rebuilds (1 sql for 2 reads)", n == 1, f"sql={n}")
|
|
print(f"RESULT mode=recovery sql_after_restart={n}")
|
|
|
|
|
|
def main():
|
|
global BASE
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--base-url", default=BASE)
|
|
ap.add_argument("--phase", required=True, choices=["off", "naive", "versioned", "recovery"])
|
|
a = ap.parse_args()
|
|
BASE = a.base_url
|
|
{"off": phase_off, "naive": phase_naive,
|
|
"versioned": phase_versioned, "recovery": phase_recovery}[a.phase]()
|
|
if FAILURES:
|
|
print(f"PHASE {a.phase} FAILED: {FAILURES}")
|
|
raise SystemExit(1)
|
|
print(f"PHASE {a.phase} DONE")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |