Files
mtusi2026lab2/scripts/cache_stampede.py
T
2026-11-06 13:11:04 +03:00

419 lines
22 KiB
Python

#!/usr/bin/env python3
"""Repeatable local cache experiment; uses Docker, Python stdlib and synthetic data.
Requires prebuilt before/after API images. Temporarily restarts local PostgreSQL
with pg_stat_statements, then restores Compose without deleting data or volumes.
"""
import argparse
import base64
import concurrent.futures as futures
import contextlib
import csv
import datetime as dt
import http.client
import hashlib
import json
import math
import pathlib
import secrets
import select
import socket
import subprocess
import tempfile
import threading
import time
import uuid
ROOT = pathlib.Path(__file__).resolve().parents[1]
NORMAL = ['docker', 'compose', '--env-file', str(ROOT / '.env'), '-f', str(ROOT / 'docker-compose.yml')]
PSQL = ['exec', '-T', 'postgres', 'sh', '-c',
'exec psql -X -q -At -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"']
def now():
return dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z')
def run(args, sql=None, timeout=90):
result = subprocess.run(args, input=sql, text=True, capture_output=True, cwd=ROOT, timeout=timeout)
if result.returncode:
# Never echo config, SQL parameters, HTTP bodies or credentials on failure.
raise RuntimeError('External command failed: ' + ' '.join(args[:3]))
return result.stdout.strip()
def percentile(values, fraction):
values = sorted(values)
return round(values[max(0, math.ceil(len(values) * fraction) - 1)], 3)
class Experiment:
def __init__(self, args, override):
self.args, self.override = args, override
self.out = ROOT / args.output
self.out.mkdir(parents=True, exist_ok=True)
cfg = json.loads(run(NORMAL + ['config', '--format', 'json']))
api = cfg['services']['api']
self.port = int(api['ports'][0]['published'])
self.db = str(api['environment']['REDIS_DB'])
self.pg_command = cfg['services']['postgres'].get('command') or ['postgres']
self.normal_image = api['image']
self.requests, self.runs, self.checks, self.logs, self.resources = [], [], [], [], []
self.summary = dict(started_utc=now(), commit=run(['git', 'rev-parse', 'HEAD']), dirty=True,
method=vars(args).copy(), images={}, runs=self.runs, checks=self.checks, resources=self.resources)
self.summary['docker_resources'] = run(['docker', 'info', '--format',
'{{.OSType}}/{{.Architecture}} CPUs={{.NCPU}} memory_bytes={{.MemTotal}}'])
self.summary['docker_version'] = run(['docker', 'version', '--format', '{{.Server.Version}}'])
for name, image in [('before', args.before_image), ('after', args.after_image)]:
self.summary['images'][name] = run(['docker', 'image', 'inspect', '--format', '{{.Id}}', image])
if self.summary['images']['after'] != run(['docker', 'image', 'inspect', '--format', '{{.Id}}', self.normal_image]):
raise RuntimeError('Build/tag the after image as the normal Compose API image before running')
self.summary['postgres_version'] = self.sql('SHOW server_version;')
self.summary['redis_version'] = self.redis('INFO', 'server').split('redis_version:')[1].splitlines()[0]
self.summary['source_sha256'] = {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in ('service/internal/service/chat/svc.go','service/go.mod','scripts/cache_stampede.py')}
self.preload = self.sql('SHOW shared_preload_libraries;')
self.had_extension = self.sql("SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements';") == '1'
libraries = [s.strip() for s in self.preload.split(',') if s.strip()]
if 'pg_stat_statements' not in libraries:
libraries.append('pg_stat_statements')
self.libraries = ','.join(libraries)
def compose(self, *args):
return run(NORMAL + ['-f', str(self.override)] + list(args))
def sql(self, sql):
return run(NORMAL + PSQL, sql=sql)
def use_image(self, image):
self.override.write_text(json.dumps({'services': {
'api': {'image': image},
'postgres': {'command': self.pg_command + ['-c', 'shared_preload_libraries=' + self.libraries]}}}))
def redis(self, *args):
return run(NORMAL + ['exec', '-T', 'redis', 'sh', '-c',
'REDISCLI_AUTH="$REDIS_PASSWORD" exec redis-cli --no-auth-warning -n "$@"',
'redis-cli', self.db] + list(args))
def wait_postgres(self):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
if self.sql('SELECT 1;') == '1':
return
except RuntimeError:
pass
time.sleep(.3)
raise RuntimeError('PostgreSQL readiness timeout')
def http(self, method, path, actor=None, body=None, basic=None, timeout=5):
conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=timeout)
headers = {}
if actor:
headers['Authorization'] = 'Bearer ' + actor['token']
if basic:
headers['Authorization'] = 'Basic ' + base64.b64encode(basic.encode()).decode()
if body is not None:
headers['Content-Type'] = 'application/json'
stamp, started = now(), time.perf_counter()
try:
conn.request(method, path, body=None if body is None else json.dumps(body), headers=headers)
response = conn.getresponse()
data = response.read()
return dict(status=response.status, data=json.loads(data) if data else None,
request_id=response.getheader('X-Request-Id', ''), timestamp_utc=stamp,
duration_ms=round((time.perf_counter() - started) * 1000, 3))
finally:
conn.close()
def ready(self):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
try:
if self.http('GET', '/health/ready')['status'] == 200:
return
except (OSError, http.client.HTTPException):
pass
time.sleep(.3)
raise RuntimeError('API readiness timeout')
def collect_logs(self, since, label):
time.sleep(.05)
lines = run(NORMAL + ['logs', '--no-color', '--timestamps', '--since', since, 'api'])
safe = [line for line in lines.splitlines() if
(self.chat['id'] in line and ('chat_cache_' in line or 'chat_db_lookup' in line or 'failed to cache chat' in line))
or 'redis unavailable at startup' in line or 'HTTP server started' in line]
self.logs.append('\n# ' + label + '\n' + '\n'.join(safe) + '\n')
self.flush()
def flush(self):
self.summary['updated_utc'] = now()
(self.out / 'summary.json').write_text(json.dumps(self.summary, indent=2) + '\n')
(self.out / 'api.log').write_text(''.join(self.logs))
if self.requests:
with (self.out / 'requests.csv').open('w', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=list(self.requests[0]), lineterminator="\n")
writer.writeheader()
writer.writerows(self.requests)
def actor(self, label):
username, password = 'stampede_' + uuid.uuid4().hex[:16], secrets.token_urlsafe(24)
user = self.http('POST', '/api/v1/users', body=dict(username=username, display_name='Experiment ' + label, password=password))
if user['status'] != 201:
raise RuntimeError('Synthetic registration failed')
session = self.http('POST', '/api/v1/auth/sessions', basic=username + ':' + password)
if session['status'] != 201:
raise RuntimeError('Synthetic login failed')
return dict(label=label, id=user['data']['id'], token=session['data']['token'])
def snapshot(self):
# Only normalized query identities and counters are exported, never SQL parameters.
data = self.sql("""SELECT coalesce(json_agg(row_to_json(s)), '[]'::json) FROM (
SELECT queryid::text AS queryid, calls, total_exec_time,
CASE WHEN query LIKE '%FROM chats c%' THEN 'chat' ELSE 'session' END AS kind
FROM pg_stat_statements WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database())
AND ((query LIKE '%FROM chats c%' AND query LIKE '%AND EXISTS%')
OR (query LIKE '%FROM refresh_tokens%' AND query LIKE '%WHERE id = $1%' AND query NOT LIKE '%FOR UPDATE%'))
) s;""")
return json.loads(data)
@staticmethod
def delta(before, after):
result = {}
for kind in ('chat', 'session'):
result[kind + '_sql'] = sum(r['calls'] for r in after if r['kind'] == kind) - sum(r['calls'] for r in before if r['kind'] == kind)
result[kind + '_sql_exec_ms'] = round(sum(r['total_exec_time'] for r in after if r['kind'] == kind) - sum(r['total_exec_time'] for r in before if r['kind'] == kind), 3)
return result
@contextlib.contextmanager
def lock(self):
proc = subprocess.Popen(NORMAL + PSQL, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, cwd=ROOT)
try:
proc.stdin.write("SET statement_timeout='5s'; SET idle_in_transaction_session_timeout='5s';\nBEGIN;\nLOCK TABLE chats IN ACCESS EXCLUSIVE MODE;\n\\echo LOCKED\n")
proc.stdin.flush()
if not select.select([proc.stdout], [], [], 8)[0] or proc.stdout.readline().strip() != 'LOCKED':
raise RuntimeError('Could not acquire experimental table lock')
yield
finally:
if proc.poll() is None:
try:
proc.communicate('ROLLBACK;\n\\q\n', timeout=8)
except (subprocess.TimeoutExpired, BrokenPipeError):
proc.kill()
proc.communicate()
def burst(self, phase, mode, repetition, actors=None):
actors = actors or [self.alice] * self.args.concurrency
if mode not in ('warm', 'mixed_warm'):
self.redis('DEL', self.key)
before, since = self.snapshot(), now()
barrier = threading.Barrier(len(actors) + 1, timeout=15)
path = '/api/v1/chats/' + self.chat['id']
def worker(index, actor):
# Establish threads and touch HTTP before releasing the simultaneous burst.
if self.http('GET', '/health/live')['status'] != 200:
raise RuntimeError('Worker warmup failed')
barrier.wait()
try:
result = self.http('GET', path, actor)
expected = 404 if actor['label'] == 'eve' else 200
ok = result['status'] == expected and (expected == 404 or result['data'] == self.chat)
status, ms, rid, stamp = result['status'], result['duration_ms'], result['request_id'], result['timestamp_utc']
except (OSError, http.client.HTTPException):
ok, status, ms, rid, stamp = False, 0, 5000, '', now()
return dict(phase=phase, mode=mode, repetition=repetition, worker=index, actor=actor['label'],
timestamp_utc=stamp, status=status, duration_ms=ms, request_id=rid, correct=ok)
with futures.ThreadPoolExecutor(max_workers=len(actors)) as pool:
jobs = [pool.submit(worker, i, actor) for i, actor in enumerate(actors)]
controlled = mode in ('cold', 'mixed_cold')
with self.lock() if controlled else contextlib.nullcontext():
barrier.wait()
started = time.perf_counter()
if controlled:
time.sleep(self.args.lock_ms / 1000)
group = [job.result(timeout=10) for job in jobs]
elapsed = time.perf_counter() - started
after = self.snapshot()
errors = sum(not row['correct'] for row in group)
durations = [row['duration_ms'] for row in group]
measured = dict(phase=phase, mode=mode, repetition=repetition, requests=len(group), errors=errors,
elapsed_ms=round(elapsed*1000, 3), successful_rps=round((len(group)-errors)/elapsed, 3),
latency_ms={f'p{p}': percentile(durations, p/100) for p in (50,95,99)},
sql_before=before, sql_after=after, **self.delta(before, after))
self.requests.extend(group)
self.runs.append(measured)
self.collect_logs(since, f'{phase}/{mode}/{repetition}')
print(f'{phase}/{mode}/{repetition}: chat SQL={measured["chat_sql"]}, session SQL={measured["session_sql"]}, p95={measured["latency_ms"]["p95"]} ms, errors={errors}', flush=True)
if errors:
raise RuntimeError('Incorrect responses during burst')
if mode == 'cold' and phase == 'after' and measured['chat_sql'] != 1:
raise RuntimeError('Expected exactly one chat SELECT with singleflight')
if mode in ('warm', 'mixed_warm') and measured['chat_sql'] != 0:
raise RuntimeError('Warm cache unexpectedly queried chats')
if measured['session_sql'] != len(actors):
raise RuntimeError('Session SQL accounting differs from request count; check concurrent traffic')
time.sleep(.3)
def checked(self, name, method, path, actor, body=None, status=200):
result = self.http(method, path, actor, body)
if result['status'] != status:
raise RuntimeError('Functional check failed: ' + name)
self.checks.append(dict(name=name, status=result['status'], duration_ms=result['duration_ms'], request_id=result['request_id']))
return result['data']
def cancellation(self):
self.redis('DEL', self.key)
before, since = self.snapshot(), now()
path = '/api/v1/chats/' + self.chat['id']
with futures.ThreadPoolExecutor(max_workers=1) as pool:
with self.lock():
leader = http.client.HTTPConnection('127.0.0.1', self.port, timeout=5)
try:
leader.request('GET', path, headers={'Authorization': 'Bearer ' + self.alice['token']})
deadline = time.monotonic() + 1
while self.sql("SELECT count(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND wait_event_type='Lock' AND query LIKE 'SELECT c.id%';") == '0':
if time.monotonic() > deadline:
raise RuntimeError('Leader did not reach SQL')
time.sleep(.01)
leader.sock.shutdown(socket.SHUT_RDWR)
finally:
leader.close()
survivor = pool.submit(self.http, 'GET', path, self.alice)
time.sleep(.1)
result = survivor.result(timeout=5)
counters = self.delta(before, self.snapshot())
if result['status'] != 200 or result['data'] != self.chat or counters['chat_sql'] != 1:
raise RuntimeError('Client cancellation broke shared loading')
self.checks.append(dict(name='leader_cancelled_survivor_succeeded', status=result['status'],
request_id=result['request_id'], duration_ms=result['duration_ms'], **counters))
self.collect_logs(since, 'Cancelled initiating client; surviving client received chat')
def functional(self):
self.cancellation()
path = '/api/v1/chats/' + self.chat['id']
self.burst('after', 'mixed_cold', 1, [self.eve, self.alice, self.bob] * 4)
self.burst('after', 'mixed_warm', 1, [self.eve, self.alice, self.bob] * 4)
message = self.checked('message_committed', 'POST', path+'/messages', self.alice,
{'text': 'Synthetic read-after-write evidence'}, status=201)
history = self.checked('read_after_write', 'GET', path+'/messages?limit=100', self.bob)
if message['id'] not in [item['id'] for item in history['items']]:
raise RuntimeError('Committed message missing from following read')
since = now()
api_before = run(NORMAL + ['ps', '-q', 'api'])
self.summary['key_before_restart'] = self.redis('EXISTS', self.key)
try:
run(NORMAL + ['stop', 'redis'])
self.checked('redis_unavailable', 'GET', path, self.alice)
finally:
run(NORMAL + ['start', 'redis'])
deadline = time.monotonic()+30
while self.redis('PING') != 'PONG':
if time.monotonic()>deadline:
raise RuntimeError('Redis recovery timeout')
time.sleep(.3)
self.summary['key_after_restart'] = self.redis('EXISTS', self.key)
self.checked('redis_recovered', 'GET', path, self.bob)
history = self.checked('message_after_restart', 'GET', path+'/messages?limit=100', self.bob)
if message['id'] not in [item['id'] for item in history['items']]:
raise RuntimeError('Message lost after Redis restart')
if api_before != run(NORMAL + ['ps', '-q', 'api']):
raise RuntimeError('API restarted during Redis recovery')
self.collect_logs(since, 'Redis stop/start and read-after-write')
self.burst('after', 'cold', self.args.repetitions+1)
@contextlib.contextmanager
def sample_resources(self, phase):
stop = threading.Event()
containers = [run(NORMAL + ['ps', '-q', name]) for name in ('api','postgres','redis')]
def sample():
while not stop.is_set():
try:
output = run(['docker','stats','--no-stream','--format','{{json .}}'] + containers, timeout=10)
values = [json.loads(line) for line in output.splitlines() if line.strip()]
self.resources.append(dict(phase=phase, timestamp_utc=now(), containers=[
{key: value[key] for key in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')} for value in values]))
except (RuntimeError, subprocess.TimeoutExpired):
self.resources.append(dict(phase=phase, timestamp_utc=now(), error='resource sampling failed'))
stop.wait(.2)
thread = threading.Thread(target=sample, daemon=True)
thread.start()
try:
yield
finally:
stop.set()
thread.join(timeout=12)
def execute(self):
self.use_image(self.args.before_image)
extension_created = False
try:
self.compose('up', '-d', '--no-build', '--force-recreate', 'postgres')
self.wait_postgres()
if not self.had_extension:
self.sql('CREATE EXTENSION pg_stat_statements;')
extension_created = True
run(NORMAL + ['start', 'redis'])
for phase, image in [('before', self.args.before_image), ('after', self.args.after_image)]:
self.use_image(image)
self.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate', 'api')
self.ready()
if phase == 'before':
self.alice, self.bob, self.eve = [self.actor(label) for label in ('alice', 'bob', 'eve')]
self.chat = self.checked('chat_created', 'POST', '/api/v1/chats', self.alice,
{'title':'Stampede experiment', 'member_ids':[self.alice['id'],self.bob['id']]}, status=201)
self.key = 'chat:' + self.chat['id']
self.summary['chat_id'] = self.chat['id']
observed = self.checked(phase+'_chat_read_after_create', 'GET', '/api/v1/chats/'+self.chat['id'], self.bob)
if observed != self.chat:
raise RuntimeError('Chat read-after-create differs')
self.summary.setdefault('container_limits', {})[phase] = json.loads(run(['docker','inspect','--format',
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}},"cpu_quota":{{.HostConfig.CpuQuota}}}',
run(NORMAL+['ps','-q','api'])]))
with self.sample_resources(phase):
for repetition in range(1, self.args.repetitions+1):
self.burst(phase, 'cold', repetition)
for repetition in range(1, 6):
self.burst(phase, 'warm', repetition)
if not any(r['chat_sql'] > 1 for r in self.runs if r['phase']=='before' and r['mode']=='cold'):
raise RuntimeError('Baseline stampede not reproduced')
self.functional()
self.summary['success'] = True
finally:
if extension_created:
self.sql('DROP EXTENSION pg_stat_statements;')
# Restore the original Compose command and fixed normal API image.
run(NORMAL + ['up','-d','--no-build','--force-recreate','postgres'])
self.wait_postgres()
run(NORMAL + ['start','redis'])
run(NORMAL + ['up','-d','--no-build','--no-deps','--force-recreate','api'])
self.ready()
self.summary['restored_preload'] = self.sql('SHOW shared_preload_libraries;')
self.summary['extension_restored'] = (self.sql("SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements';") == ('1' if self.had_extension else '0'))
self.logs.append('\n# Final Compose state\n'+run(NORMAL+['ps','-a'])+'\n')
self.summary['ended_utc'] = now()
self.flush()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--before-image', default='mtusi-messenger:stampede-before')
parser.add_argument('--after-image', default='mtusi-messenger:stampede-after')
parser.add_argument('--concurrency', type=int, default=32)
parser.add_argument('--repetitions', type=int, default=10)
parser.add_argument('--lock-ms', type=int, default=200)
parser.add_argument('--output', default='evidence/cache-stampede')
args = parser.parse_args()
if not (2 <= args.concurrency <= 64 and 1 <= args.repetitions <= 50 and 50 <= args.lock_ms <= 500):
parser.error('Use concurrency 2..64, repetitions 1..50, lock-ms 50..500')
with tempfile.TemporaryDirectory(prefix='cache-stampede-') as directory:
Experiment(args, pathlib.Path(directory)/'compose.json').execute()
print('Experiment complete; normal Compose restored.', flush=True)
if __name__ == '__main__':
main()