#!/usr/bin/env python3 """Repeatable local cache experiment; uses Docker, Python stdlib and synthetic data. Requires prebuilt before/after API images. Temporarily restarts local PostgreSQL with pg_stat_statements, then restores Compose without deleting data or volumes. """ import argparse import base64 import concurrent.futures as futures import contextlib import csv import datetime as dt import http.client import hashlib import json import math import pathlib import secrets import select import socket import subprocess import tempfile import threading import time import uuid ROOT = pathlib.Path(__file__).resolve().parents[1] NORMAL = ['docker', 'compose', '--env-file', str(ROOT / '.env'), '-f', str(ROOT / 'docker-compose.yml')] PSQL = ['exec', '-T', 'postgres', 'sh', '-c', 'exec psql -X -q -At -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"'] def now(): return dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z') def run(args, sql=None, timeout=90): result = subprocess.run(args, input=sql, text=True, capture_output=True, cwd=ROOT, timeout=timeout) if result.returncode: # Never echo config, SQL parameters, HTTP bodies or credentials on failure. raise RuntimeError('External command failed: ' + ' '.join(args[:3])) return result.stdout.strip() def percentile(values, fraction): values = sorted(values) return round(values[max(0, math.ceil(len(values) * fraction) - 1)], 3) class Experiment: def __init__(self, args, override): self.args, self.override = args, override self.out = ROOT / args.output self.out.mkdir(parents=True, exist_ok=True) cfg = json.loads(run(NORMAL + ['config', '--format', 'json'])) api = cfg['services']['api'] self.port = int(api['ports'][0]['published']) self.db = str(api['environment']['REDIS_DB']) self.pg_command = cfg['services']['postgres'].get('command') or ['postgres'] self.normal_image = api['image'] self.requests, self.runs, self.checks, self.logs, self.resources = [], [], [], [], [] self.summary = dict(started_utc=now(), commit=run(['git', 'rev-parse', 'HEAD']), dirty=True, method=vars(args).copy(), images={}, runs=self.runs, checks=self.checks, resources=self.resources) self.summary['docker_resources'] = run(['docker', 'info', '--format', '{{.OSType}}/{{.Architecture}} CPUs={{.NCPU}} memory_bytes={{.MemTotal}}']) self.summary['docker_version'] = run(['docker', 'version', '--format', '{{.Server.Version}}']) for name, image in [('before', args.before_image), ('after', args.after_image)]: self.summary['images'][name] = run(['docker', 'image', 'inspect', '--format', '{{.Id}}', image]) if self.summary['images']['after'] != run(['docker', 'image', 'inspect', '--format', '{{.Id}}', self.normal_image]): raise RuntimeError('Build/tag the after image as the normal Compose API image before running') self.summary['postgres_version'] = self.sql('SHOW server_version;') self.summary['redis_version'] = self.redis('INFO', 'server').split('redis_version:')[1].splitlines()[0] self.summary['source_sha256'] = {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in ('service/internal/service/chat/svc.go','service/go.mod','scripts/cache_stampede.py')} self.preload = self.sql('SHOW shared_preload_libraries;') self.had_extension = self.sql("SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements';") == '1' libraries = [s.strip() for s in self.preload.split(',') if s.strip()] if 'pg_stat_statements' not in libraries: libraries.append('pg_stat_statements') self.libraries = ','.join(libraries) def compose(self, *args): return run(NORMAL + ['-f', str(self.override)] + list(args)) def sql(self, sql): return run(NORMAL + PSQL, sql=sql) def use_image(self, image): self.override.write_text(json.dumps({'services': { 'api': {'image': image}, 'postgres': {'command': self.pg_command + ['-c', 'shared_preload_libraries=' + self.libraries]}}})) def redis(self, *args): return run(NORMAL + ['exec', '-T', 'redis', 'sh', '-c', 'REDISCLI_AUTH="$REDIS_PASSWORD" exec redis-cli --no-auth-warning -n "$@"', 'redis-cli', self.db] + list(args)) def wait_postgres(self): deadline = time.monotonic() + 60 while time.monotonic() < deadline: try: if self.sql('SELECT 1;') == '1': return except RuntimeError: pass time.sleep(.3) raise RuntimeError('PostgreSQL readiness timeout') def http(self, method, path, actor=None, body=None, basic=None, timeout=5): conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=timeout) headers = {} if actor: headers['Authorization'] = 'Bearer ' + actor['token'] if basic: headers['Authorization'] = 'Basic ' + base64.b64encode(basic.encode()).decode() if body is not None: headers['Content-Type'] = 'application/json' stamp, started = now(), time.perf_counter() try: conn.request(method, path, body=None if body is None else json.dumps(body), headers=headers) response = conn.getresponse() data = response.read() return dict(status=response.status, data=json.loads(data) if data else None, request_id=response.getheader('X-Request-Id', ''), timestamp_utc=stamp, duration_ms=round((time.perf_counter() - started) * 1000, 3)) finally: conn.close() def ready(self): deadline = time.monotonic() + 45 while time.monotonic() < deadline: try: if self.http('GET', '/health/ready')['status'] == 200: return except (OSError, http.client.HTTPException): pass time.sleep(.3) raise RuntimeError('API readiness timeout') def collect_logs(self, since, label): time.sleep(.05) lines = run(NORMAL + ['logs', '--no-color', '--timestamps', '--since', since, 'api']) safe = [line for line in lines.splitlines() if (self.chat['id'] in line and ('chat_cache_' in line or 'chat_db_lookup' in line or 'failed to cache chat' in line)) or 'redis unavailable at startup' in line or 'HTTP server started' in line] self.logs.append('\n# ' + label + '\n' + '\n'.join(safe) + '\n') self.flush() def flush(self): self.summary['updated_utc'] = now() (self.out / 'summary.json').write_text(json.dumps(self.summary, indent=2) + '\n') (self.out / 'api.log').write_text(''.join(self.logs)) if self.requests: with (self.out / 'requests.csv').open('w', newline='') as stream: writer = csv.DictWriter(stream, fieldnames=list(self.requests[0]), lineterminator="\n") writer.writeheader() writer.writerows(self.requests) def actor(self, label): username, password = 'stampede_' + uuid.uuid4().hex[:16], secrets.token_urlsafe(24) user = self.http('POST', '/api/v1/users', body=dict(username=username, display_name='Experiment ' + label, password=password)) if user['status'] != 201: raise RuntimeError('Synthetic registration failed') session = self.http('POST', '/api/v1/auth/sessions', basic=username + ':' + password) if session['status'] != 201: raise RuntimeError('Synthetic login failed') return dict(label=label, id=user['data']['id'], token=session['data']['token']) def snapshot(self): # Only normalized query identities and counters are exported, never SQL parameters. data = self.sql("""SELECT coalesce(json_agg(row_to_json(s)), '[]'::json) FROM ( SELECT queryid::text AS queryid, calls, total_exec_time, CASE WHEN query LIKE '%FROM chats c%' THEN 'chat' ELSE 'session' END AS kind FROM pg_stat_statements WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) AND ((query LIKE '%FROM chats c%' AND query LIKE '%AND EXISTS%') OR (query LIKE '%FROM refresh_tokens%' AND query LIKE '%WHERE id = $1%' AND query NOT LIKE '%FOR UPDATE%')) ) s;""") return json.loads(data) @staticmethod def delta(before, after): result = {} for kind in ('chat', 'session'): result[kind + '_sql'] = sum(r['calls'] for r in after if r['kind'] == kind) - sum(r['calls'] for r in before if r['kind'] == kind) result[kind + '_sql_exec_ms'] = round(sum(r['total_exec_time'] for r in after if r['kind'] == kind) - sum(r['total_exec_time'] for r in before if r['kind'] == kind), 3) return result @contextlib.contextmanager def lock(self): proc = subprocess.Popen(NORMAL + PSQL, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=ROOT) try: proc.stdin.write("SET statement_timeout='5s'; SET idle_in_transaction_session_timeout='5s';\nBEGIN;\nLOCK TABLE chats IN ACCESS EXCLUSIVE MODE;\n\\echo LOCKED\n") proc.stdin.flush() if not select.select([proc.stdout], [], [], 8)[0] or proc.stdout.readline().strip() != 'LOCKED': raise RuntimeError('Could not acquire experimental table lock') yield finally: if proc.poll() is None: try: proc.communicate('ROLLBACK;\n\\q\n', timeout=8) except (subprocess.TimeoutExpired, BrokenPipeError): proc.kill() proc.communicate() def burst(self, phase, mode, repetition, actors=None): actors = actors or [self.alice] * self.args.concurrency if mode not in ('warm', 'mixed_warm'): self.redis('DEL', self.key) before, since = self.snapshot(), now() barrier = threading.Barrier(len(actors) + 1, timeout=15) path = '/api/v1/chats/' + self.chat['id'] def worker(index, actor): # Establish threads and touch HTTP before releasing the simultaneous burst. if self.http('GET', '/health/live')['status'] != 200: raise RuntimeError('Worker warmup failed') barrier.wait() try: result = self.http('GET', path, actor) expected = 404 if actor['label'] == 'eve' else 200 ok = result['status'] == expected and (expected == 404 or result['data'] == self.chat) status, ms, rid, stamp = result['status'], result['duration_ms'], result['request_id'], result['timestamp_utc'] except (OSError, http.client.HTTPException): ok, status, ms, rid, stamp = False, 0, 5000, '', now() return dict(phase=phase, mode=mode, repetition=repetition, worker=index, actor=actor['label'], timestamp_utc=stamp, status=status, duration_ms=ms, request_id=rid, correct=ok) with futures.ThreadPoolExecutor(max_workers=len(actors)) as pool: jobs = [pool.submit(worker, i, actor) for i, actor in enumerate(actors)] controlled = mode in ('cold', 'mixed_cold') with self.lock() if controlled else contextlib.nullcontext(): barrier.wait() started = time.perf_counter() if controlled: time.sleep(self.args.lock_ms / 1000) group = [job.result(timeout=10) for job in jobs] elapsed = time.perf_counter() - started after = self.snapshot() errors = sum(not row['correct'] for row in group) durations = [row['duration_ms'] for row in group] measured = dict(phase=phase, mode=mode, repetition=repetition, requests=len(group), errors=errors, elapsed_ms=round(elapsed*1000, 3), successful_rps=round((len(group)-errors)/elapsed, 3), latency_ms={f'p{p}': percentile(durations, p/100) for p in (50,95,99)}, sql_before=before, sql_after=after, **self.delta(before, after)) self.requests.extend(group) self.runs.append(measured) self.collect_logs(since, f'{phase}/{mode}/{repetition}') print(f'{phase}/{mode}/{repetition}: chat SQL={measured["chat_sql"]}, session SQL={measured["session_sql"]}, p95={measured["latency_ms"]["p95"]} ms, errors={errors}', flush=True) if errors: raise RuntimeError('Incorrect responses during burst') if mode == 'cold' and phase == 'after' and measured['chat_sql'] != 1: raise RuntimeError('Expected exactly one chat SELECT with singleflight') if mode in ('warm', 'mixed_warm') and measured['chat_sql'] != 0: raise RuntimeError('Warm cache unexpectedly queried chats') if measured['session_sql'] != len(actors): raise RuntimeError('Session SQL accounting differs from request count; check concurrent traffic') time.sleep(.3) def checked(self, name, method, path, actor, body=None, status=200): result = self.http(method, path, actor, body) if result['status'] != status: raise RuntimeError('Functional check failed: ' + name) self.checks.append(dict(name=name, status=result['status'], duration_ms=result['duration_ms'], request_id=result['request_id'])) return result['data'] def cancellation(self): self.redis('DEL', self.key) before, since = self.snapshot(), now() path = '/api/v1/chats/' + self.chat['id'] with futures.ThreadPoolExecutor(max_workers=1) as pool: with self.lock(): leader = http.client.HTTPConnection('127.0.0.1', self.port, timeout=5) try: leader.request('GET', path, headers={'Authorization': 'Bearer ' + self.alice['token']}) deadline = time.monotonic() + 1 while self.sql("SELECT count(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND wait_event_type='Lock' AND query LIKE 'SELECT c.id%';") == '0': if time.monotonic() > deadline: raise RuntimeError('Leader did not reach SQL') time.sleep(.01) leader.sock.shutdown(socket.SHUT_RDWR) finally: leader.close() survivor = pool.submit(self.http, 'GET', path, self.alice) time.sleep(.1) result = survivor.result(timeout=5) counters = self.delta(before, self.snapshot()) if result['status'] != 200 or result['data'] != self.chat or counters['chat_sql'] != 1: raise RuntimeError('Client cancellation broke shared loading') self.checks.append(dict(name='leader_cancelled_survivor_succeeded', status=result['status'], request_id=result['request_id'], duration_ms=result['duration_ms'], **counters)) self.collect_logs(since, 'Cancelled initiating client; surviving client received chat') def functional(self): self.cancellation() path = '/api/v1/chats/' + self.chat['id'] self.burst('after', 'mixed_cold', 1, [self.eve, self.alice, self.bob] * 4) self.burst('after', 'mixed_warm', 1, [self.eve, self.alice, self.bob] * 4) message = self.checked('message_committed', 'POST', path+'/messages', self.alice, {'text': 'Synthetic read-after-write evidence'}, status=201) history = self.checked('read_after_write', 'GET', path+'/messages?limit=100', self.bob) if message['id'] not in [item['id'] for item in history['items']]: raise RuntimeError('Committed message missing from following read') since = now() api_before = run(NORMAL + ['ps', '-q', 'api']) self.summary['key_before_restart'] = self.redis('EXISTS', self.key) try: run(NORMAL + ['stop', 'redis']) self.checked('redis_unavailable', 'GET', path, self.alice) finally: run(NORMAL + ['start', 'redis']) deadline = time.monotonic()+30 while self.redis('PING') != 'PONG': if time.monotonic()>deadline: raise RuntimeError('Redis recovery timeout') time.sleep(.3) self.summary['key_after_restart'] = self.redis('EXISTS', self.key) self.checked('redis_recovered', 'GET', path, self.bob) history = self.checked('message_after_restart', 'GET', path+'/messages?limit=100', self.bob) if message['id'] not in [item['id'] for item in history['items']]: raise RuntimeError('Message lost after Redis restart') if api_before != run(NORMAL + ['ps', '-q', 'api']): raise RuntimeError('API restarted during Redis recovery') self.collect_logs(since, 'Redis stop/start and read-after-write') self.burst('after', 'cold', self.args.repetitions+1) @contextlib.contextmanager def sample_resources(self, phase): stop = threading.Event() containers = [run(NORMAL + ['ps', '-q', name]) for name in ('api','postgres','redis')] def sample(): while not stop.is_set(): try: output = run(['docker','stats','--no-stream','--format','{{json .}}'] + containers, timeout=10) values = [json.loads(line) for line in output.splitlines() if line.strip()] self.resources.append(dict(phase=phase, timestamp_utc=now(), containers=[ {key: value[key] for key in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')} for value in values])) except (RuntimeError, subprocess.TimeoutExpired): self.resources.append(dict(phase=phase, timestamp_utc=now(), error='resource sampling failed')) stop.wait(.2) thread = threading.Thread(target=sample, daemon=True) thread.start() try: yield finally: stop.set() thread.join(timeout=12) def execute(self): self.use_image(self.args.before_image) extension_created = False try: self.compose('up', '-d', '--no-build', '--force-recreate', 'postgres') self.wait_postgres() if not self.had_extension: self.sql('CREATE EXTENSION pg_stat_statements;') extension_created = True run(NORMAL + ['start', 'redis']) for phase, image in [('before', self.args.before_image), ('after', self.args.after_image)]: self.use_image(image) self.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate', 'api') self.ready() if phase == 'before': self.alice, self.bob, self.eve = [self.actor(label) for label in ('alice', 'bob', 'eve')] self.chat = self.checked('chat_created', 'POST', '/api/v1/chats', self.alice, {'title':'Stampede experiment', 'member_ids':[self.alice['id'],self.bob['id']]}, status=201) self.key = 'chat:' + self.chat['id'] self.summary['chat_id'] = self.chat['id'] observed = self.checked(phase+'_chat_read_after_create', 'GET', '/api/v1/chats/'+self.chat['id'], self.bob) if observed != self.chat: raise RuntimeError('Chat read-after-create differs') self.summary.setdefault('container_limits', {})[phase] = json.loads(run(['docker','inspect','--format', '{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}},"cpu_quota":{{.HostConfig.CpuQuota}}}', run(NORMAL+['ps','-q','api'])])) with self.sample_resources(phase): for repetition in range(1, self.args.repetitions+1): self.burst(phase, 'cold', repetition) for repetition in range(1, 6): self.burst(phase, 'warm', repetition) if not any(r['chat_sql'] > 1 for r in self.runs if r['phase']=='before' and r['mode']=='cold'): raise RuntimeError('Baseline stampede not reproduced') self.functional() self.summary['success'] = True finally: if extension_created: self.sql('DROP EXTENSION pg_stat_statements;') # Restore the original Compose command and fixed normal API image. run(NORMAL + ['up','-d','--no-build','--force-recreate','postgres']) self.wait_postgres() run(NORMAL + ['start','redis']) run(NORMAL + ['up','-d','--no-build','--no-deps','--force-recreate','api']) self.ready() self.summary['restored_preload'] = self.sql('SHOW shared_preload_libraries;') self.summary['extension_restored'] = (self.sql("SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements';") == ('1' if self.had_extension else '0')) self.logs.append('\n# Final Compose state\n'+run(NORMAL+['ps','-a'])+'\n') self.summary['ended_utc'] = now() self.flush() def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--before-image', default='mtusi-messenger:stampede-before') parser.add_argument('--after-image', default='mtusi-messenger:stampede-after') parser.add_argument('--concurrency', type=int, default=32) parser.add_argument('--repetitions', type=int, default=10) parser.add_argument('--lock-ms', type=int, default=200) parser.add_argument('--output', default='evidence/cache-stampede') args = parser.parse_args() if not (2 <= args.concurrency <= 64 and 1 <= args.repetitions <= 50 and 50 <= args.lock_ms <= 500): parser.error('Use concurrency 2..64, repetitions 1..50, lock-ms 50..500') with tempfile.TemporaryDirectory(prefix='cache-stampede-') as directory: Experiment(args, pathlib.Path(directory)/'compose.json').execute() print('Experiment complete; normal Compose restored.', flush=True) if __name__ == '__main__': main()