forked from mtusi2026/mtusi2026lab2
419 lines
22 KiB
Python
419 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable local cache experiment; uses Docker, Python stdlib and synthetic data.
|
|
|
|
Requires prebuilt before/after API images. Temporarily restarts local PostgreSQL
|
|
with pg_stat_statements, then restores Compose without deleting data or volumes.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import concurrent.futures as futures
|
|
import contextlib
|
|
import csv
|
|
import datetime as dt
|
|
import http.client
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import pathlib
|
|
import secrets
|
|
import select
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
NORMAL = ['docker', 'compose', '--env-file', str(ROOT / '.env'), '-f', str(ROOT / 'docker-compose.yml')]
|
|
PSQL = ['exec', '-T', 'postgres', 'sh', '-c',
|
|
'exec psql -X -q -At -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"']
|
|
|
|
|
|
def now():
|
|
return dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z')
|
|
|
|
|
|
def run(args, sql=None, timeout=90):
|
|
result = subprocess.run(args, input=sql, text=True, capture_output=True, cwd=ROOT, timeout=timeout)
|
|
if result.returncode:
|
|
# Never echo config, SQL parameters, HTTP bodies or credentials on failure.
|
|
raise RuntimeError('External command failed: ' + ' '.join(args[:3]))
|
|
return result.stdout.strip()
|
|
|
|
|
|
def percentile(values, fraction):
|
|
values = sorted(values)
|
|
return round(values[max(0, math.ceil(len(values) * fraction) - 1)], 3)
|
|
|
|
|
|
class Experiment:
|
|
def __init__(self, args, override):
|
|
self.args, self.override = args, override
|
|
self.out = ROOT / args.output
|
|
self.out.mkdir(parents=True, exist_ok=True)
|
|
cfg = json.loads(run(NORMAL + ['config', '--format', 'json']))
|
|
api = cfg['services']['api']
|
|
self.port = int(api['ports'][0]['published'])
|
|
self.db = str(api['environment']['REDIS_DB'])
|
|
self.pg_command = cfg['services']['postgres'].get('command') or ['postgres']
|
|
self.normal_image = api['image']
|
|
self.requests, self.runs, self.checks, self.logs, self.resources = [], [], [], [], []
|
|
self.summary = dict(started_utc=now(), commit=run(['git', 'rev-parse', 'HEAD']), dirty=True,
|
|
method=vars(args).copy(), images={}, runs=self.runs, checks=self.checks, resources=self.resources)
|
|
self.summary['docker_resources'] = run(['docker', 'info', '--format',
|
|
'{{.OSType}}/{{.Architecture}} CPUs={{.NCPU}} memory_bytes={{.MemTotal}}'])
|
|
self.summary['docker_version'] = run(['docker', 'version', '--format', '{{.Server.Version}}'])
|
|
for name, image in [('before', args.before_image), ('after', args.after_image)]:
|
|
self.summary['images'][name] = run(['docker', 'image', 'inspect', '--format', '{{.Id}}', image])
|
|
if self.summary['images']['after'] != run(['docker', 'image', 'inspect', '--format', '{{.Id}}', self.normal_image]):
|
|
raise RuntimeError('Build/tag the after image as the normal Compose API image before running')
|
|
self.summary['postgres_version'] = self.sql('SHOW server_version;')
|
|
self.summary['redis_version'] = self.redis('INFO', 'server').split('redis_version:')[1].splitlines()[0]
|
|
self.summary['source_sha256'] = {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in ('service/internal/service/chat/svc.go','service/go.mod','scripts/cache_stampede.py')}
|
|
self.preload = self.sql('SHOW shared_preload_libraries;')
|
|
self.had_extension = self.sql("SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements';") == '1'
|
|
libraries = [s.strip() for s in self.preload.split(',') if s.strip()]
|
|
if 'pg_stat_statements' not in libraries:
|
|
libraries.append('pg_stat_statements')
|
|
self.libraries = ','.join(libraries)
|
|
|
|
def compose(self, *args):
|
|
return run(NORMAL + ['-f', str(self.override)] + list(args))
|
|
|
|
def sql(self, sql):
|
|
return run(NORMAL + PSQL, sql=sql)
|
|
|
|
def use_image(self, image):
|
|
self.override.write_text(json.dumps({'services': {
|
|
'api': {'image': image},
|
|
'postgres': {'command': self.pg_command + ['-c', 'shared_preload_libraries=' + self.libraries]}}}))
|
|
|
|
def redis(self, *args):
|
|
return run(NORMAL + ['exec', '-T', 'redis', 'sh', '-c',
|
|
'REDISCLI_AUTH="$REDIS_PASSWORD" exec redis-cli --no-auth-warning -n "$@"',
|
|
'redis-cli', self.db] + list(args))
|
|
|
|
def wait_postgres(self):
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if self.sql('SELECT 1;') == '1':
|
|
return
|
|
except RuntimeError:
|
|
pass
|
|
time.sleep(.3)
|
|
raise RuntimeError('PostgreSQL readiness timeout')
|
|
|
|
def http(self, method, path, actor=None, body=None, basic=None, timeout=5):
|
|
conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=timeout)
|
|
headers = {}
|
|
if actor:
|
|
headers['Authorization'] = 'Bearer ' + actor['token']
|
|
if basic:
|
|
headers['Authorization'] = 'Basic ' + base64.b64encode(basic.encode()).decode()
|
|
if body is not None:
|
|
headers['Content-Type'] = 'application/json'
|
|
stamp, started = now(), time.perf_counter()
|
|
try:
|
|
conn.request(method, path, body=None if body is None else json.dumps(body), headers=headers)
|
|
response = conn.getresponse()
|
|
data = response.read()
|
|
return dict(status=response.status, data=json.loads(data) if data else None,
|
|
request_id=response.getheader('X-Request-Id', ''), timestamp_utc=stamp,
|
|
duration_ms=round((time.perf_counter() - started) * 1000, 3))
|
|
finally:
|
|
conn.close()
|
|
|
|
def ready(self):
|
|
deadline = time.monotonic() + 45
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if self.http('GET', '/health/ready')['status'] == 200:
|
|
return
|
|
except (OSError, http.client.HTTPException):
|
|
pass
|
|
time.sleep(.3)
|
|
raise RuntimeError('API readiness timeout')
|
|
|
|
def collect_logs(self, since, label):
|
|
time.sleep(.05)
|
|
lines = run(NORMAL + ['logs', '--no-color', '--timestamps', '--since', since, 'api'])
|
|
safe = [line for line in lines.splitlines() if
|
|
(self.chat['id'] in line and ('chat_cache_' in line or 'chat_db_lookup' in line or 'failed to cache chat' in line))
|
|
or 'redis unavailable at startup' in line or 'HTTP server started' in line]
|
|
self.logs.append('\n# ' + label + '\n' + '\n'.join(safe) + '\n')
|
|
self.flush()
|
|
|
|
def flush(self):
|
|
self.summary['updated_utc'] = now()
|
|
(self.out / 'summary.json').write_text(json.dumps(self.summary, indent=2) + '\n')
|
|
(self.out / 'api.log').write_text(''.join(self.logs))
|
|
if self.requests:
|
|
with (self.out / 'requests.csv').open('w', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=list(self.requests[0]), lineterminator="\n")
|
|
writer.writeheader()
|
|
writer.writerows(self.requests)
|
|
|
|
def actor(self, label):
|
|
username, password = 'stampede_' + uuid.uuid4().hex[:16], secrets.token_urlsafe(24)
|
|
user = self.http('POST', '/api/v1/users', body=dict(username=username, display_name='Experiment ' + label, password=password))
|
|
if user['status'] != 201:
|
|
raise RuntimeError('Synthetic registration failed')
|
|
session = self.http('POST', '/api/v1/auth/sessions', basic=username + ':' + password)
|
|
if session['status'] != 201:
|
|
raise RuntimeError('Synthetic login failed')
|
|
return dict(label=label, id=user['data']['id'], token=session['data']['token'])
|
|
|
|
def snapshot(self):
|
|
# Only normalized query identities and counters are exported, never SQL parameters.
|
|
data = self.sql("""SELECT coalesce(json_agg(row_to_json(s)), '[]'::json) FROM (
|
|
SELECT queryid::text AS queryid, calls, total_exec_time,
|
|
CASE WHEN query LIKE '%FROM chats c%' THEN 'chat' ELSE 'session' END AS kind
|
|
FROM pg_stat_statements WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database())
|
|
AND ((query LIKE '%FROM chats c%' AND query LIKE '%AND EXISTS%')
|
|
OR (query LIKE '%FROM refresh_tokens%' AND query LIKE '%WHERE id = $1%' AND query NOT LIKE '%FOR UPDATE%'))
|
|
) s;""")
|
|
return json.loads(data)
|
|
|
|
@staticmethod
|
|
def delta(before, after):
|
|
result = {}
|
|
for kind in ('chat', 'session'):
|
|
result[kind + '_sql'] = sum(r['calls'] for r in after if r['kind'] == kind) - sum(r['calls'] for r in before if r['kind'] == kind)
|
|
result[kind + '_sql_exec_ms'] = round(sum(r['total_exec_time'] for r in after if r['kind'] == kind) - sum(r['total_exec_time'] for r in before if r['kind'] == kind), 3)
|
|
return result
|
|
|
|
@contextlib.contextmanager
|
|
def lock(self):
|
|
proc = subprocess.Popen(NORMAL + PSQL, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=True, cwd=ROOT)
|
|
try:
|
|
proc.stdin.write("SET statement_timeout='5s'; SET idle_in_transaction_session_timeout='5s';\nBEGIN;\nLOCK TABLE chats IN ACCESS EXCLUSIVE MODE;\n\\echo LOCKED\n")
|
|
proc.stdin.flush()
|
|
if not select.select([proc.stdout], [], [], 8)[0] or proc.stdout.readline().strip() != 'LOCKED':
|
|
raise RuntimeError('Could not acquire experimental table lock')
|
|
yield
|
|
finally:
|
|
if proc.poll() is None:
|
|
try:
|
|
proc.communicate('ROLLBACK;\n\\q\n', timeout=8)
|
|
except (subprocess.TimeoutExpired, BrokenPipeError):
|
|
proc.kill()
|
|
proc.communicate()
|
|
|
|
def burst(self, phase, mode, repetition, actors=None):
|
|
actors = actors or [self.alice] * self.args.concurrency
|
|
if mode not in ('warm', 'mixed_warm'):
|
|
self.redis('DEL', self.key)
|
|
before, since = self.snapshot(), now()
|
|
barrier = threading.Barrier(len(actors) + 1, timeout=15)
|
|
path = '/api/v1/chats/' + self.chat['id']
|
|
|
|
def worker(index, actor):
|
|
# Establish threads and touch HTTP before releasing the simultaneous burst.
|
|
if self.http('GET', '/health/live')['status'] != 200:
|
|
raise RuntimeError('Worker warmup failed')
|
|
barrier.wait()
|
|
try:
|
|
result = self.http('GET', path, actor)
|
|
expected = 404 if actor['label'] == 'eve' else 200
|
|
ok = result['status'] == expected and (expected == 404 or result['data'] == self.chat)
|
|
status, ms, rid, stamp = result['status'], result['duration_ms'], result['request_id'], result['timestamp_utc']
|
|
except (OSError, http.client.HTTPException):
|
|
ok, status, ms, rid, stamp = False, 0, 5000, '', now()
|
|
return dict(phase=phase, mode=mode, repetition=repetition, worker=index, actor=actor['label'],
|
|
timestamp_utc=stamp, status=status, duration_ms=ms, request_id=rid, correct=ok)
|
|
|
|
with futures.ThreadPoolExecutor(max_workers=len(actors)) as pool:
|
|
jobs = [pool.submit(worker, i, actor) for i, actor in enumerate(actors)]
|
|
controlled = mode in ('cold', 'mixed_cold')
|
|
with self.lock() if controlled else contextlib.nullcontext():
|
|
barrier.wait()
|
|
started = time.perf_counter()
|
|
if controlled:
|
|
time.sleep(self.args.lock_ms / 1000)
|
|
group = [job.result(timeout=10) for job in jobs]
|
|
elapsed = time.perf_counter() - started
|
|
after = self.snapshot()
|
|
errors = sum(not row['correct'] for row in group)
|
|
durations = [row['duration_ms'] for row in group]
|
|
measured = dict(phase=phase, mode=mode, repetition=repetition, requests=len(group), errors=errors,
|
|
elapsed_ms=round(elapsed*1000, 3), successful_rps=round((len(group)-errors)/elapsed, 3),
|
|
latency_ms={f'p{p}': percentile(durations, p/100) for p in (50,95,99)},
|
|
sql_before=before, sql_after=after, **self.delta(before, after))
|
|
self.requests.extend(group)
|
|
self.runs.append(measured)
|
|
self.collect_logs(since, f'{phase}/{mode}/{repetition}')
|
|
print(f'{phase}/{mode}/{repetition}: chat SQL={measured["chat_sql"]}, session SQL={measured["session_sql"]}, p95={measured["latency_ms"]["p95"]} ms, errors={errors}', flush=True)
|
|
if errors:
|
|
raise RuntimeError('Incorrect responses during burst')
|
|
if mode == 'cold' and phase == 'after' and measured['chat_sql'] != 1:
|
|
raise RuntimeError('Expected exactly one chat SELECT with singleflight')
|
|
if mode in ('warm', 'mixed_warm') and measured['chat_sql'] != 0:
|
|
raise RuntimeError('Warm cache unexpectedly queried chats')
|
|
if measured['session_sql'] != len(actors):
|
|
raise RuntimeError('Session SQL accounting differs from request count; check concurrent traffic')
|
|
time.sleep(.3)
|
|
|
|
def checked(self, name, method, path, actor, body=None, status=200):
|
|
result = self.http(method, path, actor, body)
|
|
if result['status'] != status:
|
|
raise RuntimeError('Functional check failed: ' + name)
|
|
self.checks.append(dict(name=name, status=result['status'], duration_ms=result['duration_ms'], request_id=result['request_id']))
|
|
return result['data']
|
|
|
|
def cancellation(self):
|
|
self.redis('DEL', self.key)
|
|
before, since = self.snapshot(), now()
|
|
path = '/api/v1/chats/' + self.chat['id']
|
|
with futures.ThreadPoolExecutor(max_workers=1) as pool:
|
|
with self.lock():
|
|
leader = http.client.HTTPConnection('127.0.0.1', self.port, timeout=5)
|
|
try:
|
|
leader.request('GET', path, headers={'Authorization': 'Bearer ' + self.alice['token']})
|
|
deadline = time.monotonic() + 1
|
|
while self.sql("SELECT count(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND wait_event_type='Lock' AND query LIKE 'SELECT c.id%';") == '0':
|
|
if time.monotonic() > deadline:
|
|
raise RuntimeError('Leader did not reach SQL')
|
|
time.sleep(.01)
|
|
leader.sock.shutdown(socket.SHUT_RDWR)
|
|
finally:
|
|
leader.close()
|
|
survivor = pool.submit(self.http, 'GET', path, self.alice)
|
|
time.sleep(.1)
|
|
result = survivor.result(timeout=5)
|
|
counters = self.delta(before, self.snapshot())
|
|
if result['status'] != 200 or result['data'] != self.chat or counters['chat_sql'] != 1:
|
|
raise RuntimeError('Client cancellation broke shared loading')
|
|
self.checks.append(dict(name='leader_cancelled_survivor_succeeded', status=result['status'],
|
|
request_id=result['request_id'], duration_ms=result['duration_ms'], **counters))
|
|
self.collect_logs(since, 'Cancelled initiating client; surviving client received chat')
|
|
|
|
def functional(self):
|
|
self.cancellation()
|
|
path = '/api/v1/chats/' + self.chat['id']
|
|
self.burst('after', 'mixed_cold', 1, [self.eve, self.alice, self.bob] * 4)
|
|
self.burst('after', 'mixed_warm', 1, [self.eve, self.alice, self.bob] * 4)
|
|
message = self.checked('message_committed', 'POST', path+'/messages', self.alice,
|
|
{'text': 'Synthetic read-after-write evidence'}, status=201)
|
|
history = self.checked('read_after_write', 'GET', path+'/messages?limit=100', self.bob)
|
|
if message['id'] not in [item['id'] for item in history['items']]:
|
|
raise RuntimeError('Committed message missing from following read')
|
|
since = now()
|
|
api_before = run(NORMAL + ['ps', '-q', 'api'])
|
|
self.summary['key_before_restart'] = self.redis('EXISTS', self.key)
|
|
try:
|
|
run(NORMAL + ['stop', 'redis'])
|
|
self.checked('redis_unavailable', 'GET', path, self.alice)
|
|
finally:
|
|
run(NORMAL + ['start', 'redis'])
|
|
deadline = time.monotonic()+30
|
|
while self.redis('PING') != 'PONG':
|
|
if time.monotonic()>deadline:
|
|
raise RuntimeError('Redis recovery timeout')
|
|
time.sleep(.3)
|
|
self.summary['key_after_restart'] = self.redis('EXISTS', self.key)
|
|
self.checked('redis_recovered', 'GET', path, self.bob)
|
|
history = self.checked('message_after_restart', 'GET', path+'/messages?limit=100', self.bob)
|
|
if message['id'] not in [item['id'] for item in history['items']]:
|
|
raise RuntimeError('Message lost after Redis restart')
|
|
if api_before != run(NORMAL + ['ps', '-q', 'api']):
|
|
raise RuntimeError('API restarted during Redis recovery')
|
|
self.collect_logs(since, 'Redis stop/start and read-after-write')
|
|
self.burst('after', 'cold', self.args.repetitions+1)
|
|
|
|
@contextlib.contextmanager
|
|
def sample_resources(self, phase):
|
|
stop = threading.Event()
|
|
containers = [run(NORMAL + ['ps', '-q', name]) for name in ('api','postgres','redis')]
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
try:
|
|
output = run(['docker','stats','--no-stream','--format','{{json .}}'] + containers, timeout=10)
|
|
values = [json.loads(line) for line in output.splitlines() if line.strip()]
|
|
self.resources.append(dict(phase=phase, timestamp_utc=now(), containers=[
|
|
{key: value[key] for key in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')} for value in values]))
|
|
except (RuntimeError, subprocess.TimeoutExpired):
|
|
self.resources.append(dict(phase=phase, timestamp_utc=now(), error='resource sampling failed'))
|
|
stop.wait(.2)
|
|
|
|
thread = threading.Thread(target=sample, daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield
|
|
finally:
|
|
stop.set()
|
|
thread.join(timeout=12)
|
|
|
|
def execute(self):
|
|
self.use_image(self.args.before_image)
|
|
extension_created = False
|
|
try:
|
|
self.compose('up', '-d', '--no-build', '--force-recreate', 'postgres')
|
|
self.wait_postgres()
|
|
if not self.had_extension:
|
|
self.sql('CREATE EXTENSION pg_stat_statements;')
|
|
extension_created = True
|
|
run(NORMAL + ['start', 'redis'])
|
|
for phase, image in [('before', self.args.before_image), ('after', self.args.after_image)]:
|
|
self.use_image(image)
|
|
self.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate', 'api')
|
|
self.ready()
|
|
if phase == 'before':
|
|
self.alice, self.bob, self.eve = [self.actor(label) for label in ('alice', 'bob', 'eve')]
|
|
self.chat = self.checked('chat_created', 'POST', '/api/v1/chats', self.alice,
|
|
{'title':'Stampede experiment', 'member_ids':[self.alice['id'],self.bob['id']]}, status=201)
|
|
self.key = 'chat:' + self.chat['id']
|
|
self.summary['chat_id'] = self.chat['id']
|
|
observed = self.checked(phase+'_chat_read_after_create', 'GET', '/api/v1/chats/'+self.chat['id'], self.bob)
|
|
if observed != self.chat:
|
|
raise RuntimeError('Chat read-after-create differs')
|
|
self.summary.setdefault('container_limits', {})[phase] = json.loads(run(['docker','inspect','--format',
|
|
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}},"cpu_quota":{{.HostConfig.CpuQuota}}}',
|
|
run(NORMAL+['ps','-q','api'])]))
|
|
with self.sample_resources(phase):
|
|
for repetition in range(1, self.args.repetitions+1):
|
|
self.burst(phase, 'cold', repetition)
|
|
for repetition in range(1, 6):
|
|
self.burst(phase, 'warm', repetition)
|
|
if not any(r['chat_sql'] > 1 for r in self.runs if r['phase']=='before' and r['mode']=='cold'):
|
|
raise RuntimeError('Baseline stampede not reproduced')
|
|
self.functional()
|
|
self.summary['success'] = True
|
|
finally:
|
|
if extension_created:
|
|
self.sql('DROP EXTENSION pg_stat_statements;')
|
|
# Restore the original Compose command and fixed normal API image.
|
|
run(NORMAL + ['up','-d','--no-build','--force-recreate','postgres'])
|
|
self.wait_postgres()
|
|
run(NORMAL + ['start','redis'])
|
|
run(NORMAL + ['up','-d','--no-build','--no-deps','--force-recreate','api'])
|
|
self.ready()
|
|
self.summary['restored_preload'] = self.sql('SHOW shared_preload_libraries;')
|
|
self.summary['extension_restored'] = (self.sql("SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements';") == ('1' if self.had_extension else '0'))
|
|
self.logs.append('\n# Final Compose state\n'+run(NORMAL+['ps','-a'])+'\n')
|
|
self.summary['ended_utc'] = now()
|
|
self.flush()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--before-image', default='mtusi-messenger:stampede-before')
|
|
parser.add_argument('--after-image', default='mtusi-messenger:stampede-after')
|
|
parser.add_argument('--concurrency', type=int, default=32)
|
|
parser.add_argument('--repetitions', type=int, default=10)
|
|
parser.add_argument('--lock-ms', type=int, default=200)
|
|
parser.add_argument('--output', default='evidence/cache-stampede')
|
|
args = parser.parse_args()
|
|
if not (2 <= args.concurrency <= 64 and 1 <= args.repetitions <= 50 and 50 <= args.lock_ms <= 500):
|
|
parser.error('Use concurrency 2..64, repetitions 1..50, lock-ms 50..500')
|
|
with tempfile.TemporaryDirectory(prefix='cache-stampede-') as directory:
|
|
Experiment(args, pathlib.Path(directory)/'compose.json').execute()
|
|
print('Experiment complete; normal Compose restored.', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|