#!/usr/bin/env python3 """Track 1: stale first page, then versioned cache and singleflight. The script switches HISTORY_CACHE, recreates only the API, and always restores the previous .env. It does not print tokens. """ import base64 import json import os import subprocess import sys import threading import time import urllib.error import urllib.request import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path ROOT = Path(__file__).resolve().parents[1] BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/') ENV_PATH = ROOT / '.env' EVIDENCE_PATH = ROOT / 'evidence' / 'track1.json' COLD_CLIENTS = 32 READERS = 8 WARM_READS = 20 def call(method, path, body=None, headers=None, timeout=20): data = None if body is None else json.dumps(body).encode('utf-8') req = urllib.request.Request(BASE_URL + path, data=data, method=method) if body is not None: req.add_header('Content-Type', 'application/json') for key, value in (headers or {}).items(): req.add_header(key, value) started = time.perf_counter() try: with urllib.request.urlopen(req, timeout=timeout) as response: payload = response.read() elapsed_ms = (time.perf_counter() - started) * 1000 return response.status, payload, elapsed_ms except urllib.error.HTTPError as exc: elapsed_ms = (time.perf_counter() - started) * 1000 return exc.code, exc.read(), elapsed_ms def compose(*args, check=True): proc = subprocess.run( ['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=False, capture_output=True, text=True, ) if check and proc.returncode != 0: sys.stderr.write(proc.stderr) raise SystemExit('docker compose %s failed' % ' '.join(args)) return proc def psql(sql): proc = compose( 'exec', '-T', 'postgres', 'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'), '-d', os.environ.get('POSTGRES_DB', 'messenger'), '-v', 'ON_ERROR_STOP=1', '-tAc', sql, ) return proc.stdout.strip() def redis_cli(*args, check=True): proc = compose('exec', '-T', 'redis', 'redis-cli', *args, check=check) return proc def wait_ready(seconds=90): deadline = time.monotonic() + seconds while time.monotonic() < deadline: try: status, _, _ = call('GET', '/health/ready', timeout=3) if status == 200: return except Exception: pass time.sleep(0.5) raise SystemExit('API did not become ready') def wait_redis(seconds=40): deadline = time.monotonic() + seconds while time.monotonic() < deadline: proc = redis_cli('ping', check=False) if proc.returncode == 0 and 'PONG' in proc.stdout: return time.sleep(0.3) raise SystemExit('redis did not answer PING') def message_scans(): raw = psql( "SELECT COALESCE(idx_scan,0)+COALESCE(seq_scan,0) " "FROM pg_stat_user_tables WHERE relname = 'messages'" ) if not raw: raise SystemExit('messages stats are missing') return int(raw) def drained_message_scans(): """Backends publish table stats on exit. Drop the ones that ran the workload.""" raw = psql( "SELECT pid FROM pg_stat_activity " "WHERE datname = current_database() AND pid <> pg_backend_pid() " "AND backend_type = 'client backend'" ) pids = [line.strip() for line in raw.splitlines() if line.strip()] if pids: if not all(pid.isdigit() for pid in pids): raise SystemExit('unexpected pid list') listed = ','.join(pids) psql('SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid IN (%s)' % listed) deadline = time.monotonic() + 5 while time.monotonic() < deadline: still = psql('SELECT count(*) FROM pg_stat_activity WHERE pid IN (%s)' % listed) if still == '0': break time.sleep(0.05) else: raise SystemExit('postgres backends did not disconnect') return message_scans() def p50(samples): ordered = sorted(samples) return ordered[(len(ordered) - 1) // 2] def bearer(token): return {'Authorization': 'Bearer ' + token} def write_mode(mode): text = ENV_PATH.read_text(encoding='utf-8') newline = '\r\n' if '\r\n' in text else '\n' lines = [line for line in text.splitlines() if not line.startswith('HISTORY_CACHE=')] lines.append('HISTORY_CACHE=' + mode) ENV_PATH.write_bytes((newline.join(lines) + newline).encode('utf-8')) def recreate_api(build=False): args = ['up', '-d', '--force-recreate', '--no-deps'] if build: args = ['up', '-d', '--build', '--force-recreate', '--no-deps'] args.append('api') compose(*args) wait_ready() def flush_history(): keys = [] for pattern in ('history:*', 'hver:*'): proc = redis_cli('--scan', '--pattern', pattern) keys.extend(line.strip() for line in proc.stdout.splitlines() if line.strip()) if keys: redis_cli('DEL', *keys) def register_and_chat(): username = 'u_t1_%s' % uuid.uuid4().hex[:12] password = 'T3st-' + uuid.uuid4().hex status, raw, _ = call('POST', '/api/v1/users', { 'username': username, 'display_name': 'track1', 'password': password, }) if status != 201: raise SystemExit('register failed: %s' % status) user = json.loads(raw) basic = base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii') status, raw, _ = call( 'POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic}, ) if status != 201: raise SystemExit('login failed: %s' % status) token = json.loads(raw)['token'] status, raw, _ = call( 'POST', '/api/v1/chats', {'title': 'track1', 'member_ids': [user['id']]}, headers=bearer(token), ) if status != 201: raise SystemExit('chat failed: %s' % status) chat_id = json.loads(raw)['id'] status, _, _ = call( 'POST', '/api/v1/chats/%s/messages' % chat_id, {'text': 'seed'}, headers=bearer(token), ) if status != 201: raise SystemExit('seed message failed: %s' % status) return token, chat_id def reads_after_write(token, chat_id): status, _, _ = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) if status != 200: raise SystemExit('warm read failed: %s' % status) post_done = threading.Event() posted = {} def poster(): status, raw, _ = call( 'POST', '/api/v1/chats/%s/messages' % chat_id, {'text': 'confirmed-' + uuid.uuid4().hex[:8]}, headers=bearer(token), ) posted['status'] = status if status == 201: posted['id'] = json.loads(raw)['id'] post_done.set() def reader(_index): if not post_done.wait(20): return None status, raw, _ = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) if status != 200: return None ids = [item['id'] for item in json.loads(raw)['items']] return posted.get('id') in ids with ThreadPoolExecutor(max_workers=READERS + 1) as pool: post = pool.submit(poster) jobs = [pool.submit(reader, index) for index in range(READERS)] post.result() seen = [job.result() for job in jobs] if posted.get('status') != 201 or 'id' not in posted: raise SystemExit('post failed: %s' % posted.get('status')) present = sum(1 for item in seen if item is True) missing = sum(1 for item in seen if item is False) return { 'reads_started_after_201': present + missing, 'reads_missing_new_message': missing, 'message_id': posted['id'], } def cold_reads(token, chat_id): flush_history() before = drained_message_scans() timings = [] failures = [] def one(_index): status, _, elapsed = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) timings.append(elapsed) if status != 200: failures.append(status) with ThreadPoolExecutor(max_workers=COLD_CLIENTS) as pool: list(pool.map(one, range(COLD_CLIENTS))) if failures: raise SystemExit('cold reads failed: %s' % failures[:3]) return { 'clients': COLD_CLIENTS, 'message_scans': drained_message_scans() - before, 'p50_ms': round(p50(timings), 3), } def miss_latency(token, chat_id, samples=12): """One GET after a flush. The clock starts at the request, not at the flush.""" timings = [] for _ in range(samples): flush_history() status, _, elapsed = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) if status != 200: raise SystemExit('cold miss failed: %s' % status) timings.append(elapsed) return round(p50(timings), 3) def warm_reads(token, chat_id): status, _, _ = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) if status != 200: raise SystemExit('warm fill failed: %s' % status) before = drained_message_scans() timings = [] for _ in range(WARM_READS): status, _, elapsed = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) if status != 200: raise SystemExit('warm read failed: %s' % status) timings.append(elapsed) return { 'clients': WARM_READS, 'message_scans': drained_message_scans() - before, 'p50_ms': round(p50(timings), 3), } def redis_restart(token, chat_id, message_id): compose('exec', '-T', 'redis', 'sh', '-c', 'rm -f /data/dump.rdb /data/appendonly.aof') compose('restart', 'redis') wait_redis() time.sleep(0.5) session_status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token)) before = drained_message_scans() status, raw, _ = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) first_delta = drained_message_scans() - before present = False if status == 200: present = message_id in [item['id'] for item in json.loads(raw)['items']] before = drained_message_scans() status, _, _ = call( 'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id, headers=bearer(token), ) second_delta = drained_message_scans() - before return { 'confirmed_message_present': present and status == 200, 'first_read_message_scans': first_delta, 'second_read_message_scans': second_delta, 'session_status': session_status, } def problems_of(evidence): found = [] naive = evidence['naive'] protected = evidence['protected'] if naive['reads_missing_new_message'] < 1: found.append('naive did not show a stale read') if protected['reads_missing_new_message'] != 0: found.append('protected missed a confirmed message') if evidence['off']['message_scans'] != WARM_READS: found.append('off scans %s' % evidence['off']['message_scans']) if naive['cold']['message_scans'] < COLD_CLIENTS - 8: found.append('naive cold scans %s' % naive['cold']['message_scans']) if protected['cold']['message_scans'] > 2: found.append('protected cold scans %s' % protected['cold']['message_scans']) if protected['warm']['message_scans'] != 0: found.append('warm scans %s' % protected['warm']['message_scans']) if protected['warm']['p50_ms'] >= protected['sequential_miss_p50_ms']: found.append('warm p50 is not below a cold miss') restart = protected['after_redis_restart'] if not restart['confirmed_message_present']: found.append('message missing after redis restart') if restart['first_read_message_scans'] < 1: found.append('restart was not a miss') if restart['second_read_message_scans'] != 0: found.append('second read after restart was not a hit') if restart['session_status'] != 200: found.append('session after redis restart: %s' % restart['session_status']) return found def main(): if not ENV_PATH.is_file(): raise SystemExit('.env is missing') original = ENV_PATH.read_bytes() failure = None try: print('naive mode', flush=True) write_mode('naive') compose('up', '-d', '--build') wait_ready() token, chat_id = register_and_chat() naive_race = reads_after_write(token, chat_id) naive_cold = cold_reads(token, chat_id) print('off mode', flush=True) write_mode('off') recreate_api(build=False) off_repeat = warm_reads(token, chat_id) print('protected mode', flush=True) write_mode('protected') recreate_api(build=False) protected_race = reads_after_write(token, chat_id) protected_cold = cold_reads(token, chat_id) protected_miss_p50 = miss_latency(token, chat_id) protected_warm = warm_reads(token, chat_id) print('redis restart', flush=True) restarted = redis_restart(token, chat_id, protected_race['message_id']) evidence = { 'track': 1, 'cached_limit': 50, 'page_ttl_seconds': 30, 'naive_fill_hold_seconds': 0.2, 'naive': { 'reads_started_after_201': naive_race['reads_started_after_201'], 'reads_missing_new_message': naive_race['reads_missing_new_message'], 'stale_read_observed': naive_race['reads_missing_new_message'] > 0, 'cold': naive_cold, }, 'off': off_repeat, 'protected': { 'reads_started_after_201': protected_race['reads_started_after_201'], 'reads_missing_new_message': protected_race['reads_missing_new_message'], 'read_after_write': protected_race['reads_missing_new_message'] == 0, 'cold': protected_cold, 'sequential_miss_p50_ms': protected_miss_p50, 'warm': protected_warm, 'after_redis_restart': restarted, }, } found = problems_of(evidence) print(json.dumps(evidence, indent=2), flush=True) if found: raise SystemExit('track1 failed: ' + '; '.join(found)) EVIDENCE_PATH.parent.mkdir(parents=True, exist_ok=True) EVIDENCE_PATH.write_text(json.dumps(evidence, indent=2) + '\n', encoding='utf-8') print('wrote evidence/track1.json', flush=True) except SystemExit as exc: failure = exc finally: ENV_PATH.write_bytes(original) print('restoring protected api', flush=True) compose('up', '-d', '--force-recreate', '--no-deps', 'api') wait_ready() compose('start', 'redis') compose('start', 'postgres') if failure is not None: raise failure if __name__ == '__main__': main()