forked from mtusi2026/mtusi2026lab2
Complete lab
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
"""Track 1: deterministic stale-fill race and measured reads on real PostgreSQL/Redis.
|
||||
|
||||
The unsafe cache exists ONLY in this experiment; API always uses versioned keys.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import resource
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, event, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from src.auth import sessions
|
||||
from src.cache import HistoryCache
|
||||
from src.database import create_db_engine
|
||||
from src.models import User, Chat, ChatMember
|
||||
from src.store import PostgresStore
|
||||
from .fixtures import seed_messages
|
||||
|
||||
|
||||
class NaiveCache(HistoryCache):
|
||||
def key(self, chat_id, version, limit, after):
|
||||
return super().key(chat_id, 0, limit, after)
|
||||
|
||||
|
||||
class NoCache:
|
||||
def get_or_load(self, chat_id, version, limit, after, load):
|
||||
return load()
|
||||
|
||||
|
||||
class QueryCounter:
|
||||
lock = threading.Lock()
|
||||
counts = {'sql': 0, 'history_selects': 0}
|
||||
|
||||
@classmethod
|
||||
def record(cls, conn, cursor, statement, parameters, context, executemany):
|
||||
with cls.lock:
|
||||
cls.counts['sql'] += 1
|
||||
normalized = ' '.join(statement.split()).lower()
|
||||
if normalized.startswith('select ') and ' from messages ' in normalized:
|
||||
cls.counts['history_selects'] += 1
|
||||
|
||||
@classmethod
|
||||
def reset(cls):
|
||||
with cls.lock:
|
||||
cls.counts = {'sql': 0, 'history_selects': 0}
|
||||
|
||||
|
||||
def make_store(engine, cache):
|
||||
store = PostgresStore(cache)
|
||||
store.sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
return store
|
||||
|
||||
|
||||
def stale_race(reader, writer, actor_id, chat_id, naive=False):
|
||||
"""Pause AFTER the real history SELECT, commit a write, then finish cache SET."""
|
||||
loaded, resume = threading.Event(), threading.Event()
|
||||
original_load = reader._load_messages
|
||||
|
||||
def paused_load(*args):
|
||||
page = original_load(*args)
|
||||
loaded.set()
|
||||
if not resume.wait(10):
|
||||
raise AssertionError('Race writer did not finish')
|
||||
return page
|
||||
|
||||
reader._load_messages = paused_load
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(reader.list_messages, chat_id, actor_id, 100, None)
|
||||
try:
|
||||
assert loaded.wait(10), 'Reader did not reach the cache-fill barrier'
|
||||
message = writer.create_message(chat_id, actor_id, 'committed during cache fill')
|
||||
if naive:
|
||||
reader.cache.client.delete(reader.cache.key(chat_id, 0, 100, None))
|
||||
finally:
|
||||
resume.set()
|
||||
overlapping_page = pending.result(timeout=10)
|
||||
finally:
|
||||
reader._load_messages = original_load
|
||||
assert message['id'] not in {item['id'] for item in overlapping_page['items']}
|
||||
started = time.perf_counter()
|
||||
page = reader.list_messages(chat_id, actor_id, 100, None)
|
||||
return {'stale': message['id'] not in {item['id'] for item in page['items']},
|
||||
'read_after_write_ms': round((time.perf_counter() - started) * 1000, 3)}
|
||||
|
||||
|
||||
def benchmark(store, actor_id, chat_id, requests=300, concurrency=8):
|
||||
if isinstance(store.cache, HistoryCache):
|
||||
for key in store.cache.client.scan_iter(match=f'{store.cache.prefix}:{chat_id}:*'):
|
||||
store.cache.client.delete(key)
|
||||
expected = store.list_messages(chat_id, actor_id, 100, None)
|
||||
QueryCounter.reset()
|
||||
started = time.perf_counter()
|
||||
cpu_started = time.process_time()
|
||||
|
||||
def read(_):
|
||||
tick = time.perf_counter()
|
||||
page = store.list_messages(chat_id, actor_id, 100, None)
|
||||
assert page == expected
|
||||
return (time.perf_counter() - tick) * 1000
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
latency = sorted(executor.map(read, range(requests)))
|
||||
elapsed = time.perf_counter() - started
|
||||
result = {
|
||||
'requests': requests, 'concurrency': concurrency, 'errors': 0,
|
||||
'seconds': round(elapsed, 3), 'successful_rps': round(requests / elapsed, 2),
|
||||
'client_cpu_seconds': round(time.process_time() - cpu_started, 3),
|
||||
'client_peak_rss_mib': round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 2),
|
||||
**QueryCounter.counts,
|
||||
}
|
||||
for percentile in (50, 95, 99):
|
||||
result[f'p{percentile}_ms'] = round(latency[math.ceil(len(latency) * percentile / 100) - 1], 3)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
prefix = 'experiment:' + uuid4().hex
|
||||
engine = create_db_engine()
|
||||
event.listen(engine, 'before_cursor_execute', QueryCounter.record)
|
||||
writer = make_store(engine, NoCache())
|
||||
user = writer.create_user('exp_' + uuid4().hex[:12], 'Experiment', 'Synthetic-password-123')
|
||||
result = {'race_repeats': 5, 'race': {}, 'benchmark': {}, 'messages': 10000,
|
||||
'page_size': 100, 'warmup_reads_per_run': 1,
|
||||
'scope': 'store calls; real SQL and Redis; excludes HTTP, sessions and KDF; SQL excludes BEGIN/COMMIT'}
|
||||
try:
|
||||
for name, cache_type in [('naive', NaiveCache), ('versioned', HistoryCache)]:
|
||||
results = []
|
||||
for n in range(5):
|
||||
chat = writer.create_chat(user['id'], 'Race', [user['id']])
|
||||
reader = make_store(engine, cache_type(sessions.client, prefix=f'{prefix}:{name}:{n}'))
|
||||
QueryCounter.reset()
|
||||
row = stale_race(reader, writer, user['id'], chat['id'], naive=name == 'naive')
|
||||
row.update(QueryCounter.counts)
|
||||
assert row['stale'] == (name == 'naive')
|
||||
results.append(row)
|
||||
result['race'][name] = results
|
||||
|
||||
chat = writer.create_chat(user['id'], 'Benchmark', [user['id']])
|
||||
with writer.sessions.begin() as session:
|
||||
seed_messages(session, chat['id'], user['id'])
|
||||
modes = {
|
||||
'uncached': make_store(engine, NoCache()),
|
||||
'naive': make_store(engine, NaiveCache(sessions.client, prefix=prefix + ':bench-naive')),
|
||||
'versioned': make_store(engine, HistoryCache(sessions.client, prefix=prefix + ':bench-versioned')),
|
||||
}
|
||||
result['benchmark'] = {name: [] for name in modes}
|
||||
names = list(modes)
|
||||
for repeat in range(3):
|
||||
for name in names[repeat:] + names[:repeat]:
|
||||
result['benchmark'][name].append(benchmark(modes[name], user['id'], chat['id']))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
for key in sessions.client.scan_iter(match=prefix + ':*'):
|
||||
sessions.client.delete(key)
|
||||
with writer.sessions.begin() as session:
|
||||
session.execute(delete(Chat).where(Chat.id.in_(
|
||||
select(ChatMember.chat_id).where(ChatMember.user_id == user['id']))))
|
||||
session.execute(delete(User).where(User.id == user['id']))
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""EXPLAIN the actual ORM history query on 10,000 rows, then roll back fixtures."""
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.database import create_db_engine
|
||||
from src.models import User, Chat, Message
|
||||
from src.store import history_query
|
||||
from .fixtures import seed_messages
|
||||
|
||||
|
||||
def main():
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
user = User(username='explain_' + uuid4().hex[:12], display_name='Explain', password_hash='unused')
|
||||
session.add(user)
|
||||
session.flush()
|
||||
chat = Chat(title='Explain')
|
||||
session.add(chat)
|
||||
session.flush()
|
||||
seed_messages(session, chat.id, user.id)
|
||||
conn = session.connection()
|
||||
conn.exec_driver_sql('ANALYZE messages')
|
||||
boundary = session.execute(select(Message.created_at, Message.id).where(
|
||||
Message.chat_id == chat.id).order_by(Message.created_at, Message.id).offset(8999).limit(1)).one()
|
||||
after = (boundary.created_at.isoformat(), boundary.id)
|
||||
compiled = history_query(chat.id, 50, after).compile(dialect=engine.dialect)
|
||||
print('Dataset: 10000 messages; page after message 9000; ORM query; limit 51')
|
||||
for row in conn.exec_driver_sql('EXPLAIN (ANALYZE, BUFFERS) ' + str(compiled), compiled.params):
|
||||
print(row[0])
|
||||
session.rollback()
|
||||
print('Synthetic data rolled back.')
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Synthetic message history shared by EXPLAIN and cache measurements."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import insert, update
|
||||
|
||||
from src.models import Chat, Message
|
||||
|
||||
|
||||
def seed_messages(session, chat_id, user_id, count=10000):
|
||||
start = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
session.execute(insert(Message), [
|
||||
{'chat_id': chat_id, 'sender_id': user_id, 'text': 'synthetic ' * 100,
|
||||
'created_at': start + timedelta(milliseconds=n)}
|
||||
for n in range(1, count + 1)
|
||||
])
|
||||
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||
history_version=Chat.history_version + 1))
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\restart.py
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Restart the whole lab stack, retaining volumes and recording actual start times."""
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
|
||||
return subprocess.check_output(
|
||||
['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids],
|
||||
cwd=ROOT, text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
before = snapshot()
|
||||
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
|
||||
cwd=ROOT, check=True, timeout=60)
|
||||
after = snapshot()
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
(ROOT / 'evidence' / 'restart.txt').write_text(
|
||||
'Command: docker compose restart api postgres redis\nBefore (UTC):\n' + before
|
||||
+ 'After (UTC):\n' + after, encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/restart.py
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Run on the host: research measurements and a real Redis restart through HTTP."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8', timeout=180)
|
||||
|
||||
|
||||
def cache_state(chat_id, clear=False):
|
||||
code = """import json, sys
|
||||
from uuid import UUID
|
||||
from src.auth import sessions
|
||||
keys = list(sessions.client.scan_iter(match='history:' + str(UUID(sys.argv[1])) + ':*'))
|
||||
print(json.dumps({'keys': len(keys)}))
|
||||
if sys.argv[2] == 'clear' and keys:
|
||||
sessions.client.delete(*keys)
|
||||
"""
|
||||
return json.loads(command('docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code,
|
||||
chat_id, 'clear' if clear else 'inspect'))
|
||||
|
||||
|
||||
def restart_check():
|
||||
client = Client('http://127.0.0.1:8080')
|
||||
actor = client.user('restartcache')
|
||||
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
||||
body={'title': 'Cache restart', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
def read():
|
||||
return client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
first = client.message(actor, chat, 'before Redis restart')['json']
|
||||
assert read()['items'] == [first]
|
||||
cached_before = cache_state(chat['id'])['keys']
|
||||
assert cached_before > 0
|
||||
second = client.message(actor, chat, 'committed after cache fill')['json']
|
||||
redis_id = command('docker', 'compose', 'ps', '-q', 'redis').strip()
|
||||
before_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip()
|
||||
started = time.monotonic()
|
||||
command('docker', 'compose', 'restart', 'redis')
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call('GET', '/health/ready')
|
||||
break
|
||||
except AssertionError:
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise AssertionError('Redis recovery timed out')
|
||||
after_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip()
|
||||
assert before_start != after_start
|
||||
assert read()['items'] == [first, second]
|
||||
recovery = round(time.monotonic() - started, 3)
|
||||
cache_state(chat['id'], clear=True)
|
||||
assert cache_state(chat['id'])['keys'] == 0
|
||||
assert read()['items'] == [first, second]
|
||||
assert cache_state(chat['id'])['keys'] > 0
|
||||
third = client.message(actor, chat, 'after Redis recovery')['json']
|
||||
assert read()['items'] == [first, second, third]
|
||||
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
client.call('GET', path, 401, actor=actor)
|
||||
return {'status': 'PASS', 'same_session': True, 'confirmed_messages': 3,
|
||||
'cold_cache_rebuilt': True, 'read_after_write': True, 'logout_still_401': True,
|
||||
'recovery_seconds': recovery, 'redis_start_before_utc': before_start,
|
||||
'redis_start_after_utc': after_start}
|
||||
|
||||
|
||||
def main():
|
||||
evidence = ROOT / 'evidence'
|
||||
evidence.mkdir(exist_ok=True)
|
||||
result = json.loads(command('docker', 'compose', 'exec', '-T', 'api',
|
||||
'python', '-m', 'scripts.cache_experiment'))
|
||||
(evidence / 'cache-experiment.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
||||
result = restart_check()
|
||||
(evidence / 'cache-restart.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
||||
ids = command('docker', 'compose', 'ps', '-q').split()
|
||||
snapshot = command('docker', 'stats', '--no-stream', '--format', '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}', *ids)
|
||||
(evidence / 'cache-resources.txt').write_text('Post-experiment snapshot (not peak):\n' + snapshot, encoding='utf-8')
|
||||
print('PASS: stale race reproduced before protection and prevented after; SQL/latency measured.')
|
||||
print('PASS: Redis restart, same session, read-after-write, cold cache rebuild, logout.')
|
||||
print('Evidence: cache-experiment.json, cache-restart.json, cache-resources.txt')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -9,7 +9,6 @@ root = Path(__file__).resolve().parents[1]
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text())
|
||||
validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
|
||||
Reference in New Issue
Block a user