Complete lab

This commit is contained in:
2026-10-09 16:11:06 +03:00
parent d9fe67a8ce
commit 5de73c5475
48 changed files with 1749 additions and 203 deletions
+168
View File
@@ -0,0 +1,168 @@
"""Track 1: deterministic stale-fill race and measured reads on real PostgreSQL/Redis.
The unsafe cache exists ONLY in this experiment; API always uses versioned keys.
"""
import json
import math
import resource
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from sqlalchemy import delete, event, select
from sqlalchemy.orm import sessionmaker
from src.auth import sessions
from src.cache import HistoryCache
from src.database import create_db_engine
from src.models import User, Chat, ChatMember
from src.store import PostgresStore
from .fixtures import seed_messages
class NaiveCache(HistoryCache):
def key(self, chat_id, version, limit, after):
return super().key(chat_id, 0, limit, after)
class NoCache:
def get_or_load(self, chat_id, version, limit, after, load):
return load()
class QueryCounter:
lock = threading.Lock()
counts = {'sql': 0, 'history_selects': 0}
@classmethod
def record(cls, conn, cursor, statement, parameters, context, executemany):
with cls.lock:
cls.counts['sql'] += 1
normalized = ' '.join(statement.split()).lower()
if normalized.startswith('select ') and ' from messages ' in normalized:
cls.counts['history_selects'] += 1
@classmethod
def reset(cls):
with cls.lock:
cls.counts = {'sql': 0, 'history_selects': 0}
def make_store(engine, cache):
store = PostgresStore(cache)
store.sessions = sessionmaker(engine, expire_on_commit=False)
return store
def stale_race(reader, writer, actor_id, chat_id, naive=False):
"""Pause AFTER the real history SELECT, commit a write, then finish cache SET."""
loaded, resume = threading.Event(), threading.Event()
original_load = reader._load_messages
def paused_load(*args):
page = original_load(*args)
loaded.set()
if not resume.wait(10):
raise AssertionError('Race writer did not finish')
return page
reader._load_messages = paused_load
try:
with ThreadPoolExecutor(max_workers=1) as executor:
pending = executor.submit(reader.list_messages, chat_id, actor_id, 100, None)
try:
assert loaded.wait(10), 'Reader did not reach the cache-fill barrier'
message = writer.create_message(chat_id, actor_id, 'committed during cache fill')
if naive:
reader.cache.client.delete(reader.cache.key(chat_id, 0, 100, None))
finally:
resume.set()
overlapping_page = pending.result(timeout=10)
finally:
reader._load_messages = original_load
assert message['id'] not in {item['id'] for item in overlapping_page['items']}
started = time.perf_counter()
page = reader.list_messages(chat_id, actor_id, 100, None)
return {'stale': message['id'] not in {item['id'] for item in page['items']},
'read_after_write_ms': round((time.perf_counter() - started) * 1000, 3)}
def benchmark(store, actor_id, chat_id, requests=300, concurrency=8):
if isinstance(store.cache, HistoryCache):
for key in store.cache.client.scan_iter(match=f'{store.cache.prefix}:{chat_id}:*'):
store.cache.client.delete(key)
expected = store.list_messages(chat_id, actor_id, 100, None)
QueryCounter.reset()
started = time.perf_counter()
cpu_started = time.process_time()
def read(_):
tick = time.perf_counter()
page = store.list_messages(chat_id, actor_id, 100, None)
assert page == expected
return (time.perf_counter() - tick) * 1000
with ThreadPoolExecutor(max_workers=concurrency) as executor:
latency = sorted(executor.map(read, range(requests)))
elapsed = time.perf_counter() - started
result = {
'requests': requests, 'concurrency': concurrency, 'errors': 0,
'seconds': round(elapsed, 3), 'successful_rps': round(requests / elapsed, 2),
'client_cpu_seconds': round(time.process_time() - cpu_started, 3),
'client_peak_rss_mib': round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 2),
**QueryCounter.counts,
}
for percentile in (50, 95, 99):
result[f'p{percentile}_ms'] = round(latency[math.ceil(len(latency) * percentile / 100) - 1], 3)
return result
def main():
prefix = 'experiment:' + uuid4().hex
engine = create_db_engine()
event.listen(engine, 'before_cursor_execute', QueryCounter.record)
writer = make_store(engine, NoCache())
user = writer.create_user('exp_' + uuid4().hex[:12], 'Experiment', 'Synthetic-password-123')
result = {'race_repeats': 5, 'race': {}, 'benchmark': {}, 'messages': 10000,
'page_size': 100, 'warmup_reads_per_run': 1,
'scope': 'store calls; real SQL and Redis; excludes HTTP, sessions and KDF; SQL excludes BEGIN/COMMIT'}
try:
for name, cache_type in [('naive', NaiveCache), ('versioned', HistoryCache)]:
results = []
for n in range(5):
chat = writer.create_chat(user['id'], 'Race', [user['id']])
reader = make_store(engine, cache_type(sessions.client, prefix=f'{prefix}:{name}:{n}'))
QueryCounter.reset()
row = stale_race(reader, writer, user['id'], chat['id'], naive=name == 'naive')
row.update(QueryCounter.counts)
assert row['stale'] == (name == 'naive')
results.append(row)
result['race'][name] = results
chat = writer.create_chat(user['id'], 'Benchmark', [user['id']])
with writer.sessions.begin() as session:
seed_messages(session, chat['id'], user['id'])
modes = {
'uncached': make_store(engine, NoCache()),
'naive': make_store(engine, NaiveCache(sessions.client, prefix=prefix + ':bench-naive')),
'versioned': make_store(engine, HistoryCache(sessions.client, prefix=prefix + ':bench-versioned')),
}
result['benchmark'] = {name: [] for name in modes}
names = list(modes)
for repeat in range(3):
for name in names[repeat:] + names[:repeat]:
result['benchmark'][name].append(benchmark(modes[name], user['id'], chat['id']))
print(json.dumps(result, ensure_ascii=False, indent=2))
finally:
for key in sessions.client.scan_iter(match=prefix + ':*'):
sessions.client.delete(key)
with writer.sessions.begin() as session:
session.execute(delete(Chat).where(Chat.id.in_(
select(ChatMember.chat_id).where(ChatMember.user_id == user['id']))))
session.execute(delete(User).where(User.id == user['id']))
engine.dispose()
if __name__ == '__main__':
main()
+40
View File
@@ -0,0 +1,40 @@
"""EXPLAIN the actual ORM history query on 10,000 rows, then roll back fixtures."""
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.orm import Session
from src.database import create_db_engine
from src.models import User, Chat, Message
from src.store import history_query
from .fixtures import seed_messages
def main():
engine = create_db_engine()
try:
with Session(engine) as session:
user = User(username='explain_' + uuid4().hex[:12], display_name='Explain', password_hash='unused')
session.add(user)
session.flush()
chat = Chat(title='Explain')
session.add(chat)
session.flush()
seed_messages(session, chat.id, user.id)
conn = session.connection()
conn.exec_driver_sql('ANALYZE messages')
boundary = session.execute(select(Message.created_at, Message.id).where(
Message.chat_id == chat.id).order_by(Message.created_at, Message.id).offset(8999).limit(1)).one()
after = (boundary.created_at.isoformat(), boundary.id)
compiled = history_query(chat.id, 50, after).compile(dialect=engine.dialect)
print('Dataset: 10000 messages; page after message 9000; ORM query; limit 51')
for row in conn.exec_driver_sql('EXPLAIN (ANALYZE, BUFFERS) ' + str(compiled), compiled.params):
print(row[0])
session.rollback()
print('Synthetic data rolled back.')
finally:
engine.dispose()
if __name__ == '__main__':
main()
+17
View File
@@ -0,0 +1,17 @@
"""Synthetic message history shared by EXPLAIN and cache measurements."""
from datetime import datetime, timedelta, timezone
from sqlalchemy import insert, update
from src.models import Chat, Message
def seed_messages(session, chat_id, user_id, count=10000):
start = datetime(2026, 1, 1, tzinfo=timezone.utc)
session.execute(insert(Message), [
{'chat_id': chat_id, 'sender_id': user_id, 'text': 'synthetic ' * 100,
'created_at': start + timedelta(milliseconds=n)}
for n in range(1, count + 1)
])
session.execute(update(Chat).where(Chat.id == chat_id).values(
history_version=Chat.history_version + 1))
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
python scripts\restart.py
+28
View File
@@ -0,0 +1,28 @@
"""Restart the whole lab stack, retaining volumes and recording actual start times."""
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def snapshot():
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
return subprocess.check_output(
['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids],
cwd=ROOT, text=True,
)
def main():
before = snapshot()
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
cwd=ROOT, check=True, timeout=60)
after = snapshot()
(ROOT / 'evidence').mkdir(exist_ok=True)
(ROOT / 'evidence' / 'restart.txt').write_text(
'Command: docker compose restart api postgres redis\nBefore (UTC):\n' + before
+ 'After (UTC):\n' + after, encoding='utf-8')
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/restart.py
+91
View File
@@ -0,0 +1,91 @@
"""Run on the host: research measurements and a real Redis restart through HTTP."""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
def command(*args):
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8', timeout=180)
def cache_state(chat_id, clear=False):
code = """import json, sys
from uuid import UUID
from src.auth import sessions
keys = list(sessions.client.scan_iter(match='history:' + str(UUID(sys.argv[1])) + ':*'))
print(json.dumps({'keys': len(keys)}))
if sys.argv[2] == 'clear' and keys:
sessions.client.delete(*keys)
"""
return json.loads(command('docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code,
chat_id, 'clear' if clear else 'inspect'))
def restart_check():
client = Client('http://127.0.0.1:8080')
actor = client.user('restartcache')
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Cache restart', 'member_ids': [actor['id']]}, schema='Chat')['json']
path = f"/api/v1/chats/{chat['id']}/messages"
def read():
return client.call('GET', path, actor=actor, schema='MessagePage')['json']
first = client.message(actor, chat, 'before Redis restart')['json']
assert read()['items'] == [first]
cached_before = cache_state(chat['id'])['keys']
assert cached_before > 0
second = client.message(actor, chat, 'committed after cache fill')['json']
redis_id = command('docker', 'compose', 'ps', '-q', 'redis').strip()
before_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip()
started = time.monotonic()
command('docker', 'compose', 'restart', 'redis')
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
client.call('GET', '/health/ready')
break
except AssertionError:
time.sleep(0.5)
else:
raise AssertionError('Redis recovery timed out')
after_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip()
assert before_start != after_start
assert read()['items'] == [first, second]
recovery = round(time.monotonic() - started, 3)
cache_state(chat['id'], clear=True)
assert cache_state(chat['id'])['keys'] == 0
assert read()['items'] == [first, second]
assert cache_state(chat['id'])['keys'] > 0
third = client.message(actor, chat, 'after Redis recovery')['json']
assert read()['items'] == [first, second, third]
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
client.call('GET', path, 401, actor=actor)
return {'status': 'PASS', 'same_session': True, 'confirmed_messages': 3,
'cold_cache_rebuilt': True, 'read_after_write': True, 'logout_still_401': True,
'recovery_seconds': recovery, 'redis_start_before_utc': before_start,
'redis_start_after_utc': after_start}
def main():
evidence = ROOT / 'evidence'
evidence.mkdir(exist_ok=True)
result = json.loads(command('docker', 'compose', 'exec', '-T', 'api',
'python', '-m', 'scripts.cache_experiment'))
(evidence / 'cache-experiment.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
result = restart_check()
(evidence / 'cache-restart.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
ids = command('docker', 'compose', 'ps', '-q').split()
snapshot = command('docker', 'stats', '--no-stream', '--format', '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}', *ids)
(evidence / 'cache-resources.txt').write_text('Post-experiment snapshot (not peak):\n' + snapshot, encoding='utf-8')
print('PASS: stale race reproduced before protection and prevented after; SQL/latency measured.')
print('PASS: Redis restart, same session, read-after-write, cold cache rebuild, logout.')
print('Evidence: cache-experiment.json, cache-restart.json, cache-resources.txt')
if __name__ == '__main__':
main()
-1
View File
@@ -9,7 +9,6 @@ root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():