449 lines
15 KiB
Python
449 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 1: stale first page, then versioned cache and singleflight.
|
|
|
|
The script switches HISTORY_CACHE, recreates only the API, and always restores
|
|
the previous .env. It does not print tokens.
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
|
ENV_PATH = ROOT / '.env'
|
|
EVIDENCE_PATH = ROOT / 'evidence' / 'track1.json'
|
|
COLD_CLIENTS = 32
|
|
READERS = 8
|
|
WARM_READS = 20
|
|
|
|
|
|
def call(method, path, body=None, headers=None, timeout=20):
|
|
data = None if body is None else json.dumps(body).encode('utf-8')
|
|
req = urllib.request.Request(BASE_URL + path, data=data, method=method)
|
|
if body is not None:
|
|
req.add_header('Content-Type', 'application/json')
|
|
for key, value in (headers or {}).items():
|
|
req.add_header(key, value)
|
|
started = time.perf_counter()
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
payload = response.read()
|
|
elapsed_ms = (time.perf_counter() - started) * 1000
|
|
return response.status, payload, elapsed_ms
|
|
except urllib.error.HTTPError as exc:
|
|
elapsed_ms = (time.perf_counter() - started) * 1000
|
|
return exc.code, exc.read(), elapsed_ms
|
|
|
|
|
|
def compose(*args, check=True):
|
|
proc = subprocess.run(
|
|
['docker', 'compose', '--env-file', '.env', *args],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if check and proc.returncode != 0:
|
|
sys.stderr.write(proc.stderr)
|
|
raise SystemExit('docker compose %s failed' % ' '.join(args))
|
|
return proc
|
|
|
|
|
|
def psql(sql):
|
|
proc = compose(
|
|
'exec', '-T', 'postgres',
|
|
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
|
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
|
'-v', 'ON_ERROR_STOP=1', '-tAc', sql,
|
|
)
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def redis_cli(*args, check=True):
|
|
proc = compose('exec', '-T', 'redis', 'redis-cli', *args, check=check)
|
|
return proc
|
|
|
|
|
|
def wait_ready(seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
status, _, _ = call('GET', '/health/ready', timeout=3)
|
|
if status == 200:
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise SystemExit('API did not become ready')
|
|
|
|
|
|
def wait_redis(seconds=40):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
proc = redis_cli('ping', check=False)
|
|
if proc.returncode == 0 and 'PONG' in proc.stdout:
|
|
return
|
|
time.sleep(0.3)
|
|
raise SystemExit('redis did not answer PING')
|
|
|
|
|
|
def message_scans():
|
|
raw = psql(
|
|
"SELECT COALESCE(idx_scan,0)+COALESCE(seq_scan,0) "
|
|
"FROM pg_stat_user_tables WHERE relname = 'messages'"
|
|
)
|
|
if not raw:
|
|
raise SystemExit('messages stats are missing')
|
|
return int(raw)
|
|
|
|
|
|
def drained_message_scans():
|
|
"""Backends publish table stats on exit. Drop the ones that ran the workload."""
|
|
raw = psql(
|
|
"SELECT pid FROM pg_stat_activity "
|
|
"WHERE datname = current_database() AND pid <> pg_backend_pid() "
|
|
"AND backend_type = 'client backend'"
|
|
)
|
|
pids = [line.strip() for line in raw.splitlines() if line.strip()]
|
|
if pids:
|
|
if not all(pid.isdigit() for pid in pids):
|
|
raise SystemExit('unexpected pid list')
|
|
listed = ','.join(pids)
|
|
psql('SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid IN (%s)' % listed)
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
still = psql('SELECT count(*) FROM pg_stat_activity WHERE pid IN (%s)' % listed)
|
|
if still == '0':
|
|
break
|
|
time.sleep(0.05)
|
|
else:
|
|
raise SystemExit('postgres backends did not disconnect')
|
|
return message_scans()
|
|
|
|
|
|
def p50(samples):
|
|
ordered = sorted(samples)
|
|
return ordered[(len(ordered) - 1) // 2]
|
|
|
|
|
|
def bearer(token):
|
|
return {'Authorization': 'Bearer ' + token}
|
|
|
|
|
|
def write_mode(mode):
|
|
text = ENV_PATH.read_text(encoding='utf-8')
|
|
newline = '\r\n' if '\r\n' in text else '\n'
|
|
lines = [line for line in text.splitlines() if not line.startswith('HISTORY_CACHE=')]
|
|
lines.append('HISTORY_CACHE=' + mode)
|
|
ENV_PATH.write_bytes((newline.join(lines) + newline).encode('utf-8'))
|
|
|
|
|
|
def recreate_api(build=False):
|
|
args = ['up', '-d', '--force-recreate', '--no-deps']
|
|
if build:
|
|
args = ['up', '-d', '--build', '--force-recreate', '--no-deps']
|
|
args.append('api')
|
|
compose(*args)
|
|
wait_ready()
|
|
|
|
|
|
def flush_history():
|
|
keys = []
|
|
for pattern in ('history:*', 'hver:*'):
|
|
proc = redis_cli('--scan', '--pattern', pattern)
|
|
keys.extend(line.strip() for line in proc.stdout.splitlines() if line.strip())
|
|
if keys:
|
|
redis_cli('DEL', *keys)
|
|
|
|
|
|
def register_and_chat():
|
|
username = 'u_t1_%s' % uuid.uuid4().hex[:12]
|
|
password = 'T3st-' + uuid.uuid4().hex
|
|
status, raw, _ = call('POST', '/api/v1/users', {
|
|
'username': username,
|
|
'display_name': 'track1',
|
|
'password': password,
|
|
})
|
|
if status != 201:
|
|
raise SystemExit('register failed: %s' % status)
|
|
user = json.loads(raw)
|
|
basic = base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii')
|
|
status, raw, _ = call(
|
|
'POST', '/api/v1/auth/sessions',
|
|
headers={'Authorization': 'Basic ' + basic},
|
|
)
|
|
if status != 201:
|
|
raise SystemExit('login failed: %s' % status)
|
|
token = json.loads(raw)['token']
|
|
status, raw, _ = call(
|
|
'POST', '/api/v1/chats',
|
|
{'title': 'track1', 'member_ids': [user['id']]},
|
|
headers=bearer(token),
|
|
)
|
|
if status != 201:
|
|
raise SystemExit('chat failed: %s' % status)
|
|
chat_id = json.loads(raw)['id']
|
|
status, _, _ = call(
|
|
'POST', '/api/v1/chats/%s/messages' % chat_id,
|
|
{'text': 'seed'},
|
|
headers=bearer(token),
|
|
)
|
|
if status != 201:
|
|
raise SystemExit('seed message failed: %s' % status)
|
|
return token, chat_id
|
|
|
|
|
|
def reads_after_write(token, chat_id):
|
|
status, _, _ = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
if status != 200:
|
|
raise SystemExit('warm read failed: %s' % status)
|
|
post_done = threading.Event()
|
|
posted = {}
|
|
|
|
def poster():
|
|
status, raw, _ = call(
|
|
'POST', '/api/v1/chats/%s/messages' % chat_id,
|
|
{'text': 'confirmed-' + uuid.uuid4().hex[:8]},
|
|
headers=bearer(token),
|
|
)
|
|
posted['status'] = status
|
|
if status == 201:
|
|
posted['id'] = json.loads(raw)['id']
|
|
post_done.set()
|
|
|
|
def reader(_index):
|
|
if not post_done.wait(20):
|
|
return None
|
|
status, raw, _ = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
if status != 200:
|
|
return None
|
|
ids = [item['id'] for item in json.loads(raw)['items']]
|
|
return posted.get('id') in ids
|
|
|
|
with ThreadPoolExecutor(max_workers=READERS + 1) as pool:
|
|
post = pool.submit(poster)
|
|
jobs = [pool.submit(reader, index) for index in range(READERS)]
|
|
post.result()
|
|
seen = [job.result() for job in jobs]
|
|
if posted.get('status') != 201 or 'id' not in posted:
|
|
raise SystemExit('post failed: %s' % posted.get('status'))
|
|
present = sum(1 for item in seen if item is True)
|
|
missing = sum(1 for item in seen if item is False)
|
|
return {
|
|
'reads_started_after_201': present + missing,
|
|
'reads_missing_new_message': missing,
|
|
'message_id': posted['id'],
|
|
}
|
|
|
|
|
|
def cold_reads(token, chat_id):
|
|
flush_history()
|
|
before = drained_message_scans()
|
|
timings = []
|
|
failures = []
|
|
|
|
def one(_index):
|
|
status, _, elapsed = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
timings.append(elapsed)
|
|
if status != 200:
|
|
failures.append(status)
|
|
|
|
with ThreadPoolExecutor(max_workers=COLD_CLIENTS) as pool:
|
|
list(pool.map(one, range(COLD_CLIENTS)))
|
|
if failures:
|
|
raise SystemExit('cold reads failed: %s' % failures[:3])
|
|
return {
|
|
'clients': COLD_CLIENTS,
|
|
'message_scans': drained_message_scans() - before,
|
|
'p50_ms': round(p50(timings), 3),
|
|
}
|
|
|
|
|
|
def miss_latency(token, chat_id, samples=12):
|
|
"""One GET after a flush. The clock starts at the request, not at the flush."""
|
|
timings = []
|
|
for _ in range(samples):
|
|
flush_history()
|
|
status, _, elapsed = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
if status != 200:
|
|
raise SystemExit('cold miss failed: %s' % status)
|
|
timings.append(elapsed)
|
|
return round(p50(timings), 3)
|
|
|
|
|
|
def warm_reads(token, chat_id):
|
|
status, _, _ = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
if status != 200:
|
|
raise SystemExit('warm fill failed: %s' % status)
|
|
before = drained_message_scans()
|
|
timings = []
|
|
for _ in range(WARM_READS):
|
|
status, _, elapsed = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
if status != 200:
|
|
raise SystemExit('warm read failed: %s' % status)
|
|
timings.append(elapsed)
|
|
return {
|
|
'clients': WARM_READS,
|
|
'message_scans': drained_message_scans() - before,
|
|
'p50_ms': round(p50(timings), 3),
|
|
}
|
|
|
|
|
|
def redis_restart(token, chat_id, message_id):
|
|
compose('exec', '-T', 'redis', 'sh', '-c', 'rm -f /data/dump.rdb /data/appendonly.aof')
|
|
compose('restart', 'redis')
|
|
wait_redis()
|
|
time.sleep(0.5)
|
|
session_status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
|
before = drained_message_scans()
|
|
status, raw, _ = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
first_delta = drained_message_scans() - before
|
|
present = False
|
|
if status == 200:
|
|
present = message_id in [item['id'] for item in json.loads(raw)['items']]
|
|
before = drained_message_scans()
|
|
status, _, _ = call(
|
|
'GET', '/api/v1/chats/%s/messages?limit=50' % chat_id,
|
|
headers=bearer(token),
|
|
)
|
|
second_delta = drained_message_scans() - before
|
|
return {
|
|
'confirmed_message_present': present and status == 200,
|
|
'first_read_message_scans': first_delta,
|
|
'second_read_message_scans': second_delta,
|
|
'session_status': session_status,
|
|
}
|
|
|
|
|
|
def problems_of(evidence):
|
|
found = []
|
|
naive = evidence['naive']
|
|
protected = evidence['protected']
|
|
if naive['reads_missing_new_message'] < 1:
|
|
found.append('naive did not show a stale read')
|
|
if protected['reads_missing_new_message'] != 0:
|
|
found.append('protected missed a confirmed message')
|
|
if evidence['off']['message_scans'] != WARM_READS:
|
|
found.append('off scans %s' % evidence['off']['message_scans'])
|
|
if naive['cold']['message_scans'] < COLD_CLIENTS - 8:
|
|
found.append('naive cold scans %s' % naive['cold']['message_scans'])
|
|
if protected['cold']['message_scans'] > 2:
|
|
found.append('protected cold scans %s' % protected['cold']['message_scans'])
|
|
if protected['warm']['message_scans'] != 0:
|
|
found.append('warm scans %s' % protected['warm']['message_scans'])
|
|
if protected['warm']['p50_ms'] >= protected['sequential_miss_p50_ms']:
|
|
found.append('warm p50 is not below a cold miss')
|
|
restart = protected['after_redis_restart']
|
|
if not restart['confirmed_message_present']:
|
|
found.append('message missing after redis restart')
|
|
if restart['first_read_message_scans'] < 1:
|
|
found.append('restart was not a miss')
|
|
if restart['second_read_message_scans'] != 0:
|
|
found.append('second read after restart was not a hit')
|
|
if restart['session_status'] != 200:
|
|
found.append('session after redis restart: %s' % restart['session_status'])
|
|
return found
|
|
|
|
|
|
def main():
|
|
if not ENV_PATH.is_file():
|
|
raise SystemExit('.env is missing')
|
|
original = ENV_PATH.read_bytes()
|
|
failure = None
|
|
try:
|
|
print('naive mode', flush=True)
|
|
write_mode('naive')
|
|
compose('up', '-d', '--build')
|
|
wait_ready()
|
|
token, chat_id = register_and_chat()
|
|
naive_race = reads_after_write(token, chat_id)
|
|
naive_cold = cold_reads(token, chat_id)
|
|
print('off mode', flush=True)
|
|
write_mode('off')
|
|
recreate_api(build=False)
|
|
off_repeat = warm_reads(token, chat_id)
|
|
print('protected mode', flush=True)
|
|
write_mode('protected')
|
|
recreate_api(build=False)
|
|
protected_race = reads_after_write(token, chat_id)
|
|
protected_cold = cold_reads(token, chat_id)
|
|
protected_miss_p50 = miss_latency(token, chat_id)
|
|
protected_warm = warm_reads(token, chat_id)
|
|
print('redis restart', flush=True)
|
|
restarted = redis_restart(token, chat_id, protected_race['message_id'])
|
|
evidence = {
|
|
'track': 1,
|
|
'cached_limit': 50,
|
|
'page_ttl_seconds': 30,
|
|
'naive_fill_hold_seconds': 0.2,
|
|
'naive': {
|
|
'reads_started_after_201': naive_race['reads_started_after_201'],
|
|
'reads_missing_new_message': naive_race['reads_missing_new_message'],
|
|
'stale_read_observed': naive_race['reads_missing_new_message'] > 0,
|
|
'cold': naive_cold,
|
|
},
|
|
'off': off_repeat,
|
|
'protected': {
|
|
'reads_started_after_201': protected_race['reads_started_after_201'],
|
|
'reads_missing_new_message': protected_race['reads_missing_new_message'],
|
|
'read_after_write': protected_race['reads_missing_new_message'] == 0,
|
|
'cold': protected_cold,
|
|
'sequential_miss_p50_ms': protected_miss_p50,
|
|
'warm': protected_warm,
|
|
'after_redis_restart': restarted,
|
|
},
|
|
}
|
|
found = problems_of(evidence)
|
|
print(json.dumps(evidence, indent=2), flush=True)
|
|
if found:
|
|
raise SystemExit('track1 failed: ' + '; '.join(found))
|
|
EVIDENCE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
EVIDENCE_PATH.write_text(json.dumps(evidence, indent=2) + '\n', encoding='utf-8')
|
|
print('wrote evidence/track1.json', flush=True)
|
|
except SystemExit as exc:
|
|
failure = exc
|
|
finally:
|
|
ENV_PATH.write_bytes(original)
|
|
print('restoring protected api', flush=True)
|
|
compose('up', '-d', '--force-recreate', '--no-deps', 'api')
|
|
wait_ready()
|
|
compose('start', 'redis')
|
|
compose('start', 'postgres')
|
|
if failure is not None:
|
|
raise failure
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|