test: cover persistence failover and graceful draining

Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
2026-10-29 11:47:46 +03:00
parent 676b38c18b
commit 5c8bcf0f80
37 changed files with 5538 additions and 0 deletions
+257
View File
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""Run the lab3 migration while a lab2 API uses an isolated temporary database."""
import base64
import json
from pathlib import Path
import secrets
import subprocess
import sys
import threading
import time
import urllib.error
from urllib.parse import quote
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
NETWORK = 'mtusi2026lab3-fork_default'
OLD_API = 'mtusi2026lab2-fork-backend:latest'
OLD_MIGRATE = 'mtusi2026lab2-fork-migrate:latest'
NEW_API = 'mtusi2026lab3-fork-api:latest'
NEW_MIGRATE = 'mtusi2026lab3-fork-migrate:latest'
def docker(*args, timeout=60):
result = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
text=True, timeout=timeout, check=False)
if result.returncode:
raise RuntimeError(f'docker {args[0]} failed with code {result.returncode}')
return result.stdout.strip()
def request(base, method, path, body=None, token=None, basic=None, key=None):
headers = {'Accept': 'application/json'}
if token:
headers['Authorization'] = 'Bearer ' + token
if basic:
headers['Authorization'] = 'Basic ' + base64.b64encode(
(basic[0] + ':' + basic[1]).encode()).decode()
if key:
headers['Idempotency-Key'] = key
data = None
if body is not None:
data = json.dumps(body).encode()
headers['Content-Type'] = 'application/json'
req = urllib.request.Request(base + path, data=data, headers=headers,
method=method)
try:
with urllib.request.urlopen(req, timeout=3) as response:
raw = response.read()
return response.status, json.loads(raw) if raw else None
except urllib.error.HTTPError as exc:
raise RuntimeError(f'{method} {path}: HTTP {exc.code}') from None
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise RuntimeError(f'{method} {path}: {type(exc).__name__}') from None
def wait_http(base):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
try:
if request(base, 'GET', '/health/ready')[0] == 200:
return
except RuntimeError:
pass
time.sleep(0.2)
raise RuntimeError('temporary API did not become ready')
def publish_port(name):
value = docker('port', name, '8080/tcp')
return 'http://127.0.0.1:' + value.rsplit(':', 1)[1]
def make_user(base, label):
username = 'compat_' + label + '_' + uuid.uuid4().hex[:8]
password = 'T3st-' + uuid.uuid4().hex
status, user = request(base, 'POST', '/api/v1/users',
{'username': username, 'display_name': label,
'password': password})
assert status == 201
status, session = request(base, 'POST', '/api/v1/auth/sessions',
basic=(username, password))
assert status == 201
return user, session['token']
def message(base, token, chat_id, text, key=None):
status, item = request(base, 'POST',
f'/api/v1/chats/{chat_id}/messages',
{'text': text}, token=token, key=key)
assert status == 201
return item
def history(base, token, chat_id):
items = []
cursor = None
for _ in range(100):
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
if cursor:
path += '&cursor=' + quote(cursor, safe='')
status, page = request(base, 'GET', path, token=token)
assert status == 200
items.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return items
raise RuntimeError('history pagination exceeded safety bound')
def run():
suffix = uuid.uuid4().hex[:10]
names = {label: 'lab3-compat-' + suffix + '-' + label
for label in ('pg', 'redis', 'old', 'new')}
password = secrets.token_urlsafe(18)
db_url = f'postgresql://app:{password}@{names["pg"]}:5432/compat'
cache_url = f'redis://{names["redis"]}:6379/0'
created = []
try:
print('Starting isolated PostgreSQL and Redis...', flush=True)
docker('run', '--rm', '-d', '--name', names['pg'], '--network', NETWORK,
'--tmpfs', '/var/lib/postgresql/data:rw,mode=0700',
'-e', 'POSTGRES_DB=compat', '-e', 'POSTGRES_USER=app',
'-e', 'POSTGRES_PASSWORD=' + password,
'postgres:17.6-bookworm')
created.append(names['pg'])
docker('run', '--rm', '-d', '--name', names['redis'], '--network', NETWORK,
'redis:7.4.5-alpine', 'redis-server', '--save', '',
'--appendonly', 'no')
created.append(names['redis'])
deadline = time.monotonic() + 30
while True:
try:
docker('exec', names['pg'], 'pg_isready', '-U', 'app',
'-d', 'compat', timeout=5)
break
except RuntimeError:
if time.monotonic() >= deadline:
raise RuntimeError('temporary PostgreSQL did not start')
time.sleep(0.3)
docker('run', '--rm', '--network', NETWORK, '-e', 'DATABASE_URL=' + db_url,
OLD_MIGRATE, timeout=90)
print('Lab2 schema applied.', flush=True)
before_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
'compat', '-tAc',
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
assert before_table == 'f', 'new table exists before migration'
base_env = ['-e', 'DATABASE_URL=' + db_url,
'-e', 'CACHE_URL=' + cache_url,
'-e', 'CACHE_PREFIX=compat:' + suffix,
'-e', 'APP_HOST=0.0.0.0', '-e', 'APP_PORT=8080',
'-e', 'APP_ENV=lab', '-e', 'SESSION_STORE=postgres']
docker('run', '--rm', '-d', '--name', names['old'], '--network', NETWORK,
'-p', '127.0.0.1::8080', *base_env, OLD_API)
created.append(names['old'])
old_base = publish_port(names['old'])
wait_http(old_base)
print('Lab2 API ready; preparing synthetic chat.', flush=True)
alice, token = make_user(old_base, 'alice')
bob, _ = make_user(old_base, 'bob')
status, chat = request(old_base, 'POST', '/api/v1/chats',
{'title': 'Migration compatibility',
'member_ids': [alice['id'], bob['id']]},
token=token)
assert status == 201
pre = message(old_base, token, chat['id'], 'before migration')
stop = threading.Event()
events = []
def old_writer():
index = 0
while not stop.is_set():
at = time.monotonic()
try:
item = message(old_base, token, chat['id'],
f'during migration {index}')
events.append((at, True, item['id']))
except (RuntimeError, AssertionError):
events.append((at, False, None))
index += 1
stop.wait(0.3)
thread = threading.Thread(target=old_writer, daemon=True)
thread.start()
time.sleep(0.5)
started = time.monotonic()
try:
print('Applying lab3 migration while lab2 API writes...', flush=True)
docker('run', '--rm', '--network', NETWORK,
'-e', 'DATABASE_URL=' + db_url,
NEW_MIGRATE, timeout=90)
finally:
migrated = time.monotonic()
time.sleep(0.7)
stop.set()
thread.join(timeout=5)
after_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
'compat', '-tAc',
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
assert after_table == 't', 'new table was not created'
assert any(ok and at < started for at, ok, _ in events), 'no old writes before migration'
assert any(ok and at > migrated for at, ok, _ in events), 'no old writes after migration'
assert all(ok for _, ok, _ in events), 'old API write failed near migration'
after = message(old_base, token, chat['id'], 'after migration')
print('Lab2 API still reads and writes; starting lab3 API.', flush=True)
docker('run', '--rm', '-d', '--name', names['new'], '--network', NETWORK,
'-p', '127.0.0.1::8080', *base_env,
'-e', 'INSTANCE_ID=compat-new', NEW_API)
created.append(names['new'])
new_base = publish_port(names['new'])
wait_http(new_base)
print('Lab3 API ready; checking mixed-version reads and writes.', flush=True)
status, profile = request(new_base, 'GET', '/api/v1/users/me', token=token)
assert status == 200 and profile['id'] == alice['id'], 'session not shared'
key = uuid.uuid4().hex
new_message = message(new_base, token, chat['id'], 'from new version', key)
replay = message(new_base, token, chat['id'], 'from new version', key)
assert replay == new_message, 'new API idempotent replay changed response'
old_after_new = message(old_base, token, chat['id'], 'from old version')
for version, base in (('old', old_base), ('new', new_base)):
items = history(base, token, chat['id'])
ids = [item['id'] for item in items]
for label, item in (('pre', pre), ('after', after),
('new', new_message), ('old-after-new', old_after_new)):
assert ids.count(item['id']) == 1, f'{version} API cannot see {label} message'
result = {
'old_image': OLD_API, 'new_image': NEW_API,
'migration_image': NEW_MIGRATE,
'migration_seconds': round(migrated - started, 3),
'old_writes_near_migration': len(events),
'old_write_errors': sum(not ok for _, ok, _ in events),
'old_and_new_read_each_others_messages': True,
'existing_session_accepted_by_new_api': True,
'new_api_idempotent_replay': True,
'temporary_database': True,
}
print(json.dumps(result, indent=2))
print('PASS: additive migration compatible with running lab2 API')
finally:
print('Stopping isolated test containers...', flush=True)
for name in reversed(created):
try:
docker('stop', name, timeout=30)
except (RuntimeError, subprocess.TimeoutExpired):
print(f'WARNING: stop temporary container {name} manually',
file=sys.stderr)
if __name__ == '__main__':
try:
run()
except (RuntimeError, AssertionError, OSError, subprocess.TimeoutExpired) as exc:
print(f'FAIL: {exc}', file=sys.stderr)
raise SystemExit(1) from None