forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the lab3 migration while a lab2 API uses an isolated temporary database."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
from urllib.parse import quote
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NETWORK = 'mtusi2026lab3-fork_default'
|
||||
OLD_API = 'mtusi2026lab2-fork-backend:latest'
|
||||
OLD_MIGRATE = 'mtusi2026lab2-fork-migrate:latest'
|
||||
NEW_API = 'mtusi2026lab3-fork-api:latest'
|
||||
NEW_MIGRATE = 'mtusi2026lab3-fork-migrate:latest'
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
result = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'docker {args[0]} failed with code {result.returncode}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def request(base, method, path, body=None, token=None, basic=None, key=None):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
if basic:
|
||||
headers['Authorization'] = 'Basic ' + base64.b64encode(
|
||||
(basic[0] + ':' + basic[1]).encode()).decode()
|
||||
if key:
|
||||
headers['Idempotency-Key'] = key
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
headers['Content-Type'] = 'application/json'
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers,
|
||||
method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=3) as response:
|
||||
raw = response.read()
|
||||
return response.status, json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(f'{method} {path}: HTTP {exc.code}') from None
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise RuntimeError(f'{method} {path}: {type(exc).__name__}') from None
|
||||
|
||||
|
||||
def wait_http(base):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if request(base, 'GET', '/health/ready')[0] == 200:
|
||||
return
|
||||
except RuntimeError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError('temporary API did not become ready')
|
||||
|
||||
|
||||
def publish_port(name):
|
||||
value = docker('port', name, '8080/tcp')
|
||||
return 'http://127.0.0.1:' + value.rsplit(':', 1)[1]
|
||||
|
||||
|
||||
def make_user(base, label):
|
||||
username = 'compat_' + label + '_' + uuid.uuid4().hex[:8]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
status, user = request(base, 'POST', '/api/v1/users',
|
||||
{'username': username, 'display_name': label,
|
||||
'password': password})
|
||||
assert status == 201
|
||||
status, session = request(base, 'POST', '/api/v1/auth/sessions',
|
||||
basic=(username, password))
|
||||
assert status == 201
|
||||
return user, session['token']
|
||||
|
||||
|
||||
def message(base, token, chat_id, text, key=None):
|
||||
status, item = request(base, 'POST',
|
||||
f'/api/v1/chats/{chat_id}/messages',
|
||||
{'text': text}, token=token, key=key)
|
||||
assert status == 201
|
||||
return item
|
||||
|
||||
|
||||
def history(base, token, chat_id):
|
||||
items = []
|
||||
cursor = None
|
||||
for _ in range(100):
|
||||
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
|
||||
if cursor:
|
||||
path += '&cursor=' + quote(cursor, safe='')
|
||||
status, page = request(base, 'GET', path, token=token)
|
||||
assert status == 200
|
||||
items.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return items
|
||||
raise RuntimeError('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def run():
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
names = {label: 'lab3-compat-' + suffix + '-' + label
|
||||
for label in ('pg', 'redis', 'old', 'new')}
|
||||
password = secrets.token_urlsafe(18)
|
||||
db_url = f'postgresql://app:{password}@{names["pg"]}:5432/compat'
|
||||
cache_url = f'redis://{names["redis"]}:6379/0'
|
||||
created = []
|
||||
try:
|
||||
print('Starting isolated PostgreSQL and Redis...', flush=True)
|
||||
docker('run', '--rm', '-d', '--name', names['pg'], '--network', NETWORK,
|
||||
'--tmpfs', '/var/lib/postgresql/data:rw,mode=0700',
|
||||
'-e', 'POSTGRES_DB=compat', '-e', 'POSTGRES_USER=app',
|
||||
'-e', 'POSTGRES_PASSWORD=' + password,
|
||||
'postgres:17.6-bookworm')
|
||||
created.append(names['pg'])
|
||||
docker('run', '--rm', '-d', '--name', names['redis'], '--network', NETWORK,
|
||||
'redis:7.4.5-alpine', 'redis-server', '--save', '',
|
||||
'--appendonly', 'no')
|
||||
created.append(names['redis'])
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
docker('exec', names['pg'], 'pg_isready', '-U', 'app',
|
||||
'-d', 'compat', timeout=5)
|
||||
break
|
||||
except RuntimeError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('temporary PostgreSQL did not start')
|
||||
time.sleep(0.3)
|
||||
docker('run', '--rm', '--network', NETWORK, '-e', 'DATABASE_URL=' + db_url,
|
||||
OLD_MIGRATE, timeout=90)
|
||||
print('Lab2 schema applied.', flush=True)
|
||||
before_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
|
||||
'compat', '-tAc',
|
||||
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
|
||||
assert before_table == 'f', 'new table exists before migration'
|
||||
base_env = ['-e', 'DATABASE_URL=' + db_url,
|
||||
'-e', 'CACHE_URL=' + cache_url,
|
||||
'-e', 'CACHE_PREFIX=compat:' + suffix,
|
||||
'-e', 'APP_HOST=0.0.0.0', '-e', 'APP_PORT=8080',
|
||||
'-e', 'APP_ENV=lab', '-e', 'SESSION_STORE=postgres']
|
||||
docker('run', '--rm', '-d', '--name', names['old'], '--network', NETWORK,
|
||||
'-p', '127.0.0.1::8080', *base_env, OLD_API)
|
||||
created.append(names['old'])
|
||||
old_base = publish_port(names['old'])
|
||||
wait_http(old_base)
|
||||
print('Lab2 API ready; preparing synthetic chat.', flush=True)
|
||||
alice, token = make_user(old_base, 'alice')
|
||||
bob, _ = make_user(old_base, 'bob')
|
||||
status, chat = request(old_base, 'POST', '/api/v1/chats',
|
||||
{'title': 'Migration compatibility',
|
||||
'member_ids': [alice['id'], bob['id']]},
|
||||
token=token)
|
||||
assert status == 201
|
||||
pre = message(old_base, token, chat['id'], 'before migration')
|
||||
|
||||
stop = threading.Event()
|
||||
events = []
|
||||
|
||||
def old_writer():
|
||||
index = 0
|
||||
while not stop.is_set():
|
||||
at = time.monotonic()
|
||||
try:
|
||||
item = message(old_base, token, chat['id'],
|
||||
f'during migration {index}')
|
||||
events.append((at, True, item['id']))
|
||||
except (RuntimeError, AssertionError):
|
||||
events.append((at, False, None))
|
||||
index += 1
|
||||
stop.wait(0.3)
|
||||
|
||||
thread = threading.Thread(target=old_writer, daemon=True)
|
||||
thread.start()
|
||||
time.sleep(0.5)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
print('Applying lab3 migration while lab2 API writes...', flush=True)
|
||||
docker('run', '--rm', '--network', NETWORK,
|
||||
'-e', 'DATABASE_URL=' + db_url,
|
||||
NEW_MIGRATE, timeout=90)
|
||||
finally:
|
||||
migrated = time.monotonic()
|
||||
time.sleep(0.7)
|
||||
stop.set()
|
||||
thread.join(timeout=5)
|
||||
after_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
|
||||
'compat', '-tAc',
|
||||
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
|
||||
assert after_table == 't', 'new table was not created'
|
||||
assert any(ok and at < started for at, ok, _ in events), 'no old writes before migration'
|
||||
assert any(ok and at > migrated for at, ok, _ in events), 'no old writes after migration'
|
||||
assert all(ok for _, ok, _ in events), 'old API write failed near migration'
|
||||
after = message(old_base, token, chat['id'], 'after migration')
|
||||
print('Lab2 API still reads and writes; starting lab3 API.', flush=True)
|
||||
|
||||
docker('run', '--rm', '-d', '--name', names['new'], '--network', NETWORK,
|
||||
'-p', '127.0.0.1::8080', *base_env,
|
||||
'-e', 'INSTANCE_ID=compat-new', NEW_API)
|
||||
created.append(names['new'])
|
||||
new_base = publish_port(names['new'])
|
||||
wait_http(new_base)
|
||||
print('Lab3 API ready; checking mixed-version reads and writes.', flush=True)
|
||||
status, profile = request(new_base, 'GET', '/api/v1/users/me', token=token)
|
||||
assert status == 200 and profile['id'] == alice['id'], 'session not shared'
|
||||
key = uuid.uuid4().hex
|
||||
new_message = message(new_base, token, chat['id'], 'from new version', key)
|
||||
replay = message(new_base, token, chat['id'], 'from new version', key)
|
||||
assert replay == new_message, 'new API idempotent replay changed response'
|
||||
old_after_new = message(old_base, token, chat['id'], 'from old version')
|
||||
for version, base in (('old', old_base), ('new', new_base)):
|
||||
items = history(base, token, chat['id'])
|
||||
ids = [item['id'] for item in items]
|
||||
for label, item in (('pre', pre), ('after', after),
|
||||
('new', new_message), ('old-after-new', old_after_new)):
|
||||
assert ids.count(item['id']) == 1, f'{version} API cannot see {label} message'
|
||||
result = {
|
||||
'old_image': OLD_API, 'new_image': NEW_API,
|
||||
'migration_image': NEW_MIGRATE,
|
||||
'migration_seconds': round(migrated - started, 3),
|
||||
'old_writes_near_migration': len(events),
|
||||
'old_write_errors': sum(not ok for _, ok, _ in events),
|
||||
'old_and_new_read_each_others_messages': True,
|
||||
'existing_session_accepted_by_new_api': True,
|
||||
'new_api_idempotent_replay': True,
|
||||
'temporary_database': True,
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
print('PASS: additive migration compatible with running lab2 API')
|
||||
finally:
|
||||
print('Stopping isolated test containers...', flush=True)
|
||||
for name in reversed(created):
|
||||
try:
|
||||
docker('stop', name, timeout=30)
|
||||
except (RuntimeError, subprocess.TimeoutExpired):
|
||||
print(f'WARNING: stop temporary container {name} manually',
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
run()
|
||||
except (RuntimeError, AssertionError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f'FAIL: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
Reference in New Issue
Block a user