Files
mtusi2026lab3/tests/rolling_check.py
T
2026-10-26 21:32:36 +03:00

225 lines
11 KiB
Python

#!/usr/bin/env python3
"""Sequential replacement of TWO API containers with the SAME image under load.
Put beside level4_check.py, client.py and resilience.py in tests/.
Does not build images, run migrations, modify dependencies or delete volumes.
On failure it stops replacing containers and preserves the remaining replica.
"""
import csv
import hashlib
import json
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from client import Client, Failure, check
from resilience import history
from level4_check import docker, ready, observe
ROOT = Path(__file__).resolve().parents[1]
def inspect(cid):
# Only output safe fields; never persist container environment/secrets.
return {'id': cid,
'hostname': docker('inspect', '--format', '{{.Config.Hostname}}', cid),
'image': docker('inspect', '--format', '{{.Image}}', cid)}
def containers():
return docker('compose', 'ps', '-a', '-q', 'api').splitlines()
def main():
original = containers()
check(len(original) == 2, 'Expected exactly two API containers')
for cid in original:
ready(cid)
info = {cid: inspect(cid) for cid in original}
check(len({v['image'] for v in info.values()}) == 1, 'Both replicas must use the same image')
original_image = info[original[0]]['image']
# Verify the configured image tag still points at that same immutable image.
config = json.loads(docker('compose', 'config', '--format', 'json'))
api = config['services']['api']
image_ref = api.get('image')
check(bool(image_ref), 'API image must be explicitly configured')
check(int(api.get('deploy', {}).get('replicas', 2)) == 2, 'Configure two replicas')
check(docker('image', 'inspect', '--format', '{{.Id}}', image_ref) == original_image,
'Image tag changed. This test expects replacement with the existing image')
del config, api
output = ROOT / 'results' / ('rolling-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
output.mkdir(parents=True)
report = {'script_version': 3, 'scope': 'Sequential replacement with the same image; no schema migration or version upgrade',
'before': list(info.values()), 'steps': [], 'pass': False,
'workload': '4 closed-loop workers; POST then GET; 100ms pause; HTTP timeout 2s',
'migration_sha256': {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted((ROOT / 'java/src/main/resources/db/migration').glob('*.sql'))}}
c = Client(timeout=2)
alice, bob, _, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
observe(c, alice, {v['hostname'] for v in info.values()})
seed_key = uuid.uuid4().hex
seed_text = 'rolling-seed-' + seed_key
seed = c.message(alice, chat, seed_text, key=seed_key)['json']
confirmed = {seed['id']: seed}
rows = []
halt = threading.Event()
lock = threading.Lock()
beginning = time.monotonic()
def worker():
local = Client(timeout=2)
key = uuid.uuid4().hex
while not halt.is_set():
for method in ('POST', 'GET'):
row = {'method': method, 'start': time.monotonic()-beginning,
'ok': False, 'instance': '', 'error': ''}
try:
if method == 'POST':
r = local.message(alice, chat, 'rolling-' + key, key=key)
with lock:
confirmed[r['json']['id']] = r['json']
key = uuid.uuid4().hex
else:
r = local.call('GET', path, actor=alice, schema='MessagePage')
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
except Exception as exc:
row['error'] = str(exc) if isinstance(exc, AssertionError) else type(exc).__name__
row['end'] = time.monotonic()-beginning
with lock:
rows.append(row)
halt.wait(.1)
pool = ThreadPoolExecutor(max_workers=4)
jobs = [pool.submit(worker) for _ in range(4)]
try:
time.sleep(5)
for old in original:
current = containers()
check(len(current) == 2 and old in current, 'Unexpected API container set')
survivor = next(cid for cid in current if cid != old)
ready(survivor)
step = {'old': info[old], 'survivor': inspect(survivor),
'start': time.monotonic()-beginning}
report['steps'].append(step)
print('Replacing ' + info[old]['hostname'], flush=True)
docker('stop', '--timeout', '20', old)
check(not json.loads(docker('inspect', '--format', '{{json .State.Running}}', old)),
'Old container is still running')
docker('rm', old)
# --no-recreate preserves the survivor and creates only the missing replica.
# --no-deps prevents restarting migrate, DB, Redis or LB.
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build',
'--pull', 'never', '--scale', 'api=2', 'api')
current = containers()
added = set(current)-set([old, survivor])
check(len(current) == 2 and survivor in current and len(added) == 1,
'Replacement affected an unexpected container')
new = added.pop()
step['new'] = inspect(new)
check(step['new']['image'] == original_image, 'Replacement image differs')
ready(new)
ready(survivor)
# Wait until the replacement actually receives traffic through LB.
observe(c, alice, {step['new']['hostname'], step['survivor']['hostname']})
step['ready_through_lb'] = time.monotonic()-beginning
# A control POST can time out just like a background POST.
# Retry the SAME key/body, record all attempts, and keep a hard deadline.
replay_started = time.monotonic()
replay_deadline = replay_started + 10
step['control_replay'] = {'attempts': 0, 'errors': [], 'pass': False}
while True:
remaining = replay_deadline - time.monotonic()
check(remaining > 0, 'Control replay did not recover within 10s')
step['control_replay']['attempts'] += 1
try:
control = Client(timeout=min(2, remaining))
replay = control.message(alice, chat, seed_text, key=seed_key)
except Failure as exc:
detail = str(exc)
step['control_replay']['errors'].append(detail)
# Contract errors, authorization errors and conflicts are not retried.
if not ('connection/TLS failure' in detail
or any('received ' + str(code) in detail for code in (502, 503, 504))):
raise
time.sleep(min(.1, max(0, replay_deadline-time.monotonic())))
continue
check(replay['json'] == seed,
'Idempotency replay changed after replacement')
step['control_replay']['pass'] = True
step['control_replay']['elapsed_seconds'] = round(time.monotonic()-replay_started, 3)
break
# Readiness is not proof of timely application traffic under load.
# Keep the original 2s client timeout; retain every timeout/error.
deadline = time.monotonic() + 30
while True:
with lock:
counts = {method: sum(1 for r in rows
if r['ok'] and r['method'] == method
and r['instance'] == step['new']['hostname']
and r['start'] >= step['ready_through_lb'])
for method in ('POST', 'GET')}
if (min(counts.values()) >= 3
and time.monotonic()-beginning >= step['ready_through_lb']+5):
break
if time.monotonic() >= deadline:
break
time.sleep(.1)
step['new_replica_successes_after_ready'] = counts
step['end'] = time.monotonic()-beginning
with lock:
selected = [r.copy() for r in rows if step['start'] <= r['start']
and r['end'] <= step['end']]
duration = step['end']-step['start']
good = [r for r in selected if r['ok']]
step['completed_requests'] = len(selected)
step['errors'] = len(selected)-len(good)
step['success_rps'] = round(len(good)/duration, 2)
step['replacement_seconds'] = round(step['ready_through_lb']-step['start'], 3)
for method in ('POST', 'GET'):
check(any(r['method'] == method and r['instance'] == step['survivor']['hostname']
and r['end'] < step['ready_through_lb'] for r in good),
'No successful ' + method + ' on survivor during replacement')
check(any(r['method'] == method and r['instance'] == step['new']['hostname']
for r in good), 'New replica did not serve ' + method)
check(min(counts.values()) >= 3,
'New replica did not complete 3 POST and 3 GET within 30s after readiness')
print(json.dumps(step, indent=2), flush=True)
halt.set()
pool.shutdown(wait=True)
for job in jobs:
job.result()
messages = history(c, bob, chat)
by_id = {m['id']: m for m in messages}
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed data lost')
texts = [m['text'] for m in messages]
check(len(texts) == len(set(texts)), 'Retry created duplicate messages')
report['confirmed_messages_preserved'] = len(confirmed)
report['after'] = [inspect(cid) for cid in containers()]
check(not set(original).intersection(v['id'] for v in report['after']),
'An original container remains')
report['pass'] = True
except Exception as exc:
report['error'] = str(exc)
raise
finally:
halt.set()
pool.shutdown(wait=True)
if rows:
with (output / 'requests.csv').open('w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(sorted(rows, key=lambda r: r['start']))
(output / 'summary.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
print('Results:', output, flush=True)
print('PASS: both containers replaced, survivor served reads/writes, session and data preserved')
if __name__ == '__main__':
try:
main()
except Exception as exc:
print('FAIL:', str(exc))
print('Stop the experiment. Inspect docker compose ps -a and logs before continuing.')
raise SystemExit(1)