#!/usr/bin/env python3 """Sequential replacement of TWO API containers with the SAME image under load. Put beside level4_check.py, client.py and resilience.py in tests/. Does not build images, run migrations, modify dependencies or delete volumes. On failure it stops replacing containers and preserves the remaining replica. """ import csv import hashlib import json import subprocess import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path from client import Client, Failure, check from resilience import history from level4_check import docker, ready, observe ROOT = Path(__file__).resolve().parents[1] def inspect(cid): # Only output safe fields; never persist container environment/secrets. return {'id': cid, 'hostname': docker('inspect', '--format', '{{.Config.Hostname}}', cid), 'image': docker('inspect', '--format', '{{.Image}}', cid)} def containers(): return docker('compose', 'ps', '-a', '-q', 'api').splitlines() def main(): original = containers() check(len(original) == 2, 'Expected exactly two API containers') for cid in original: ready(cid) info = {cid: inspect(cid) for cid in original} check(len({v['image'] for v in info.values()}) == 1, 'Both replicas must use the same image') original_image = info[original[0]]['image'] # Verify the configured image tag still points at that same immutable image. config = json.loads(docker('compose', 'config', '--format', 'json')) api = config['services']['api'] image_ref = api.get('image') check(bool(image_ref), 'API image must be explicitly configured') check(int(api.get('deploy', {}).get('replicas', 2)) == 2, 'Configure two replicas') check(docker('image', 'inspect', '--format', '{{.Id}}', image_ref) == original_image, 'Image tag changed. This test expects replacement with the existing image') del config, api output = ROOT / 'results' / ('rolling-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')) output.mkdir(parents=True) report = {'script_version': 3, 'scope': 'Sequential replacement with the same image; no schema migration or version upgrade', 'before': list(info.values()), 'steps': [], 'pass': False, 'workload': '4 closed-loop workers; POST then GET; 100ms pause; HTTP timeout 2s', 'migration_sha256': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted((ROOT / 'java/src/main/resources/db/migration').glob('*.sql'))}} c = Client(timeout=2) alice, bob, _, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' observe(c, alice, {v['hostname'] for v in info.values()}) seed_key = uuid.uuid4().hex seed_text = 'rolling-seed-' + seed_key seed = c.message(alice, chat, seed_text, key=seed_key)['json'] confirmed = {seed['id']: seed} rows = [] halt = threading.Event() lock = threading.Lock() beginning = time.monotonic() def worker(): local = Client(timeout=2) key = uuid.uuid4().hex while not halt.is_set(): for method in ('POST', 'GET'): row = {'method': method, 'start': time.monotonic()-beginning, 'ok': False, 'instance': '', 'error': ''} try: if method == 'POST': r = local.message(alice, chat, 'rolling-' + key, key=key) with lock: confirmed[r['json']['id']] = r['json'] key = uuid.uuid4().hex else: r = local.call('GET', path, actor=alice, schema='MessagePage') row.update(ok=True, instance=r['headers']['X-Instance-Id']) except Exception as exc: row['error'] = str(exc) if isinstance(exc, AssertionError) else type(exc).__name__ row['end'] = time.monotonic()-beginning with lock: rows.append(row) halt.wait(.1) pool = ThreadPoolExecutor(max_workers=4) jobs = [pool.submit(worker) for _ in range(4)] try: time.sleep(5) for old in original: current = containers() check(len(current) == 2 and old in current, 'Unexpected API container set') survivor = next(cid for cid in current if cid != old) ready(survivor) step = {'old': info[old], 'survivor': inspect(survivor), 'start': time.monotonic()-beginning} report['steps'].append(step) print('Replacing ' + info[old]['hostname'], flush=True) docker('stop', '--timeout', '20', old) check(not json.loads(docker('inspect', '--format', '{{json .State.Running}}', old)), 'Old container is still running') docker('rm', old) # --no-recreate preserves the survivor and creates only the missing replica. # --no-deps prevents restarting migrate, DB, Redis or LB. docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build', '--pull', 'never', '--scale', 'api=2', 'api') current = containers() added = set(current)-set([old, survivor]) check(len(current) == 2 and survivor in current and len(added) == 1, 'Replacement affected an unexpected container') new = added.pop() step['new'] = inspect(new) check(step['new']['image'] == original_image, 'Replacement image differs') ready(new) ready(survivor) # Wait until the replacement actually receives traffic through LB. observe(c, alice, {step['new']['hostname'], step['survivor']['hostname']}) step['ready_through_lb'] = time.monotonic()-beginning # A control POST can time out just like a background POST. # Retry the SAME key/body, record all attempts, and keep a hard deadline. replay_started = time.monotonic() replay_deadline = replay_started + 10 step['control_replay'] = {'attempts': 0, 'errors': [], 'pass': False} while True: remaining = replay_deadline - time.monotonic() check(remaining > 0, 'Control replay did not recover within 10s') step['control_replay']['attempts'] += 1 try: control = Client(timeout=min(2, remaining)) replay = control.message(alice, chat, seed_text, key=seed_key) except Failure as exc: detail = str(exc) step['control_replay']['errors'].append(detail) # Contract errors, authorization errors and conflicts are not retried. if not ('connection/TLS failure' in detail or any('received ' + str(code) in detail for code in (502, 503, 504))): raise time.sleep(min(.1, max(0, replay_deadline-time.monotonic()))) continue check(replay['json'] == seed, 'Idempotency replay changed after replacement') step['control_replay']['pass'] = True step['control_replay']['elapsed_seconds'] = round(time.monotonic()-replay_started, 3) break # Readiness is not proof of timely application traffic under load. # Keep the original 2s client timeout; retain every timeout/error. deadline = time.monotonic() + 30 while True: with lock: counts = {method: sum(1 for r in rows if r['ok'] and r['method'] == method and r['instance'] == step['new']['hostname'] and r['start'] >= step['ready_through_lb']) for method in ('POST', 'GET')} if (min(counts.values()) >= 3 and time.monotonic()-beginning >= step['ready_through_lb']+5): break if time.monotonic() >= deadline: break time.sleep(.1) step['new_replica_successes_after_ready'] = counts step['end'] = time.monotonic()-beginning with lock: selected = [r.copy() for r in rows if step['start'] <= r['start'] and r['end'] <= step['end']] duration = step['end']-step['start'] good = [r for r in selected if r['ok']] step['completed_requests'] = len(selected) step['errors'] = len(selected)-len(good) step['success_rps'] = round(len(good)/duration, 2) step['replacement_seconds'] = round(step['ready_through_lb']-step['start'], 3) for method in ('POST', 'GET'): check(any(r['method'] == method and r['instance'] == step['survivor']['hostname'] and r['end'] < step['ready_through_lb'] for r in good), 'No successful ' + method + ' on survivor during replacement') check(any(r['method'] == method and r['instance'] == step['new']['hostname'] for r in good), 'New replica did not serve ' + method) check(min(counts.values()) >= 3, 'New replica did not complete 3 POST and 3 GET within 30s after readiness') print(json.dumps(step, indent=2), flush=True) halt.set() pool.shutdown(wait=True) for job in jobs: job.result() messages = history(c, bob, chat) by_id = {m['id']: m for m in messages} check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed data lost') texts = [m['text'] for m in messages] check(len(texts) == len(set(texts)), 'Retry created duplicate messages') report['confirmed_messages_preserved'] = len(confirmed) report['after'] = [inspect(cid) for cid in containers()] check(not set(original).intersection(v['id'] for v in report['after']), 'An original container remains') report['pass'] = True except Exception as exc: report['error'] = str(exc) raise finally: halt.set() pool.shutdown(wait=True) if rows: with (output / 'requests.csv').open('w', newline='', encoding='utf-8') as f: writer = csv.DictWriter(f, fieldnames=list(rows[0])) writer.writeheader() writer.writerows(sorted(rows, key=lambda r: r['start'])) (output / 'summary.json').write_text(json.dumps(report, indent=2), encoding='utf-8') print('Results:', output, flush=True) print('PASS: both containers replaced, survivor served reads/writes, session and data preserved') if __name__ == '__main__': try: main() except Exception as exc: print('FAIL:', str(exc)) print('Stop the experiment. Inspect docker compose ps -a and logs before continuing.') raise SystemExit(1)