225 lines
11 KiB
Python
225 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Sequential replacement of TWO API containers with the SAME image under load.
|
|
Put beside level4_check.py, client.py and resilience.py in tests/.
|
|
Does not build images, run migrations, modify dependencies or delete volumes.
|
|
On failure it stops replacing containers and preserves the remaining replica.
|
|
"""
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from client import Client, Failure, check
|
|
from resilience import history
|
|
from level4_check import docker, ready, observe
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def inspect(cid):
|
|
# Only output safe fields; never persist container environment/secrets.
|
|
return {'id': cid,
|
|
'hostname': docker('inspect', '--format', '{{.Config.Hostname}}', cid),
|
|
'image': docker('inspect', '--format', '{{.Image}}', cid)}
|
|
|
|
|
|
def containers():
|
|
return docker('compose', 'ps', '-a', '-q', 'api').splitlines()
|
|
|
|
|
|
def main():
|
|
original = containers()
|
|
check(len(original) == 2, 'Expected exactly two API containers')
|
|
for cid in original:
|
|
ready(cid)
|
|
info = {cid: inspect(cid) for cid in original}
|
|
check(len({v['image'] for v in info.values()}) == 1, 'Both replicas must use the same image')
|
|
original_image = info[original[0]]['image']
|
|
# Verify the configured image tag still points at that same immutable image.
|
|
config = json.loads(docker('compose', 'config', '--format', 'json'))
|
|
api = config['services']['api']
|
|
image_ref = api.get('image')
|
|
check(bool(image_ref), 'API image must be explicitly configured')
|
|
check(int(api.get('deploy', {}).get('replicas', 2)) == 2, 'Configure two replicas')
|
|
check(docker('image', 'inspect', '--format', '{{.Id}}', image_ref) == original_image,
|
|
'Image tag changed. This test expects replacement with the existing image')
|
|
del config, api
|
|
output = ROOT / 'results' / ('rolling-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
|
|
output.mkdir(parents=True)
|
|
report = {'script_version': 3, 'scope': 'Sequential replacement with the same image; no schema migration or version upgrade',
|
|
'before': list(info.values()), 'steps': [], 'pass': False,
|
|
'workload': '4 closed-loop workers; POST then GET; 100ms pause; HTTP timeout 2s',
|
|
'migration_sha256': {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted((ROOT / 'java/src/main/resources/db/migration').glob('*.sql'))}}
|
|
c = Client(timeout=2)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
|
observe(c, alice, {v['hostname'] for v in info.values()})
|
|
seed_key = uuid.uuid4().hex
|
|
seed_text = 'rolling-seed-' + seed_key
|
|
seed = c.message(alice, chat, seed_text, key=seed_key)['json']
|
|
confirmed = {seed['id']: seed}
|
|
rows = []
|
|
halt = threading.Event()
|
|
lock = threading.Lock()
|
|
beginning = time.monotonic()
|
|
def worker():
|
|
local = Client(timeout=2)
|
|
key = uuid.uuid4().hex
|
|
while not halt.is_set():
|
|
for method in ('POST', 'GET'):
|
|
row = {'method': method, 'start': time.monotonic()-beginning,
|
|
'ok': False, 'instance': '', 'error': ''}
|
|
try:
|
|
if method == 'POST':
|
|
r = local.message(alice, chat, 'rolling-' + key, key=key)
|
|
with lock:
|
|
confirmed[r['json']['id']] = r['json']
|
|
key = uuid.uuid4().hex
|
|
else:
|
|
r = local.call('GET', path, actor=alice, schema='MessagePage')
|
|
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
|
|
except Exception as exc:
|
|
row['error'] = str(exc) if isinstance(exc, AssertionError) else type(exc).__name__
|
|
row['end'] = time.monotonic()-beginning
|
|
with lock:
|
|
rows.append(row)
|
|
halt.wait(.1)
|
|
pool = ThreadPoolExecutor(max_workers=4)
|
|
jobs = [pool.submit(worker) for _ in range(4)]
|
|
try:
|
|
time.sleep(5)
|
|
for old in original:
|
|
current = containers()
|
|
check(len(current) == 2 and old in current, 'Unexpected API container set')
|
|
survivor = next(cid for cid in current if cid != old)
|
|
ready(survivor)
|
|
step = {'old': info[old], 'survivor': inspect(survivor),
|
|
'start': time.monotonic()-beginning}
|
|
report['steps'].append(step)
|
|
print('Replacing ' + info[old]['hostname'], flush=True)
|
|
docker('stop', '--timeout', '20', old)
|
|
check(not json.loads(docker('inspect', '--format', '{{json .State.Running}}', old)),
|
|
'Old container is still running')
|
|
docker('rm', old)
|
|
# --no-recreate preserves the survivor and creates only the missing replica.
|
|
# --no-deps prevents restarting migrate, DB, Redis or LB.
|
|
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build',
|
|
'--pull', 'never', '--scale', 'api=2', 'api')
|
|
current = containers()
|
|
added = set(current)-set([old, survivor])
|
|
check(len(current) == 2 and survivor in current and len(added) == 1,
|
|
'Replacement affected an unexpected container')
|
|
new = added.pop()
|
|
step['new'] = inspect(new)
|
|
check(step['new']['image'] == original_image, 'Replacement image differs')
|
|
ready(new)
|
|
ready(survivor)
|
|
# Wait until the replacement actually receives traffic through LB.
|
|
observe(c, alice, {step['new']['hostname'], step['survivor']['hostname']})
|
|
step['ready_through_lb'] = time.monotonic()-beginning
|
|
# A control POST can time out just like a background POST.
|
|
# Retry the SAME key/body, record all attempts, and keep a hard deadline.
|
|
replay_started = time.monotonic()
|
|
replay_deadline = replay_started + 10
|
|
step['control_replay'] = {'attempts': 0, 'errors': [], 'pass': False}
|
|
while True:
|
|
remaining = replay_deadline - time.monotonic()
|
|
check(remaining > 0, 'Control replay did not recover within 10s')
|
|
step['control_replay']['attempts'] += 1
|
|
try:
|
|
control = Client(timeout=min(2, remaining))
|
|
replay = control.message(alice, chat, seed_text, key=seed_key)
|
|
except Failure as exc:
|
|
detail = str(exc)
|
|
step['control_replay']['errors'].append(detail)
|
|
# Contract errors, authorization errors and conflicts are not retried.
|
|
if not ('connection/TLS failure' in detail
|
|
or any('received ' + str(code) in detail for code in (502, 503, 504))):
|
|
raise
|
|
time.sleep(min(.1, max(0, replay_deadline-time.monotonic())))
|
|
continue
|
|
check(replay['json'] == seed,
|
|
'Idempotency replay changed after replacement')
|
|
step['control_replay']['pass'] = True
|
|
step['control_replay']['elapsed_seconds'] = round(time.monotonic()-replay_started, 3)
|
|
break
|
|
# Readiness is not proof of timely application traffic under load.
|
|
# Keep the original 2s client timeout; retain every timeout/error.
|
|
deadline = time.monotonic() + 30
|
|
while True:
|
|
with lock:
|
|
counts = {method: sum(1 for r in rows
|
|
if r['ok'] and r['method'] == method
|
|
and r['instance'] == step['new']['hostname']
|
|
and r['start'] >= step['ready_through_lb'])
|
|
for method in ('POST', 'GET')}
|
|
if (min(counts.values()) >= 3
|
|
and time.monotonic()-beginning >= step['ready_through_lb']+5):
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
break
|
|
time.sleep(.1)
|
|
step['new_replica_successes_after_ready'] = counts
|
|
step['end'] = time.monotonic()-beginning
|
|
with lock:
|
|
selected = [r.copy() for r in rows if step['start'] <= r['start']
|
|
and r['end'] <= step['end']]
|
|
duration = step['end']-step['start']
|
|
good = [r for r in selected if r['ok']]
|
|
step['completed_requests'] = len(selected)
|
|
step['errors'] = len(selected)-len(good)
|
|
step['success_rps'] = round(len(good)/duration, 2)
|
|
step['replacement_seconds'] = round(step['ready_through_lb']-step['start'], 3)
|
|
for method in ('POST', 'GET'):
|
|
check(any(r['method'] == method and r['instance'] == step['survivor']['hostname']
|
|
and r['end'] < step['ready_through_lb'] for r in good),
|
|
'No successful ' + method + ' on survivor during replacement')
|
|
check(any(r['method'] == method and r['instance'] == step['new']['hostname']
|
|
for r in good), 'New replica did not serve ' + method)
|
|
check(min(counts.values()) >= 3,
|
|
'New replica did not complete 3 POST and 3 GET within 30s after readiness')
|
|
print(json.dumps(step, indent=2), flush=True)
|
|
halt.set()
|
|
pool.shutdown(wait=True)
|
|
for job in jobs:
|
|
job.result()
|
|
messages = history(c, bob, chat)
|
|
by_id = {m['id']: m for m in messages}
|
|
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed data lost')
|
|
texts = [m['text'] for m in messages]
|
|
check(len(texts) == len(set(texts)), 'Retry created duplicate messages')
|
|
report['confirmed_messages_preserved'] = len(confirmed)
|
|
report['after'] = [inspect(cid) for cid in containers()]
|
|
check(not set(original).intersection(v['id'] for v in report['after']),
|
|
'An original container remains')
|
|
report['pass'] = True
|
|
except Exception as exc:
|
|
report['error'] = str(exc)
|
|
raise
|
|
finally:
|
|
halt.set()
|
|
pool.shutdown(wait=True)
|
|
if rows:
|
|
with (output / 'requests.csv').open('w', newline='', encoding='utf-8') as f:
|
|
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(sorted(rows, key=lambda r: r['start']))
|
|
(output / 'summary.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
|
|
print('Results:', output, flush=True)
|
|
print('PASS: both containers replaced, survivor served reads/writes, session and data preserved')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as exc:
|
|
print('FAIL:', str(exc))
|
|
print('Stop the experiment. Inspect docker compose ps -a and logs before continuing.')
|
|
raise SystemExit(1)
|