#!/usr/bin/env python3 import argparse import json import os import time from pathlib import Path from lab3lib import WriteLoad, api_containers, ps_snapshot, sh, wait_healthy def image_label(cid): return sh('docker', 'inspect', '-f', '{{index .Config.Labels "org.opencontainers.image.revision"}}', cid) def main(): p = argparse.ArgumentParser() p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--build-label', default=time.strftime('rollout-%Y%m%dT%H%M%S')) p.add_argument('--workers', type=int, default=8) p.add_argument('--output', default=None) a = p.parse_args() old = [r for r in api_containers() if r['state'] == 'running'] n = len(old) assert n >= 2, 'need >=2 running replicas' log = [] say = lambda *x: (print(*x), log.append(' '.join(map(str, x)))) say('old replicas:', [(r['name'], r['id'], image_label(r['id'])) for r in old]) load = WriteLoad(a.base_url, workers=a.workers) load.start() time.sleep(5) t_begin = time.monotonic() env = dict(os.environ, APP_BUILD=a.build_label) t = time.monotonic() import subprocess subprocess.run(['docker', 'compose', 'build', '-q', 'migrate'], check=True, env=env, cwd=Path(__file__).resolve().parents[1]) say('built image with label', a.build_label, 'in %.1fs' % (time.monotonic() - t)) sh('docker', 'compose', 'run', '--rm', '--no-deps', 'migrate') say('migrations applied once (advisory lock, idempotent)') for r in old: before = {x['id'] for x in api_containers()} sh('docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (n + 1), 'api') new = [x for x in api_containers() if x['id'] not in before] assert len(new) == 1, 'expected exactly one new replica, got %s' % new cid = new[0]['id'] assert wait_healthy(cid, 60), 'new replica not healthy' t_h = time.monotonic() while not any(e[1] and e[2] == cid and e[0] > t_h for e in list(load.events)): assert time.monotonic() - t_h < 15, 'new replica not in LB rotation' time.sleep(0.1) say('new %s (%s) healthy and serving; label=%s' % (new[0]['name'], cid, image_label(cid))) t_s = time.monotonic() sh('docker', 'stop', '-t', '15', r['id']) sh('docker', 'rm', r['id']) say('old %s (%s) drained+removed in %.1fs' % (r['name'], r['id'], time.monotonic() - t_s)) t_end = time.monotonic() time.sleep(5) load.stop() now = api_containers() say('new replicas:', [(x['name'], x['id'], image_label(x['id'])) for x in now]) result = { 'build_label': a.build_label, 'replicas': n, 'workers': a.workers, 'before_rollout_5s': load.window(t_begin - 5, t_begin), 'during_rollout': load.window(t_begin, t_end), 'after_rollout_5s': load.window(t_end, t_end + 5), 'client_retries': load.retries, 'error_samples': sorted({e[2] for e in load.events if not e[1]})[:5], 'all_new_label': all(image_label(x['id']) == a.build_label for x in now), 'data_check': load.verify_history(), 'log': log, 'docker_ps_end': ps_snapshot().splitlines(), } print(json.dumps({k: result[k] for k in ('before_rollout_5s', 'during_rollout', 'after_rollout_5s', 'client_retries', 'all_new_label', 'data_check')}, indent=1)) ok = result['all_new_label'] and result['data_check']['lost_confirmed'] == 0 \ and result['data_check']['duplicate_texts'] == 0 and len(now) == n print('PASS' if ok else 'FAIL', 'rolling update') if a.output: Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') raise SystemExit(0 if ok else 1) if __name__ == '__main__': main()