Files
mtusi2026lab3/scripts/rolling_update.py
2026-10-16 01:30:10 +03:00

88 lines
3.8 KiB
Python

#!/usr/bin/env python3
import argparse
import json
import os
import time
from pathlib import Path
from lab3lib import WriteLoad, api_containers, ps_snapshot, sh, wait_healthy
def image_label(cid):
return sh('docker', 'inspect', '-f', '{{index .Config.Labels "org.opencontainers.image.revision"}}', cid)
def main():
p = argparse.ArgumentParser()
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--build-label', default=time.strftime('rollout-%Y%m%dT%H%M%S'))
p.add_argument('--workers', type=int, default=8)
p.add_argument('--output', default=None)
a = p.parse_args()
old = [r for r in api_containers() if r['state'] == 'running']
n = len(old)
assert n >= 2, 'need >=2 running replicas'
log = []
say = lambda *x: (print(*x), log.append(' '.join(map(str, x))))
say('old replicas:', [(r['name'], r['id'], image_label(r['id'])) for r in old])
load = WriteLoad(a.base_url, workers=a.workers)
load.start()
time.sleep(5)
t_begin = time.monotonic()
env = dict(os.environ, APP_BUILD=a.build_label)
t = time.monotonic()
import subprocess
subprocess.run(['docker', 'compose', 'build', '-q', 'migrate'], check=True, env=env,
cwd=Path(__file__).resolve().parents[1])
say('built image with label', a.build_label, 'in %.1fs' % (time.monotonic() - t))
sh('docker', 'compose', 'run', '--rm', '--no-deps', 'migrate')
say('migrations applied once (advisory lock, idempotent)')
for r in old:
before = {x['id'] for x in api_containers()}
sh('docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (n + 1), 'api')
new = [x for x in api_containers() if x['id'] not in before]
assert len(new) == 1, 'expected exactly one new replica, got %s' % new
cid = new[0]['id']
assert wait_healthy(cid, 60), 'new replica not healthy'
t_h = time.monotonic()
while not any(e[1] and e[2] == cid and e[0] > t_h for e in list(load.events)):
assert time.monotonic() - t_h < 15, 'new replica not in LB rotation'
time.sleep(0.1)
say('new %s (%s) healthy and serving; label=%s' % (new[0]['name'], cid, image_label(cid)))
t_s = time.monotonic()
sh('docker', 'stop', '-t', '15', r['id'])
sh('docker', 'rm', r['id'])
say('old %s (%s) drained+removed in %.1fs' % (r['name'], r['id'], time.monotonic() - t_s))
t_end = time.monotonic()
time.sleep(5)
load.stop()
now = api_containers()
say('new replicas:', [(x['name'], x['id'], image_label(x['id'])) for x in now])
result = {
'build_label': a.build_label, 'replicas': n, 'workers': a.workers,
'before_rollout_5s': load.window(t_begin - 5, t_begin),
'during_rollout': load.window(t_begin, t_end),
'after_rollout_5s': load.window(t_end, t_end + 5),
'client_retries': load.retries,
'error_samples': sorted({e[2] for e in load.events if not e[1]})[:5],
'all_new_label': all(image_label(x['id']) == a.build_label for x in now),
'data_check': load.verify_history(), 'log': log, 'docker_ps_end': ps_snapshot().splitlines(),
}
print(json.dumps({k: result[k] for k in ('before_rollout_5s', 'during_rollout', 'after_rollout_5s',
'client_retries', 'all_new_label', 'data_check')}, indent=1))
ok = result['all_new_label'] and result['data_check']['lost_confirmed'] == 0 \
and result['data_check']['duplicate_texts'] == 0 and len(now) == n
print('PASS' if ok else 'FAIL', 'rolling update')
if a.output:
Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
raise SystemExit(0 if ok else 1)
if __name__ == '__main__':
main()