88 lines
3.8 KiB
Python
88 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from lab3lib import WriteLoad, api_containers, ps_snapshot, sh, wait_healthy
|
|
|
|
|
|
def image_label(cid):
|
|
return sh('docker', 'inspect', '-f', '{{index .Config.Labels "org.opencontainers.image.revision"}}', cid)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--build-label', default=time.strftime('rollout-%Y%m%dT%H%M%S'))
|
|
p.add_argument('--workers', type=int, default=8)
|
|
p.add_argument('--output', default=None)
|
|
a = p.parse_args()
|
|
|
|
old = [r for r in api_containers() if r['state'] == 'running']
|
|
n = len(old)
|
|
assert n >= 2, 'need >=2 running replicas'
|
|
log = []
|
|
say = lambda *x: (print(*x), log.append(' '.join(map(str, x))))
|
|
say('old replicas:', [(r['name'], r['id'], image_label(r['id'])) for r in old])
|
|
|
|
load = WriteLoad(a.base_url, workers=a.workers)
|
|
load.start()
|
|
time.sleep(5)
|
|
t_begin = time.monotonic()
|
|
|
|
env = dict(os.environ, APP_BUILD=a.build_label)
|
|
t = time.monotonic()
|
|
import subprocess
|
|
subprocess.run(['docker', 'compose', 'build', '-q', 'migrate'], check=True, env=env,
|
|
cwd=Path(__file__).resolve().parents[1])
|
|
say('built image with label', a.build_label, 'in %.1fs' % (time.monotonic() - t))
|
|
sh('docker', 'compose', 'run', '--rm', '--no-deps', 'migrate')
|
|
say('migrations applied once (advisory lock, idempotent)')
|
|
|
|
for r in old:
|
|
before = {x['id'] for x in api_containers()}
|
|
sh('docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (n + 1), 'api')
|
|
new = [x for x in api_containers() if x['id'] not in before]
|
|
assert len(new) == 1, 'expected exactly one new replica, got %s' % new
|
|
cid = new[0]['id']
|
|
assert wait_healthy(cid, 60), 'new replica not healthy'
|
|
t_h = time.monotonic()
|
|
while not any(e[1] and e[2] == cid and e[0] > t_h for e in list(load.events)):
|
|
assert time.monotonic() - t_h < 15, 'new replica not in LB rotation'
|
|
time.sleep(0.1)
|
|
say('new %s (%s) healthy and serving; label=%s' % (new[0]['name'], cid, image_label(cid)))
|
|
t_s = time.monotonic()
|
|
sh('docker', 'stop', '-t', '15', r['id'])
|
|
sh('docker', 'rm', r['id'])
|
|
say('old %s (%s) drained+removed in %.1fs' % (r['name'], r['id'], time.monotonic() - t_s))
|
|
|
|
t_end = time.monotonic()
|
|
time.sleep(5)
|
|
load.stop()
|
|
now = api_containers()
|
|
say('new replicas:', [(x['name'], x['id'], image_label(x['id'])) for x in now])
|
|
result = {
|
|
'build_label': a.build_label, 'replicas': n, 'workers': a.workers,
|
|
'before_rollout_5s': load.window(t_begin - 5, t_begin),
|
|
'during_rollout': load.window(t_begin, t_end),
|
|
'after_rollout_5s': load.window(t_end, t_end + 5),
|
|
'client_retries': load.retries,
|
|
'error_samples': sorted({e[2] for e in load.events if not e[1]})[:5],
|
|
'all_new_label': all(image_label(x['id']) == a.build_label for x in now),
|
|
'data_check': load.verify_history(), 'log': log, 'docker_ps_end': ps_snapshot().splitlines(),
|
|
}
|
|
print(json.dumps({k: result[k] for k in ('before_rollout_5s', 'during_rollout', 'after_rollout_5s',
|
|
'client_retries', 'all_new_label', 'data_check')}, indent=1))
|
|
ok = result['all_new_label'] and result['data_check']['lost_confirmed'] == 0 \
|
|
and result['data_check']['duplicate_texts'] == 0 and len(now) == n
|
|
print('PASS' if ok else 'FAIL', 'rolling update')
|
|
if a.output:
|
|
Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
raise SystemExit(0 if ok else 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|