Files
mtusi2026lab5/scripts/rolling-update.py
T

46 lines
2.5 KiB
Python

#!/usr/bin/env python3
"""Drain and replace API processes individually, retaining other container IDs."""
import json,os,subprocess,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def run(args,input=None):
p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100)
if p.returncode:raise RuntimeError(p.stderr.strip()[:300])
return p.stdout.strip()
def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n')
def ids():return run(['./scripts/compose.sh','ps','-q','api']).split()
def states(lb):
lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split()
return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()]
def main():
initial=ids()
if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas')
lb=run(['./scripts/compose.sh','ps','-q','lb'])
for old in initial:
info=json.loads(run(['docker','inspect',old]))[0]
ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress']
backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api')
runtime(lb,f'set server api/{backend} state drain')
try:
deadline=time.monotonic()+10
while True:
# Existing transactions get a chance to finish before SIGTERM.
import csv,io
raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv'])
row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend)
if int(row['scur'])==0:break
if time.monotonic()>deadline:raise RuntimeError('backend did not drain')
time.sleep(.1)
before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old])
run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api'])
after=set(ids())
if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly')
new=(after-before).pop();deadline=time.monotonic()+60
while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy':
if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness')
time.sleep(.5)
print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True)
finally:runtime(lb,f'set server api/{backend} state ready')
time.sleep(3) # allow DNS refresh and a successful active readiness check
if __name__=='__main__':main()