forked from mtusi2026/mtusi2026lab3
46 lines
2.5 KiB
Python
46 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Drain and replace API processes individually, retaining other container IDs."""
|
|
import json,os,subprocess,time
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
def run(args,input=None):
|
|
p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100)
|
|
if p.returncode:raise RuntimeError(p.stderr.strip()[:300])
|
|
return p.stdout.strip()
|
|
def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n')
|
|
def ids():return run(['./scripts/compose.sh','ps','-q','api']).split()
|
|
def states(lb):
|
|
lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split()
|
|
return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()]
|
|
def main():
|
|
initial=ids()
|
|
if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas')
|
|
lb=run(['./scripts/compose.sh','ps','-q','lb'])
|
|
for old in initial:
|
|
info=json.loads(run(['docker','inspect',old]))[0]
|
|
ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress']
|
|
backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api')
|
|
runtime(lb,f'set server api/{backend} state drain')
|
|
try:
|
|
deadline=time.monotonic()+10
|
|
while True:
|
|
# Existing transactions get a chance to finish before SIGTERM.
|
|
import csv,io
|
|
raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv'])
|
|
row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend)
|
|
if int(row['scur'])==0:break
|
|
if time.monotonic()>deadline:raise RuntimeError('backend did not drain')
|
|
time.sleep(.1)
|
|
before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old])
|
|
run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api'])
|
|
after=set(ids())
|
|
if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly')
|
|
new=(after-before).pop();deadline=time.monotonic()+60
|
|
while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy':
|
|
if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness')
|
|
time.sleep(.5)
|
|
print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True)
|
|
finally:runtime(lb,f'set server api/{backend} state ready')
|
|
time.sleep(3) # allow DNS refresh and a successful active readiness check
|
|
if __name__=='__main__':main()
|