#!/usr/bin/env python3 """Drain and replace API processes individually, retaining other container IDs.""" import json,os,subprocess,time from pathlib import Path ROOT=Path(__file__).resolve().parents[1] def run(args,input=None): p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100) if p.returncode:raise RuntimeError(p.stderr.strip()[:300]) return p.stdout.strip() def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n') def ids():return run(['./scripts/compose.sh','ps','-q','api']).split() def states(lb): lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split() return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()] def main(): initial=ids() if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas') lb=run(['./scripts/compose.sh','ps','-q','lb']) for old in initial: info=json.loads(run(['docker','inspect',old]))[0] ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress'] backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api') runtime(lb,f'set server api/{backend} state drain') try: deadline=time.monotonic()+10 while True: # Existing transactions get a chance to finish before SIGTERM. import csv,io raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv']) row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend) if int(row['scur'])==0:break if time.monotonic()>deadline:raise RuntimeError('backend did not drain') time.sleep(.1) before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old]) run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api']) after=set(ids()) if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly') new=(after-before).pop();deadline=time.monotonic()+60 while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy': if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness') time.sleep(.5) print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True) finally:runtime(lb,f'set server api/{backend} state ready') time.sleep(3) # allow DNS refresh and a successful active readiness check if __name__=='__main__':main()