72 lines
4.2 KiB
Python
72 lines
4.2 KiB
Python
"""Actual Docker failures, scoped by Compose service labels."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
|
from client import Client, check, Failure
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
BASE=['docker','compose','--project-directory',str(ROOT),'-p','lyakhonov-lab4']
|
|
def dc(*args): return subprocess.check_output(BASE+list(args),text=True).strip()
|
|
def info(cid):
|
|
x=json.loads(subprocess.check_output(['docker','inspect',cid],text=True))[0]
|
|
return {'id':x['Id'][:12],'hostname':x['Config']['Hostname'],'name':x['Name'].lstrip('/'),'running':x['State']['Running'],'started_at':x['State']['StartedAt'],'project':x['Config']['Labels']['com.docker.compose.project']}
|
|
def wait(c):
|
|
for _ in range(120):
|
|
try: c.call('GET','/health/ready'); return
|
|
except Failure: time.sleep(.5)
|
|
raise AssertionError('readiness did not recover')
|
|
def main():
|
|
c=Client('http://localhost:8084',timeout=8)
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
records=[]
|
|
ids=dc('ps','-q','api').splitlines()
|
|
for cid in ids:
|
|
target=info(cid)
|
|
check(target['project']=='lyakhonov-lab4','wrong project')
|
|
for _ in range(20):
|
|
result=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
if result['headers']['X-Instance-Id'].startswith(target['hostname']+'-'): break
|
|
else: raise AssertionError('target never uploaded')
|
|
started=time.monotonic()
|
|
try:
|
|
subprocess.check_call(['docker','stop','--time','1',cid],stdout=subprocess.DEVNULL)
|
|
check(not info(cid)['running'],'target still running')
|
|
for _ in range(40):
|
|
try:
|
|
downloaded=c.call('GET','/api/v1/attachments/'+result['json']['id']+'/content',actor=bob)
|
|
break
|
|
except Failure: time.sleep(.25)
|
|
else: raise AssertionError('download unavailable')
|
|
check(downloaded['bytes']==data,'original changed')
|
|
check(not downloaded['headers']['X-Instance-Id'].startswith(target['hostname']+'-'),'same replica')
|
|
records.append({'stopped':target,'after':info(cid),'download_instance':downloaded['headers']['X-Instance-Id'],'sha256':hashlib.sha256(downloaded['bytes']).hexdigest(),'recovery_seconds':round(time.monotonic()-started,3)})
|
|
finally:
|
|
subprocess.check_call(['docker','start',cid],stdout=subprocess.DEVNULL)
|
|
wait(c); time.sleep(3)
|
|
attachment,_=c.attachment(alice,chat)
|
|
# S3 failure must bound the wait and preserve the existing DB/session.
|
|
try:
|
|
dc('stop','--timeout','1','s3')
|
|
started=time.monotonic()
|
|
c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',503,actor=bob)
|
|
failure_seconds=time.monotonic()-started
|
|
finally:
|
|
dc('start','s3'); wait(c)
|
|
check(c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)['bytes']==data,'S3 recovery lost original')
|
|
before={service:[info(x) for x in dc('ps','-q',service).splitlines()] for service in ('api','db','redis','s3','lb')}
|
|
dc('restart','api','db','redis','s3','lb'); wait(c)
|
|
after={service:[info(x) for x in dc('ps','-q',service).splitlines()] for service in before}
|
|
for service in before:
|
|
check(all(a['started_at']!=b['started_at'] for a,b in zip(before[service],after[service])),'service not restarted: '+service)
|
|
check(c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)['bytes']==data,'restart lost original')
|
|
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
|
out={'result':'PASS','each_uploading_replica_stopped':records,'s3_503_seconds':failure_seconds,'restart_before':before,'restart_after':after}
|
|
(ROOT/'evidence/operations.json').write_text(json.dumps(out,indent=2),encoding='utf-8')
|
|
print('PASS: both uploading API replicas stopped; S3 failure/recovery; all services really restarted; original and session preserved')
|
|
if __name__=='__main__': main()
|