Files
mtusi2026lab4-fork/student-tests/test_operations.py
T

72 lines
4.2 KiB
Python

"""Actual Docker failures, scoped by Compose service labels."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client, check, Failure
ROOT=Path(__file__).resolve().parents[1]
BASE=['docker','compose','--project-directory',str(ROOT),'-p','lyakhonov-lab4']
def dc(*args): return subprocess.check_output(BASE+list(args),text=True).strip()
def info(cid):
x=json.loads(subprocess.check_output(['docker','inspect',cid],text=True))[0]
return {'id':x['Id'][:12],'hostname':x['Config']['Hostname'],'name':x['Name'].lstrip('/'),'running':x['State']['Running'],'started_at':x['State']['StartedAt'],'project':x['Config']['Labels']['com.docker.compose.project']}
def wait(c):
for _ in range(120):
try: c.call('GET','/health/ready'); return
except Failure: time.sleep(.5)
raise AssertionError('readiness did not recover')
def main():
c=Client('http://localhost:8084',timeout=8)
alice,bob,eve,chat=c.bootstrap()
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
records=[]
ids=dc('ps','-q','api').splitlines()
for cid in ids:
target=info(cid)
check(target['project']=='lyakhonov-lab4','wrong project')
for _ in range(20):
result=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')
if result['headers']['X-Instance-Id'].startswith(target['hostname']+'-'): break
else: raise AssertionError('target never uploaded')
started=time.monotonic()
try:
subprocess.check_call(['docker','stop','--time','1',cid],stdout=subprocess.DEVNULL)
check(not info(cid)['running'],'target still running')
for _ in range(40):
try:
downloaded=c.call('GET','/api/v1/attachments/'+result['json']['id']+'/content',actor=bob)
break
except Failure: time.sleep(.25)
else: raise AssertionError('download unavailable')
check(downloaded['bytes']==data,'original changed')
check(not downloaded['headers']['X-Instance-Id'].startswith(target['hostname']+'-'),'same replica')
records.append({'stopped':target,'after':info(cid),'download_instance':downloaded['headers']['X-Instance-Id'],'sha256':hashlib.sha256(downloaded['bytes']).hexdigest(),'recovery_seconds':round(time.monotonic()-started,3)})
finally:
subprocess.check_call(['docker','start',cid],stdout=subprocess.DEVNULL)
wait(c); time.sleep(3)
attachment,_=c.attachment(alice,chat)
# S3 failure must bound the wait and preserve the existing DB/session.
try:
dc('stop','--timeout','1','s3')
started=time.monotonic()
c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',503,actor=bob)
failure_seconds=time.monotonic()-started
finally:
dc('start','s3'); wait(c)
check(c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)['bytes']==data,'S3 recovery lost original')
before={service:[info(x) for x in dc('ps','-q',service).splitlines()] for service in ('api','db','redis','s3','lb')}
dc('restart','api','db','redis','s3','lb'); wait(c)
after={service:[info(x) for x in dc('ps','-q',service).splitlines()] for service in before}
for service in before:
check(all(a['started_at']!=b['started_at'] for a,b in zip(before[service],after[service])),'service not restarted: '+service)
check(c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)['bytes']==data,'restart lost original')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
out={'result':'PASS','each_uploading_replica_stopped':records,'s3_503_seconds':failure_seconds,'restart_before':before,'restart_after':after}
(ROOT/'evidence/operations.json').write_text(json.dumps(out,indent=2),encoding='utf-8')
print('PASS: both uploading API replicas stopped; S3 failure/recovery; all services really restarted; original and session preserved')
if __name__=='__main__': main()