Files
mtusi2026lab4/scripts/attachment_failures.py

58 lines
2.6 KiB
Python

#!/usr/bin/env python3
"""Actual dependency outages. Runs inside ops with an operator-authored hook."""
import argparse
import json
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client,Failure,check
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--dependency',choices=['s3','db'],required=True)
args=parser.parse_args()
# Host orchestrates this protocol; credentials/tokens remain inside this process.
c=Client('http://lb:8080',timeout=15)
direct=Client('http://api:8080',timeout=10)
alice,bob,eve,chat=c.bootstrap()
att,data=c.attachment(alice,chat)
from app import object_store
before=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0)
print('READY',flush=True)
assert input().strip()=='STOPPED'
result={'dependency':args.dependency}
direct.call('GET','/health/ready',503,schema='Health')
result['readiness']=503
if args.dependency=='s3':
direct.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment')
direct.call('GET','/api/v1/attachments/'+att['id']+'/content',503,actor=bob)
direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'})
result.update(metadata_available=True,download=503,upload=503)
else:
direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'})
after=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0)
check(before==after,'DB failure created an untracked S3 object')
result.update(upload=503,no_new_s3_objects=True)
print('CHECKED',flush=True)
assert input().strip()=='RESTORED'
deadline=time.monotonic()+60
while True:
try:
direct.call('GET','/health/ready',schema='Health')
break
except Failure:
check(time.monotonic()<deadline,'dependency did not recover')
time.sleep(.5)
restored=direct.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(restored['bytes']==data,'original changed after dependency restart')
result.update(status='PASS',confirmed_original_preserved=True)
object_store.close()
print(json.dumps(result),flush=True)
if __name__=='__main__':
main()