forked from mtusi2026/mtusi2026lab4
58 lines
2.6 KiB
Python
58 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Actual dependency outages. Runs inside ops with an operator-authored hook."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
|
from client import Client,Failure,check
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--dependency',choices=['s3','db'],required=True)
|
|
args=parser.parse_args()
|
|
# Host orchestrates this protocol; credentials/tokens remain inside this process.
|
|
c=Client('http://lb:8080',timeout=15)
|
|
direct=Client('http://api:8080',timeout=10)
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
att,data=c.attachment(alice,chat)
|
|
from app import object_store
|
|
before=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0)
|
|
print('READY',flush=True)
|
|
assert input().strip()=='STOPPED'
|
|
result={'dependency':args.dependency}
|
|
direct.call('GET','/health/ready',503,schema='Health')
|
|
result['readiness']=503
|
|
if args.dependency=='s3':
|
|
direct.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment')
|
|
direct.call('GET','/api/v1/attachments/'+att['id']+'/content',503,actor=bob)
|
|
direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'})
|
|
result.update(metadata_available=True,download=503,upload=503)
|
|
else:
|
|
direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'})
|
|
after=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0)
|
|
check(before==after,'DB failure created an untracked S3 object')
|
|
result.update(upload=503,no_new_s3_objects=True)
|
|
print('CHECKED',flush=True)
|
|
assert input().strip()=='RESTORED'
|
|
deadline=time.monotonic()+60
|
|
while True:
|
|
try:
|
|
direct.call('GET','/health/ready',schema='Health')
|
|
break
|
|
except Failure:
|
|
check(time.monotonic()<deadline,'dependency did not recover')
|
|
time.sleep(.5)
|
|
restored=direct.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
|
check(restored['bytes']==data,'original changed after dependency restart')
|
|
result.update(status='PASS',confirmed_original_preserved=True)
|
|
object_store.close()
|
|
print(json.dumps(result),flush=True)
|
|
|
|
if __name__=='__main__':
|
|
main()
|