#!/usr/bin/env python3 """Actual dependency outages. Runs inside ops with an operator-authored hook.""" import argparse import json from pathlib import Path import subprocess import sys import time sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) sys.path.insert(0,str(Path(__file__).resolve().parents[1])) from client import Client,Failure,check def main(): parser=argparse.ArgumentParser() parser.add_argument('--dependency',choices=['s3','db'],required=True) args=parser.parse_args() # Host orchestrates this protocol; credentials/tokens remain inside this process. c=Client('http://lb:8080',timeout=15) direct=Client('http://api:8080',timeout=10) alice,bob,eve,chat=c.bootstrap() att,data=c.attachment(alice,chat) from app import object_store before=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0) print('READY',flush=True) assert input().strip()=='STOPPED' result={'dependency':args.dependency} direct.call('GET','/health/ready',503,schema='Health') result['readiness']=503 if args.dependency=='s3': direct.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment') direct.call('GET','/api/v1/attachments/'+att['id']+'/content',503,actor=bob) direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'}) result.update(metadata_available=True,download=503,upload=503) else: direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'}) after=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0) check(before==after,'DB failure created an untracked S3 object') result.update(upload=503,no_new_s3_objects=True) print('CHECKED',flush=True) assert input().strip()=='RESTORED' deadline=time.monotonic()+60 while True: try: direct.call('GET','/health/ready',schema='Health') break except Failure: check(time.monotonic()