Files
mtusi2026lab5/tests/pipeline_resilience.py

109 lines
4.6 KiB
Python

#!/usr/bin/env python3
"""Explicit operator evidence for Lab5 queue durability and failure windows.
Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test
preserves user/session secrets strictly in process memory and prints no tokens.
"""
import argparse
import hashlib
import os
import subprocess
import sys
import time
from pathlib import Path
from client import Client, Failure, check, run
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,)):
p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240)
if p.returncode not in expected:
raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs'
% (' '.join(args[:2]), p.returncode))
return p
def wait_api(timeout=60):
import urllib.request
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as r:
if r.status==200:
return
except Exception:
pass
time.sleep(.5)
raise Failure('API did not become ready')
def make_queued(client):
alice,bob,eve,chat=client.bootstrap()
att,original=client.attachment(alice,chat)
check(att['status']=='queued','202 must return queued snapshot')
c=client.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
attached=client.message(alice,chat,'',attachments=[att['id']])['json']
check(attached['attachment_ids']==[att['id']],'queued attachment cannot be linked')
return alice,bob,att,original
def verify_ready(client,actor,att,original,seconds=180):
r=client.wait_ready(actor,att,wait_seconds=seconds)
check(len(r['variants']) == 3, 'expected exactly three variants, no duplicate entries')
variants={v['name']:v for v in r['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'missing/extra variants')
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==(256,160), 'thumbnail dimensions')
check((variants['crop']['width'],variants['crop']['height'])==(128,128), 'crop dimensions')
raw=client.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
check(raw['bytes']==original,'original changed across asynchronous pipeline')
for name in ('thumbnail','crop','optimized'):
v=variants[name]
response=client.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],
'variant checksum mismatch: '+name)
return r
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--scenario',choices=['paused','outbox-crash','worker-crash'],required=True)
args=parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST')!='yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes in PowerShell to run destructive failure test')
c=Client('http://localhost:8080')
# All tests make a fresh attachment while dispatcher/worker are stopped.
compose('stop','worker','dispatcher')
try:
alice,bob,att,original=make_queued(c)
print('Created durable queued attachment',att['id'],flush=True)
if args.scenario=='paused':
compose('restart','api1','api2','broker')
compose('up','-d','--wait','broker')
wait_api()
elif args.scenario=='outbox-crash':
# Transaction committed, dispatcher crashes before publish.
compose('run','--rm','-e','LAB5_FAULT_BEFORE_PUBLISH=once',
'dispatcher',expected=(78,))
print('Injected dispatcher exit 78 after committed outbox, before publish')
else:
compose('up','-d','dispatcher')
# Fault after all S3 writes and before PostgreSQL result COMMIT and ack.
compose('run','--rm','-e','LAB5_FAULT_AFTER_S3=once',
'worker',expected=(77,))
print('Injected worker exit 77 after S3 effects, before result/ack')
compose('up','-d','dispatcher','worker')
wait_api()
ready=verify_ready(c,bob,att,original)
check(ready['status']=='ready','attachment not ready')
print('PASS:',args.scenario,'durable queued->ready; original/3 variants verified; no duplicates')
finally:
# Restore the normal topology even after an assertion failure.
compose('up','-d','dispatcher','worker')
if __name__=='__main__':
run(main)