141 lines
7.6 KiB
Python
141 lines
7.6 KiB
Python
"""Reproducible level-3 failure scenarios; synthetic data, no tokens in output."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT/'tests'))
|
|
from client import Client, check
|
|
from image_probe import jpeg_size
|
|
|
|
def compose(*args):
|
|
result = subprocess.run(['docker','compose','--env-file','.env',*args], cwd=ROOT,
|
|
capture_output=True, text=True)
|
|
if result.returncode:
|
|
raise RuntimeError('compose action failed: '+ ' '.join(args[:3]))
|
|
return result.stdout.strip()
|
|
|
|
def internal(code):
|
|
return compose('exec','-T','dispatcher','python','-c',code)
|
|
|
|
def queue_count():
|
|
output = compose('exec','-T','broker','rabbitmqctl','list_queues','name','messages_ready','--silent')
|
|
for line in output.splitlines():
|
|
fields = line.split()
|
|
if len(fields)==2 and fields[0]=='attachments.v1':
|
|
return int(fields[1])
|
|
return 0
|
|
|
|
def started_at(service):
|
|
ident = compose('ps','-q',service)
|
|
return subprocess.check_output(['docker','inspect','--format','{{.State.StartedAt}}',ident],
|
|
text=True).strip()
|
|
|
|
def eventually(predicate, seconds=65):
|
|
deadline = time.monotonic()+seconds
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise AssertionError('condition did not recover before deadline')
|
|
|
|
def main():
|
|
import argparse
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', default='http://localhost:8085')
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url)
|
|
results = []
|
|
compose('stop','worker')
|
|
try:
|
|
alice,bob,eve,chat = client.bootstrap()
|
|
attachment, raw = client.attachment(alice,chat)
|
|
ident = attachment['id']
|
|
path = '/api/v1/attachments/'+ident
|
|
check(attachment['status']=='queued', 'upload snapshot must be queued')
|
|
client.call('GET',path+'/content',409,actor=alice)
|
|
linked = client.message(alice,chat,'',attachments=[ident])['json']
|
|
client.message(bob,chat,'Worker is stopped; messages still work')
|
|
eventually(lambda: queue_count()>0)
|
|
# Freeze publisher so broker persistence is observed without republishing.
|
|
compose('stop','dispatcher')
|
|
queued_before = queue_count()
|
|
started_before = {name:started_at(name) for name in ('api1','api2','broker')}
|
|
compose('restart','api1','api2','broker')
|
|
eventually(lambda: client.call('GET','/health/ready',schema='Health'))
|
|
queued_after = eventually(lambda: queue_count())
|
|
started_after = {name:started_at(name) for name in ('api1','api2','broker')}
|
|
check(all(started_before[name]!=started_after[name] for name in started_before), 'process did not actually restart')
|
|
check(queued_after==queued_before, 'broker lost durable queued messages')
|
|
pending = client.call('GET',path,actor=alice,schema='Attachment')['json']
|
|
check(pending['status']=='queued', 'API performed synchronous processing')
|
|
compose('start','dispatcher','worker')
|
|
start = time.monotonic()
|
|
ready = client.wait_ready(bob,attachment)
|
|
check(client.call('GET',path+'/content',actor=alice)['bytes']==raw, 'original changed')
|
|
thumbnail = client.call('GET',path+'/content?variant=thumbnail',actor=bob)['bytes']
|
|
check(jpeg_size(thumbnail)==(256,160), 'invalid thumbnail')
|
|
results.append(dict(scenario='stopped_worker_and_api_broker_restart', attachment_id=ident,
|
|
message_id=linked['id'], queued_before=queued_before, queued_after=queued_after,
|
|
broker_restart_verified=True, started_before=started_before, started_after=started_after,
|
|
ready_seconds=round(time.monotonic()-start,3),
|
|
thumbnail=[256,160], result='PASS'))
|
|
# Publish the exact same logical job five more times, with confirms.
|
|
snapshot = internal(f"""
|
|
import json
|
|
from app.storage import db
|
|
from app.broker import connect,QUEUE
|
|
from app.models.time_format import utc_z
|
|
import pika
|
|
with db() as c:
|
|
j=c.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s',('{ident}',)).fetchone()
|
|
body=dict(schema_version=1,job_id=str(j['job_id']),type='attachment.process.v1',attachment_id='{ident}',pipeline_version=1,attempt=1,created_at=utc_z(j['created_at']))
|
|
connection,ch=connect()
|
|
for _ in range(5):
|
|
ch.basic_publish('',QUEUE,json.dumps(body),pika.BasicProperties(delivery_mode=2),mandatory=True)
|
|
connection.close()
|
|
print(j['attempts'])
|
|
""")
|
|
eventually(lambda: queue_count()==0)
|
|
# Queue empty can mean delivered but unacked; wait for all acknowledgements.
|
|
eventually(lambda: 'attachments.v1\t0' in compose('exec','-T','broker','rabbitmqctl','list_queues','name','messages','--silent'))
|
|
check(client.call('GET',path,actor=alice,schema='Attachment')['json']==ready, 'duplicate changed ready metadata')
|
|
attempts = internal(f"from app.storage import db\nwith db() as c:\n print(c.execute('SELECT attempts FROM attachment_jobs WHERE attachment_id=%s',('{ident}',)).fetchone()['attempts'])")
|
|
check(attempts==snapshot, 'duplicate reran successful job')
|
|
check(client.call('GET',path+'/content?variant=thumbnail',actor=bob)['bytes']==thumbnail,'duplicate changed variant')
|
|
objects = json.loads(internal(f"from app.s3 import s3_client,bucket\nimport json\nprint(json.dumps([o['Key'] for o in s3_client().list_objects_v2(Bucket=bucket(),Prefix='results/{ident}/')['Contents']]))"))
|
|
check(len(objects)==2,'duplicate created extra objects')
|
|
results.append(dict(scenario='five_duplicate_deliveries',attachment_id=ident,attempts=int(attempts),
|
|
final_objects=objects, unchanged_metadata=True, result='PASS'))
|
|
# Transient storage failure: controlled, nonexistent staging key of a new test job.
|
|
compose('stop','worker')
|
|
failed, _ = client.attachment(alice,chat)
|
|
fid = failed['id']
|
|
internal(f"from app.storage import db\nwith db() as c:\n c.execute(\"UPDATE attachments SET object_key=%s WHERE id=%s\",('staging/missing-{fid}','{fid}'))")
|
|
compose('start','worker')
|
|
fpath = '/api/v1/attachments/'+fid
|
|
final = eventually(lambda: (lambda a:a if a['status']=='failed' else None)(client.call('GET',fpath,actor=alice,schema='Attachment')['json']))
|
|
check(final['error_code']=='retry_exhausted','unsafe or wrong failure code')
|
|
attempts = int(internal(f"from app.storage import db\nwith db() as c:\n print(c.execute('SELECT attempts FROM attachment_jobs WHERE attachment_id=%s',('{fid}',)).fetchone()['attempts'])"))
|
|
check(attempts==5,'retry bound not respected')
|
|
client.call('GET',fpath+'/content',409,actor=alice)
|
|
client.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
|
body={'text':'','attachment_ids':[fid]},headers={'Idempotency-Key':'failed-'+fid})
|
|
client.call('GET',fpath,404,actor=eve)
|
|
results.append(dict(scenario='bounded_retry',attachment_id=fid,attempts=attempts,
|
|
error_code=final['error_code'], download_status=409, attach_status=409,result='PASS'))
|
|
finally:
|
|
compose('start','dispatcher','worker')
|
|
(ROOT/'evidence'/'pipeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(results,indent=2))
|
|
print('PASS: lab5 durable queue, restart, redelivery and retry scenarios')
|
|
|
|
if __name__=='__main__':
|
|
main()
|