Files
mtusi2026lab5/scripts/check_async_grade3.py
T

91 lines
4.9 KiB
Python

#!/usr/bin/env python3
"""Synthetic lab5 recovery checks; restarts only this Compose project."""
import json
from pathlib import Path
import subprocess
import sys
import time
from local_config import api_base_url
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check, Failure
def compose(*args):
return subprocess.check_output(['docker', 'compose', '--project-name', 'highload-lab5', *args], cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
def sql(query):
return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query)
def until(fn, seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
value = fn()
if value:
return value
except (OSError, Failure, subprocess.CalledProcessError):
pass
time.sleep(.5)
raise AssertionError('condition timed out')
def queue_depth():
rows=json.loads(compose('exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages_ready','messages_unacknowledged','--formatter','json'))
return sum(row['messages_ready']+row['messages_unacknowledged'] for row in rows if row['name']=='attachments.v1')
def main():
c = Client(sys.argv[1] if len(sys.argv)>1 else api_base_url())
alice, bob, _, chat = c.bootstrap()
evidence = {}
compose('stop', 'worker')
try:
att, original = c.attachment(alice, chat)
c.message(alice, chat, '', attachments=[att['id']])
c.call('GET', '/api/v1/attachments/'+att['id']+'/content', 409, actor=bob)
until(lambda: queue_depth() > 0)
before = queue_depth()
compose('stop','dispatcher')
compose('restart', 'api', 'broker')
until(lambda: c.call('GET','/health/ready')['status']==200)
until(lambda: queue_depth() >= int(before))
check(queue_depth() == before, 'broker lost queued entries')
queued = c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
check(queued['status']=='queued','API processed work without worker')
compose('start','dispatcher')
compose('start','worker')
ready = c.wait_ready(bob, att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original lost')
evidence['restart_recovery'] = {'attachment_id':att['id'], 'status':ready['status'], 'rabbitmq_messages_before_and_after':int(before)}
job = json.loads(sql("SELECT json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))
attempts = sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")
for _ in range(3):
compose('exec','-T','dispatcher','./messenger','broker-publish',json.dumps(job))
until(lambda: queue_depth()==0)
check(sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")==attempts,'terminal duplicate reprocessed')
check(c.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment')['json']==ready,'duplicate changed ready metadata')
evidence['duplicate_delivery']={'duplicates':3,'unchanged':True}
compose('stop','worker')
broken,_=c.attachment(alice,chat)
# Only this newly created synthetic job is changed to inject S3 failure.
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE attachment_id='"+broken['id']+"'")
compose('start','worker')
def failed():
value=c.call('GET','/api/v1/attachments/'+broken['id'],actor=alice,schema='Attachment')['json']
return value if value['status']=='failed' else None
result=until(failed,90)
count=int(sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+broken['id']+"'"))
check(count==5 and result['error_code']=='retry_exhausted','retry not bounded to configured five attempts')
c.call('GET','/api/v1/attachments/'+broken['id']+'/content',409,actor=bob)
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[broken['id']]},headers={'Idempotency-Key':'failed-link-'+broken['id']})
evidence['retry_exhaustion']={'attachment_id':broken['id'],'attempts':count,'error_code':result['error_code']}
target=ROOT/'.memory/grade3-results.json'
target.parent.mkdir(exist_ok=True)
target.write_text(json.dumps(evidence,ensure_ascii=False,indent=2)+'\n')
print('PASS: worker-off upload, broker/API restart, ready recovery, three duplicates, bounded retry.')
finally:
compose('start','dispatcher','worker')
if __name__=='__main__':
main()