#!/usr/bin/env python3 """Synthetic lab5 recovery checks; restarts only this Compose project.""" import json from pathlib import Path import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check, Failure def compose(*args): return subprocess.check_output(['docker', 'compose', '--project-name', 'highload-lab5', *args], cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip() def sql(query): return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query) def until(fn, seconds=60): deadline = time.monotonic() + seconds while time.monotonic() < deadline: try: value = fn() if value: return value except (OSError, Failure, subprocess.CalledProcessError): pass time.sleep(.5) raise AssertionError('condition timed out') def queue_depth(): rows=json.loads(compose('exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages_ready','messages_unacknowledged','--formatter','json')) return sum(row['messages_ready']+row['messages_unacknowledged'] for row in rows if row['name']=='attachments.v1') def main(): c = Client(sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085') alice, bob, _, chat = c.bootstrap() evidence = {} compose('stop', 'worker') try: att, original = c.attachment(alice, chat) c.message(alice, chat, '', attachments=[att['id']]) c.call('GET', '/api/v1/attachments/'+att['id']+'/content', 409, actor=bob) until(lambda: queue_depth() > 0) before = queue_depth() compose('stop','dispatcher') compose('restart', 'api', 'broker') until(lambda: c.call('GET','/health/ready')['status']==200) until(lambda: queue_depth() >= int(before)) check(queue_depth() == before, 'broker lost queued entries') queued = c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json'] check(queued['status']=='queued','API processed work without worker') compose('start','dispatcher') compose('start','worker') ready = c.wait_ready(bob, att) check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original lost') evidence['restart_recovery'] = {'attachment_id':att['id'], 'status':ready['status'], 'rabbitmq_messages_before_and_after':int(before)} job = json.loads(sql("SELECT json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")) attempts = sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+att['id']+"'") for _ in range(3): compose('exec','-T','dispatcher','./messenger','broker-publish',json.dumps(job)) until(lambda: queue_depth()==0) check(sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")==attempts,'terminal duplicate reprocessed') check(c.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment')['json']==ready,'duplicate changed ready metadata') evidence['duplicate_delivery']={'duplicates':3,'unchanged':True} compose('stop','worker') broken,_=c.attachment(alice,chat) # Only this newly created synthetic job is changed to inject S3 failure. sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE attachment_id='"+broken['id']+"'") compose('start','worker') def failed(): value=c.call('GET','/api/v1/attachments/'+broken['id'],actor=alice,schema='Attachment')['json'] return value if value['status']=='failed' else None result=until(failed,90) count=int(sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+broken['id']+"'")) check(count==5 and result['error_code']=='retry_exhausted','retry not bounded to configured five attempts') c.call('GET','/api/v1/attachments/'+broken['id']+'/content',409,actor=bob) c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[broken['id']]},headers={'Idempotency-Key':'failed-link-'+broken['id']}) evidence['retry_exhaustion']={'attachment_id':broken['id'],'attempts':count,'error_code':result['error_code']} target=ROOT/'.memory/grade3-results.json' target.parent.mkdir(exist_ok=True) target.write_text(json.dumps(evidence,ensure_ascii=False,indent=2)+'\n') print('PASS: worker-off upload, broker/API restart, ready recovery, three duplicates, bounded retry.') finally: compose('start','dispatcher','worker') if __name__=='__main__': main()