Добавить устойчивый приём с ответом 202, durable jobs, dispatcher с publisher confirms и отдельный worker с ручным ACK. Формировать thumbnail, ограничить retry и защитить результат от повторной доставки. Добавить проверки очереди и адаптировать клиент к асинхронным состояниям.
90 lines
4.9 KiB
Python
90 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Synthetic lab5 recovery checks; restarts only this Compose project."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check, Failure
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(['docker', 'compose', '--project-name', 'highload-lab5', *args], cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query)
|
|
|
|
def until(fn, seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = fn()
|
|
if value:
|
|
return value
|
|
except (OSError, Failure, subprocess.CalledProcessError):
|
|
pass
|
|
time.sleep(.5)
|
|
raise AssertionError('condition timed out')
|
|
|
|
def queue_depth():
|
|
rows=json.loads(compose('exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages_ready','messages_unacknowledged','--formatter','json'))
|
|
return sum(row['messages_ready']+row['messages_unacknowledged'] for row in rows if row['name']=='attachments.v1')
|
|
|
|
|
|
def main():
|
|
c = Client(sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085')
|
|
alice, bob, _, chat = c.bootstrap()
|
|
evidence = {}
|
|
compose('stop', 'worker')
|
|
try:
|
|
att, original = c.attachment(alice, chat)
|
|
c.message(alice, chat, '', attachments=[att['id']])
|
|
c.call('GET', '/api/v1/attachments/'+att['id']+'/content', 409, actor=bob)
|
|
until(lambda: queue_depth() > 0)
|
|
before = queue_depth()
|
|
compose('stop','dispatcher')
|
|
compose('restart', 'api', 'broker')
|
|
until(lambda: c.call('GET','/health/ready')['status']==200)
|
|
until(lambda: queue_depth() >= int(before))
|
|
check(queue_depth() == before, 'broker lost queued entries')
|
|
queued = c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
|
|
check(queued['status']=='queued','API processed work without worker')
|
|
compose('start','dispatcher')
|
|
compose('start','worker')
|
|
ready = c.wait_ready(bob, att)
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original lost')
|
|
evidence['restart_recovery'] = {'attachment_id':att['id'], 'status':ready['status'], 'rabbitmq_messages_before_and_after':int(before)}
|
|
job = json.loads(sql("SELECT json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))
|
|
attempts = sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")
|
|
for _ in range(3):
|
|
compose('exec','-T','dispatcher','./messenger','broker-publish',json.dumps(job))
|
|
until(lambda: queue_depth()==0)
|
|
check(sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")==attempts,'terminal duplicate reprocessed')
|
|
check(c.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment')['json']==ready,'duplicate changed ready metadata')
|
|
evidence['duplicate_delivery']={'duplicates':3,'unchanged':True}
|
|
compose('stop','worker')
|
|
broken,_=c.attachment(alice,chat)
|
|
# Only this newly created synthetic job is changed to inject S3 failure.
|
|
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE attachment_id='"+broken['id']+"'")
|
|
compose('start','worker')
|
|
def failed():
|
|
value=c.call('GET','/api/v1/attachments/'+broken['id'],actor=alice,schema='Attachment')['json']
|
|
return value if value['status']=='failed' else None
|
|
result=until(failed,90)
|
|
count=int(sql("SELECT attempts FROM attachment_jobs WHERE attachment_id='"+broken['id']+"'"))
|
|
check(count==5 and result['error_code']=='retry_exhausted','retry not bounded to configured five attempts')
|
|
c.call('GET','/api/v1/attachments/'+broken['id']+'/content',409,actor=bob)
|
|
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[broken['id']]},headers={'Idempotency-Key':'failed-link-'+broken['id']})
|
|
evidence['retry_exhaustion']={'attachment_id':broken['id'],'attempts':count,'error_code':result['error_code']}
|
|
target=ROOT/'.memory/grade3-results.json'
|
|
target.parent.mkdir(exist_ok=True)
|
|
target.write_text(json.dumps(evidence,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS: worker-off upload, broker/API restart, ready recovery, three duplicates, bounded retry.')
|
|
finally:
|
|
compose('start','dispatcher','worker')
|
|
|
|
if __name__=='__main__':
|
|
main()
|