Files
mtusi2026lab5-fork/tests/queue_recovery.py
T

69 lines
3.7 KiB
Python
Executable File

#!/usr/bin/env python3
import json
import time
from client import Client, arguments, check, run
from image_probe import jpeg_size
from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status
def stream_has(job_id):
return job_id in broker('XRANGE', 'attachments.jobs', '-', '+')
def main():
args = arguments('Worker stop, API/broker restart and redelivery scenario')
c = Client(args.base_url, args.ca_file, timeout=15)
alice, bob, eve, chat = c.bootstrap()
compose('stop', 'worker')
try:
att = upload(c, alice, chat)['json']
check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot')
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment')
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob)
c.message(bob, chat, 'text works while worker is down')
job_id = psql("select id from jobs where attachment_id='%s'" % att['id'])
deadline = time.monotonic() + 10
while not stream_has(job_id):
check(time.monotonic() < deadline, 'job was not published to broker')
time.sleep(0.5)
check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked')
print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id))
compose('restart', 'broker', 'api')
compose('up', '-d', '--wait', '--no-deps', 'broker', 'api')
compose('restart', 'lb')
wait_api(c)
check(stream_has(job_id), 'job lost after broker restart')
check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker')
print('restarted broker and api: job still in stream, attachment still queued')
finally:
compose('start', 'worker')
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
check(ready['status'] == 'ready', 'job did not finish after worker start')
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)
check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size')
check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants')
print('worker started: attachment ready, attempt=1, variants=3')
since = now_iso()
before = psql("select updated_at from attachments where id='%s'" % att['id'])
payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'],
'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']})
broker('XADD', 'attachments.jobs', '*', 'job', payload)
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30)
time.sleep(1)
check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment')
check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects')
pending = broker('XPENDING', 'attachments.jobs', 'workers')
check(pending.splitlines()[0] == '0', 'duplicate message was not acked')
print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged')
print('PASS: worker stop, broker/api restart, redelivery idempotency')
if __name__ == '__main__':
run(main)