forked from mtusi2026/mtusi2026lab5
69 lines
3.7 KiB
Python
Executable File
69 lines
3.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import json
|
|
import time
|
|
from client import Client, arguments, check, run
|
|
from image_probe import jpeg_size
|
|
from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status
|
|
|
|
|
|
def stream_has(job_id):
|
|
return job_id in broker('XRANGE', 'attachments.jobs', '-', '+')
|
|
|
|
|
|
def main():
|
|
args = arguments('Worker stop, API/broker restart and redelivery scenario')
|
|
c = Client(args.base_url, args.ca_file, timeout=15)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
|
|
compose('stop', 'worker')
|
|
try:
|
|
att = upload(c, alice, chat)['json']
|
|
check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot')
|
|
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
|
|
check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment')
|
|
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
|
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob)
|
|
c.message(bob, chat, 'text works while worker is down')
|
|
job_id = psql("select id from jobs where attachment_id='%s'" % att['id'])
|
|
deadline = time.monotonic() + 10
|
|
while not stream_has(job_id):
|
|
check(time.monotonic() < deadline, 'job was not published to broker')
|
|
time.sleep(0.5)
|
|
check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked')
|
|
print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id))
|
|
|
|
compose('restart', 'broker', 'api')
|
|
compose('up', '-d', '--wait', '--no-deps', 'broker', 'api')
|
|
compose('restart', 'lb')
|
|
wait_api(c)
|
|
check(stream_has(job_id), 'job lost after broker restart')
|
|
check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker')
|
|
print('restarted broker and api: job still in stream, attachment still queued')
|
|
finally:
|
|
compose('start', 'worker')
|
|
|
|
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
|
check(ready['status'] == 'ready', 'job did not finish after worker start')
|
|
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)
|
|
check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size')
|
|
check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants')
|
|
print('worker started: attachment ready, attempt=1, variants=3')
|
|
|
|
since = now_iso()
|
|
before = psql("select updated_at from attachments where id='%s'" % att['id'])
|
|
payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'],
|
|
'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']})
|
|
broker('XADD', 'attachments.jobs', '*', 'job', payload)
|
|
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30)
|
|
time.sleep(1)
|
|
check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment')
|
|
check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects')
|
|
pending = broker('XPENDING', 'attachments.jobs', 'workers')
|
|
check(pending.splitlines()[0] == '0', 'duplicate message was not acked')
|
|
print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged')
|
|
print('PASS: worker stop, broker/api restart, redelivery idempotency')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|