#!/usr/bin/env python3 import json import time from client import Client, arguments, check, run from image_probe import jpeg_size from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status def stream_has(job_id): return job_id in broker('XRANGE', 'attachments.jobs', '-', '+') def main(): args = arguments('Worker stop, API/broker restart and redelivery scenario') c = Client(args.base_url, args.ca_file, timeout=15) alice, bob, eve, chat = c.bootstrap() compose('stop', 'worker') try: att = upload(c, alice, chat)['json'] check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot') msg = c.message(alice, chat, '', attachments=[att['id']])['json'] check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment') c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob) c.message(bob, chat, 'text works while worker is down') job_id = psql("select id from jobs where attachment_id='%s'" % att['id']) deadline = time.monotonic() + 10 while not stream_has(job_id): check(time.monotonic() < deadline, 'job was not published to broker') time.sleep(0.5) check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked') print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id)) compose('restart', 'broker', 'api') compose('up', '-d', '--wait', '--no-deps', 'broker', 'api') compose('restart', 'lb') wait_api(c) check(stream_has(job_id), 'job lost after broker restart') check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker') print('restarted broker and api: job still in stream, attachment still queued') finally: compose('start', 'worker') ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) check(ready['status'] == 'ready', 'job did not finish after worker start') thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob) check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size') check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants') print('worker started: attachment ready, attempt=1, variants=3') since = now_iso() before = psql("select updated_at from attachments where id='%s'" % att['id']) payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'], 'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']}) broker('XADD', 'attachments.jobs', '*', 'job', payload) wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30) time.sleep(1) check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment') check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects') pending = broker('XPENDING', 'attachments.jobs', 'workers') check(pending.splitlines()[0] == '0', 'duplicate message was not acked') print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged') print('PASS: worker stop, broker/api restart, redelivery idempotency') if __name__ == '__main__': run(main)