#!/usr/bin/env python3 """Level-3 demo: work accepted while the worker is down survives restarts.""" import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from stand import ( # noqa: E402 Failure, check, cli, compose, new_client, print_step, rabbit_queues, sample_bytes, sha256, upload, wait_for, wait_ready, ) from client import arguments, run # noqa: E402 from image_probe import jpeg_size # noqa: E402 def main(): args = arguments('lab5 worker-down durability demo') client = new_client(args.base_url, args.ca_file) alice, _bob, _eve, chat = client.bootstrap() data = sample_bytes() print_step(1, 'compose ps') print(compose('ps', '--format', 'table {{.Service}}\t{{.Status}}').stdout.strip()) print_step(2, 'stop worker') compose('stop', 'worker') print_step(3, 'upload while worker is down (expect 202 queued)') attachment = upload(client, alice, chat, data, 'durability.png') attachment_id = attachment['id'] print('attachment=' + attachment_id + ' status=' + attachment['status']) print_step(4, 'content and thumbnail must answer 409 before ready') for query in ('', '?variant=thumbnail'): result = client.call( 'GET', '/api/v1/attachments/' + attachment_id + '/content' + query, 409, actor=alice, ) check(result['status'] == 409, 'content was served before ready') print_step(5, 'link the queued attachment to messages') with_attachment = client.message( alice, chat, '', attachments=[attachment_id], )['json'] plain = client.message(alice, chat, 'plain while worker is down')['json'] check(with_attachment['attachment_ids'] == [attachment_id], 'message lost the queued attachment') check(plain['attachment_ids'] == [], 'plain message gained an attachment') print('message=' + with_attachment['id']) print_step(6, 'wait for outbox publish and a message in the queue') def published(): _code, status = cli('status', attachment_id) queues = rabbit_queues() return ( status['outbox']['pending'] == 0 and queues.get('attachments.process', {}).get('messages', 0) >= 1 ) wait_for(published, 30, 'outbox publish and queue depth >= 1') queues = rabbit_queues() print('queue=' + str(queues.get('attachments.process'))) print_step(7, 'restart broker and API with volumes') compose('restart', 'broker', 'api') compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api') print_step(8, 'the quorum message survived the broker restart') def survived(): depth = rabbit_queues().get('attachments.process', {}).get('messages', 0) return depth >= 1 wait_for(survived, 60, 'queue depth >= 1 after broker restart') print('queue=' + str(rabbit_queues().get('attachments.process'))) print_step(9, 'start worker') compose('start', 'worker') print_step(10, 'wait ready and verify the thumbnail') ready = wait_ready(client, alice, attachment_id, 60) thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] content = client.call( 'GET', '/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail', actor=alice, ) check(jpeg_size(content['bytes']) == (256, 160), 'thumbnail is not 256x160 JPEG') check(len(content['bytes']) == thumbnail['size_bytes'], 'thumbnail size differs from metadata') check(sha256(content['bytes']) == thumbnail['sha256'], 'thumbnail checksum differs from metadata') history = client.call( 'GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage', )['json'] check( any(attachment_id in item['attachment_ids'] for item in history['items']), 'message history lost the attachment', ) print('ready attachment=' + attachment_id + ' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...') print_step(11, 'compose ps') print(compose('ps', '--format', 'table {{.Service}}\t{{.Status}}').stdout.strip()) print('PASS: worker-down durability demo') if __name__ == '__main__': run(main)