forked from mtusi2026/mtusi2026lab5
127 lines
4.3 KiB
Python
Executable File
127 lines
4.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Level-3 demo: work accepted while the worker is down survives restarts."""
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from stand import ( # noqa: E402
|
|
Failure,
|
|
check,
|
|
cli,
|
|
compose,
|
|
new_client,
|
|
print_step,
|
|
rabbit_queues,
|
|
sample_bytes,
|
|
sha256,
|
|
upload,
|
|
wait_for,
|
|
wait_ready,
|
|
)
|
|
from client import arguments, run # noqa: E402
|
|
from image_probe import jpeg_size # noqa: E402
|
|
|
|
|
|
def main():
|
|
args = arguments('lab5 worker-down durability demo')
|
|
client = new_client(args.base_url, args.ca_file)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
data = sample_bytes()
|
|
|
|
print_step(1, 'compose ps')
|
|
print(compose('ps', '--format',
|
|
'table {{.Service}}\t{{.Status}}').stdout.strip())
|
|
|
|
print_step(2, 'stop worker')
|
|
compose('stop', 'worker')
|
|
|
|
print_step(3, 'upload while worker is down (expect 202 queued)')
|
|
attachment = upload(client, alice, chat, data, 'durability.png')
|
|
attachment_id = attachment['id']
|
|
print('attachment=' + attachment_id + ' status=' + attachment['status'])
|
|
|
|
print_step(4, 'content and thumbnail must answer 409 before ready')
|
|
for query in ('', '?variant=thumbnail'):
|
|
result = client.call(
|
|
'GET',
|
|
'/api/v1/attachments/' + attachment_id + '/content' + query,
|
|
409,
|
|
actor=alice,
|
|
)
|
|
check(result['status'] == 409, 'content was served before ready')
|
|
|
|
print_step(5, 'link the queued attachment to messages')
|
|
with_attachment = client.message(
|
|
alice, chat, '', attachments=[attachment_id],
|
|
)['json']
|
|
plain = client.message(alice, chat, 'plain while worker is down')['json']
|
|
check(with_attachment['attachment_ids'] == [attachment_id],
|
|
'message lost the queued attachment')
|
|
check(plain['attachment_ids'] == [], 'plain message gained an attachment')
|
|
print('message=' + with_attachment['id'])
|
|
|
|
print_step(6, 'wait for outbox publish and a message in the queue')
|
|
def published():
|
|
_code, status = cli('status', attachment_id)
|
|
queues = rabbit_queues()
|
|
return (
|
|
status['outbox']['pending'] == 0
|
|
and queues.get('attachments.process', {}).get('messages', 0) >= 1
|
|
)
|
|
wait_for(published, 30, 'outbox publish and queue depth >= 1')
|
|
queues = rabbit_queues()
|
|
print('queue=' + str(queues.get('attachments.process')))
|
|
|
|
print_step(7, 'restart broker and API with volumes')
|
|
compose('restart', 'broker', 'api')
|
|
compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api')
|
|
|
|
print_step(8, 'the quorum message survived the broker restart')
|
|
def survived():
|
|
depth = rabbit_queues().get('attachments.process', {}).get('messages', 0)
|
|
return depth >= 1
|
|
wait_for(survived, 60, 'queue depth >= 1 after broker restart')
|
|
print('queue=' + str(rabbit_queues().get('attachments.process')))
|
|
|
|
print_step(9, 'start worker')
|
|
compose('start', 'worker')
|
|
|
|
print_step(10, 'wait ready and verify the thumbnail')
|
|
ready = wait_ready(client, alice, attachment_id, 60)
|
|
thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
|
|
content = client.call(
|
|
'GET',
|
|
'/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail',
|
|
actor=alice,
|
|
)
|
|
check(jpeg_size(content['bytes']) == (256, 160),
|
|
'thumbnail is not 256x160 JPEG')
|
|
check(len(content['bytes']) == thumbnail['size_bytes'],
|
|
'thumbnail size differs from metadata')
|
|
check(sha256(content['bytes']) == thumbnail['sha256'],
|
|
'thumbnail checksum differs from metadata')
|
|
history = client.call(
|
|
'GET',
|
|
'/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
|
actor=alice,
|
|
schema='MessagePage',
|
|
)['json']
|
|
check(
|
|
any(attachment_id in item['attachment_ids']
|
|
for item in history['items']),
|
|
'message history lost the attachment',
|
|
)
|
|
print('ready attachment=' + attachment_id +
|
|
' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...')
|
|
|
|
print_step(11, 'compose ps')
|
|
print(compose('ps', '--format',
|
|
'table {{.Service}}\t{{.Status}}').stdout.strip())
|
|
print('PASS: worker-down durability demo')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|