forked from mtusi2026/mtusi2026lab5
API принимает 202 после durable staging и TX intent (job+outbox); отдельные relay и worker публикуют/обрабатывают задания с publisher confirm, lease, retry/backoff, DLQ и replay. Добавлены версии pipeline, варианты thumbnail/crop/optimized, presign в verified staging, status-aware sweeper и admission control.
93 lines
4.3 KiB
Python
Executable File
93 lines
4.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Lab5: accepted work survives a stopped worker and a broker restart.
|
|
|
|
Run against a live stand:
|
|
python3 student-tests/lab5_async.py --base-url http://localhost:8080
|
|
"""
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
from tests_helpers import ROOT, compose, psql # noqa: E402
|
|
|
|
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
|
|
|
|
|
|
def wait_state(client, actor, attachment, expected, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
|
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
|
|
if current['status'] == expected:
|
|
return current
|
|
time.sleep(0.2)
|
|
raise Failure('attachment did not reach ' + expected)
|
|
|
|
|
|
def main():
|
|
args = arguments('Lab5 async worker/broker durability')
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
content = FIXTURE.read_bytes()
|
|
|
|
# 1. Worker down: API still accepts 202, content is 409, the message links.
|
|
compose('stop', 'worker')
|
|
try:
|
|
attached, _ = client.attachment(alice, chat)
|
|
check(attached['status'] == 'queued', '202 snapshot must be queued')
|
|
queued = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
|
|
check(queued['status'] == 'queued', 'worker stopped but attachment is not queued')
|
|
client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice)
|
|
client.message(alice, chat, '', attachments=[attached['id']])
|
|
time.sleep(2)
|
|
still = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
|
|
check(still['status'] == 'queued', 'worker stopped but job was processed synchronously')
|
|
print('worker down: 202 queued, content 409, queued attachment linked')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', check=False)
|
|
|
|
ready = client.wait_ready(alice, attached)
|
|
thumb = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=thumbnail', actor=bob)
|
|
check(thumb['headers'].get_content_type() == 'image/jpeg', 'thumbnail content type mismatch')
|
|
download = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
|
|
check(download['bytes'] == content, 'original bytes changed')
|
|
check(psql("select coalesce(staging_key, '') from attachments where id='%s'" % ready['id']) == '',
|
|
'staging key was not cleared after ready')
|
|
print('worker restarted: ready, thumbnail and original downloadable, staging cleared')
|
|
|
|
# 2. Broker restart with its volume: an intent accepted while the broker is
|
|
# down is published by the relay after recovery.
|
|
compose('stop', 'relay')
|
|
compose('stop', 'broker')
|
|
try:
|
|
during, _ = client.attachment(alice, chat)
|
|
check(during['status'] == 'queued', 'upload during broker outage must be queued')
|
|
print('broker down: upload accepted into outbox')
|
|
finally:
|
|
compose('up', '-d', '--wait', '--no-deps', 'broker', check=False)
|
|
compose('up', '-d', '--no-deps', 'relay', check=False)
|
|
|
|
ready = client.wait_ready(alice, during, wait_seconds=90)
|
|
print('broker + relay recovered: queued job reached ready')
|
|
|
|
# 3. A full stack restart with volumes keeps ready data and the worker path.
|
|
compose('restart', 'api', 'worker', 'relay')
|
|
recovered = client.call('GET', '/api/v1/attachments/' + ready['id'], actor=bob, schema='Attachment')['json']
|
|
check(recovered['status'] == 'ready', 'ready attachment lost across restart')
|
|
body = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
|
|
check(body['bytes'] == content, 'original lost across restart')
|
|
print('stack restart: ready attachment and original preserved')
|
|
|
|
print('PASS: lab5 async worker/broker durability')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
run(main)
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', 'relay', check=False)
|