Files
mtusi2026lab5/student-tests/lab5_async.py
T
Vladislav Kunetskiy 01b885e846 lab5: очереди и асинхронная обработка изображений
Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
2026-10-31 14:34:24 +03:00

154 lines
7.9 KiB
Python
Executable File

#!/usr/bin/env python3
"""Lab5: accepted work survives a stopped worker and a broker restart.
Run against a live stand:
python3 student-tests/lab5_async.py --base-url http://localhost:8080
"""
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from client import Client, Failure, arguments, check, run # noqa: E402
from tests_helpers import ROOT, compose, jobctl, psql # noqa: E402
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
def wait_api_ready(client, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
client.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.2)
raise Failure('API did not become ready after restart')
def wait_state(client, actor, attachment, expected, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
if current['status'] == expected:
return current
time.sleep(0.2)
raise Failure('attachment did not reach ' + expected)
def main():
args = arguments('Lab5 async worker/broker durability')
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
content = FIXTURE.read_bytes()
# 1. Worker down: API still accepts 202, content is 409, the message links.
compose('stop', 'worker')
try:
attached, _ = client.attachment(alice, chat)
check(attached['status'] == 'queued', '202 snapshot must be queued')
queued = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
check(queued['status'] == 'queued', 'worker stopped but attachment is not queued')
client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice)
client.message(alice, chat, '', attachments=[attached['id']])
time.sleep(2)
still = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
check(still['status'] == 'queued', 'worker stopped but job was processed synchronously')
# The README demo restarts the API after the 202 and before the worker.
compose('restart', 'api')
wait_api_ready(client)
print('worker down: 202 queued, content 409, queued attachment linked, API restarted')
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
ready = client.wait_ready(alice, attached)
thumb = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=thumbnail', actor=bob)
check(thumb['headers'].get_content_type() == 'image/jpeg', 'thumbnail content type mismatch')
download = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
check(download['bytes'] == content, 'original bytes changed')
check(psql("select coalesce(staging_key, '') from attachments where id='%s'" % ready['id']) == '',
'staging key was not cleared after ready')
print('worker restarted: ready, thumbnail and original downloadable, staging cleared')
# 2. Broker restart with its volume: an intent accepted while the broker is
# down is published by the relay after recovery.
compose('stop', 'relay')
compose('stop', 'broker')
try:
during, _ = client.attachment(alice, chat)
check(during['status'] == 'queued', 'upload during broker outage must be queued')
print('broker down: upload accepted into outbox')
finally:
compose('up', '-d', '--wait', '--no-deps', 'broker', check=False)
compose('up', '-d', '--no-deps', 'relay', check=False)
ready = client.wait_ready(alice, during, wait_seconds=90)
print('broker + relay recovered: queued job reached ready')
# 3. A full stack restart with volumes keeps ready data and the worker path.
compose('restart', 'api', 'worker', 'relay')
wait_api_ready(client)
recovered = client.call('GET', '/api/v1/attachments/' + ready['id'], actor=bob, schema='Attachment')['json']
check(recovered['status'] == 'ready', 'ready attachment lost across restart')
body = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
check(body['bytes'] == content, 'original lost across restart')
print('stack restart: ready attachment and original preserved')
# 4. Reprocess of a queued upload while the worker is down. The final
# original has never been written, so pipeline 2 must source from staging;
# pointing at the not-yet-existing final original superseded v1 and left
# the attachment queued forever with v2 failed (source_missing).
compose('stop', 'worker')
try:
pinned, _ = client.attachment(alice, chat)
check(pinned['status'] == 'queued', 'reprocess fixture must be queued')
result = jobctl('reprocess', '-attachment', pinned['id'], '-pipeline', '2')
check('SourceKey' in result.stdout, 'reprocess did not return the created job')
queued2 = client.call('GET', '/api/v1/attachments/' + pinned['id'], actor=alice, schema='Attachment')['json']
check(queued2['status'] == 'queued', 'reprocessed upload is not queued')
check(psql("select count(*) from attachment_job_events e join attachment_jobs j on j.job_id=e.job_id where j.attachment_id='%s' and j.pipeline_version=1 and e.event='superseded'" % pinned['id']) == '1',
'reprocess did not record the v1 superseded event')
print('worker down: reprocess pipeline 2 accepted, v1 audit event recorded, upload still queued')
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
ready2 = client.wait_ready(alice, pinned, wait_seconds=90)
check(psql("select pipeline_version from attachments where id='%s'" % pinned['id']) == '2',
'serving pipeline version is not 2 after reprocess')
check(psql("select status from attachment_jobs where attachment_id='%s' order by pipeline_version" % pinned['id']) == 'superseded\nready',
'old job not superseded or new job not ready')
print('worker up: reprocessed upload reached ready at pipeline 2, v1 superseded')
# 5. A pipeline version the worker cannot execute must be refused by the
# operator CLI before any state change; the accepted v1 upload keeps its
# job and finishes normally instead of being superseded into queued.
compose('stop', 'worker')
try:
preserved, _ = client.attachment(alice, chat)
check(preserved['status'] == 'queued', 'unsupported-pipeline fixture must be queued')
refused = jobctl('reprocess', '-attachment', preserved['id'], '-pipeline', '3', check=False)
check(refused.returncode != 0 and 'unsupported pipeline version 3' in refused.stderr,
'unsupported pipeline version was not rejected')
check(psql("select status from attachment_jobs where attachment_id='%s' and pipeline_version=1" % preserved['id']) == 'queued',
'v1 job changed by rejected reprocess')
print('worker down: reprocess pipeline 3 refused, v1 job untouched')
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
client.wait_ready(alice, preserved, wait_seconds=90)
check(psql("select pipeline_version from attachments where id='%s'" % preserved['id']) == '1',
'rejected reprocess changed serving version')
print('worker up: preserved upload reached ready at pipeline 1')
print('PASS: lab5 async worker/broker durability')
if __name__ == '__main__':
try:
run(main)
finally:
compose('up', '-d', '--no-deps', 'worker', 'relay', check=False)