Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
154 lines
7.9 KiB
Python
Executable File
154 lines
7.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Lab5: accepted work survives a stopped worker and a broker restart.
|
|
|
|
Run against a live stand:
|
|
python3 student-tests/lab5_async.py --base-url http://localhost:8080
|
|
"""
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
from tests_helpers import ROOT, compose, jobctl, psql # noqa: E402
|
|
|
|
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
|
|
|
|
|
|
def wait_api_ready(client, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Failure:
|
|
time.sleep(0.2)
|
|
raise Failure('API did not become ready after restart')
|
|
|
|
|
|
def wait_state(client, actor, attachment, expected, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
|
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
|
|
if current['status'] == expected:
|
|
return current
|
|
time.sleep(0.2)
|
|
raise Failure('attachment did not reach ' + expected)
|
|
|
|
|
|
def main():
|
|
args = arguments('Lab5 async worker/broker durability')
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
content = FIXTURE.read_bytes()
|
|
|
|
# 1. Worker down: API still accepts 202, content is 409, the message links.
|
|
compose('stop', 'worker')
|
|
try:
|
|
attached, _ = client.attachment(alice, chat)
|
|
check(attached['status'] == 'queued', '202 snapshot must be queued')
|
|
queued = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
|
|
check(queued['status'] == 'queued', 'worker stopped but attachment is not queued')
|
|
client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice)
|
|
client.message(alice, chat, '', attachments=[attached['id']])
|
|
time.sleep(2)
|
|
still = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
|
|
check(still['status'] == 'queued', 'worker stopped but job was processed synchronously')
|
|
# The README demo restarts the API after the 202 and before the worker.
|
|
compose('restart', 'api')
|
|
wait_api_ready(client)
|
|
print('worker down: 202 queued, content 409, queued attachment linked, API restarted')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', check=False)
|
|
|
|
ready = client.wait_ready(alice, attached)
|
|
thumb = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content?variant=thumbnail', actor=bob)
|
|
check(thumb['headers'].get_content_type() == 'image/jpeg', 'thumbnail content type mismatch')
|
|
download = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
|
|
check(download['bytes'] == content, 'original bytes changed')
|
|
check(psql("select coalesce(staging_key, '') from attachments where id='%s'" % ready['id']) == '',
|
|
'staging key was not cleared after ready')
|
|
print('worker restarted: ready, thumbnail and original downloadable, staging cleared')
|
|
|
|
# 2. Broker restart with its volume: an intent accepted while the broker is
|
|
# down is published by the relay after recovery.
|
|
compose('stop', 'relay')
|
|
compose('stop', 'broker')
|
|
try:
|
|
during, _ = client.attachment(alice, chat)
|
|
check(during['status'] == 'queued', 'upload during broker outage must be queued')
|
|
print('broker down: upload accepted into outbox')
|
|
finally:
|
|
compose('up', '-d', '--wait', '--no-deps', 'broker', check=False)
|
|
compose('up', '-d', '--no-deps', 'relay', check=False)
|
|
|
|
ready = client.wait_ready(alice, during, wait_seconds=90)
|
|
print('broker + relay recovered: queued job reached ready')
|
|
|
|
# 3. A full stack restart with volumes keeps ready data and the worker path.
|
|
compose('restart', 'api', 'worker', 'relay')
|
|
wait_api_ready(client)
|
|
recovered = client.call('GET', '/api/v1/attachments/' + ready['id'], actor=bob, schema='Attachment')['json']
|
|
check(recovered['status'] == 'ready', 'ready attachment lost across restart')
|
|
body = client.call('GET', '/api/v1/attachments/' + ready['id'] + '/content', actor=bob)
|
|
check(body['bytes'] == content, 'original lost across restart')
|
|
print('stack restart: ready attachment and original preserved')
|
|
|
|
# 4. Reprocess of a queued upload while the worker is down. The final
|
|
# original has never been written, so pipeline 2 must source from staging;
|
|
# pointing at the not-yet-existing final original superseded v1 and left
|
|
# the attachment queued forever with v2 failed (source_missing).
|
|
compose('stop', 'worker')
|
|
try:
|
|
pinned, _ = client.attachment(alice, chat)
|
|
check(pinned['status'] == 'queued', 'reprocess fixture must be queued')
|
|
result = jobctl('reprocess', '-attachment', pinned['id'], '-pipeline', '2')
|
|
check('SourceKey' in result.stdout, 'reprocess did not return the created job')
|
|
queued2 = client.call('GET', '/api/v1/attachments/' + pinned['id'], actor=alice, schema='Attachment')['json']
|
|
check(queued2['status'] == 'queued', 'reprocessed upload is not queued')
|
|
check(psql("select count(*) from attachment_job_events e join attachment_jobs j on j.job_id=e.job_id where j.attachment_id='%s' and j.pipeline_version=1 and e.event='superseded'" % pinned['id']) == '1',
|
|
'reprocess did not record the v1 superseded event')
|
|
print('worker down: reprocess pipeline 2 accepted, v1 audit event recorded, upload still queued')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', check=False)
|
|
|
|
ready2 = client.wait_ready(alice, pinned, wait_seconds=90)
|
|
check(psql("select pipeline_version from attachments where id='%s'" % pinned['id']) == '2',
|
|
'serving pipeline version is not 2 after reprocess')
|
|
check(psql("select status from attachment_jobs where attachment_id='%s' order by pipeline_version" % pinned['id']) == 'superseded\nready',
|
|
'old job not superseded or new job not ready')
|
|
print('worker up: reprocessed upload reached ready at pipeline 2, v1 superseded')
|
|
|
|
# 5. A pipeline version the worker cannot execute must be refused by the
|
|
# operator CLI before any state change; the accepted v1 upload keeps its
|
|
# job and finishes normally instead of being superseded into queued.
|
|
compose('stop', 'worker')
|
|
try:
|
|
preserved, _ = client.attachment(alice, chat)
|
|
check(preserved['status'] == 'queued', 'unsupported-pipeline fixture must be queued')
|
|
refused = jobctl('reprocess', '-attachment', preserved['id'], '-pipeline', '3', check=False)
|
|
check(refused.returncode != 0 and 'unsupported pipeline version 3' in refused.stderr,
|
|
'unsupported pipeline version was not rejected')
|
|
check(psql("select status from attachment_jobs where attachment_id='%s' and pipeline_version=1" % preserved['id']) == 'queued',
|
|
'v1 job changed by rejected reprocess')
|
|
print('worker down: reprocess pipeline 3 refused, v1 job untouched')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', check=False)
|
|
|
|
client.wait_ready(alice, preserved, wait_seconds=90)
|
|
check(psql("select pipeline_version from attachments where id='%s'" % preserved['id']) == '1',
|
|
'rejected reprocess changed serving version')
|
|
print('worker up: preserved upload reached ready at pipeline 1')
|
|
|
|
print('PASS: lab5 async worker/broker durability')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
run(main)
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', 'relay', check=False)
|