Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
326 lines
14 KiB
Python
Executable File
326 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Lab5 level 4: retry, DLQ/replay, crash windows, lease takeover.
|
|
|
|
The stand must be running. The script temporarily recreates worker/relay with
|
|
FAULTS_ENABLED=1 and targeted failpoints, then restores the normal services.
|
|
|
|
python3 student-tests/lab5_faults.py --base-url http://localhost:8080
|
|
"""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
from tests_helpers import compose, jobctl, psql # noqa: E402
|
|
|
|
FIXTURE = Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png'
|
|
WORKER = 'mtusi-messenger-lab5-worker-1'
|
|
RELAY = 'mtusi-messenger-lab5-relay-1'
|
|
|
|
|
|
def worker_with(**env):
|
|
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=env)
|
|
|
|
|
|
def relay_with(**env):
|
|
compose('up', '-d', '--force-recreate', '--no-deps', 'relay', env=env)
|
|
|
|
|
|
def reset_worker():
|
|
worker_with()
|
|
|
|
|
|
def reset_relay():
|
|
relay_with()
|
|
|
|
|
|
def job_id(attachment_id):
|
|
value = psql("select job_id from attachment_jobs where attachment_id = '%s'" % attachment_id)
|
|
check(bool(value), 'job row is missing for ' + attachment_id)
|
|
|
|
return value.splitlines()[0]
|
|
|
|
|
|
def wait_db(attachment_id, predicate, timeout=120, description=''):
|
|
deadline = time.monotonic() + timeout
|
|
last = ''
|
|
while time.monotonic() < deadline:
|
|
rows = psql("select a.status || '|' || coalesce(a.error_code,'') || '|' || j.attempt from attachments a join attachment_jobs j on j.attachment_id=a.id where a.id='%s'" % attachment_id)
|
|
if rows:
|
|
last = rows.splitlines()[0]
|
|
status, error, attempt = last.split('|')
|
|
if predicate(status, error, int(attempt)):
|
|
return status, error, int(attempt)
|
|
time.sleep(0.2)
|
|
raise Failure('timeout waiting for ' + description + ' (last=' + last + ')')
|
|
|
|
|
|
def queue_messages(name):
|
|
project = 'mtusi-messenger-lab5'
|
|
result = subprocess.run(
|
|
['docker', 'exec', project + '-broker-1', 'rabbitmqctl', '-p', 'messenger', 'list_queues', 'name', 'messages'],
|
|
capture_output=True, text=True, check=True,
|
|
)
|
|
for line in result.stdout.splitlines():
|
|
if line.startswith(name):
|
|
return int(line.split()[1])
|
|
return 0
|
|
|
|
|
|
def scenario_permanent_failure_and_replay(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
|
|
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='permanent')
|
|
status, code, attempts = wait_db(attached['id'], lambda s, e, a: s == 'failed', 40, 'permanent failure')
|
|
check(code == 'injected_failure', 'unexpected permanent error_code: ' + code)
|
|
check(attempts == 1, 'a permanent error must not consume retries')
|
|
meta = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment')['json']
|
|
check(meta['status'] == 'failed' and meta['error_code'] == code, 'API did not expose failed/error_code')
|
|
client.call('GET', '/api/v1/attachments/' + attached['id'] + '/content', 409, actor=alice)
|
|
client.call(
|
|
'POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
|
body={'text': '', 'attachment_ids': [attached['id']]},
|
|
headers={'Idempotency-Key': 'failed-link-' + attached['id']},
|
|
)
|
|
failed_view = jobctl('failed', '-limit', '20').stdout
|
|
check(jid in failed_view, 'failed job is missing from the operator registry')
|
|
print('permanent failure: failed/' + code + ', content 409, failed attachment not linkable, registry visible')
|
|
|
|
reset_worker()
|
|
replay = jobctl('replay', '-job', jid)
|
|
check(replay.returncode == 0, 'jobctl replay failed: ' + replay.stderr)
|
|
ready = client.wait_ready(alice, attached, wait_seconds=60)
|
|
check(len([v for v in ready['variants'] if v['name'] == 'thumbnail']) == 1, 'replay created a duplicate thumbnail')
|
|
print('replay: failed -> queued -> ready, one thumbnail')
|
|
|
|
# Replay of a ready job must be refused and must not create another effect.
|
|
again = jobctl('replay', '-job', jid, check=False)
|
|
check(again.returncode != 0, 'replay of a ready job must be refused')
|
|
print('replay ready job: refused')
|
|
|
|
|
|
def scenario_transient_exhaustion(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
|
|
dlq_before = queue_messages('attachment.process.dead')
|
|
|
|
worker_with(
|
|
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='transient',
|
|
PROCESSING_RETRY_BASE='100ms', PROCESSING_RETRY_CAP='500ms',
|
|
)
|
|
status, code, attempts = wait_db(attached['id'], lambda s, e, a: s == 'failed', 120, 'retry exhaustion')
|
|
check(attempts == 5, 'retry cap must be exactly 5, got ' + str(attempts))
|
|
check(code == 'retry_exhausted', 'unexpected exhaustion code: ' + code)
|
|
events = psql("select event || ':' || coalesce(attempt,0) from attachment_job_events where job_id='%s' order by id" % jid)
|
|
check(events.count('retried:') == 4, 'expected four managed retries: ' + events)
|
|
check(events.count('failed:') == 1, 'expected one terminal failure: ' + events)
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
if queue_messages('attachment.process.dead') >= dlq_before + 1:
|
|
break
|
|
time.sleep(0.2)
|
|
|
|
check(queue_messages('attachment.process.dead') >= dlq_before + 1,
|
|
'terminal failure did not add a DLQ message')
|
|
print('transient exhaustion: attempts=5, retry_exhausted, four retries, DLQ message present')
|
|
|
|
reset_worker()
|
|
|
|
|
|
def scenario_crash_after_first_variant(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
|
|
relay_with(OUTBOX_REAPER_INTERVAL='2s')
|
|
worker_with(
|
|
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_FIRST_VARIANT='exit',
|
|
PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='10s',
|
|
)
|
|
|
|
ready = client.wait_ready(alice, attached, wait_seconds=90)
|
|
status, _, attempts = wait_db(attached['id'], lambda s, e, a: s == 'ready', 10, 'crash recovery')
|
|
check(attempts >= 2, 'crash recovery must use a new managed attempt, got ' + str(attempts))
|
|
names = sorted(v['name'] for v in ready['variants'])
|
|
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate/partial variant set: ' + str(names))
|
|
print('crash after first variant: redelivered, same three variants, attempts=' + str(attempts))
|
|
|
|
reset_worker()
|
|
reset_relay()
|
|
|
|
|
|
def scenario_crash_after_commit(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
|
|
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_COMMIT='exit')
|
|
ready = client.wait_ready(alice, attached, wait_seconds=60)
|
|
check(ready['status'] == 'ready', 'job did not stay ready after commit/ack crash')
|
|
check(psql("select attempt from attachment_jobs where job_id='%s'" % jid) == '1',
|
|
'redelivery after ready must be a no-op, not a new attempt')
|
|
print('crash after commit before ack: ready remains, redelivery is a no-op')
|
|
|
|
reset_worker()
|
|
|
|
|
|
def scenario_crash_before_side_effect(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
|
|
relay_with(OUTBOX_REAPER_INTERVAL='2s')
|
|
worker_with(
|
|
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_AFTER_CLAIM='exit',
|
|
PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='12s',
|
|
)
|
|
|
|
ready = client.wait_ready(alice, attached, wait_seconds=90)
|
|
status, _, attempts = wait_db(attached['id'], lambda s, e, a: s == 'ready', 10, 'pre-effect crash recovery')
|
|
check(attempts >= 2, 'pre-effect crash must use a new managed attempt, got ' + str(attempts))
|
|
check(sorted(v['name'] for v in ready['variants']) == ['crop', 'optimized', 'thumbnail'],
|
|
'pre-effect crash produced an incomplete variant set')
|
|
print('crash after claim before side effect: requeued, attempts=' + str(attempts) + ', ready')
|
|
|
|
reset_worker()
|
|
reset_relay()
|
|
|
|
|
|
def scenario_outbox_gap(client, alice, chat):
|
|
compose('stop', 'relay')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
unpublished = psql("select count(*) from outbox_messages where job_id='%s' and published_at is null" % jid)
|
|
check(unpublished == '1', 'upload did not leave a durable outbox intent')
|
|
|
|
relay_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_OUTBOX_AFTER_COMMIT='exit')
|
|
ready = client.wait_ready(alice, attached, wait_seconds=90)
|
|
check(psql("select count(*) from outbox_messages where job_id='%s' and published_at is null" % jid) == '0',
|
|
'outbox row stayed unpublished after recovery')
|
|
print('outbox gap: crash before publish recovered through the relay, outbox published, ready')
|
|
reset_relay()
|
|
|
|
|
|
def scenario_lease_takeover(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid = job_id(attached['id'])
|
|
|
|
relay_with(OUTBOX_REAPER_INTERVAL='1s')
|
|
worker_with(
|
|
FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_SLEEP_AFTER_CLAIM='8s',
|
|
PROCESSING_LEASE_TTL='15s', PROCESSING_HEARTBEAT='3s', PROCESSING_JOB_TIMEOUT='12s',
|
|
)
|
|
|
|
wait_db(attached['id'], lambda s, e, a: s == 'processing', 20, 'processing claim')
|
|
subprocess.run(['docker', 'pause', WORKER], check=True, capture_output=True)
|
|
wait_db(attached['id'], lambda s, e, a: s == 'queued' and a >= 2, 30, 'lease expiry requeue')
|
|
subprocess.run(['docker', 'unpause', WORKER], check=True, capture_output=True)
|
|
ready = client.wait_ready(alice, attached, wait_seconds=60)
|
|
check(len(ready['variants']) == 3, 'lease takeover produced an incomplete variant set')
|
|
print('lease expiry: paused worker, reaper requeued, stale attempt fenced, ready')
|
|
|
|
reset_worker()
|
|
reset_relay()
|
|
|
|
|
|
def scenario_replay_reprocessed_version(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
reprocessed = json.loads(jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2').stdout)
|
|
jid = reprocessed['ID']
|
|
|
|
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid, FAULT_WORKER_FAIL='permanent')
|
|
|
|
deadline = time.monotonic() + 40
|
|
last = ''
|
|
while time.monotonic() < deadline:
|
|
job_status = psql("select status from attachment_jobs where job_id='%s'" % jid)
|
|
last = psql("select status || '|' || coalesce(error_code,'') from attachments where id='%s'" % attached['id'])
|
|
if job_status == 'failed' and last.startswith('failed|'):
|
|
break
|
|
time.sleep(0.2)
|
|
|
|
check(job_status == 'failed' and last == 'failed|injected_failure',
|
|
'reprocessed failure did not settle: ' + last)
|
|
|
|
compose('stop', 'worker')
|
|
jobctl('replay', '-job', jid)
|
|
projection = client.call(
|
|
'GET', '/api/v1/attachments/' + attached['id'], actor=alice, schema='Attachment',
|
|
)['json']
|
|
check(projection['status'] == 'queued' and projection['error_code'] is None,
|
|
'replay left stale terminal projection: ' + str(projection['status']))
|
|
|
|
reset_worker()
|
|
client.wait_ready(alice, attached, wait_seconds=90)
|
|
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
|
|
'replayed reprocess did not publish pipeline 2')
|
|
print('replay of failed reprocess: queued projection restored, then ready at pipeline 2')
|
|
|
|
|
|
def scenario_replay_older_version_refused(client, alice, chat):
|
|
compose('stop', 'worker')
|
|
attached, _ = client.attachment(alice, chat)
|
|
jid_v1 = job_id(attached['id'])
|
|
|
|
worker_with(FAULTS_ENABLED='1', FAULT_TARGET_JOB=jid_v1, FAULT_WORKER_FAIL='permanent')
|
|
status, code, _ = wait_db(attached['id'], lambda s, e, a: s == 'failed', 40, 'failed v1')
|
|
check(code == 'injected_failure', 'unexpected v1 failure: ' + code)
|
|
|
|
compose('stop', 'worker')
|
|
reprocessed = json.loads(jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2').stdout)
|
|
refused = jobctl('replay', '-job', jid_v1, check=False)
|
|
check(refused.returncode != 0, 'older failed version was replayed while v2 exists')
|
|
jobs = psql("select pipeline_version || ':' || status from attachment_jobs where attachment_id='%s' order by pipeline_version" % attached['id'])
|
|
check(jobs == '1:failed\n2:queued', 'older replay changed job state: ' + jobs.replace('\n', ', '))
|
|
print('replay older failed version while v2 queued: refused, state intact')
|
|
|
|
reset_worker()
|
|
client.wait_ready(alice, attached, wait_seconds=90)
|
|
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
|
|
'v2 did not publish after older replay was refused')
|
|
check(psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=2" % attached['id']) == reprocessed['ID'],
|
|
'v2 job identity changed')
|
|
print('v2 published after older replay refusal')
|
|
|
|
|
|
def main():
|
|
args = arguments('Lab5 level 4 fault injection')
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, _, _, chat = client.bootstrap()
|
|
|
|
scenarios = [
|
|
scenario_permanent_failure_and_replay,
|
|
scenario_replay_older_version_refused,
|
|
scenario_replay_reprocessed_version,
|
|
scenario_transient_exhaustion,
|
|
scenario_crash_after_first_variant,
|
|
scenario_crash_after_commit,
|
|
scenario_crash_before_side_effect,
|
|
scenario_outbox_gap,
|
|
scenario_lease_takeover,
|
|
]
|
|
|
|
try:
|
|
for scenario in scenarios:
|
|
scenario(client, alice, chat)
|
|
finally:
|
|
reset_worker()
|
|
reset_relay()
|
|
|
|
print('PASS: lab5 level 4 fault injection')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|