Files
mtusi2026lab6-fork/tests/faults.py

158 lines
7.9 KiB
Python
Executable File

#!/usr/bin/env python3
import json
import re
import time
import uuid
from client import Client, Failure, arguments, check, run
from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload,
variant_count, wait_api, wait_log, wait_status)
def job_id_of(att_id):
return psql("select id from jobs where attachment_id='%s'" % att_id)
def worker_with(fault, **extra):
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra))
def wait_s3(timeout=90):
deadline = time.monotonic() + timeout
while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout:
check(time.monotonic() < deadline, 's3 did not recover')
time.sleep(1)
def ops(*args):
out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout
return [json.loads(line) for line in out.splitlines() if line.startswith('{')]
def crash_after_commit(c, alice, bob, chat):
since = now_iso()
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'})
compose('restart', 'lb')
wait_api(c)
name = 'crash-' + uuid.uuid4().hex[:12] + '.png'
answered = None
try:
answered = upload(c, alice, chat, filename=name)
except Failure:
pass
check(answered is None, 'api answered 202 despite injected crash')
wait_log('api', since, '"point":"after_commit"', 20)
att_id = psql("select id from attachments where filename='%s'" % name)
check(att_id, 'committed attachment not found')
restore_api(c)
ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60)
check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job')
published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id))
print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1'
% (att_id, published))
def crash_after_store(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
since = now_iso()
worker_with('after_store', JOB_TIMEOUT_SECONDS='10')
wait_log('worker', since, '"point":"after_store"', 60)
status, attempt, _ = job_row(att['id'])
check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease')
since = now_iso()
worker_with('', JOB_TIMEOUT_SECONDS='10')
wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90)
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
status, attempt, _ = job_row(att['id'])
check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry')
check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants')
thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata')
restore_worker()
print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt)
def crash_before_ack(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
job_id = job_id_of(att['id'])
since = now_iso()
worker_with('before_ack')
wait_log('worker', since, '"point":"before_ack"', 60)
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60)
status, attempt, _ = job_row(att['id'])
check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result')
check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked')
check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready')
restore_worker()
print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3')
def retries_and_replay(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
job_id = job_id_of(att['id'])
compose('stop', 's3')
since = now_iso()
try:
compose('up', '-d', '--no-deps', 'worker')
check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped')
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted')
text = logs('worker', since)
delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)]
check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays))
for n, d in enumerate(delays):
base = 1000 * 2 ** n
check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1))
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})
check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job')
print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays)
finally:
compose('start', 's3')
wait_s3()
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120)
check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish')
check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op')
check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing')
check(variant_count(att['id']) == 3, 'second replay created new effects')
print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3')
def permanent_error(c, alice, bob, chat):
data = bytearray(SAMPLE)
mid = len(data) // 2
data[mid:mid + 64] = b'\x00' * 64
att = upload(c, alice, chat, bytes(data))['json']
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60)
status, attempt, _ = job_row(att['id'])
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries')
check(failed['variants'] == [], 'failed attachment has variants')
print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries')
def main():
args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay')
c = Client(args.base_url, args.ca_file, timeout=15)
alice, bob, eve, chat = c.bootstrap()
try:
crash_after_commit(c, alice, bob, chat)
crash_after_store(c, alice, bob, chat)
crash_before_ack(c, alice, bob, chat)
retries_and_replay(c, alice, bob, chat)
permanent_error(c, alice, bob, chat)
finally:
compose('up', '-d', '--no-deps', 's3')
restore_worker()
if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout:
restore_api(c)
print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error')
if __name__ == '__main__':
run(main)