158 lines
7.9 KiB
Python
Executable File
158 lines
7.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import json
|
|
import re
|
|
import time
|
|
import uuid
|
|
from client import Client, Failure, arguments, check, run
|
|
from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload,
|
|
variant_count, wait_api, wait_log, wait_status)
|
|
|
|
|
|
def job_id_of(att_id):
|
|
return psql("select id from jobs where attachment_id='%s'" % att_id)
|
|
|
|
|
|
def worker_with(fault, **extra):
|
|
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra))
|
|
|
|
|
|
def wait_s3(timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout:
|
|
check(time.monotonic() < deadline, 's3 did not recover')
|
|
time.sleep(1)
|
|
|
|
|
|
def ops(*args):
|
|
out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout
|
|
return [json.loads(line) for line in out.splitlines() if line.startswith('{')]
|
|
|
|
|
|
def crash_after_commit(c, alice, bob, chat):
|
|
since = now_iso()
|
|
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'})
|
|
compose('restart', 'lb')
|
|
wait_api(c)
|
|
name = 'crash-' + uuid.uuid4().hex[:12] + '.png'
|
|
answered = None
|
|
try:
|
|
answered = upload(c, alice, chat, filename=name)
|
|
except Failure:
|
|
pass
|
|
check(answered is None, 'api answered 202 despite injected crash')
|
|
wait_log('api', since, '"point":"after_commit"', 20)
|
|
att_id = psql("select id from attachments where filename='%s'" % name)
|
|
check(att_id, 'committed attachment not found')
|
|
restore_api(c)
|
|
ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60)
|
|
check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job')
|
|
published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id))
|
|
print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1'
|
|
% (att_id, published))
|
|
|
|
|
|
def crash_after_store(c, alice, bob, chat):
|
|
compose('stop', 'worker')
|
|
att = upload(c, alice, chat)['json']
|
|
since = now_iso()
|
|
worker_with('after_store', JOB_TIMEOUT_SECONDS='10')
|
|
wait_log('worker', since, '"point":"after_store"', 60)
|
|
status, attempt, _ = job_row(att['id'])
|
|
check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease')
|
|
since = now_iso()
|
|
worker_with('', JOB_TIMEOUT_SECONDS='10')
|
|
wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90)
|
|
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
|
status, attempt, _ = job_row(att['id'])
|
|
check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry')
|
|
check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants')
|
|
thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
|
|
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
|
|
check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata')
|
|
restore_worker()
|
|
print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt)
|
|
|
|
|
|
def crash_before_ack(c, alice, bob, chat):
|
|
compose('stop', 'worker')
|
|
att = upload(c, alice, chat)['json']
|
|
job_id = job_id_of(att['id'])
|
|
since = now_iso()
|
|
worker_with('before_ack')
|
|
wait_log('worker', since, '"point":"before_ack"', 60)
|
|
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60)
|
|
status, attempt, _ = job_row(att['id'])
|
|
check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result')
|
|
check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked')
|
|
check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready')
|
|
restore_worker()
|
|
print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3')
|
|
|
|
|
|
def retries_and_replay(c, alice, bob, chat):
|
|
compose('stop', 'worker')
|
|
att = upload(c, alice, chat)['json']
|
|
job_id = job_id_of(att['id'])
|
|
compose('stop', 's3')
|
|
since = now_iso()
|
|
try:
|
|
compose('up', '-d', '--no-deps', 'worker')
|
|
check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped')
|
|
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
|
|
check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted')
|
|
text = logs('worker', since)
|
|
delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)]
|
|
check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays))
|
|
for n, d in enumerate(delays):
|
|
base = 1000 * 2 ** n
|
|
check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1))
|
|
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]},
|
|
headers={'Idempotency-Key': uuid.uuid4().hex})
|
|
check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job')
|
|
print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays)
|
|
finally:
|
|
compose('start', 's3')
|
|
wait_s3()
|
|
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
|
|
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120)
|
|
check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish')
|
|
check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op')
|
|
check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing')
|
|
check(variant_count(att['id']) == 3, 'second replay created new effects')
|
|
print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3')
|
|
|
|
|
|
def permanent_error(c, alice, bob, chat):
|
|
data = bytearray(SAMPLE)
|
|
mid = len(data) // 2
|
|
data[mid:mid + 64] = b'\x00' * 64
|
|
att = upload(c, alice, chat, bytes(data))['json']
|
|
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60)
|
|
status, attempt, _ = job_row(att['id'])
|
|
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries')
|
|
check(failed['variants'] == [], 'failed attachment has variants')
|
|
print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries')
|
|
|
|
|
|
def main():
|
|
args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay')
|
|
c = Client(args.base_url, args.ca_file, timeout=15)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
try:
|
|
crash_after_commit(c, alice, bob, chat)
|
|
crash_after_store(c, alice, bob, chat)
|
|
crash_before_ack(c, alice, bob, chat)
|
|
retries_and_replay(c, alice, bob, chat)
|
|
permanent_error(c, alice, bob, chat)
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 's3')
|
|
restore_worker()
|
|
if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout:
|
|
restore_api(c)
|
|
print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|