#!/usr/bin/env python3 """Automated fault injection for lab5 levels 4 and 5 (track 2).""" import argparse import json import re import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4')) from stand import ( # noqa: E402 ROOT, Failure, check, clear_fault, clear_faults, cli, compose, docker, new_client, rabbit_queues, sample_bytes, set_fault, sha256, upload, variant_sha, wait_for, wait_ready, worker_env, worker_start_count, ) from pngkit import worst_decode_png # noqa: E402 HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 ' 'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 ' 'make stand-up') LEVEL4 = [ 'crash_after_commit_before_publish', 'crash_after_side_effect', 'crash_after_commit_before_ack', 'retry_and_replay', 'hung_worker', 'poison', 'big_image', ] TRACK2 = [ 'duplicate', 'reorder', 'crash_before_side_effect', 'crash_after_side_effect', 'crash_after_commit_before_ack', 'relay_crash_after_publish', 'stale_version', ] class Tracker(threading.Thread): """Polls metadata while a scenario runs to prove readiness never regresses.""" def __init__(self, base_url, actor, ca_file=None): super().__init__(daemon=True) self.client = new_client(base_url, ca_file) self.actor = actor self.ids = [] self.records = {} self.stop_event = threading.Event() def track(self, attachment_id): if attachment_id not in self.records: self.ids.append(attachment_id) self.records[attachment_id] = [] def run(self): while not self.stop_event.is_set(): for attachment_id in list(self.ids): try: meta = self.client.call( 'GET', '/api/v1/attachments/' + attachment_id, actor=self.actor, schema='Attachment', )['json'] self.records[attachment_id].append( (time.monotonic(), meta['status'], meta.get('ready_pipeline_version')) ) except Exception: pass self.stop_event.wait(0.2) def stop(self): self.stop_event.set() self.join(timeout=5) def build_message(status, job, attempt=None): created = next( event for event in status['events'] if event['job_id'] == job['job_id'] and event['event'] == 'created' ) return { 'schema_version': 1, 'job_id': job['job_id'], 'type': 'attachment.process.v1', 'attachment_id': status['attachment']['id'], 'pipeline_version': job['pipeline_version'], 'attempt': attempt if attempt is not None else job['attempt'], 'created_at': created['created_at'], } def publish_raw(messages): text = ''.join(json.dumps(message, separators=(',', ':')) + '\n' for message in messages) code, payload = cli('publish-raw', stdin=text) check(code == 0, 'publish-raw failed: ' + str(payload)) return payload def status_of(attachment_id): code, payload = cli('status', attachment_id) check(code == 0, 'status failed: ' + str(payload)) return payload def flush_outbox(attachment_id): wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0, 30, 'outbox publish', 1.0) def new_attachment_id(chat_id, before): rows = cli('find-by-chat', chat_id)[1] for row in reversed(rows): if row['id'] not in before: return row['id'] return None def evaluate(tracker, attachment_id, baseline, expect_failed=False, compare_baseline=True): if attachment_id is None: return {'I1': True, 'I2': True, 'I3': True, 'I4': True, 'I5': True, 'I6': True} status = status_of(attachment_id) attachment = status['attachment'] jobs = status['jobs'] variants = status['variants'] ready_version = attachment['ready_pipeline_version'] records = tracker.records.get(attachment_id, []) reached = any(record[1] == ('failed' if expect_failed else 'ready') for record in records) if not records: reached = attachment['status'] == ('failed' if expect_failed else 'ready') i1 = reached or attachment['status'] in ('ready', 'failed') ready_variants = [v for v in variants if v['pipeline_version'] == ready_version] succeeded_ids = {job['job_id'] for job in jobs if job['status'] == 'succeeded'} succeeded_events = {} for event in status['events']: if event['event'] == 'succeeded': succeeded_events[event['job_id']] = ( succeeded_events.get(event['job_id'], 0) + 1 ) i2 = (len(ready_variants) == 3 and all(succeeded_events.get(job_id) == 1 for job_id in succeeded_ids)) i3 = True seen_ready = False last_version = 0 for _at, state, version in records: if seen_ready and state != 'ready': i3 = False if state == 'ready': seen_ready = True if version is not None: if version < last_version: i3 = False last_version = version i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready' keys = {(variant['pipeline_version'], variant['variant']) for variant in variants} i5 = len(keys) == len(variants) if compare_baseline and ready_version == 1 and ready_variants: observed = {variant['variant']: variant['sha256'] for variant in ready_variants} i6 = all(observed.get(name) == digest for name, digest in baseline.items()) else: i6 = True return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4, 'I5': i5, 'I6': i6} # --- scenarios ----------------------------------------------------------- def scenario_duplicate(ctx): attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], sample_bytes(), 'duplicate.png') ctx['attachment_id'] = attachment['id'] ctx['tracker'].track(attachment['id']) wait_ready(ctx['client'], ctx['alice'], attachment['id']) status = status_of(attachment['id']) before = {variant['variant']: variant['s3'] for variant in status['variants']} message = build_message(status, status['jobs'][0]) publish_raw([message] * 5) wait_for( lambda: len([event for event in status_of(attachment['id'])['events'] if event['event'] == 'duplicate_skipped' and event['detail'].get('reason') == 'already_succeeded']) == 5, 20, 'five duplicate_skipped events', 1.0, ) after_status = status_of(attachment['id']) after = {variant['variant']: variant['s3'] for variant in after_status['variants']} check(before == after, 'duplicate delivery rewrote variant objects') return {'attachment_id': attachment['id'], 'job_attempts': [job['attempt'] for job in after_status['jobs']], 'notes': '5 duplicate deliveries were absorbed by the DB fence'} def scenario_reorder(ctx): compose('stop', 'worker') try: attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], sample_bytes(), 'reorder.png') ctx['attachment_id'] = attachment['id'] ctx['tracker'].track(attachment['id']) flush_outbox(attachment['id']) status = status_of(attachment['id']) message = build_message(status, status['jobs'][0]) stale = dict(message, attempt=99) publish_raw([stale, message]) compose('start', 'worker') wait_ready(ctx['client'], ctx['alice'], attachment['id']) finally: compose('start', 'worker') after_status = status_of(attachment['id']) skipped = [event for event in after_status['events'] if event['event'] == 'duplicate_skipped'] check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages') check(len([event for event in after_status['events'] if event['event'] == 'succeeded']) == 1, 'expected exactly one successful effect') return {'attachment_id': attachment['id'], 'job_attempts': [job['attempt'] for job in after_status['jobs']], 'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'} def crash_scenario(ctx, point, filename, expected_attempt, expect_lease_expired=False, expect_duplicate=False): compose('stop', 'worker') try: attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], sample_bytes(), filename) ctx['attachment_id'] = attachment['id'] ctx['tracker'].track(attachment['id']) starts = worker_start_count() set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash') compose('start', 'worker') wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90) finally: compose('start', 'worker') status = status_of(attachment['id']) job = status['jobs'][0] check(job['attempt'] == expected_attempt, 'expected attempt ' + str(expected_attempt) + ', got ' + str(job['attempt'])) check(worker_start_count() >= starts + 2, 'worker process did not restart after the injected crash') if expect_lease_expired: check(any(event['event'] == 'lease_expired' for event in status['events']), 'missing lease_expired event') if expect_duplicate: check(any(event['event'] == 'duplicate_skipped' and event['detail'].get('reason') == 'already_succeeded' for event in status['events']), 'missing already_succeeded duplicate after redelivery') return {'attachment_id': attachment['id'], 'job_attempts': [job['attempt']], 'notes': 'crashed at ' + point} def scenario_crash_before_side_effect(ctx): return crash_scenario(ctx, 'worker.before_side_effect', 'crash-before.png', 2, expect_lease_expired=True) def scenario_crash_after_side_effect(ctx): return crash_scenario(ctx, 'worker.after_side_effect', 'crash-after.png', 2, expect_lease_expired=True) def scenario_crash_after_commit_before_ack(ctx): return crash_scenario(ctx, 'worker.after_commit_before_ack', 'crash-ack.png', 1, expect_duplicate=True) def scenario_crash_after_commit_before_publish(ctx): before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]} set_fault('fault:api.after_commit_before_publish', 'crash') try: try: ctx['client'].call( 'POST', '/api/v1/chats/' + ctx['chat']['id'] + '/attachments?filename=crash-publish.png', (202, 502, 503, 504), actor=ctx['alice'], raw=sample_bytes(), headers={'Content-Type': 'image/png'}, ) except Failure: # A killed replica may close the connection instead of answering. pass finally: clear_fault('fault:api.after_commit_before_publish') attachment_id = wait_for( lambda: new_attachment_id(ctx['chat']['id'], before), 30, 'attachment row after API crash', 1.0, ) ctx['attachment_id'] = attachment_id ctx['tracker'].track(attachment_id) wait_ready(ctx['client'], ctx['alice'], attachment_id, 90) status = status_of(attachment_id) check(len([event for event in status['events'] if event['event'] == 'created']) == 1, 'expected exactly one created event') return {'attachment_id': attachment_id, 'job_attempts': [job['attempt'] for job in status['jobs']], 'notes': 'outbox relay published the committed job after API restart'} def scenario_relay_crash_after_publish(ctx): compose('stop', 'worker') before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]} try: set_fault('fault:relay.after_publish_before_mark', 'crash') try: ctx['client'].call( 'POST', '/api/v1/chats/' + ctx['chat']['id'] + '/attachments?filename=relay-crash.png', (202, 502, 503, 504), actor=ctx['alice'], raw=sample_bytes(), headers={'Content-Type': 'image/png'}, ) except Failure: # The relay may crash the handling replica before the reply flushes. pass clear_fault('fault:relay.after_publish_before_mark') attachment_id = wait_for( lambda: new_attachment_id(ctx['chat']['id'], before), 30, 'attachment row after relay crash', 1.0, ) ctx['attachment_id'] = attachment_id ctx['tracker'].track(attachment_id) wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0, 60, 'outbox published after relay crash', 1.0) queue_depth = rabbit_queues().get('attachments.process', {}).get( 'messages', 0) compose('start', 'worker') wait_ready(ctx['client'], ctx['alice'], attachment_id, 90) finally: clear_fault('fault:relay.after_publish_before_mark') compose('start', 'worker') status = status_of(attachment_id) check(len([event for event in status['events'] if event['event'] == 'succeeded']) == 1, 'expected exactly one succeeded event') duplicated = any(event['event'] == 'duplicate_skipped' for event in status['events']) note = ('relay republished the same message; duplicate absorbed' if duplicated else 'relay published once in this run; I1/I2 still hold') if queue_depth >= 2: check(duplicated, 'two queued copies but no duplicate_skipped event') return {'attachment_id': attachment_id, 'job_attempts': [job['attempt'] for job in status['jobs']], 'notes': note} def scenario_stale_version(ctx): check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2, 'stale_version needs WORKER_CONCURRENCY>=2') compose('stop', 'worker') try: attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], sample_bytes(), 'stale.png') attachment_id = attachment['id'] ctx['attachment_id'] = attachment_id ctx['tracker'].track(attachment_id) flush_outbox(attachment_id) code, payload = cli('reprocess', attachment_id, '--pipeline-version', '2', '--reason', 'suite') check(code == 0, 'reprocess v2 failed: ' + str(payload)) set_fault('fault:worker.after_side_effect:' + attachment_id + ':1', 'delay:8000') compose('start', 'worker') v2_etags = None def both_terminal(): nonlocal v2_etags status = status_of(attachment_id) jobs = {(job['pipeline_version']): job for job in status['jobs']} if '2' in jobs or 2 in jobs: v2_job = jobs.get(2) if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None: v2_etags = {variant['variant']: variant['s3'] for variant in status['variants'] if variant['pipeline_version'] == 2} v1_job = jobs.get(1) v2_job = jobs.get(2) return (v1_job and v2_job and v1_job['status'] == 'superseded' and v2_job['status'] == 'succeeded') wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0) status = status_of(attachment_id) check(status['attachment']['ready_pipeline_version'] == 2, 'ready pipeline version is not 2') check(all(variant['pipeline_version'] == 2 for variant in status['variants']), 'variant rows from v1 survived') after_etags = {variant['variant']: variant['s3'] for variant in status['variants'] if variant['pipeline_version'] == 2} check(v2_etags == after_etags, 'v1 overwrote v2 variant objects') served = ctx['client'].call( 'GET', '/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail', actor=ctx['alice'], ) v2_thumbnail = next(variant for variant in status['variants'] if variant['variant'] == 'thumbnail') check(sha256(served['bytes']) == v2_thumbnail['sha256'], 'served thumbnail is not the v2 object') check(sha256(served['bytes']) != ctx['baseline']['thumbnail'], 'served thumbnail equals the v1 baseline') v1_job = next(job for job in status['jobs'] if job['pipeline_version'] == 1) publish_raw([build_message(status, v1_job)]) wait_for( lambda: any(event['event'] == 'duplicate_skipped' and event['detail'].get('reason') == 'superseded' for event in status_of(attachment_id)['events']), 20, 'superseded duplicate skip', 1.0, ) finally: clear_fault('fault:worker.after_side_effect:' + (attachment['id'] if 'attachment' in locals() else '')) compose('start', 'worker') return {'attachment_id': attachment_id, 'job_attempts': [job['attempt'] for job in status['jobs']], 'notes': 'v1 finished after v2 and was recorded as superseded'} def scenario_poison(ctx): before = rabbit_queues().get('attachments.process.dlq', {}).get( 'messages', 0) publish_raw(['not-json', '{"schema_version":1}']) wait_for( lambda: rabbit_queues().get('attachments.process.dlq', {}).get( 'messages', 0) >= before + 2, 15, 'two poison messages in DLQ', 0.5, ) return {'attachment_id': None, 'job_attempts': [], 'notes': 'malformed messages moved to the DLQ without touching jobs'} def scenario_retry_and_replay(ctx): max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5') base = int(worker_env('JOB_RETRY_BASE_MS') or '500') cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000') compose('stop', 'worker') try: attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], sample_bytes(), 'retry.png') attachment_id = attachment['id'] ctx['attachment_id'] = attachment_id ctx['tracker'].track(attachment_id) set_fault('fault:worker.before_side_effect:' + attachment_id + ':1', 'transient') compose('start', 'worker') failed = wait_ready(ctx['client'], ctx['alice'], attachment_id, 90, allow_failed=True) finally: compose('start', 'worker') check(failed['status'] == 'failed', 'transient fault was not exhausted') check(failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted, got ' + str(failed['error_code'])) status = status_of(attachment_id) retries = [event for event in status['events'] if event['event'] == 'retry_scheduled'] check(len(retries) == max_attempts - 1, 'expected ' + str(max_attempts - 1) + ' retries, got ' + str(len(retries))) for index, event in enumerate(retries): limit = min(cap, base * 2 ** index) check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit, 'retry delay exceeds its jitter cap') ctx['client'].call( 'GET', '/api/v1/attachments/' + attachment_id + '/content', 409, actor=ctx['alice'], ) code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite') check(code == 3 and payload.get('reason') in ('no_failed_job', 'already_ready'), 'replay of a ready attachment was not refused') clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1') code, payload = cli('replay', attachment_id, '--reason', 'suite') check(code == 0, 'operator replay failed: ' + str(payload)) wait_ready(ctx['client'], ctx['alice'], attachment_id, 60) status = status_of(attachment_id) replayed = [event for event in status['events'] if event['event'] == 'replayed'] check(len(replayed) == 1 and replayed[0]['detail'].get('operator'), 'missing replayed event or operator audit') return {'attachment_id': attachment_id, 'job_attempts': [job['attempt'] for job in status['jobs']], 'notes': 'transient injected fault exhausted retries; replay recovered'} def scenario_hung_worker(ctx): timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10') compose('stop', 'worker') try: attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], sample_bytes(), 'hung.png') attachment_id = attachment['id'] ctx['attachment_id'] = attachment_id ctx['tracker'].track(attachment_id) set_fault('fault:worker.after_side_effect:' + attachment_id + ':1', 'delay:' + str((timeout_seconds + 5) * 1000)) compose('start', 'worker') wait_ready(ctx['client'], ctx['alice'], attachment_id, 90) finally: compose('start', 'worker') status = status_of(attachment_id) check(any(event['event'] == 'retry_scheduled' and event['detail'].get('code') == 'processing_timeout' for event in status['events']), 'missing processing_timeout retry') return {'attachment_id': attachment_id, 'job_attempts': [job['attempt'] for job in status['jobs']], 'notes': 'hard timeout aborted the hung render and retried'} def scenario_big_image(ctx): ctx['compare_baseline'] = False data = worst_decode_png() containers = compose('ps', '-q', 'worker').stdout.split() stop = threading.Event() peak = {'mb': 0.0} def sample(): while not stop.is_set(): if containers: result = docker( 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *containers, check_result=False, ) for line in result.stdout.splitlines(): parts = line.split() if len(parts) < 2: continue match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1]) if not match: continue value = float(match.group(1)) if match.group(2) == 'GiB': value *= 1024 elif match.group(2) == 'KiB': value /= 1024 peak['mb'] = max(peak['mb'], value) stop.wait(0.3) sampler = threading.Thread(target=sample, daemon=True) sampler.start() try: def do_upload(index): client = new_client(ctx['base_url']) return upload(client, ctx['alice'], ctx['chat'], data, 'big-' + str(index) + '.png') with ThreadPoolExecutor(max_workers=2) as pool: attachments = list(pool.map(do_upload, (1, 2))) ctx['attachment_id'] = attachments[0]['id'] for attachment in attachments: ctx['tracker'].track(attachment['id']) results = [wait_ready(ctx['client'], ctx['alice'], attachment['id'], 120, allow_failed=True) for attachment in attachments] finally: stop.set() sampler.join(timeout=5) for meta in results: check(meta['status'] in ('ready', 'failed'), 'big image stuck') if meta['status'] == 'failed': check(meta['error_code'] in ('image_too_large', 'invalid_image'), 'unexpected big-image error code') for container in containers: result = docker('inspect', '-f', '{{.State.OOMKilled}}', container, check_result=False) check(result.stdout.strip() == 'false', 'worker was OOM-killed') return {'attachment_id': attachments[0]['id'], 'job_attempts': [], 'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']} SCENARIOS = { 'duplicate': scenario_duplicate, 'reorder': scenario_reorder, 'crash_before_side_effect': scenario_crash_before_side_effect, 'crash_after_side_effect': scenario_crash_after_side_effect, 'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack, 'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish, 'relay_crash_after_publish': scenario_relay_crash_after_publish, 'stale_version': scenario_stale_version, 'poison': scenario_poison, 'retry_and_replay': scenario_retry_and_replay, 'hung_worker': scenario_hung_worker, 'big_image': scenario_big_image, } def select(name): if name == 'all': return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4] if name == 'level4': return LEVEL4 if name == 'track2': return TRACK2 if name in SCENARIOS: return [name] raise SystemExit('unknown scenario: ' + name) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default=None) parser.add_argument('--ca-file', default=None) parser.add_argument('--scenario', default='level4') parser.add_argument('--repeat', type=int, default=1) parser.add_argument('--output', default=None) args = parser.parse_args() if worker_env('FAULT_INJECTION_ENABLED') != '1': print('fault injection is disabled; restart the stand with:\n ' + HINT, file=sys.stderr) raise SystemExit(2) client = new_client(args.base_url, args.ca_file) alice, _bob, _eve, chat = client.bootstrap() clear_faults() reference = upload(client, alice, chat, sample_bytes(), 'reference.png') ready = wait_ready(client, alice, reference['id'], 60) baseline = variant_sha(ready) selected = select(args.scenario) results = [] failed = 0 for name in selected: for repeat in range(1, args.repeat + 1): clear_faults() tracker = Tracker(args.base_url or 'http://localhost:8080', alice, args.ca_file) tracker.start() started = time.monotonic() ctx = { 'client': client, 'base_url': args.base_url or 'http://localhost:8080', 'alice': alice, 'chat': chat, 'tracker': tracker, 'baseline': baseline, 'reference_id': reference['id'], 'ca_file': args.ca_file, } scenario = {'attachment_id': None, 'job_attempts': [], 'notes': ''} passed = True try: scenario.update(SCENARIOS[name](ctx)) except Exception as error: # noqa: BLE001 - scenario boundary passed = False scenario['notes'] = type(error).__name__ + ': ' + str(error) finally: tracker.stop() clear_faults() compose('start', 'worker', check_result=False) attachment_id = scenario['attachment_id'] or ctx.get('attachment_id') try: invariants = evaluate( tracker, attachment_id, baseline, compare_baseline=ctx.get('compare_baseline', True), ) except Exception: # noqa: BLE001 invariants = {key: False for key in ('I1', 'I2', 'I3', 'I4', 'I5', 'I6')} passed = passed and all(invariants.values()) if not passed: failed += 1 result = { 'name': name, 'repeat': repeat, 'passed': passed, 'duration_s': round(time.monotonic() - started, 3), 'attachment_id': attachment_id, 'job_attempts': scenario['job_attempts'], 'events_summary': scenario.get('events_summary', {}), 'invariants': invariants, 'notes': scenario['notes'], } results.append(result) print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) + ' ' + str(result['duration_s']) + 's ' + json.dumps(invariants) + ' ' + result['notes'], flush=True) if args.output: path = Path(args.output) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n') print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed') raise SystemExit(1 if failed else 0) if __name__ == '__main__': main()