#!/usr/bin/env python3 """Additional *live* Lab5 fault tests; operator-controlled, synthetic records only. broker-persistence: require a publisher-confirmed RabbitMQ message to survive broker restart without a running dispatcher or worker. This differs from the outbox-restoration tests where dispatcher publishes *after* broker recovery. failed-replay: safely inject a deterministic checksum mismatch into one fresh synthetic attachment's DB metadata, verify permanent failure, restore the original hash, replay via the operator-only CLI, and verify success plus audit. staging-cleanup: verify the staged S3 object is absent after ready (with a matching cleared staging_key in the DB). No volumes are deleted. The test may stop/restart worker/dispatcher/broker; run only against a disposable LAB5 environment and never concurrently. """ import argparse import hashlib import os import re import subprocess import time import uuid from client import Client, Failure, check, run from pipeline_resilience import verify_ready, wait_api from pathlib import Path ROOT = Path(__file__).resolve().parents[1] def compose(*args, expected=(0,), timeout=240): proc = subprocess.run( ['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, ) if proc.returncode not in expected: # Don't print env vars, tokens or command stdout/stderr by default. raise Failure('docker compose %s failed (exit %d); inspect ps/logs' % (' '.join(args[:2]), proc.returncode)) return proc.stdout.strip() def pg(sql): """Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here.""" user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip() dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip() return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql) def scalar(sql): return pg(sql).splitlines()[-1].strip() def wait_for(label, action, predicate, seconds=30, interval=0.5): deadline = time.monotonic() + seconds last = None while time.monotonic() < deadline: try: last = action() if predicate(last): return last except Failure: pass time.sleep(interval) raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100]) def job_query(att_id, fields): uuid.UUID(att_id) return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' % (fields, att_id)) def queue_ready(): out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues', 'name', 'messages_ready', 'messages_unacknowledged') for line in out.splitlines(): parts = re.split(r'\s+', line.strip()) if parts[0] == 'attachment-processing.v1' and len(parts) >= 3: return int(parts[1]), int(parts[2]) raise Failure('RabbitMQ durable queue missing') def make_upload(c): alice, bob, _eve, chat = c.bootstrap() att, original = c.attachment(alice, chat) check(att['status'] == 'queued', 'upload must return 202 queued') uuid.UUID(att['id']) return alice, bob, att, original def wait_staging_cleared(att_id, seconds=20): return wait_for('staging_key cleared after ready', lambda: scalar("SELECT COALESCE(staging_key,'') FROM attachments WHERE id='%s'" % att_id), lambda value: value == '', seconds=seconds) def verify_staging_absent(att_id): # This executes *inside* the worker container with its normal S3 credentials. # Never expose S3 secrets to the host or write them to evidence files. code = '''import sys from botocore.exceptions import ClientError from app.attachments import s3_client, BUCKET key='staging/'+sys.argv[1]+'/original' try: s3_client().head_object(Bucket=BUCKET,Key=key) except ClientError as exc: if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404: print('ABSENT') else: raise else: print('STILL_PRESENT')''' result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id) check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70]) def broker_persistence(c): compose('stop', 'worker') compose('up', '-d', 'dispatcher') baseline = queue_ready()[0] alice, _bob, att, original = make_upload(c) print('202 queued:', att['id'], flush=True) sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id'] wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30) wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0], lambda n: n >= baseline + 1, seconds=15) compose('stop', 'dispatcher') before = queue_ready()[0] check(before >= baseline + 1, 'published message disappeared before restart') print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True) compose('restart', 'broker') compose('up', '-d', '--wait', 'broker') after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0], lambda n: n >= before, seconds=40, interval=1) check(scalar(sql) == 't', 'outbox publish confirmation lost') print('Broker restarted; ready messages persisted=%d' % after, flush=True) # CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message # can deliver this job to the worker. compose('up', '-d', 'worker') verify_ready(c, alice, att, original, seconds=180) check(job_query(att['id'], 'status') == 'ready', 'job not committed ready') print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish') def failed_replay(c): compose('stop', 'worker', 'dispatcher') alice, _bob, att, original = make_upload(c) ident = att['id'] original_sha = hashlib.sha256(original).hexdigest() check(original_sha == att['sha256'], 'fixture hash mismatch') check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha, 'DB hash unexpected before controlled fault') fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64 changed = False try: updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" % (fault_sha, ident)) check(ident in updated, 'failed to inject synthetic checksum mismatch') changed = True compose('up', '-d', 'dispatcher', 'worker') def get_state(): return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json'] failed = wait_for('permanent failed status', get_state, lambda a: a['status'] == 'failed', seconds=90) check(failed['error_code'] == 'invalid_image', 'expected permanent invalid_image, got ' + str(failed['error_code'])) check(job_query(ident, 'status') == 'failed', 'processing job must fail') check(job_query(ident, 'attempt') == '1', 'permanent error must not retry') check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0', 'failed job created variants') print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True) # Repair only the generated synthetic row before explicit operator replay. restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" % (original_sha, ident)) check(ident in restored, 'synthetic metadata could not be restored') changed = False compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident) verify_ready(c, alice, att, original, seconds=180) audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident) check(audit == '1', 'operator replay audit count incorrect: ' + audit) check(job_query(ident, 'status') == 'ready', 'replayed job not ready') attempt = job_query(ident, 'attempt') check(attempt == '1', 'replay did not reset attempt: ' + attempt) invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident, expected=(1,)) print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected') finally: if changed: # Never leave the synthetic attachment with altered metadata. pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident)) def staging_cleanup(c): compose('up', '-d', 'dispatcher', 'worker') alice, _bob, att, original = make_upload(c) verify_ready(c, alice, att, original, seconds=180) wait_staging_cleared(att['id'], seconds=30) verify_staging_absent(att['id']) print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3') def main(): parser = argparse.ArgumentParser() parser.add_argument('--scenario', required=True, choices=['broker-persistence', 'failed-replay', 'staging-cleanup']) args = parser.parse_args() if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes': raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack') c = Client('http://localhost:8080') try: wait_api() if args.scenario == 'broker-persistence': broker_persistence(c) elif args.scenario == 'failed-replay': failed_replay(c) else: staging_cleanup(c) finally: # Restore normal services even when an assertion fails. NO volume deletion. try: compose('up', '-d', '--wait', 'broker') compose('up', '-d', 'dispatcher', 'worker') except Failure: print('WARNING: inspect docker compose ps and restore dispatcher/worker manually') if __name__ == '__main__': run(main)