Files
mtusi2026lab5/tests/pipeline_extended.py

234 lines
10 KiB
Python

#!/usr/bin/env python3
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
broker restart without a running dispatcher or worker. This differs from the
outbox-restoration tests where dispatcher publishes *after* broker recovery.
failed-replay: safely inject a deterministic checksum mismatch into one fresh
synthetic attachment's DB metadata, verify permanent failure, restore the
original hash, replay via the operator-only CLI, and verify success plus audit.
staging-cleanup: verify the staged S3 object is absent after ready (with a
matching cleared staging_key in the DB).
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
run only against a disposable LAB5 environment and never concurrently.
"""
import argparse
import hashlib
import os
import re
import subprocess
import time
import uuid
from client import Client, Failure, check, run
from pipeline_resilience import verify_ready, wait_api
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,), timeout=240):
proc = subprocess.run(
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
)
if proc.returncode not in expected:
# Don't print env vars, tokens or command stdout/stderr by default.
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
(' '.join(args[:2]), proc.returncode))
return proc.stdout.strip()
def pg(sql):
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
def scalar(sql):
return pg(sql).splitlines()[-1].strip()
def wait_for(label, action, predicate, seconds=30, interval=0.5):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
last = action()
if predicate(last):
return last
except Failure:
pass
time.sleep(interval)
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
def job_query(att_id, fields):
uuid.UUID(att_id)
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
(fields, att_id))
def queue_ready():
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
'name', 'messages_ready', 'messages_unacknowledged')
for line in out.splitlines():
parts = re.split(r'\s+', line.strip())
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
return int(parts[1]), int(parts[2])
raise Failure('RabbitMQ durable queue missing')
def make_upload(c):
alice, bob, _eve, chat = c.bootstrap()
att, original = c.attachment(alice, chat)
check(att['status'] == 'queued', 'upload must return 202 queued')
uuid.UUID(att['id'])
return alice, bob, att, original
def wait_staging_cleared(att_id, seconds=20):
return wait_for('staging_key cleared after ready',
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
lambda value: value == '<NULL>', seconds=seconds)
def verify_staging_absent(att_id):
# This executes *inside* the worker container with its normal S3 credentials.
# Never expose S3 secrets to the host or write them to evidence files.
code = '''import sys
from botocore.exceptions import ClientError
from app.attachments import s3_client, BUCKET
key='staging/'+sys.argv[1]+'/original'
try:
s3_client().head_object(Bucket=BUCKET,Key=key)
except ClientError as exc:
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
print('ABSENT')
else:
raise
else:
print('STILL_PRESENT')'''
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
def broker_persistence(c):
compose('stop', 'worker')
compose('up', '-d', 'dispatcher')
baseline = queue_ready()[0]
alice, _bob, att, original = make_upload(c)
print('202 queued:', att['id'], flush=True)
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
lambda n: n >= baseline + 1, seconds=15)
compose('stop', 'dispatcher')
before = queue_ready()[0]
check(before >= baseline + 1, 'published message disappeared before restart')
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
compose('restart', 'broker')
compose('up', '-d', '--wait', 'broker')
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
lambda n: n >= before, seconds=40, interval=1)
check(scalar(sql) == 't', 'outbox publish confirmation lost')
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
# can deliver this job to the worker.
compose('up', '-d', 'worker')
verify_ready(c, alice, att, original, seconds=180)
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
def failed_replay(c):
compose('stop', 'worker', 'dispatcher')
alice, _bob, att, original = make_upload(c)
ident = att['id']
original_sha = hashlib.sha256(original).hexdigest()
check(original_sha == att['sha256'], 'fixture hash mismatch')
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
'DB hash unexpected before controlled fault')
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
changed = False
try:
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
(fault_sha, ident))
check(ident in updated, 'failed to inject synthetic checksum mismatch')
changed = True
compose('up', '-d', 'dispatcher', 'worker')
def get_state():
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
failed = wait_for('permanent failed status', get_state,
lambda a: a['status'] == 'failed', seconds=90)
check(failed['error_code'] == 'invalid_image',
'expected permanent invalid_image, got ' + str(failed['error_code']))
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
'failed job created variants')
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
# Repair only the generated synthetic row before explicit operator replay.
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
(original_sha, ident))
check(ident in restored, 'synthetic metadata could not be restored')
changed = False
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
verify_ready(c, alice, att, original, seconds=180)
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
attempt = job_query(ident, 'attempt')
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
'replay', '--id', ident, expected=(1,))
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
finally:
if changed:
# Never leave the synthetic attachment with altered metadata.
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
def staging_cleanup(c):
compose('up', '-d', 'dispatcher', 'worker')
alice, _bob, att, original = make_upload(c)
verify_ready(c, alice, att, original, seconds=180)
wait_staging_cleared(att['id'], seconds=30)
verify_staging_absent(att['id'])
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--scenario', required=True,
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
args = parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
c = Client('http://localhost:8080')
try:
wait_api()
if args.scenario == 'broker-persistence':
broker_persistence(c)
elif args.scenario == 'failed-replay':
failed_replay(c)
else:
staging_cleanup(c)
finally:
# Restore normal services even when an assertion fails. NO volume deletion.
try:
compose('up', '-d', '--wait', 'broker')
compose('up', '-d', 'dispatcher', 'worker')
except Failure:
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
if __name__ == '__main__':
run(main)