forked from mtusi2026/mtusi2026lab5
234 lines
10 KiB
Python
234 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
|
|
|
|
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
|
|
broker restart without a running dispatcher or worker. This differs from the
|
|
outbox-restoration tests where dispatcher publishes *after* broker recovery.
|
|
|
|
failed-replay: safely inject a deterministic checksum mismatch into one fresh
|
|
synthetic attachment's DB metadata, verify permanent failure, restore the
|
|
original hash, replay via the operator-only CLI, and verify success plus audit.
|
|
|
|
staging-cleanup: verify the staged S3 object is absent after ready (with a
|
|
matching cleared staging_key in the DB).
|
|
|
|
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
|
|
run only against a disposable LAB5 environment and never concurrently.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
from client import Client, Failure, check, run
|
|
from pipeline_resilience import verify_ready, wait_api
|
|
|
|
from pathlib import Path
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args, expected=(0,), timeout=240):
|
|
proc = subprocess.run(
|
|
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
|
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
|
|
)
|
|
if proc.returncode not in expected:
|
|
# Don't print env vars, tokens or command stdout/stderr by default.
|
|
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
|
|
(' '.join(args[:2]), proc.returncode))
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def pg(sql):
|
|
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
|
|
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
|
|
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
|
|
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
|
|
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
|
|
|
|
|
|
def scalar(sql):
|
|
return pg(sql).splitlines()[-1].strip()
|
|
|
|
|
|
def wait_for(label, action, predicate, seconds=30, interval=0.5):
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
last = action()
|
|
if predicate(last):
|
|
return last
|
|
except Failure:
|
|
pass
|
|
time.sleep(interval)
|
|
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
|
|
|
|
|
|
def job_query(att_id, fields):
|
|
uuid.UUID(att_id)
|
|
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
|
|
(fields, att_id))
|
|
|
|
|
|
def queue_ready():
|
|
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
|
|
'name', 'messages_ready', 'messages_unacknowledged')
|
|
for line in out.splitlines():
|
|
parts = re.split(r'\s+', line.strip())
|
|
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
|
|
return int(parts[1]), int(parts[2])
|
|
raise Failure('RabbitMQ durable queue missing')
|
|
|
|
|
|
def make_upload(c):
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
att, original = c.attachment(alice, chat)
|
|
check(att['status'] == 'queued', 'upload must return 202 queued')
|
|
uuid.UUID(att['id'])
|
|
return alice, bob, att, original
|
|
|
|
|
|
def wait_staging_cleared(att_id, seconds=20):
|
|
return wait_for('staging_key cleared after ready',
|
|
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
|
|
lambda value: value == '<NULL>', seconds=seconds)
|
|
|
|
|
|
def verify_staging_absent(att_id):
|
|
# This executes *inside* the worker container with its normal S3 credentials.
|
|
# Never expose S3 secrets to the host or write them to evidence files.
|
|
code = '''import sys
|
|
from botocore.exceptions import ClientError
|
|
from app.attachments import s3_client, BUCKET
|
|
key='staging/'+sys.argv[1]+'/original'
|
|
try:
|
|
s3_client().head_object(Bucket=BUCKET,Key=key)
|
|
except ClientError as exc:
|
|
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
|
|
print('ABSENT')
|
|
else:
|
|
raise
|
|
else:
|
|
print('STILL_PRESENT')'''
|
|
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
|
|
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
|
|
|
|
|
|
def broker_persistence(c):
|
|
compose('stop', 'worker')
|
|
compose('up', '-d', 'dispatcher')
|
|
baseline = queue_ready()[0]
|
|
alice, _bob, att, original = make_upload(c)
|
|
print('202 queued:', att['id'], flush=True)
|
|
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
|
|
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
|
|
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
|
|
lambda n: n >= baseline + 1, seconds=15)
|
|
compose('stop', 'dispatcher')
|
|
before = queue_ready()[0]
|
|
check(before >= baseline + 1, 'published message disappeared before restart')
|
|
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
|
|
compose('restart', 'broker')
|
|
compose('up', '-d', '--wait', 'broker')
|
|
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
|
|
lambda n: n >= before, seconds=40, interval=1)
|
|
check(scalar(sql) == 't', 'outbox publish confirmation lost')
|
|
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
|
|
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
|
|
# can deliver this job to the worker.
|
|
compose('up', '-d', 'worker')
|
|
verify_ready(c, alice, att, original, seconds=180)
|
|
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
|
|
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
|
|
|
|
|
|
def failed_replay(c):
|
|
compose('stop', 'worker', 'dispatcher')
|
|
alice, _bob, att, original = make_upload(c)
|
|
ident = att['id']
|
|
original_sha = hashlib.sha256(original).hexdigest()
|
|
check(original_sha == att['sha256'], 'fixture hash mismatch')
|
|
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
|
|
'DB hash unexpected before controlled fault')
|
|
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
|
|
changed = False
|
|
try:
|
|
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
|
|
(fault_sha, ident))
|
|
check(ident in updated, 'failed to inject synthetic checksum mismatch')
|
|
changed = True
|
|
compose('up', '-d', 'dispatcher', 'worker')
|
|
|
|
def get_state():
|
|
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
|
|
failed = wait_for('permanent failed status', get_state,
|
|
lambda a: a['status'] == 'failed', seconds=90)
|
|
check(failed['error_code'] == 'invalid_image',
|
|
'expected permanent invalid_image, got ' + str(failed['error_code']))
|
|
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
|
|
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
|
|
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
|
|
'failed job created variants')
|
|
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
|
|
# Repair only the generated synthetic row before explicit operator replay.
|
|
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
|
|
(original_sha, ident))
|
|
check(ident in restored, 'synthetic metadata could not be restored')
|
|
changed = False
|
|
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
|
|
verify_ready(c, alice, att, original, seconds=180)
|
|
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
|
|
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
|
|
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
|
|
attempt = job_query(ident, 'attempt')
|
|
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
|
|
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
|
|
'replay', '--id', ident, expected=(1,))
|
|
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
|
|
finally:
|
|
if changed:
|
|
# Never leave the synthetic attachment with altered metadata.
|
|
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
|
|
|
|
|
|
def staging_cleanup(c):
|
|
compose('up', '-d', 'dispatcher', 'worker')
|
|
alice, _bob, att, original = make_upload(c)
|
|
verify_ready(c, alice, att, original, seconds=180)
|
|
wait_staging_cleared(att['id'], seconds=30)
|
|
verify_staging_absent(att['id'])
|
|
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--scenario', required=True,
|
|
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
|
|
args = parser.parse_args()
|
|
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
|
|
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
|
|
c = Client('http://localhost:8080')
|
|
try:
|
|
wait_api()
|
|
if args.scenario == 'broker-persistence':
|
|
broker_persistence(c)
|
|
elif args.scenario == 'failed-replay':
|
|
failed_replay(c)
|
|
else:
|
|
staging_cleanup(c)
|
|
finally:
|
|
# Restore normal services even when an assertion fails. NO volume deletion.
|
|
try:
|
|
compose('up', '-d', '--wait', 'broker')
|
|
compose('up', '-d', 'dispatcher', 'worker')
|
|
except Failure:
|
|
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|