Files
mtusi2026lab5/scripts/lab5_checks.py

488 lines
25 KiB
Python

#!/usr/bin/env python3
"""Repeatable grade 3/4 failure checks for the local synthetic lab5 Compose stand.
Run separately from other tests: scenarios deliberately restart/stop services.
Every scenario restores the original services in finally. No volumes, secrets or
application records are removed. Tokens live only in process memory.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
def command(*args):
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace')
if result.returncode:
# Never print environment, response bodies, credentials or arbitrary Docker logs.
raise Failure('command failed: ' + ' '.join(args[:4]) + ' (exit %d)' % result.returncode)
return result.stdout.strip()
def compose(*args):
return command('docker', 'compose', *args)
def sql(query):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
def rows(query):
value = sql("SELECT COALESCE(json_agg(t), '[]'::json) FROM (%s) t" % query)
return json.loads(value)
def identifier(value):
return str(uuid.UUID(value))
def job(attachment_id):
data = rows("SELECT * FROM attachment_jobs WHERE attachment_id='%s'" % identifier(attachment_id))
check(len(data) == 1, 'attachment must have exactly one logical job')
return data[0]
def audit(job_id):
return rows("SELECT action,actor,reason,details,created_at FROM job_audit WHERE job_id='%s' ORDER BY id"
% identifier(job_id))
def cli(*args):
# The explicit operator service has broker/worker permissions. The API does
# not receive broker credentials merely to make an evidence script easier.
output = compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'jobs', *args)
return json.loads(output.splitlines()[-1]) if output else None
def inject(job_id, action, count=1):
return cli('inject', identifier(job_id), '--action', action, '--count', str(count),
'--actor', 'lab5-evidence', '--reason', 'repeatable synthetic grade 4 failure check')
def wait_until(predicate, description, seconds=180):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(0.4)
raise Failure('deadline exceeded: ' + description)
def ready(client):
def probe():
try:
client.call('GET', '/health/ready', schema='Health')
return True
except Failure:
return False
wait_until(probe, 'API ready', 90)
def restore(client, *services):
# start preserves stopped container configuration and all named volumes.
compose('start', *services)
ready(client)
def upload(client, actor, chat, label):
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + label + '.png',
202, actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
attachment = result['json']
check(attachment['status'] == 'queued', '202 must carry a queued snapshot')
check(attachment['variants'] == [] and attachment['error_code'] is None, 'queued result fields')
check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'accepted original checksum')
return attachment, content
def metadata(client, actor, attachment):
return client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
def wait_state(client, actor, attachment, status):
def probe():
current = metadata(client, actor, attachment)
if current['status'] == status:
return current
check(current['status'] != 'failed' or status == 'failed', 'unexpected terminal failed state')
return wait_until(probe, 'attachment ' + status)
def verified_ready(client, actor, attachment, content):
result = wait_state(client, actor, attachment, 'ready')
check(result['id'] == attachment['id'], 'logical attachment identity changed')
check(result['error_code'] is None, 'ready error_code must be null')
check({item['name'] for item in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'grade 4 variants')
check(len(result['variants']) == 3, 'duplicate result variants')
for name in ['original', 'thumbnail', 'crop', 'optimized']:
data = client.call('GET', '/api/v1/attachments/' + result['id'] + '/content?variant=' + name, actor=actor)
record = result if name == 'original' else next(v for v in result['variants'] if v['name'] == name)
check(len(data['bytes']) == record['size_bytes'], 'published object size mismatch')
check(hashlib.sha256(data['bytes']).hexdigest() == record['sha256'], 'published object checksum mismatch')
if name == 'original':
check(data['bytes'] == content, 'original bytes changed')
count = int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % result['id']))
check(count == 3, 'exactly three persistent variant records required')
return result
def object_state(attachment_id):
# Runs with the API's restricted bucket credentials; only selected facts return.
aid = identifier(attachment_id)
code = """import json
from messenger.config import read_secret
from messenger.db import create_pool
from messenger.objects import ObjectStore
from botocore.exceptions import ClientError
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True)
with p.connection() as c:
r=c.execute('SELECT object_key,staging_key,staging_deleted_at,status,pipeline_version FROM attachments WHERE id=%%s',(%r,)).fetchone()
s=ObjectStore()
def exists(key):
try: s.client.head_object(Bucket=s.bucket,Key=key);return True
except ClientError as e:
if e.response['ResponseMetadata']['HTTPStatusCode']==404:return False
raise
prefix='variants/'+%r.replace('-','')+'/'
objects=s.client.list_objects_v2(Bucket=s.bucket,Prefix=prefix).get('Contents',[])
print(json.dumps(dict(final=exists(r['object_key']),staging=exists(r['staging_key']),status=r['status'],staging_deleted=r['staging_deleted_at'] is not None,variant_objects=len(objects))))
p.close()
""" % (aid, aid)
return json.loads(compose('exec', '-T', 'api', 'python', '-c', code).splitlines()[-1])
def effect_snapshot(attachment_id):
aid = identifier(attachment_id)
return rows("SELECT a.status,a.object_key,a.sha256,a.updated_at,j.id AS job_id,j.attempt,j.updated_at AS job_updated_at,"
"(SELECT json_agg(v ORDER BY v.name) FROM attachment_variants v WHERE v.attachment_id=a.id) AS variants "
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id='%s'" % aid)[0]
def fault_consumed(job_id, action):
return bool(rows("SELECT 1 AS consumed FROM job_faults WHERE job_id='%s' AND action='%s' AND remaining=0"
% (identifier(job_id), action)))
def stale_lease_fenced(attachment_id, old_job):
"""A dead invocation cannot commit success or failure over the recovered result."""
old_claim = dict(job_id=old_job['id'], lease_token=old_job['lease_token'],
attempt=old_job['attempt'], pipeline_version=old_job['pipeline_version'])
code = """import json
from messenger.config import read_secret
from messenger.db import create_pool
from messenger.jobs import Jobs
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True);j=Jobs(p)
claim=json.loads(%r)
with p.connection() as c:
a=c.execute('SELECT object_key FROM attachments WHERE id=%%s',(%r,)).fetchone()
variants=c.execute('SELECT * FROM attachment_variants WHERE attachment_id=%%s ORDER BY name',(%r,)).fetchall()
assert j.finish(claim,a['object_key'],variants) is False,'old lease committed success'
assert j.fail(claim,'stale_worker',False,5,0) is False,'old lease committed failure'
p.close();print(json.dumps(dict(old_finish=False,old_fail=False)))
""" % (json.dumps(old_claim), identifier(attachment_id), identifier(attachment_id))
return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code).splitlines()[-1])
def stack(client):
config = json.loads(compose('config', '--format', 'json'))
services = config['services']
for name in ('api', 'worker', 'dispatcher', 'postgres', 'valkey', 'broker', 's3'):
check(name in services, 'missing independent service ' + name)
check(not services[name].get('ports'), name + ': internal ports must not be published')
check(services['api']['deploy']['replicas'] >= 2, 'two API replicas required')
check(any(v.get('type') == 'volume' for v in services['broker'].get('volumes', [])), 'broker needs persistent volume')
check(services['worker'].get('deploy', {}).get('resources', {}).get('limits', {}).get('memory'),
'worker memory limit required')
print('PASS stack: separate durable broker/worker/dispatcher, private internal ports, bounded worker memory', flush=True)
def durability(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'worker-down')
logical_job = job(attachment['id'])['id']
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
message = client.message(member, chat, '', attachments=[attachment['id']])['json']
check(message['attachment_ids'] == [attachment['id']], 'queued attachment cannot be linked')
check(object_state(attachment['id'])['staging'], '202 lost durable staging')
wait_until(lambda: bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NOT NULL"
% logical_job)), 'broker publication before restart')
compose('restart', 'api', 'broker')
ready(client)
check(metadata(client, member, attachment)['status'] == 'queued', 'worker-down status must remain queued')
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=member, schema='MessagePage')['json']
check(any(item['id'] == message['id'] for item in history['items']), 'linked message lost across restart')
compose('start', 'worker')
verified_ready(client, member, attachment, content)
check(job(attachment['id'])['id'] == logical_job, 'restart created a replacement logical job')
print('PASS durability: worker down -> 202+linked message+409 -> API/broker restart with volumes -> same job ready; job_id=' + logical_job, flush=True)
finally:
restore(client, 'api', 'broker', 'worker')
def outbox(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'dispatcher', 'worker')
attachment, content = upload(client, actor, chat, 'commit-publish-gap')
logical_job = job(attachment['id'])['id']
pending = rows("SELECT attempt,published_at FROM attachment_outbox WHERE job_id='%s'" % logical_job)
check(len(pending) == 1 and pending[0]['published_at'] is None, 'COMMIT/publish gap was not isolated')
# API is killed after its successful COMMIT while publisher is absent.
compose('kill', '-s', 'SIGKILL', 'api')
compose('start', 'api')
ready(client)
check(job(attachment['id'])['id'] == logical_job, 'COMMIT intent lost after API crash')
compose('start', 'dispatcher', 'worker')
verified_ready(client, member, attachment, content)
check(job(attachment['id'])['id'] == logical_job, 'outbox recovery recreated the job')
print('PASS outbox: COMMIT persisted while dispatcher stopped -> kill API before publish -> restart -> same job ready; job_id=' + logical_job, flush=True)
finally:
restore(client, 'api', 'dispatcher', 'worker')
def broker_outage(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'broker', 'worker')
ready(client)
attachment, content = upload(client, actor, chat, 'broker-down')
logical_job = job(attachment['id'])['id']
client.message(member, chat, 'broker outage does not stop messages', attachments=[attachment['id']])
check(bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NULL" % logical_job)),
'broker-down upload lacks durable outbox intent')
compose('start', 'broker', 'worker')
verified_ready(client, member, attachment, content)
print('PASS broker outage: API remains ready, upload 202 and messages commit, durable outbox later reaches ready; job_id=' + logical_job, flush=True)
finally:
restore(client, 'broker', 'dispatcher', 'worker')
def wait_delivery_ack(message_id):
code = """import json
from messenger.broker import Broker
b=Broker();target=tuple(map(int,%r.split('-')))
groups=b.client.xinfo_groups(b.stream)
group=next((g for g in groups if g['name']==b.group),None)
delivered=bool(group and tuple(map(int,group['last-delivered-id'].split('-')))>=target)
pending=b.client.xpending_range(b.stream,b.group,%r,%r,1)
print(json.dumps(dict(done=delivered and not pending)));b.close()
""" % (message_id, message_id, message_id)
wait_until(lambda: json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code))['done'], 'redelivery ack')
def redelivery(client):
actor, member, _, chat = client.bootstrap()
attachment, content = upload(client, actor, chat, 'redelivery')
completed = verified_ready(client, member, attachment, content)
before = effect_snapshot(attachment['id'])
published = cli('redeliver', before['job_id'])
message_id = published['message_id']
wait_delivery_ack(message_id)
after = effect_snapshot(attachment['id'])
check(before == after, 'ready duplicate mutated persistent result, job attempt or timestamps')
check(metadata(client, member, attachment) == completed, 'ready duplicate changed public metadata')
print('PASS redelivery: same ready job acked without mutation, same 3 variants/checksums; job_id=' + before['job_id'], flush=True)
def crash(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'crash-after-objects')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'crash_after_objects')
compose('start', 'worker')
wait_until(lambda: fault_consumed(logical_job, 'crash_after_objects'), 'crash fault consumed')
at_crash = object_state(attachment['id'])
check(at_crash['final'] and at_crash['variant_objects'] == 3,
'injected crash occurred before all four final objects existed')
check(at_crash['status'] == 'processing' and at_crash['staging'],
'crash window must precede ready commit and retain staging')
check(not rows("SELECT 1 FROM attachment_variants WHERE attachment_id='%s'" % attachment['id']),
'crash window must precede publication of variant metadata')
# The durable hook is consumed before exit, so container restart cannot loop it.
compose('start', 'worker')
verified_ready(client, member, attachment, content)
result = job(attachment['id'])
check(result['id'] == logical_job and result['attempt'] >= 2, 'crashed job did not recover its expired lease')
check(object_state(attachment['id'])['variant_objects'] == 3, 'redelivery duplicated S3 variants')
print('PASS crash after 4 S3 writes before DB result/ack: same job recovered; no duplicate variants; job_id=%s attempts=%s window=%s'
% (logical_job, result['attempt'], json.dumps(at_crash, sort_keys=True)), flush=True)
finally:
restore(client, 'worker')
def lease(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'expired-processing')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'stall_processing')
compose('start', 'worker')
wait_until(lambda: fault_consumed(logical_job, 'stall_processing'), 'processing stall hook')
held = job(attachment['id'])
check(held['status'] == 'processing' and held['lease_until'], 'stall did not hold a processing lease')
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
compose('kill', '-s', 'SIGKILL', 'worker')
compose('start', 'worker')
verified_ready(client, member, attachment, content)
recovered = job(attachment['id'])
check(recovered['id'] == logical_job and recovered['attempt'] > held['attempt'], 'expired processing was not reclaimed')
snapshot = effect_snapshot(attachment['id'])
stale_lease_fenced(attachment['id'], held)
check(effect_snapshot(attachment['id']) == snapshot, 'stale lease changed the recovered ready result')
print('PASS expired lease: killed stalled processing worker -> automatic recovery, same job; job_id=%s old_lease_until=%s attempts=%d'
% (logical_job, held['lease_until'], recovered['attempt']), flush=True)
print('PASS fencing: old lease cannot finish or fail after recovered ready commit', flush=True)
finally:
restore(client, 'worker')
def failed_contract(client, actor, chat, attachment):
failed = wait_state(client, actor, attachment, 'failed')
check(bool(re.fullmatch(r'[a-z][a-z0-9_]{0,63}', failed['error_code'] or '')), 'safe terminal error code required')
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=actor)
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=actor,
body={'text': '', 'attachment_ids': [attachment['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
return failed
def retries(client):
actor, member, _, chat = client.bootstrap()
config = json.loads(compose('config', '--format', 'json'))['services']['worker'].get('environment', {})
max_attempts = int(config.get('JOB_MAX_ATTEMPTS', 5))
retry_base = float(config.get('JOB_RETRY_BASE_SECONDS', 1))
retry_max = float(config.get('JOB_RETRY_MAX_SECONDS', 30))
check(max_attempts >= 3, 'retry demonstration requires a budget of at least three attempts')
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'transient-retry')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'transient', 2)
compose('start', 'worker')
verified_ready(client, member, attachment, content)
retried = job(attachment['id'])
check(retried['attempt'] >= 2, 'transient failure was not retried')
events = audit(logical_job)
retry_events = [event for event in events if event['action'] == 'retry' and event['reason'] == 'injected_transient']
check(len(retry_events) == 2, 'both retry delays must be observable in durable audit')
randomized = False
for event in retry_events:
details = event['details']
cap = min(retry_max, retry_base * 2.0 ** (details['run_attempts'] - 1))
check(cap / 2 <= details['delay_seconds'] <= cap, 'retry is outside its equal-jitter exponential interval')
randomized |= details['delay_seconds'] not in (cap / 2, cap)
created = dt.datetime.fromisoformat(event['created_at'].replace('Z', '+00:00'))
available = dt.datetime.fromisoformat(details['available_at'].replace('Z', '+00:00'))
check(abs((available - created).total_seconds() - details['delay_seconds']) < 0.25,
'retry due time does not persist the chosen jitter delay')
check(randomized, 'retry evidence contains no randomized delay')
print('PASS transient errors retry with durable backoff/jitter; job_id=%s audit=%s'
% (logical_job, json.dumps(retry_events, sort_keys=True)), flush=True)
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'retry-exhausted')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'transient', max_attempts)
compose('start', 'worker')
failed = failed_contract(client, member, chat, attachment)
check(failed['error_code'] == 'retry_exhausted', 'exhausted job needs a distinct terminal error code')
exhausted = job(attachment['id'])
check(exhausted['run_attempts'] == max_attempts, 'retry budget not enforced')
check(fault_consumed(logical_job, 'transient'), 'retry fault count does not match exhausted attempts')
failed_jobs = cli('failed', '--limit', '1000')
check(any(item['job_id'] == logical_job for item in failed_jobs), 'failed job missing from operator registry')
check(object_state(attachment['id'])['staging'], 'failed job lost original needed for replay')
cli('cleanup')
check(object_state(attachment['id'])['staging'], 'cleanup removed failed input required for replay')
replay = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'operator resolved synthetic failure')
check(replay['job_id'] == logical_job, 'operator replay created a different logical job')
verified_ready(client, member, attachment, content)
before = effect_snapshot(attachment['id'])
noop = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'verify successful replay is a no-op')
check(effect_snapshot(attachment['id']) == before, 'replay re-executed a successful job')
actions = {event['action'] for event in audit(logical_job) if event['actor'] == 'lab5-evidence'}
check({'replay', 'replay_noop'} <= actions, 'explicit replay and successful no-op need operator audit records')
print('PASS retry exhaustion -> failed+409+retained staging -> explicit audited replay -> ready -> ready replay no-op; job_id=%s max_attempts=%d error_code=%s replay=%s'
% (logical_job, max_attempts, failed['error_code'], json.dumps(noop, sort_keys=True)), flush=True)
finally:
restore(client, 'worker')
def permanent(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, _ = upload(client, actor, chat, 'permanent-failure')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'permanent')
compose('start', 'worker')
failed = failed_contract(client, member, chat, attachment)
check(failed['error_code'] == 'invalid_image', 'permanent errors need a distinct safe error code')
check(job(attachment['id'])['run_attempts'] == 1, 'permanent errors must not consume all retries')
print('PASS permanent failure: failed after one attempt; content/message rejected with409; job_id=%s error_code=%s'
% (logical_job, failed['error_code']), flush=True)
finally:
restore(client, 'worker')
def staging(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'staging-safety')
before = object_state(attachment['id'])
check(before['staging'] and not before['final'], 'staging/final publication boundary is not visible')
cli('cleanup')
check(object_state(attachment['id']) == before, 'cleanup removed current queued staging')
compose('start', 'worker')
verified_ready(client, member, attachment, content)
cli('cleanup')
after = wait_until(lambda: (lambda s: s if not s['staging'] and s['staging_deleted'] else None)
(object_state(attachment['id'])), 'post-commit staging cleanup')
check(after['final'] and after['status'] == 'ready', 'cleanup removed committed original')
print('PASS staging cleanup: queued protected; staging deleted only after ready commit; final original retained; attachment_id=' + attachment['id'], flush=True)
finally:
restore(client, 'worker')
SCENARIOS = ['stack', 'durability', 'outbox', 'broker_outage', 'redelivery', 'crash', 'lease', 'retries', 'permanent', 'staging']
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
parser.add_argument('--only', nargs='+', choices=SCENARIOS)
args = parser.parse_args()
client = Client(args.base_url, timeout=15)
ready(client)
for name in args.only or SCENARIOS:
ready(client)
print('START ' + name + ' ' + dt.datetime.now(dt.timezone.utc).isoformat(), flush=True)
globals()[name](client)
print('PASS selected lab5 grade 3/4 checks; no grade 5 track claimed', flush=True)
if __name__ == '__main__':
try:
main()
except (Failure, ValueError, OSError) as exc:
print('FAIL: ' + str(exc), flush=True)
sys.exit(1)