forked from mtusi2026/mtusi2026lab5
205 lines
8.8 KiB
Python
205 lines
8.8 KiB
Python
"""Integration/fault checks for the local lab5 Compose project; restores stopped services."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def compose(*args, input=None):
|
|
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
|
|
text=True, encoding='utf-8', capture_output=True)
|
|
if result.returncode:
|
|
raise AssertionError('Compose failed: ' + ' '.join(args[:3]) + '\n' + result.stderr)
|
|
return result.stdout
|
|
|
|
|
|
PRELUDE = '''
|
|
import json
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import sessionmaker
|
|
from src.database import create_db_engine
|
|
from src.models import Attachment, AttachmentJob
|
|
from src.jobs import broker, envelope, GROUP, STREAM
|
|
from src.object_storage import BUCKET, s3_client
|
|
engine = create_db_engine()
|
|
sessions = sessionmaker(engine, expire_on_commit=False)
|
|
'''
|
|
|
|
|
|
def remote(code, fault=None):
|
|
options = ['exec', '-T']
|
|
if fault:
|
|
options += ['-e', 'JOB_FAULT=' + fault]
|
|
return compose(*options, 'api', 'python', '-', input=PRELUDE + code).strip()
|
|
|
|
|
|
def state(attachment_id):
|
|
return json.loads(remote(f'''
|
|
with sessions() as session:
|
|
a = session.get(Attachment, {attachment_id!r})
|
|
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == a.id))
|
|
print(json.dumps(dict(status=a.status, error=a.error_code, job_id=j.id,
|
|
attempt=j.attempt, failures=j.failures, published=bool(j.published_at),
|
|
staging=a.staging_key, variants=a.variants, audit=j.replay_audit)))
|
|
'''))
|
|
|
|
|
|
def until(fn, timeout=130):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
result = fn()
|
|
if result:
|
|
return result
|
|
time.sleep(1)
|
|
raise AssertionError('Timed out waiting for durable state')
|
|
|
|
|
|
def ready(client):
|
|
def probe():
|
|
try:
|
|
return client.call('GET', '/health/ready')['status'] == 200
|
|
except Exception:
|
|
return False
|
|
until(probe, 40)
|
|
|
|
|
|
def inject(attachment_id, fault):
|
|
# Use a real consumer-group delivery, leaving the message pending at the crash point.
|
|
return remote(f'''
|
|
from src.worker import handle
|
|
client = broker()
|
|
with sessions() as session:
|
|
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {attachment_id!r}))
|
|
target = j.id
|
|
for _ in range(100):
|
|
batches = client.xreadgroup(GROUP, 'fault-test', {{STREAM: '>'}}, count=1, block=1000)
|
|
if batches:
|
|
mid, fields = batches[0][1][0]
|
|
handle(sessions, client, mid, fields)
|
|
if json.loads(fields['job'])['job_id'] == target:
|
|
print(mid)
|
|
break
|
|
else:
|
|
raise AssertionError('No delivery for target job')
|
|
''', fault=fault)
|
|
|
|
|
|
def main(outbox_only=False):
|
|
client = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'worker', 'dispatcher', 'cleanup')
|
|
attachment, content = client.attachment(alice, chat)
|
|
aid = attachment['id']
|
|
check(attachment['status'] == 'queued', 'upload must return queued')
|
|
check(not state(aid)['published'], 'outbox was already published')
|
|
client.message(alice, chat, '', attachments=[aid])
|
|
client.call('GET', f'/api/v1/attachments/{aid}/content', 409, actor=bob)
|
|
# Hard crash after durable COMMIT while dispatcher is stopped: no publish yet.
|
|
compose('kill', '-s', 'SIGKILL', 'api')
|
|
compose('start', 'api')
|
|
ready(client)
|
|
check(state(aid)['status'] == 'queued', 'API restart lost upload')
|
|
compose('start', 'dispatcher')
|
|
until(lambda: state(aid)['published'])
|
|
compose('restart', 'broker')
|
|
compose('start', 'worker')
|
|
result = client.wait_ready(bob, attachment)
|
|
check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing variants')
|
|
for variant in result['variants']:
|
|
download = client.call('GET', f'/api/v1/attachments/{aid}/content?variant=' + variant['name'], actor=bob)
|
|
check(hashlib.sha256(download['bytes']).hexdigest() == variant['sha256'], 'variant checksum mismatch')
|
|
check(len(download['bytes']) == variant['size_bytes'], 'variant size mismatch')
|
|
check(download['headers'].get_content_type() == variant['content_type'], 'variant MIME mismatch')
|
|
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'original changed')
|
|
print('PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart', flush=True)
|
|
if outbox_only:
|
|
return
|
|
|
|
for fault in ('after_objects', 'after_commit'):
|
|
compose('stop', 'worker')
|
|
att, _ = client.attachment(alice, chat)
|
|
aid = att['id']
|
|
until(lambda: state(aid)['published'])
|
|
pending_id = inject(aid, fault)
|
|
snapshot = state(aid)
|
|
check(snapshot['status'] == ('processing' if fault == 'after_objects' else 'ready'), 'wrong crash state')
|
|
compose('start', 'worker')
|
|
until(lambda: state(aid)['status'] == 'ready')
|
|
until(lambda: remote(f"print(int(not broker().xpending_range(STREAM, GROUP, {pending_id!r}, {pending_id!r}, 1)))\n") == '1')
|
|
if fault == 'after_objects':
|
|
check(state(aid)['attempt'] == 2, 'expired lease was not retried')
|
|
else:
|
|
time.sleep(2)
|
|
check(state(aid)['attempt'] == 1, 'ready job was reprocessed')
|
|
# Force duplicate delivery; stable object keys and metadata must remain identical.
|
|
before = state(aid)
|
|
remote(f'''
|
|
with sessions() as session:
|
|
j = session.get(AttachmentJob, {before['job_id']!r})
|
|
broker().xadd(STREAM, {{'job': json.dumps(envelope(j))}})
|
|
''')
|
|
time.sleep(2)
|
|
after = state(aid)
|
|
check(before['variants'] == after['variants'] and before['attempt'] == after['attempt'], 'duplicate changed result')
|
|
count = remote(f"print(s3_client().list_objects_v2(Bucket=BUCKET, Prefix='variants/{aid}/')['KeyCount'])\n")
|
|
check(count == '3', 'redelivery produced extra S3 objects')
|
|
print('PASS: crash ' + fault + ', redelivery and stable variants', flush=True)
|
|
|
|
compose('stop', 'worker')
|
|
att, content = client.attachment(alice, chat)
|
|
aid = att['id']
|
|
until(lambda: state(aid)['published'])
|
|
# Remove only this test's staging object to cause a real temporary S3 error.
|
|
staging = state(aid)['staging']
|
|
remote(f"s3_client().delete_object(Bucket=BUCKET, Key={staging!r})\n")
|
|
compose('start', 'worker')
|
|
until(lambda: state(aid)['status'] == 'failed', 150)
|
|
failed = state(aid)
|
|
check(failed['error'] == 'retry_exhausted' and failed['attempt'] == 5, 'retry budget incorrect')
|
|
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
|
body={'text': '', 'attachment_ids': [aid]}, headers={'Idempotency-Key': aid})
|
|
registry = compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'failed')
|
|
check(failed['job_id'] in registry, 'failed job not visible')
|
|
remote(f'''
|
|
import base64
|
|
s3_client().put_object(Bucket=BUCKET, Key={staging!r},
|
|
Body=base64.b64decode({base64.b64encode(content).decode()!r}), ContentType='image/png')
|
|
''')
|
|
compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'replay', failed['job_id'],
|
|
'--reason', 'synthetic test: restored staging')
|
|
client.wait_ready(bob, att)
|
|
check(len(state(aid)['audit']) == 1 and state(aid)['attempt'] == 6, 'replay lost audit or attempt')
|
|
remote(f'''
|
|
from src.jobs import replay
|
|
try:
|
|
replay(sessions, {failed['job_id']!r}, 'must reject ready')
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
raise AssertionError('Replayed a ready job')
|
|
''')
|
|
print('PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected', flush=True)
|
|
|
|
compose('exec', '-T', 'api', 'python', '-m', 'src.cleanup', '--once')
|
|
check(state(aid)['staging'] is None, 'completed staging was not cleaned')
|
|
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'cleanup deleted original')
|
|
print('PASS: staging cleanup preserves ready objects', flush=True)
|
|
finally:
|
|
compose('start', 'broker', 'dispatcher', 'worker', 'cleanup')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--outbox-only', action='store_true', help='Only the COMMIT/publish crash scenario')
|
|
main(parser.parse_args().outbox_only)
|