Files
2026-11-06 19:22:27 +03:00

205 lines
8.8 KiB
Python

"""Integration/fault checks for the local lab5 Compose project; restores stopped services."""
import base64
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def compose(*args, input=None):
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
text=True, encoding='utf-8', capture_output=True)
if result.returncode:
raise AssertionError('Compose failed: ' + ' '.join(args[:3]) + '\n' + result.stderr)
return result.stdout
PRELUDE = '''
import json
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from src.database import create_db_engine
from src.models import Attachment, AttachmentJob
from src.jobs import broker, envelope, GROUP, STREAM
from src.object_storage import BUCKET, s3_client
engine = create_db_engine()
sessions = sessionmaker(engine, expire_on_commit=False)
'''
def remote(code, fault=None):
options = ['exec', '-T']
if fault:
options += ['-e', 'JOB_FAULT=' + fault]
return compose(*options, 'api', 'python', '-', input=PRELUDE + code).strip()
def state(attachment_id):
return json.loads(remote(f'''
with sessions() as session:
a = session.get(Attachment, {attachment_id!r})
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == a.id))
print(json.dumps(dict(status=a.status, error=a.error_code, job_id=j.id,
attempt=j.attempt, failures=j.failures, published=bool(j.published_at),
staging=a.staging_key, variants=a.variants, audit=j.replay_audit)))
'''))
def until(fn, timeout=130):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = fn()
if result:
return result
time.sleep(1)
raise AssertionError('Timed out waiting for durable state')
def ready(client):
def probe():
try:
return client.call('GET', '/health/ready')['status'] == 200
except Exception:
return False
until(probe, 40)
def inject(attachment_id, fault):
# Use a real consumer-group delivery, leaving the message pending at the crash point.
return remote(f'''
from src.worker import handle
client = broker()
with sessions() as session:
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {attachment_id!r}))
target = j.id
for _ in range(100):
batches = client.xreadgroup(GROUP, 'fault-test', {{STREAM: '>'}}, count=1, block=1000)
if batches:
mid, fields = batches[0][1][0]
handle(sessions, client, mid, fields)
if json.loads(fields['job'])['job_id'] == target:
print(mid)
break
else:
raise AssertionError('No delivery for target job')
''', fault=fault)
def main(outbox_only=False):
client = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
alice, bob, eve, chat = client.bootstrap()
try:
compose('stop', 'worker', 'dispatcher', 'cleanup')
attachment, content = client.attachment(alice, chat)
aid = attachment['id']
check(attachment['status'] == 'queued', 'upload must return queued')
check(not state(aid)['published'], 'outbox was already published')
client.message(alice, chat, '', attachments=[aid])
client.call('GET', f'/api/v1/attachments/{aid}/content', 409, actor=bob)
# Hard crash after durable COMMIT while dispatcher is stopped: no publish yet.
compose('kill', '-s', 'SIGKILL', 'api')
compose('start', 'api')
ready(client)
check(state(aid)['status'] == 'queued', 'API restart lost upload')
compose('start', 'dispatcher')
until(lambda: state(aid)['published'])
compose('restart', 'broker')
compose('start', 'worker')
result = client.wait_ready(bob, attachment)
check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing variants')
for variant in result['variants']:
download = client.call('GET', f'/api/v1/attachments/{aid}/content?variant=' + variant['name'], actor=bob)
check(hashlib.sha256(download['bytes']).hexdigest() == variant['sha256'], 'variant checksum mismatch')
check(len(download['bytes']) == variant['size_bytes'], 'variant size mismatch')
check(download['headers'].get_content_type() == variant['content_type'], 'variant MIME mismatch')
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'original changed')
print('PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart', flush=True)
if outbox_only:
return
for fault in ('after_objects', 'after_commit'):
compose('stop', 'worker')
att, _ = client.attachment(alice, chat)
aid = att['id']
until(lambda: state(aid)['published'])
pending_id = inject(aid, fault)
snapshot = state(aid)
check(snapshot['status'] == ('processing' if fault == 'after_objects' else 'ready'), 'wrong crash state')
compose('start', 'worker')
until(lambda: state(aid)['status'] == 'ready')
until(lambda: remote(f"print(int(not broker().xpending_range(STREAM, GROUP, {pending_id!r}, {pending_id!r}, 1)))\n") == '1')
if fault == 'after_objects':
check(state(aid)['attempt'] == 2, 'expired lease was not retried')
else:
time.sleep(2)
check(state(aid)['attempt'] == 1, 'ready job was reprocessed')
# Force duplicate delivery; stable object keys and metadata must remain identical.
before = state(aid)
remote(f'''
with sessions() as session:
j = session.get(AttachmentJob, {before['job_id']!r})
broker().xadd(STREAM, {{'job': json.dumps(envelope(j))}})
''')
time.sleep(2)
after = state(aid)
check(before['variants'] == after['variants'] and before['attempt'] == after['attempt'], 'duplicate changed result')
count = remote(f"print(s3_client().list_objects_v2(Bucket=BUCKET, Prefix='variants/{aid}/')['KeyCount'])\n")
check(count == '3', 'redelivery produced extra S3 objects')
print('PASS: crash ' + fault + ', redelivery and stable variants', flush=True)
compose('stop', 'worker')
att, content = client.attachment(alice, chat)
aid = att['id']
until(lambda: state(aid)['published'])
# Remove only this test's staging object to cause a real temporary S3 error.
staging = state(aid)['staging']
remote(f"s3_client().delete_object(Bucket=BUCKET, Key={staging!r})\n")
compose('start', 'worker')
until(lambda: state(aid)['status'] == 'failed', 150)
failed = state(aid)
check(failed['error'] == 'retry_exhausted' and failed['attempt'] == 5, 'retry budget incorrect')
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text': '', 'attachment_ids': [aid]}, headers={'Idempotency-Key': aid})
registry = compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'failed')
check(failed['job_id'] in registry, 'failed job not visible')
remote(f'''
import base64
s3_client().put_object(Bucket=BUCKET, Key={staging!r},
Body=base64.b64decode({base64.b64encode(content).decode()!r}), ContentType='image/png')
''')
compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'replay', failed['job_id'],
'--reason', 'synthetic test: restored staging')
client.wait_ready(bob, att)
check(len(state(aid)['audit']) == 1 and state(aid)['attempt'] == 6, 'replay lost audit or attempt')
remote(f'''
from src.jobs import replay
try:
replay(sessions, {failed['job_id']!r}, 'must reject ready')
except ValueError:
pass
else:
raise AssertionError('Replayed a ready job')
''')
print('PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected', flush=True)
compose('exec', '-T', 'api', 'python', '-m', 'src.cleanup', '--once')
check(state(aid)['staging'] is None, 'completed staging was not cleaned')
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'cleanup deleted original')
print('PASS: staging cleanup preserves ready objects', flush=True)
finally:
compose('start', 'broker', 'dispatcher', 'worker', 'cleanup')
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--outbox-only', action='store_true', help='Only the COMMIT/publish crash scenario')
main(parser.parse_args().outbox_only)