Files
mtusi2026lab6/student-tests/pipeline_integration.py
T
2026-11-05 21:17:50 +03:00

206 lines
9.0 KiB
Python

#!/usr/bin/env python3
"""Destructive-to-availability tests on the local lab stack; volumes preserved.
Only synthetic attachments created here are mutated. Tokens stay in memory.
Run sequentially, never alongside load or another fault test.
"""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
RESULTS = []
def compose(*args, env=None, source=None):
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT,
env=dict(os.environ, **(env or {})), input=source,
text=True, capture_output=True, timeout=150)
check(result.returncode == 0, 'compose failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1000:])
return result.stdout.strip()
def remote(body, **values):
source = '''import asyncio, json, os
from uuid import UUID
from src.config import load_settings
from src.storage.postgres import PostgresStore
from src.storage.s3 import S3Store, S3Settings
from src.broker import publish
V = json.loads(%r)
async def main():
db = PostgresStore(load_settings().database_url)
await db.connect()
s3 = S3Store(S3Settings.load())
try:
%s
finally:
s3.client.close()
await db.close()
asyncio.run(main())
''' % (json.dumps(values), '\n'.join(' ' + line for line in body.splitlines()))
return json.loads(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-', source=source))
def record(name, **facts):
RESULTS.append(dict(test=name, status='PASS', **facts))
print('PASS: ' + name, flush=True)
(ROOT / 'evidence/pipeline-integration.json').write_text(json.dumps(RESULTS, indent=2) + '\n')
def ready(c):
deadline = time.monotonic() + 90
successes = 0
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready')
successes += 1
if successes >= 5:
return
time.sleep(.5)
except Failure:
successes = 0
time.sleep(.5)
raise Failure('API did not recover')
def wait(c, actor, att, status='ready', timeout=120):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
value = c.call('GET', '/api/v1/attachments/' + att['id'], actor=actor, schema='Attachment')['json']
except Failure:
time.sleep(.25)
continue
if value['status'] == status:
return value
check(value['status'] != 'failed' or status == 'failed', 'unexpected failed attachment')
time.sleep(.25)
raise Failure('attachment did not reach ' + status)
def snapshot(att):
return remote('''row = await db.pool.fetchrow("SELECT a.status, j.attempt, j.job_id, a.original_key, a.staging_key FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1", UUID(V['id']))
objects = sorted(list(await s3.list('originals/' + V['id'] + '/')) + list(await s3.list('variants/' + V['id'] + '/')))
print(json.dumps(dict(row) | {'objects': objects}, default=str))''', id=att['id'])
def duplicate(att):
return remote('''payload = await db.pool.fetchval("SELECT o.payload FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE j.attachment_id=$1 ORDER BY o.created_at LIMIT 1", UUID(V['id']))
await publish(json.loads(payload) if isinstance(payload, str) else payload)
print('true')''', id=att['id'])
def main():
c = Client('http://localhost:8080', timeout=30)
alice, bob, _, chat = c.bootstrap()
# No worker: HTTP admission and message creation must remain asynchronous.
compose('stop', '-t', '2', 'worker')
att, data = c.attachment(alice, chat)
check(att['status'] == 'queued', 'upload is not queued with stopped worker')
c.message(alice, chat, 'queued attachment', attachments=[att['id']])
time.sleep(2)
compose('restart', 'broker')
compose('up', '-d', '--no-deps', '--wait', 'broker')
compose('restart', 'api-a', 'api-b')
ready(c)
compose('start', 'worker')
completed = wait(c, bob, att)
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed')
check({v['name'] for v in completed['variants']} == {'thumbnail', 'crop', 'optimized'}, 'variants missing')
record('stopped worker, queued message, broker/API restart, ready', attachment_id=att['id'])
before = snapshot(att)
duplicate(att)
duplicate(att)
time.sleep(3)
check(snapshot(att) == before, 'duplicate changed result or created S3 objects')
refused = remote("print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))", id=att['id'])
check(refused is False, 'ready replay accepted')
record('duplicate delivery and replay of ready are no-ops', object_count=len(before['objects']))
# Pause publication so the API crash is demonstrably before any publish.
compose('stop', '-t', '2', 'outbox')
compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', env={'OUTBOX_FAULT_AFTER_COMMIT': '1'})
ready(c)
fault_actor, _, _, fault_chat = c.bootstrap()
try:
c.attachment(fault_actor, fault_chat)
raise AssertionError('crash hook did not interrupt upload')
except Failure:
pass
compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b')
ready(c)
row = remote('''rows = await db.pool.fetch("SELECT a.id, o.published_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN attachment_outbox o ON o.job_id=j.job_id WHERE a.chat_id=$1", UUID(V['chat']))
assert len(rows) == 1 and rows[0]['published_at'] is None
print(json.dumps({'id': str(rows[0]['id'])}))''', chat=fault_chat['id'])
compose('start', 'outbox')
wait(c, fault_actor, row)
record('crash after COMMIT before publish recovered from existing outbox', attachment_id=row['id'])
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_FAULT_AFTER_S3_PUT': '1', 'JOB_TIMEOUT_SECONDS': '10'})
att, _ = c.attachment(alice, chat)
wait(c, alice, att)
snap = snapshot(att)
check(snap['attempt'] >= 2 and len(snap['objects']) >= 8, 'side effect crash/lease retry not observed')
marker = compose('exec', '-T', 'worker', 'cat', '/tmp/worker-fault-after-s3')
check(marker == att['id'], 'wrong fault target')
record('worker crash after S3 before commit/ack, lease recovery', attempt=snap['attempt'], objects_before_cleanup=len(snap['objects']))
compose('up', '-d', '--no-deps', '--wait', 'worker')
# Select only this test's unreferenced attempt keys for cleanup.
cleaned = remote('''from argparse import Namespace
from src.reconcile import run
keys = V['keys']
result = await run(Namespace(apply=True, grace_seconds=0, object_key=keys))
print(json.dumps(result['deleted']))''', keys=snap['objects'])
check(len(cleaned) == 4, 'stale attempt objects not cleaned safely')
check(len(snapshot(att)['objects']) == 4, 'live attempt affected by cleanup')
wait(c, alice, att)
record('targeted orphan cleanup preserves committed attempt', deleted=len(cleaned))
# Lose only this synthetic staging object to exercise transient retry exhaustion.
compose('stop', '-t', '2', 'worker')
att, data = c.attachment(alice, chat)
remote("await s3.delete('staging/' + V['id'] + '/original')\nprint('true')", id=att['id'])
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'JOB_MAX_ATTEMPTS': '2'})
failed = wait(c, alice, att, 'failed')
snap = snapshot(att)
check(snap['attempt'] == 2 and failed['error_code'] == 'retry_exhausted', 'unbounded/wrong retry')
# Restore exactly the bytes originally accepted, then explicit failed-only replay.
import base64
replayed = remote('''from io import BytesIO
import base64
content = base64.b64decode(V['data'])
await s3.put('staging/' + V['id'] + '/original', BytesIO(content), len(content), 'image/png')
print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))''', id=att['id'], data=base64.b64encode(data).decode())
check(replayed, 'failed replay refused')
wait(c, alice, att)
record('bounded transient retry, failed registry and explicit replay', attempts_before_replay=snap['attempt'])
compose('up', '-d', '--no-deps', '--wait', 'worker')
# Real Linux child with deliberately insufficient address space.
memory = remote('''import subprocess
from io import BytesIO
from PIL import Image
buf = BytesIO(); Image.new('RGB', (2400, 2400)).save(buf, 'PNG')
env = dict(os.environ, RENDER_MEMORY_MB='120')
p = subprocess.run(['/app/.venv/bin/python', '-m', 'src.render'], input=buf.getvalue(), capture_output=True, env=env, timeout=30)
print(json.dumps({'returncode': p.returncode}))''')
check(memory['returncode'] == 21, 'Linux memory limit did not produce bounded resource failure')
record('Linux renderer memory limit', **memory)
if __name__ == '__main__':
try:
main()
finally:
compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', 'worker', 'outbox')