206 lines
9.0 KiB
Python
206 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Destructive-to-availability tests on the local lab stack; volumes preserved.
|
|
|
|
Only synthetic attachments created here are mutated. Tokens stay in memory.
|
|
Run sequentially, never alongside load or another fault test.
|
|
"""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
|
|
RESULTS = []
|
|
|
|
|
|
def compose(*args, env=None, source=None):
|
|
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT,
|
|
env=dict(os.environ, **(env or {})), input=source,
|
|
text=True, capture_output=True, timeout=150)
|
|
check(result.returncode == 0, 'compose failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1000:])
|
|
return result.stdout.strip()
|
|
|
|
|
|
def remote(body, **values):
|
|
source = '''import asyncio, json, os
|
|
from uuid import UUID
|
|
from src.config import load_settings
|
|
from src.storage.postgres import PostgresStore
|
|
from src.storage.s3 import S3Store, S3Settings
|
|
from src.broker import publish
|
|
V = json.loads(%r)
|
|
async def main():
|
|
db = PostgresStore(load_settings().database_url)
|
|
await db.connect()
|
|
s3 = S3Store(S3Settings.load())
|
|
try:
|
|
%s
|
|
finally:
|
|
s3.client.close()
|
|
await db.close()
|
|
asyncio.run(main())
|
|
''' % (json.dumps(values), '\n'.join(' ' + line for line in body.splitlines()))
|
|
return json.loads(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-', source=source))
|
|
|
|
|
|
def record(name, **facts):
|
|
RESULTS.append(dict(test=name, status='PASS', **facts))
|
|
print('PASS: ' + name, flush=True)
|
|
(ROOT / 'evidence/pipeline-integration.json').write_text(json.dumps(RESULTS, indent=2) + '\n')
|
|
|
|
|
|
def ready(c):
|
|
deadline = time.monotonic() + 90
|
|
successes = 0
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
c.call('GET', '/health/ready')
|
|
successes += 1
|
|
if successes >= 5:
|
|
return
|
|
time.sleep(.5)
|
|
except Failure:
|
|
successes = 0
|
|
time.sleep(.5)
|
|
raise Failure('API did not recover')
|
|
|
|
|
|
def wait(c, actor, att, status='ready', timeout=120):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = c.call('GET', '/api/v1/attachments/' + att['id'], actor=actor, schema='Attachment')['json']
|
|
except Failure:
|
|
time.sleep(.25)
|
|
continue
|
|
if value['status'] == status:
|
|
return value
|
|
check(value['status'] != 'failed' or status == 'failed', 'unexpected failed attachment')
|
|
time.sleep(.25)
|
|
raise Failure('attachment did not reach ' + status)
|
|
|
|
|
|
def snapshot(att):
|
|
return remote('''row = await db.pool.fetchrow("SELECT a.status, j.attempt, j.job_id, a.original_key, a.staging_key FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1", UUID(V['id']))
|
|
objects = sorted(list(await s3.list('originals/' + V['id'] + '/')) + list(await s3.list('variants/' + V['id'] + '/')))
|
|
print(json.dumps(dict(row) | {'objects': objects}, default=str))''', id=att['id'])
|
|
|
|
|
|
def duplicate(att):
|
|
return remote('''payload = await db.pool.fetchval("SELECT o.payload FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE j.attachment_id=$1 ORDER BY o.created_at LIMIT 1", UUID(V['id']))
|
|
await publish(json.loads(payload) if isinstance(payload, str) else payload)
|
|
print('true')''', id=att['id'])
|
|
|
|
|
|
def main():
|
|
c = Client('http://localhost:8080', timeout=30)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
|
|
# No worker: HTTP admission and message creation must remain asynchronous.
|
|
compose('stop', '-t', '2', 'worker')
|
|
att, data = c.attachment(alice, chat)
|
|
check(att['status'] == 'queued', 'upload is not queued with stopped worker')
|
|
c.message(alice, chat, 'queued attachment', attachments=[att['id']])
|
|
time.sleep(2)
|
|
compose('restart', 'broker')
|
|
compose('up', '-d', '--no-deps', '--wait', 'broker')
|
|
compose('restart', 'api-a', 'api-b')
|
|
ready(c)
|
|
compose('start', 'worker')
|
|
completed = wait(c, bob, att)
|
|
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed')
|
|
check({v['name'] for v in completed['variants']} == {'thumbnail', 'crop', 'optimized'}, 'variants missing')
|
|
record('stopped worker, queued message, broker/API restart, ready', attachment_id=att['id'])
|
|
|
|
before = snapshot(att)
|
|
duplicate(att)
|
|
duplicate(att)
|
|
time.sleep(3)
|
|
check(snapshot(att) == before, 'duplicate changed result or created S3 objects')
|
|
refused = remote("print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))", id=att['id'])
|
|
check(refused is False, 'ready replay accepted')
|
|
record('duplicate delivery and replay of ready are no-ops', object_count=len(before['objects']))
|
|
|
|
# Pause publication so the API crash is demonstrably before any publish.
|
|
compose('stop', '-t', '2', 'outbox')
|
|
compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', env={'OUTBOX_FAULT_AFTER_COMMIT': '1'})
|
|
ready(c)
|
|
fault_actor, _, _, fault_chat = c.bootstrap()
|
|
try:
|
|
c.attachment(fault_actor, fault_chat)
|
|
raise AssertionError('crash hook did not interrupt upload')
|
|
except Failure:
|
|
pass
|
|
compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b')
|
|
ready(c)
|
|
row = remote('''rows = await db.pool.fetch("SELECT a.id, o.published_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN attachment_outbox o ON o.job_id=j.job_id WHERE a.chat_id=$1", UUID(V['chat']))
|
|
assert len(rows) == 1 and rows[0]['published_at'] is None
|
|
print(json.dumps({'id': str(rows[0]['id'])}))''', chat=fault_chat['id'])
|
|
compose('start', 'outbox')
|
|
wait(c, fault_actor, row)
|
|
record('crash after COMMIT before publish recovered from existing outbox', attachment_id=row['id'])
|
|
|
|
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_FAULT_AFTER_S3_PUT': '1', 'JOB_TIMEOUT_SECONDS': '10'})
|
|
att, _ = c.attachment(alice, chat)
|
|
wait(c, alice, att)
|
|
snap = snapshot(att)
|
|
check(snap['attempt'] >= 2 and len(snap['objects']) >= 8, 'side effect crash/lease retry not observed')
|
|
marker = compose('exec', '-T', 'worker', 'cat', '/tmp/worker-fault-after-s3')
|
|
check(marker == att['id'], 'wrong fault target')
|
|
record('worker crash after S3 before commit/ack, lease recovery', attempt=snap['attempt'], objects_before_cleanup=len(snap['objects']))
|
|
compose('up', '-d', '--no-deps', '--wait', 'worker')
|
|
|
|
# Select only this test's unreferenced attempt keys for cleanup.
|
|
cleaned = remote('''from argparse import Namespace
|
|
from src.reconcile import run
|
|
keys = V['keys']
|
|
result = await run(Namespace(apply=True, grace_seconds=0, object_key=keys))
|
|
print(json.dumps(result['deleted']))''', keys=snap['objects'])
|
|
check(len(cleaned) == 4, 'stale attempt objects not cleaned safely')
|
|
check(len(snapshot(att)['objects']) == 4, 'live attempt affected by cleanup')
|
|
wait(c, alice, att)
|
|
record('targeted orphan cleanup preserves committed attempt', deleted=len(cleaned))
|
|
|
|
# Lose only this synthetic staging object to exercise transient retry exhaustion.
|
|
compose('stop', '-t', '2', 'worker')
|
|
att, data = c.attachment(alice, chat)
|
|
remote("await s3.delete('staging/' + V['id'] + '/original')\nprint('true')", id=att['id'])
|
|
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'JOB_MAX_ATTEMPTS': '2'})
|
|
failed = wait(c, alice, att, 'failed')
|
|
snap = snapshot(att)
|
|
check(snap['attempt'] == 2 and failed['error_code'] == 'retry_exhausted', 'unbounded/wrong retry')
|
|
# Restore exactly the bytes originally accepted, then explicit failed-only replay.
|
|
import base64
|
|
replayed = remote('''from io import BytesIO
|
|
import base64
|
|
content = base64.b64decode(V['data'])
|
|
await s3.put('staging/' + V['id'] + '/original', BytesIO(content), len(content), 'image/png')
|
|
print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))''', id=att['id'], data=base64.b64encode(data).decode())
|
|
check(replayed, 'failed replay refused')
|
|
wait(c, alice, att)
|
|
record('bounded transient retry, failed registry and explicit replay', attempts_before_replay=snap['attempt'])
|
|
compose('up', '-d', '--no-deps', '--wait', 'worker')
|
|
|
|
# Real Linux child with deliberately insufficient address space.
|
|
memory = remote('''import subprocess
|
|
from io import BytesIO
|
|
from PIL import Image
|
|
buf = BytesIO(); Image.new('RGB', (2400, 2400)).save(buf, 'PNG')
|
|
env = dict(os.environ, RENDER_MEMORY_MB='120')
|
|
p = subprocess.run(['/app/.venv/bin/python', '-m', 'src.render'], input=buf.getvalue(), capture_output=True, env=env, timeout=30)
|
|
print(json.dumps({'returncode': p.returncode}))''')
|
|
check(memory['returncode'] == 21, 'Linux memory limit did not produce bounded resource failure')
|
|
record('Linux renderer memory limit', **memory)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
finally:
|
|
compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', 'worker', 'outbox')
|