"""Additional real-container checks: validation, permanent failure, timeout, retention.""" import os from pipeline import Client, check, compose, remote, state, until def main(): c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080')) alice, bob, eve, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] + '/attachments' c.call('POST', path + '?filename=bad.png', 415, actor=alice, raw=b'not a PNG', headers={'Content-Type': 'image/png'}) c.call('POST', path + '?filename=../bad.png', 400, actor=alice, raw=b'bad', headers={'Content-Type': 'image/png'}) try: compose('stop', 'worker', 'cleanup') att, data = c.attachment(alice, chat) aid = att['id'] before = state(aid) # A queued record cannot be collected even when its staging is old. remote(f''' from datetime import datetime, timedelta, timezone from types import SimpleNamespace from src.cleanup import cleanup client = s3_client() class SingleObject: def __getattr__(self, name): return getattr(client, name) def get_paginator(self, name): obj = dict(client.head_object(Bucket=BUCKET, Key={before['staging']!r}), Key={before['staging']!r}) return SimpleNamespace(paginate=lambda **kw: [{{'Contents': [obj]}}]) cleanup(sessions, SingleObject(), now=datetime.now(timezone.utc) + timedelta(days=2)) s3_client().head_object(Bucket=BUCKET, Key={before['staging']!r}) ''') # Force a real checksum failure after upload, without changing the public contract. remote(f"s3_client().put_object(Bucket=BUCKET, Key={before['staging']!r}, Body=b'corrupted')\n") key = 'queued-before-failure-' + aid message = c.message(alice, chat, '', key=key, attachments=[aid])['json'] compose('start', 'worker') until(lambda: state(aid)['status'] == 'failed') check(state(aid)['attempt'] == 1 and state(aid)['error'] == 'invalid_image', 'permanent error retried') c.call('GET', '/api/v1/attachments/' + aid + '/content', 409, actor=bob) check(c.message(alice, chat, '', key=key, attachments=[aid])['json'] == message, 'failed processing broke message idempotency') # Expire this test job's failed retention; cleanup must remove its staging. remote(f''' from datetime import timedelta from src.jobs import now, replay from src.cleanup import cleanup with sessions.begin() as session: a = session.get(Attachment, {aid!r}) a.updated_at = now() - timedelta(days=2) cleanup(sessions, s3_client()) try: replay(sessions, {before['job_id']!r}, 'expired staging must reject') except ValueError: pass else: raise AssertionError('Replay accepted missing staging') ''') check(state(aid)['staging'] is None, 'failed staging retention did not expire') print('PASS: upload validation, active staging retention, permanent failure, message replay, expired staging', flush=True) compose('stop', 'worker') att, _ = c.attachment(alice, chat) aid = att['id'] until(lambda: state(aid)['published']) # Substitute a real sleeping child for the decoder, keeping the real timeout/kill path. remote(f''' import subprocess, sys import src.worker as worker from unittest.mock import patch real_run = subprocess.run def stalled(*args, **kwargs): return real_run([sys.executable, '-c', 'import time; time.sleep(30)'], timeout=1) with sessions() as session: j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r})) fields = {{'job': json.dumps(envelope(j))}} with patch.object(worker.subprocess, 'run', stalled): worker.handle(sessions, broker(), '0-0', fields) ''') check(state(aid)['status'] == 'queued' and state(aid)['failures'] == 1, 'timeout did not schedule retry') remote(f''' with sessions() as session: j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r})) assert j.last_error == 'processing_timeout' ''') compose('start', 'worker') c.wait_ready(bob, att) check(state(aid)['attempt'] == 2, 'timeout recovery did not complete') print('PASS: real subprocess timeout, termination and successful retry', flush=True) finally: compose('start', 'worker', 'cleanup') if __name__ == '__main__': main()