import asyncio from collections.abc import AsyncIterator from typing import Any from uuid import uuid4 from httpx import AsyncClient from starlette.types import Message from test_attachments import storage, trio, upload from conftest import APPS, actor, keyed, resolve, running_app, scalar, sql_connection from messenger.attachments.domain.events import AttachmentDiscarded from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.shared.application.messagebus import EventBus from pictures import SAMPLE from pipeline import settle AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1" async def execute(sql: str, *args: Any) -> None: async with sql_connection() as connection: await connection.exec_driver_sql(sql, args) def sweeper(http: AsyncClient) -> UploadSweeper: return APPS[id(http)].state.sweeper async def discarded(http: AsyncClient) -> asyncio.Queue[AttachmentDiscarded]: seen: asyncio.Queue[AttachmentDiscarded] = asyncio.Queue() (await resolve(http, EventBus)).subscribe(AttachmentDiscarded, seen.put) return seen async def delivered(seen: asyncio.Queue[AttachmentDiscarded], count: int) -> list[str]: async with asyncio.timeout(2): events = [await seen.get() for _ in range(count)] assert seen.empty() return sorted(f"{event.reason} {event.attachment_id}" for event in events) async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: store = await storage(http) store.failing = True name = f"intent-{uuid4().hex}.png" try: assert (await upload(http, user, chat, filename=name)).status_code == 503 finally: store.failing = False found = await scalar( "SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name ) assert found is not None store.objects[f"staging/{found}"] = (SAMPLE, "image/png") return found async def exists(attachment_id: str) -> bool: return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1 async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) store = await storage(client) name = f"fail-{uuid4().hex}.png" trigger = f"lab4_fail_{uuid4().hex[:8]}" await execute( f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$""" ) await execute( f"""CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()""" ) before, seen = set(store.objects), await discarded(client) try: response = await upload(client, alice, chat, filename=name) finally: await execute(f"DROP TRIGGER {trigger} ON attachments") await execute(f"DROP FUNCTION {trigger}()") assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 assert ( await scalar( "SELECT count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id " "WHERE a.filename = $1", name, ) == 0 ) assert set(store.objects) == before assert [event.split()[0] for event in await delivered(seen, 1)] == ["compensation"] async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) store = await storage(client) async def swept_meanwhile(key: str) -> None: await execute("DELETE FROM attachments WHERE staging_key = $1", key) store.after_put = swept_meanwhile before = set(store.objects) try: response = await upload(client, alice, chat, filename="late.png") finally: store.after_put = None assert response.status_code == 503 assert set(store.objects) == before async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) pending = await intent(client, alice, chat) await sweeper(client).sweep() assert await exists(pending) await execute(AGED, pending) assert await sweeper(client).sweep() >= 1 assert not await exists(pending) assert f"staging/{pending}" not in (await storage(client)).objects async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None: alice, bob, _, chat = await trio(client) ready = (await upload(client, alice, chat)).json() await settle(client) sent = await client.post( f"/api/v1/chats/{chat['id']}/messages", json={"text": "", "attachment_ids": [ready["id"]]}, headers=keyed(alice), ) assert sent.status_code == 201 old, fresh = await intent(client, alice, chat), await intent(client, alice, chat) await execute(AGED, old) await execute(AGED, ready["id"]) assert await sweeper(client).sweep() == 1 objects = (await storage(client)).objects assert not await exists(old) assert f"staging/{old}" not in objects assert await exists(fresh) assert f"staging/{fresh}" in objects download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob)) assert download.status_code == 200 assert download.content == SAMPLE assert await sweeper(client).sweep() == 0 async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) pending = [await intent(client, alice, chat) for _ in range(5)] for attachment_id in pending: await execute(AGED, attachment_id) seen = await discarded(client) first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep()) assert first + second == 5 assert await delivered(seen, 5) == sorted(f"abandoned {item}" for item in pending) assert await sweeper(client).sweep() == 0 objects = (await storage(client)).objects assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending) async def test_background_sweeper_removes_abandoned_intents() -> None: async with running_app(upload_sweep_interval_seconds=0.05) as http: alice, _, _, chat = await trio(http) pending = await intent(http, alice, chat) await execute(AGED, pending) for _ in range(100): if not await exists(pending): break await asyncio.sleep(0.05) assert not await exists(pending) async def test_upload_waits_for_a_free_slot() -> None: async with running_app(upload_concurrency=1) as http: alice, _, _, chat = await trio(http) gate, holding, read = asyncio.Event(), asyncio.Event(), [] async def held() -> AsyncIterator[bytes]: read.append("first") holding.set() yield SAMPLE[:1000] await gate.wait() yield SAMPLE[1000:] async def queued() -> AsyncIterator[bytes]: read.append("second") yield SAMPLE first = asyncio.create_task(upload(http, alice, chat, held())) await holding.wait() second = asyncio.create_task(upload(http, alice, chat, queued())) await asyncio.sleep(0.3) assert read == ["first"] gate.set() assert [(await first).status_code, (await second).status_code] == [202, 202] assert read == ["first", "second"] async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) name = f"abort-{uuid4().hex}.png" headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()] scope = { "type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "POST", "scheme": "http", "path": f"/api/v1/chats/{chat['id']}/attachments", "raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(), "query_string": f"filename={name}".encode(), "root_path": "", "headers": [ *headers, (b"content-type", b"image/png"), (b"content-length", b"1000000"), ], "client": ("127.0.0.1", 50000), "server": ("testserver", 80), } messages = iter( [ {"type": "http.request", "body": SAMPLE[:1000], "more_body": True}, {"type": "http.disconnect"}, ] ) sent: list[Message] = [] async def receive() -> dict[str, Any]: return next(messages, {"type": "http.disconnect"}) async def send(message: Message) -> None: sent.append(message) await APPS[id(client)](scope, receive, send) assert sent[0]["status"] == 400 assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 async def test_stalled_upload_times_out_and_frees_the_slot() -> None: async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http: alice, _, _, chat = await trio(http) name = f"stalled-{uuid4().hex}.png" async def stalled() -> AsyncIterator[bytes]: yield SAMPLE[:1000] await asyncio.Event().wait() response = await upload(http, alice, chat, stalled(), filename=name) assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 assert (await upload(http, alice, chat)).status_code == 202