Upload now stages the checked original under staging/, and one transaction moves the intent to queued, inserts the job (pipeline version 1, attempt 1), its outbox row and an audit event; the 202 is the in-memory snapshot, never a reread. Content answers 409 until the attachment is ready, the thumbnail variant is served from the published version and absent variants are 404. The relay publishes outbox rows through FastStream with publisher confirms, mandatory routing and persistent delivery, marks a row only after the broker acked it, returns jobs whose lease expired and deletes the staging object of ready attachments. The worker consumes the quorum queue through a FastStream subscriber that declares nothing, takes prefetch from WORKER_CONCURRENCY and acks manually after the result commit, rejects messages off jobs.schema.json to the dead letter exchange and requeues after an infrastructure error. A job is claimed under a lease token; the original is copied to originals/ and a 256 px JPEG thumbnail is written under a content-addressed key; the commit is fenced by the token, inserts variants with ON CONFLICT DO NOTHING and publishes only a newer pipeline version. A transient error reschedules the same job with an exponential delay and a new outbox row, the last attempt fails with retry_exhausted, an undecodable image fails at once with invalid_image, and a redelivered finished job is a recorded no-op. The worker and the relay are assembled in worker.py and relay.py with the dishka FastStream integration and the platform providers of the API; the shared core imports neither FastStream nor aio-pika. Tests cover the thumbnail geometry and rendering, the job message against the contract, the pipeline against PostgreSQL (duplicates, parallel copies, retries, the attempt bound, leases and fencing) and the broker adapters against a throwaway RabbitMQ with the stand's topology and permissions. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
255 lines
9.3 KiB
Python
255 lines
9.3 KiB
Python
import asyncio
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
from httpx import AsyncClient
|
|
from starlette.types import Message
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import APPS, DATABASE, actor, keyed, resolve, running_app, scalar
|
|
from messenger.attachments.domain.events import AttachmentDiscarded
|
|
from messenger.attachments.entrypoints.sweeper import UploadSweeper
|
|
from messenger.shared.application.messagebus import EventBus
|
|
from pictures import SAMPLE
|
|
from pipeline import settle
|
|
|
|
AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1"
|
|
|
|
|
|
async def execute(sql: str, *args: Any) -> None:
|
|
connection = await asyncpg.connect(DATABASE["url"])
|
|
try:
|
|
await connection.execute(sql, *args)
|
|
finally:
|
|
await connection.close()
|
|
|
|
|
|
def sweeper(http: AsyncClient) -> UploadSweeper:
|
|
return APPS[id(http)].state.sweeper
|
|
|
|
|
|
async def discarded(http: AsyncClient) -> list[AttachmentDiscarded]:
|
|
seen: list[AttachmentDiscarded] = []
|
|
|
|
async def keep(event: AttachmentDiscarded) -> None:
|
|
seen.append(event)
|
|
|
|
(await resolve(http, EventBus)).project(AttachmentDiscarded, keep)
|
|
return seen
|
|
|
|
|
|
async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str:
|
|
store = await storage(http)
|
|
store.failing = True
|
|
name = f"intent-{uuid4().hex}.png"
|
|
try:
|
|
assert (await upload(http, user, chat, filename=name)).status_code == 503
|
|
finally:
|
|
store.failing = False
|
|
found = await scalar(
|
|
"SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name
|
|
)
|
|
assert found is not None
|
|
store.objects[f"staging/{found}"] = (SAMPLE, "image/png")
|
|
return found
|
|
|
|
|
|
async def exists(attachment_id: str) -> bool:
|
|
return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1
|
|
|
|
|
|
async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
store = await storage(client)
|
|
name = f"fail-{uuid4().hex}.png"
|
|
trigger = f"lab4_fail_{uuid4().hex[:8]}"
|
|
await execute(
|
|
f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$;
|
|
CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW
|
|
WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()"""
|
|
)
|
|
before, seen = set(store.objects), await discarded(client)
|
|
try:
|
|
response = await upload(client, alice, chat, filename=name)
|
|
finally:
|
|
await execute(f"DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}()")
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "unavailable"
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
assert (
|
|
await scalar(
|
|
"SELECT count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id "
|
|
"WHERE a.filename = $1",
|
|
name,
|
|
)
|
|
== 0
|
|
)
|
|
assert set(store.objects) == before
|
|
assert [event.reason for event in seen] == ["compensation"]
|
|
|
|
|
|
async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
store = await storage(client)
|
|
|
|
async def swept_meanwhile(key: str) -> None:
|
|
await execute("DELETE FROM attachments WHERE staging_key = $1", key)
|
|
|
|
store.after_put = swept_meanwhile
|
|
before = set(store.objects)
|
|
try:
|
|
response = await upload(client, alice, chat, filename="late.png")
|
|
finally:
|
|
store.after_put = None
|
|
assert response.status_code == 503
|
|
assert set(store.objects) == before
|
|
|
|
|
|
async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
pending = await intent(client, alice, chat)
|
|
await sweeper(client).sweep()
|
|
assert await exists(pending)
|
|
await execute(AGED, pending)
|
|
assert await sweeper(client).sweep() >= 1
|
|
assert not await exists(pending)
|
|
assert f"staging/{pending}" not in (await storage(client)).objects
|
|
|
|
|
|
async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None:
|
|
alice, bob, _, chat = await trio(client)
|
|
ready = (await upload(client, alice, chat)).json()
|
|
await settle(client)
|
|
sent = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [ready["id"]]},
|
|
headers=keyed(alice),
|
|
)
|
|
assert sent.status_code == 201
|
|
old, fresh = await intent(client, alice, chat), await intent(client, alice, chat)
|
|
await execute(AGED, old)
|
|
await execute(AGED, ready["id"])
|
|
assert await sweeper(client).sweep() == 1
|
|
objects = (await storage(client)).objects
|
|
assert not await exists(old)
|
|
assert f"staging/{old}" not in objects
|
|
assert await exists(fresh)
|
|
assert f"staging/{fresh}" in objects
|
|
download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob))
|
|
assert download.status_code == 200
|
|
assert download.content == SAMPLE
|
|
assert await sweeper(client).sweep() == 0
|
|
|
|
|
|
async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
pending = [await intent(client, alice, chat) for _ in range(5)]
|
|
for attachment_id in pending:
|
|
await execute(AGED, attachment_id)
|
|
seen = await discarded(client)
|
|
first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep())
|
|
assert first + second == 5
|
|
assert sorted(str(event.attachment_id) for event in seen) == sorted(pending)
|
|
assert {event.reason for event in seen} == {"abandoned"}
|
|
assert await sweeper(client).sweep() == 0
|
|
objects = (await storage(client)).objects
|
|
assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending)
|
|
|
|
|
|
async def test_background_sweeper_removes_abandoned_intents() -> None:
|
|
async with running_app(upload_sweep_interval_seconds=0.05) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
pending = await intent(http, alice, chat)
|
|
await execute(AGED, pending)
|
|
for _ in range(100):
|
|
if not await exists(pending):
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
assert not await exists(pending)
|
|
|
|
|
|
async def test_upload_waits_for_a_free_slot() -> None:
|
|
async with running_app(upload_concurrency=1) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
gate, holding, read = asyncio.Event(), asyncio.Event(), []
|
|
|
|
async def held() -> AsyncIterator[bytes]:
|
|
read.append("first")
|
|
holding.set()
|
|
yield SAMPLE[:1000]
|
|
await gate.wait()
|
|
yield SAMPLE[1000:]
|
|
|
|
async def queued() -> AsyncIterator[bytes]:
|
|
read.append("second")
|
|
yield SAMPLE
|
|
|
|
first = asyncio.create_task(upload(http, alice, chat, held()))
|
|
await holding.wait()
|
|
second = asyncio.create_task(upload(http, alice, chat, queued()))
|
|
await asyncio.sleep(0.3)
|
|
assert read == ["first"]
|
|
gate.set()
|
|
assert [(await first).status_code, (await second).status_code] == [202, 202]
|
|
assert read == ["first", "second"]
|
|
|
|
|
|
async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
name = f"abort-{uuid4().hex}.png"
|
|
headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()]
|
|
scope = {
|
|
"type": "http",
|
|
"asgi": {"version": "3.0"},
|
|
"http_version": "1.1",
|
|
"method": "POST",
|
|
"scheme": "http",
|
|
"path": f"/api/v1/chats/{chat['id']}/attachments",
|
|
"raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(),
|
|
"query_string": f"filename={name}".encode(),
|
|
"root_path": "",
|
|
"headers": [
|
|
*headers,
|
|
(b"content-type", b"image/png"),
|
|
(b"content-length", b"1000000"),
|
|
],
|
|
"client": ("127.0.0.1", 50000),
|
|
"server": ("testserver", 80),
|
|
}
|
|
messages = iter(
|
|
[
|
|
{"type": "http.request", "body": SAMPLE[:1000], "more_body": True},
|
|
{"type": "http.disconnect"},
|
|
]
|
|
)
|
|
sent: list[Message] = []
|
|
|
|
async def receive() -> dict[str, Any]:
|
|
return next(messages, {"type": "http.disconnect"})
|
|
|
|
async def send(message: Message) -> None:
|
|
sent.append(message)
|
|
|
|
await APPS[id(client)](scope, receive, send)
|
|
assert sent[0]["status"] == 400
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
|
|
|
|
async def test_stalled_upload_times_out_and_frees_the_slot() -> None:
|
|
async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
name = f"stalled-{uuid4().hex}.png"
|
|
|
|
async def stalled() -> AsyncIterator[bytes]:
|
|
yield SAMPLE[:1000]
|
|
await asyncio.Event().wait()
|
|
|
|
response = await upload(http, alice, chat, stalled(), filename=name)
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "unavailable"
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
assert (await upload(http, alice, chat)).status_code == 202
|