256 lines
9.5 KiB
Python
256 lines
9.5 KiB
Python
import asyncio
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from httpx import AsyncClient
|
|
from starlette.types import Message
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import APPS, actor, keyed, resolve, running_app, scalar, sql_connection
|
|
from messenger.attachments.domain.events import AttachmentDiscarded
|
|
from messenger.attachments.entrypoints.sweeper import UploadSweeper
|
|
from messenger.shared.application.messagebus import EventBus
|
|
from pictures import SAMPLE
|
|
from pipeline import settle
|
|
|
|
AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1"
|
|
|
|
|
|
async def execute(sql: str, *args: Any) -> None:
|
|
async with sql_connection() as connection:
|
|
await connection.exec_driver_sql(sql, args)
|
|
|
|
|
|
def sweeper(http: AsyncClient) -> UploadSweeper:
|
|
return APPS[id(http)].state.sweeper
|
|
|
|
|
|
async def discarded(http: AsyncClient) -> asyncio.Queue[AttachmentDiscarded]:
|
|
seen: asyncio.Queue[AttachmentDiscarded] = asyncio.Queue()
|
|
(await resolve(http, EventBus)).subscribe(AttachmentDiscarded, seen.put)
|
|
return seen
|
|
|
|
|
|
async def delivered(seen: asyncio.Queue[AttachmentDiscarded], count: int) -> list[str]:
|
|
async with asyncio.timeout(2):
|
|
events = [await seen.get() for _ in range(count)]
|
|
assert seen.empty()
|
|
return sorted(f"{event.reason} {event.attachment_id}" for event in events)
|
|
|
|
|
|
async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str:
|
|
store = await storage(http)
|
|
store.failing = True
|
|
name = f"intent-{uuid4().hex}.png"
|
|
try:
|
|
assert (await upload(http, user, chat, filename=name)).status_code == 503
|
|
finally:
|
|
store.failing = False
|
|
found = await scalar(
|
|
"SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name
|
|
)
|
|
assert found is not None
|
|
store.objects[f"staging/{found}"] = (SAMPLE, "image/png")
|
|
return found
|
|
|
|
|
|
async def exists(attachment_id: str) -> bool:
|
|
return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1
|
|
|
|
|
|
async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
store = await storage(client)
|
|
name = f"fail-{uuid4().hex}.png"
|
|
trigger = f"lab4_fail_{uuid4().hex[:8]}"
|
|
await execute(
|
|
f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$"""
|
|
)
|
|
await execute(
|
|
f"""CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW
|
|
WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()"""
|
|
)
|
|
before, seen = set(store.objects), await discarded(client)
|
|
try:
|
|
response = await upload(client, alice, chat, filename=name)
|
|
finally:
|
|
await execute(f"DROP TRIGGER {trigger} ON attachments")
|
|
await execute(f"DROP FUNCTION {trigger}()")
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "unavailable"
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
assert (
|
|
await scalar(
|
|
"SELECT count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id "
|
|
"WHERE a.filename = $1",
|
|
name,
|
|
)
|
|
== 0
|
|
)
|
|
assert set(store.objects) == before
|
|
assert [event.split()[0] for event in await delivered(seen, 1)] == ["compensation"]
|
|
|
|
|
|
async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
store = await storage(client)
|
|
|
|
async def swept_meanwhile(key: str) -> None:
|
|
await execute("DELETE FROM attachments WHERE staging_key = $1", key)
|
|
|
|
store.after_put = swept_meanwhile
|
|
before = set(store.objects)
|
|
try:
|
|
response = await upload(client, alice, chat, filename="late.png")
|
|
finally:
|
|
store.after_put = None
|
|
assert response.status_code == 503
|
|
assert set(store.objects) == before
|
|
|
|
|
|
async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
pending = await intent(client, alice, chat)
|
|
await sweeper(client).sweep()
|
|
assert await exists(pending)
|
|
await execute(AGED, pending)
|
|
assert await sweeper(client).sweep() >= 1
|
|
assert not await exists(pending)
|
|
assert f"staging/{pending}" not in (await storage(client)).objects
|
|
|
|
|
|
async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None:
|
|
alice, bob, _, chat = await trio(client)
|
|
ready = (await upload(client, alice, chat)).json()
|
|
await settle(client)
|
|
sent = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [ready["id"]]},
|
|
headers=keyed(alice),
|
|
)
|
|
assert sent.status_code == 201
|
|
old, fresh = await intent(client, alice, chat), await intent(client, alice, chat)
|
|
await execute(AGED, old)
|
|
await execute(AGED, ready["id"])
|
|
assert await sweeper(client).sweep() == 1
|
|
objects = (await storage(client)).objects
|
|
assert not await exists(old)
|
|
assert f"staging/{old}" not in objects
|
|
assert await exists(fresh)
|
|
assert f"staging/{fresh}" in objects
|
|
download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob))
|
|
assert download.status_code == 200
|
|
assert download.content == SAMPLE
|
|
assert await sweeper(client).sweep() == 0
|
|
|
|
|
|
async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
pending = [await intent(client, alice, chat) for _ in range(5)]
|
|
for attachment_id in pending:
|
|
await execute(AGED, attachment_id)
|
|
seen = await discarded(client)
|
|
first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep())
|
|
assert first + second == 5
|
|
assert await delivered(seen, 5) == sorted(f"abandoned {item}" for item in pending)
|
|
assert await sweeper(client).sweep() == 0
|
|
objects = (await storage(client)).objects
|
|
assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending)
|
|
|
|
|
|
async def test_background_sweeper_removes_abandoned_intents() -> None:
|
|
async with running_app(upload_sweep_interval_seconds=0.05) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
pending = await intent(http, alice, chat)
|
|
await execute(AGED, pending)
|
|
for _ in range(100):
|
|
if not await exists(pending):
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
assert not await exists(pending)
|
|
|
|
|
|
async def test_upload_waits_for_a_free_slot() -> None:
|
|
async with running_app(upload_concurrency=1) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
gate, holding, read = asyncio.Event(), asyncio.Event(), []
|
|
|
|
async def held() -> AsyncIterator[bytes]:
|
|
read.append("first")
|
|
holding.set()
|
|
yield SAMPLE[:1000]
|
|
await gate.wait()
|
|
yield SAMPLE[1000:]
|
|
|
|
async def queued() -> AsyncIterator[bytes]:
|
|
read.append("second")
|
|
yield SAMPLE
|
|
|
|
first = asyncio.create_task(upload(http, alice, chat, held()))
|
|
await holding.wait()
|
|
second = asyncio.create_task(upload(http, alice, chat, queued()))
|
|
await asyncio.sleep(0.3)
|
|
assert read == ["first"]
|
|
gate.set()
|
|
assert [(await first).status_code, (await second).status_code] == [202, 202]
|
|
assert read == ["first", "second"]
|
|
|
|
|
|
async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
name = f"abort-{uuid4().hex}.png"
|
|
headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()]
|
|
scope = {
|
|
"type": "http",
|
|
"asgi": {"version": "3.0"},
|
|
"http_version": "1.1",
|
|
"method": "POST",
|
|
"scheme": "http",
|
|
"path": f"/api/v1/chats/{chat['id']}/attachments",
|
|
"raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(),
|
|
"query_string": f"filename={name}".encode(),
|
|
"root_path": "",
|
|
"headers": [
|
|
*headers,
|
|
(b"content-type", b"image/png"),
|
|
(b"content-length", b"1000000"),
|
|
],
|
|
"client": ("127.0.0.1", 50000),
|
|
"server": ("testserver", 80),
|
|
}
|
|
messages = iter(
|
|
[
|
|
{"type": "http.request", "body": SAMPLE[:1000], "more_body": True},
|
|
{"type": "http.disconnect"},
|
|
]
|
|
)
|
|
sent: list[Message] = []
|
|
|
|
async def receive() -> dict[str, Any]:
|
|
return next(messages, {"type": "http.disconnect"})
|
|
|
|
async def send(message: Message) -> None:
|
|
sent.append(message)
|
|
|
|
await APPS[id(client)](scope, receive, send)
|
|
assert sent[0]["status"] == 400
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
|
|
|
|
async def test_stalled_upload_times_out_and_frees_the_slot() -> None:
|
|
async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
name = f"stalled-{uuid4().hex}.png"
|
|
|
|
async def stalled() -> AsyncIterator[bytes]:
|
|
yield SAMPLE[:1000]
|
|
await asyncio.Event().wait()
|
|
|
|
response = await upload(http, alice, chat, stalled(), filename=name)
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "unavailable"
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
assert (await upload(http, alice, chat)).status_code == 202
|