forked from mtusi2026/mtusi2026lab4
Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4 code, resolving each conflict by meaning and keeping everything lab 4 added: - a message is posted with one SQL statement under the chat row lock, and the attachment links go into the same statement; a foreign key violation on the attachment link still answers 404 and leaves nothing behind. Text and the attachment list are checked before the write, an outsider gets 404 for both, as before the carry over; - the request pipeline counts the body bytes actually received against the limit of the route, so a chunked body over 1 MiB gets 413 and the upload route keeps its MAX_UPLOAD_BYTES limit; - a request cancelled at shutdown gets 503 unavailable in the contract format, the worker waits a second less than the rest of the shutdown budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s; - readiness sits behind a Readiness port with the draining flag in shared infrastructure, and the drain window is logged through Logger; - logout revokes through SessionRevocations and withdraws the mark when PostgreSQL rejects the delete; an empty cursor is 400; - EventBus.project is gone, the sweep test subscribes and waits on a queue; domain tests check attachment_list instead of Chat.post; - make up waits for every replica in the balancer, BASE_URL follows the lb port, the image is tagged by the compose project, retry-check and faults-evidence are added, detection runs three times, failover-each and rolling-update record the stand usage, CI checks numeric uids. down also takes the tools and reconcile profiles of lab 4. Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the database size itself instead of the track 1 sampler. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
247 lines
9.3 KiB
Python
247 lines
9.3 KiB
Python
import asyncio
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
from httpx import AsyncClient
|
|
from starlette.types import Message
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import APPS, DATABASE, actor, keyed, resolve, running_app, scalar
|
|
from messenger.attachments.domain.events import AttachmentDiscarded
|
|
from messenger.attachments.entrypoints.sweeper import UploadSweeper
|
|
from messenger.shared.application.messagebus import EventBus
|
|
from pictures import SAMPLE
|
|
|
|
AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1"
|
|
|
|
|
|
async def execute(sql: str, *args: Any) -> None:
|
|
connection = await asyncpg.connect(DATABASE["url"])
|
|
try:
|
|
await connection.execute(sql, *args)
|
|
finally:
|
|
await connection.close()
|
|
|
|
|
|
def sweeper(http: AsyncClient) -> UploadSweeper:
|
|
return APPS[id(http)].state.sweeper
|
|
|
|
|
|
async def discarded(http: AsyncClient) -> asyncio.Queue[AttachmentDiscarded]:
|
|
seen: asyncio.Queue[AttachmentDiscarded] = asyncio.Queue()
|
|
(await resolve(http, EventBus)).subscribe(AttachmentDiscarded, seen.put)
|
|
return seen
|
|
|
|
|
|
async def delivered(seen: asyncio.Queue[AttachmentDiscarded], count: int) -> list[str]:
|
|
async with asyncio.timeout(2):
|
|
events = [await seen.get() for _ in range(count)]
|
|
assert seen.empty()
|
|
return sorted(f"{event.reason} {event.attachment_id}" for event in events)
|
|
|
|
|
|
async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str:
|
|
store = await storage(http)
|
|
store.failing = True
|
|
name = f"intent-{uuid4().hex}.png"
|
|
try:
|
|
assert (await upload(http, user, chat, filename=name)).status_code == 503
|
|
finally:
|
|
store.failing = False
|
|
found = await scalar(
|
|
"SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name
|
|
)
|
|
assert found is not None
|
|
store.objects[f"attachments/{found}"] = (SAMPLE, "image/png")
|
|
return found
|
|
|
|
|
|
async def exists(attachment_id: str) -> bool:
|
|
return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1
|
|
|
|
|
|
async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
store = await storage(client)
|
|
name = f"fail-{uuid4().hex}.png"
|
|
trigger = f"lab4_fail_{uuid4().hex[:8]}"
|
|
await execute(
|
|
f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$;
|
|
CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW
|
|
WHEN (NEW.filename = '{name}' AND NEW.status = 'ready') EXECUTE FUNCTION {trigger}()"""
|
|
)
|
|
before, seen = set(store.objects), await discarded(client)
|
|
try:
|
|
response = await upload(client, alice, chat, filename=name)
|
|
finally:
|
|
await execute(f"DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}()")
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "unavailable"
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
assert set(store.objects) == before
|
|
assert [event.split()[0] for event in await delivered(seen, 1)] == ["compensation"]
|
|
|
|
|
|
async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
store = await storage(client)
|
|
|
|
async def swept_meanwhile(key: str) -> None:
|
|
await execute("DELETE FROM attachments WHERE object_key = $1", key)
|
|
|
|
store.after_put = swept_meanwhile
|
|
before = set(store.objects)
|
|
try:
|
|
response = await upload(client, alice, chat, filename="late.png")
|
|
finally:
|
|
store.after_put = None
|
|
assert response.status_code == 503
|
|
assert set(store.objects) == before
|
|
|
|
|
|
async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
pending = await intent(client, alice, chat)
|
|
await sweeper(client).sweep()
|
|
assert await exists(pending)
|
|
await execute(AGED, pending)
|
|
assert await sweeper(client).sweep() >= 1
|
|
assert not await exists(pending)
|
|
assert f"attachments/{pending}" not in (await storage(client)).objects
|
|
|
|
|
|
async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None:
|
|
alice, bob, _, chat = await trio(client)
|
|
ready = (await upload(client, alice, chat)).json()
|
|
sent = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [ready["id"]]},
|
|
headers=keyed(alice),
|
|
)
|
|
assert sent.status_code == 201
|
|
old, fresh = await intent(client, alice, chat), await intent(client, alice, chat)
|
|
await execute(AGED, old)
|
|
await execute(AGED, ready["id"])
|
|
assert await sweeper(client).sweep() == 1
|
|
objects = (await storage(client)).objects
|
|
assert not await exists(old)
|
|
assert f"attachments/{old}" not in objects
|
|
assert await exists(fresh)
|
|
assert f"attachments/{fresh}" in objects
|
|
download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob))
|
|
assert download.status_code == 200
|
|
assert download.content == SAMPLE
|
|
assert await sweeper(client).sweep() == 0
|
|
|
|
|
|
async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
pending = [await intent(client, alice, chat) for _ in range(5)]
|
|
for attachment_id in pending:
|
|
await execute(AGED, attachment_id)
|
|
seen = await discarded(client)
|
|
first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep())
|
|
assert first + second == 5
|
|
assert await delivered(seen, 5) == sorted(f"abandoned {item}" for item in pending)
|
|
assert await sweeper(client).sweep() == 0
|
|
objects = (await storage(client)).objects
|
|
assert not any(f"attachments/{attachment_id}" in objects for attachment_id in pending)
|
|
|
|
|
|
async def test_background_sweeper_removes_abandoned_intents() -> None:
|
|
async with running_app(upload_sweep_interval_seconds=0.05) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
pending = await intent(http, alice, chat)
|
|
await execute(AGED, pending)
|
|
for _ in range(100):
|
|
if not await exists(pending):
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
assert not await exists(pending)
|
|
|
|
|
|
async def test_upload_waits_for_a_free_slot() -> None:
|
|
async with running_app(upload_concurrency=1) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
gate, holding, read = asyncio.Event(), asyncio.Event(), []
|
|
|
|
async def held() -> AsyncIterator[bytes]:
|
|
read.append("first")
|
|
holding.set()
|
|
yield SAMPLE[:1000]
|
|
await gate.wait()
|
|
yield SAMPLE[1000:]
|
|
|
|
async def queued() -> AsyncIterator[bytes]:
|
|
read.append("second")
|
|
yield SAMPLE
|
|
|
|
first = asyncio.create_task(upload(http, alice, chat, held()))
|
|
await holding.wait()
|
|
second = asyncio.create_task(upload(http, alice, chat, queued()))
|
|
await asyncio.sleep(0.3)
|
|
assert read == ["first"]
|
|
gate.set()
|
|
assert [(await first).status_code, (await second).status_code] == [201, 201]
|
|
assert read == ["first", "second"]
|
|
|
|
|
|
async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
name = f"abort-{uuid4().hex}.png"
|
|
headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()]
|
|
scope = {
|
|
"type": "http",
|
|
"asgi": {"version": "3.0"},
|
|
"http_version": "1.1",
|
|
"method": "POST",
|
|
"scheme": "http",
|
|
"path": f"/api/v1/chats/{chat['id']}/attachments",
|
|
"raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(),
|
|
"query_string": f"filename={name}".encode(),
|
|
"root_path": "",
|
|
"headers": [
|
|
*headers,
|
|
(b"content-type", b"image/png"),
|
|
(b"content-length", b"1000000"),
|
|
],
|
|
"client": ("127.0.0.1", 50000),
|
|
"server": ("testserver", 80),
|
|
}
|
|
messages = iter(
|
|
[
|
|
{"type": "http.request", "body": SAMPLE[:1000], "more_body": True},
|
|
{"type": "http.disconnect"},
|
|
]
|
|
)
|
|
sent: list[Message] = []
|
|
|
|
async def receive() -> dict[str, Any]:
|
|
return next(messages, {"type": "http.disconnect"})
|
|
|
|
async def send(message: Message) -> None:
|
|
sent.append(message)
|
|
|
|
await APPS[id(client)](scope, receive, send)
|
|
assert sent[0]["status"] == 400
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
|
|
|
|
async def test_stalled_upload_times_out_and_frees_the_slot() -> None:
|
|
async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http:
|
|
alice, _, _, chat = await trio(http)
|
|
name = f"stalled-{uuid4().hex}.png"
|
|
|
|
async def stalled() -> AsyncIterator[bytes]:
|
|
yield SAMPLE[:1000]
|
|
await asyncio.Event().wait()
|
|
|
|
response = await upload(http, alice, chat, stalled(), filename=name)
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "unavailable"
|
|
assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0
|
|
assert (await upload(http, alice, chat)).status_code == 201
|