forked from mtusi2026/mtusi2026lab5
Upload now stages the checked original under staging/, and one transaction moves the intent to queued, inserts the job (pipeline version 1, attempt 1), its outbox row and an audit event; the 202 is the in-memory snapshot, never a reread. Content answers 409 until the attachment is ready, the thumbnail variant is served from the published version and absent variants are 404. The relay publishes outbox rows through FastStream with publisher confirms, mandatory routing and persistent delivery, marks a row only after the broker acked it, returns jobs whose lease expired and deletes the staging object of ready attachments. The worker consumes the quorum queue through a FastStream subscriber that declares nothing, takes prefetch from WORKER_CONCURRENCY and acks manually after the result commit, rejects messages off jobs.schema.json to the dead letter exchange and requeues after an infrastructure error. A job is claimed under a lease token; the original is copied to originals/ and a 256 px JPEG thumbnail is written under a content-addressed key; the commit is fenced by the token, inserts variants with ON CONFLICT DO NOTHING and publishes only a newer pipeline version. A transient error reschedules the same job with an exponential delay and a new outbox row, the last attempt fails with retry_exhausted, an undecodable image fails at once with invalid_image, and a redelivered finished job is a recorded no-op. The worker and the relay are assembled in worker.py and relay.py with the dishka FastStream integration and the platform providers of the API; the shared core imports neither FastStream nor aio-pika. Tests cover the thumbnail geometry and rendering, the job message against the contract, the pipeline against PostgreSQL (duplicates, parallel copies, retries, the attempt bound, leases and fencing) and the broker adapters against a throwaway RabbitMQ with the stand's topology and permissions. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
313 lines
12 KiB
Python
313 lines
12 KiB
Python
from collections.abc import AsyncIterator
|
|
from hashlib import sha256
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient, Response
|
|
|
|
from conftest import actor, make_chat, register, resolve, scalar
|
|
from fakes import MemoryObjectStorage
|
|
from messenger.attachments.application.ports import ObjectStorage
|
|
from pictures import GIF, JPEG, SAMPLE, WEBP, png
|
|
from pipeline import settle
|
|
|
|
LIMIT = 10_485_760
|
|
PNG = {"Content-Type": "image/png"}
|
|
|
|
|
|
async def upload(
|
|
http: AsyncClient,
|
|
user: dict[str, Any],
|
|
chat: dict[str, Any],
|
|
content: bytes | AsyncIterator[bytes] = SAMPLE,
|
|
*,
|
|
filename: str | None = "sample.png",
|
|
headers: dict[str, str] | None = None,
|
|
) -> Response:
|
|
query = "" if filename is None else f"?filename={quote(filename, safe='')}"
|
|
return await http.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments{query}",
|
|
content=content,
|
|
headers={**actor(user), **(PNG if headers is None else headers)},
|
|
)
|
|
|
|
|
|
async def storage(http: AsyncClient) -> MemoryObjectStorage:
|
|
found = await resolve(http, ObjectStorage)
|
|
assert isinstance(found, MemoryObjectStorage)
|
|
return found
|
|
|
|
|
|
async def trio(http: AsyncClient) -> tuple[dict[str, Any], ...]:
|
|
alice, bob, eve = [await register(http, label) for label in ("alice", "bob", "eve")]
|
|
return alice, bob, eve, await make_chat(http, alice, bob)
|
|
|
|
|
|
async def test_member_uploads_and_another_member_downloads(client: AsyncClient) -> None:
|
|
alice, bob, eve, chat = await trio(client)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
attachment = response.json()
|
|
assert set(attachment) == {
|
|
"id",
|
|
"chat_id",
|
|
"owner_id",
|
|
"filename",
|
|
"content_type",
|
|
"size_bytes",
|
|
"sha256",
|
|
"status",
|
|
"variants",
|
|
"error_code",
|
|
"created_at",
|
|
"updated_at",
|
|
}
|
|
assert attachment["chat_id"] == chat["id"]
|
|
assert attachment["owner_id"] == alice["id"]
|
|
assert attachment["filename"] == "sample.png"
|
|
assert attachment["content_type"] == "image/png"
|
|
assert attachment["size_bytes"] == len(SAMPLE) == 111817
|
|
assert attachment["sha256"] == sha256(SAMPLE).hexdigest()
|
|
assert (attachment["status"], attachment["variants"], attachment["error_code"]) == (
|
|
"queued",
|
|
[],
|
|
None,
|
|
)
|
|
assert attachment["created_at"].endswith("Z")
|
|
path = f"/api/v1/attachments/{attachment['id']}"
|
|
assert (await client.get(path, headers=actor(bob))).json() == attachment
|
|
await settle(client)
|
|
ready = (await client.get(path, headers=actor(bob))).json()
|
|
assert (ready["status"], ready["error_code"], ready["sha256"]) == (
|
|
"ready",
|
|
None,
|
|
attachment["sha256"],
|
|
)
|
|
assert [variant["name"] for variant in ready["variants"]] == ["thumbnail"]
|
|
content = await client.get(
|
|
f"{path}/content", headers={**actor(bob), "Accept": "application/json"}
|
|
)
|
|
assert content.status_code == 200
|
|
assert content.content == SAMPLE
|
|
assert content.headers["content-type"] == "image/png"
|
|
assert content.headers["content-length"] == str(len(SAMPLE))
|
|
assert content.headers["x-content-type-options"] == "nosniff"
|
|
assert content.headers["x-request-id"]
|
|
assert content.headers["x-instance-id"]
|
|
for suffix in ("", "/content"):
|
|
stranger = await client.get(f"{path}{suffix}", headers=actor(eve))
|
|
assert stranger.status_code == 404
|
|
assert stranger.json()["error"]["code"] == "not_found"
|
|
|
|
|
|
async def test_object_keys_are_assigned_by_server(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
attachment = (await upload(client, alice, chat, filename="a.png")).json()
|
|
staging = await scalar("SELECT staging_key FROM attachments WHERE id = $1", attachment["id"])
|
|
assert staging == f"staging/{attachment['id']}"
|
|
assert set((await storage(client)).objects) >= {staging}
|
|
await settle(client)
|
|
original = await scalar("SELECT original_key FROM attachments WHERE id = $1", attachment["id"])
|
|
assert original == f"originals/{attachment['id']}"
|
|
assert (await storage(client)).objects[original][0] == SAMPLE
|
|
|
|
|
|
async def test_jpeg_is_accepted_by_content(client: AsyncClient) -> None:
|
|
alice, bob, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, JPEG, headers={"Content-Type": "IMAGE/JPEG"})
|
|
assert response.status_code == 202, response.text
|
|
assert response.json()["content_type"] == "image/jpeg"
|
|
await settle(client)
|
|
download = await client.get(
|
|
f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(bob)
|
|
)
|
|
assert (download.content, download.headers["content-type"]) == (JPEG, "image/jpeg")
|
|
|
|
|
|
async def test_access_is_checked_before_content(client: AsyncClient) -> None:
|
|
alice, _, eve, chat = await trio(client)
|
|
assert (await upload(client, eve, chat, b"not an image", filename="../x")).status_code == 404
|
|
missing = {"id": str(uuid4())}
|
|
assert (await upload(client, alice, missing)).status_code == 404
|
|
anonymous = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", content=SAMPLE, headers=PNG
|
|
)
|
|
assert anonymous.status_code == 401
|
|
assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"'
|
|
|
|
|
|
async def test_exact_limit_is_accepted(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, png(16, 16, size=LIMIT))
|
|
assert response.status_code == 202, response.text
|
|
assert response.json()["size_bytes"] == LIMIT
|
|
|
|
|
|
async def test_one_byte_over_with_content_length(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, png(16, 16, size=LIMIT + 1))
|
|
assert response.status_code == 413
|
|
assert response.json()["error"]["code"] == "payload_too_large"
|
|
|
|
|
|
async def test_one_byte_over_chunked(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
body = png(16, 16, size=LIMIT + 1)
|
|
|
|
async def chunked() -> AsyncIterator[bytes]:
|
|
for start in range(0, len(body), 1 << 20):
|
|
yield body[start : start + (1 << 20)]
|
|
|
|
before = await scalar("SELECT count(*) FROM attachments")
|
|
response = await upload(client, alice, chat, chunked())
|
|
assert response.status_code == 413
|
|
assert await scalar("SELECT count(*) FROM attachments") == before
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("width", "height", "status"),
|
|
[(8000, 5000, 202), (8001, 5000, 413), (100_000, 100_000, 413)],
|
|
)
|
|
async def test_pixel_limit_is_read_from_header(
|
|
client: AsyncClient, width: int, height: int, status: int
|
|
) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
body = png(width, height, rows=width * height <= 40_000_000)
|
|
assert len(body) < 64_000
|
|
response = await upload(client, alice, chat, body)
|
|
assert response.status_code == status, response.text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("content", "header"),
|
|
[
|
|
(JPEG, "image/png"),
|
|
(GIF, "image/png"),
|
|
(WEBP, "image/png"),
|
|
(b"plain text, not an image", "image/png"),
|
|
(SAMPLE, "image/jpeg"),
|
|
(SAMPLE[: len(SAMPLE) // 2], "image/png"),
|
|
(JPEG[: len(JPEG) // 2], "image/jpeg"),
|
|
(b"", "image/png"),
|
|
(SAMPLE, "application/octet-stream"),
|
|
(SAMPLE, "image/gif"),
|
|
],
|
|
ids=[
|
|
"jpeg-as-png",
|
|
"gif-as-png",
|
|
"webp-as-png",
|
|
"text-as-png",
|
|
"png-as-jpeg",
|
|
"truncated-png",
|
|
"truncated-jpeg",
|
|
"empty",
|
|
"octet-stream",
|
|
"gif-type",
|
|
],
|
|
)
|
|
async def test_type_is_checked_by_content(client: AsyncClient, content: bytes, header: str) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, content, headers={"Content-Type": header})
|
|
assert response.status_code == 415, response.text
|
|
assert response.json()["error"]["code"] == "unsupported_media_type"
|
|
|
|
|
|
async def test_missing_content_type_is_unsupported(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, headers={})
|
|
assert response.status_code == 415
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"filename",
|
|
[
|
|
*("../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\x01.png", "a\x7f.png"),
|
|
*("", " ", "x" * 256, None),
|
|
],
|
|
)
|
|
async def test_dangerous_filename_is_rejected(client: AsyncClient, filename: str | None) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, filename=filename)
|
|
assert response.status_code == 400, response.text
|
|
assert response.json()["error"]["code"] == "invalid_request"
|
|
|
|
|
|
@pytest.mark.parametrize("filename", ["фото 1.png", "я" * 255, "%41.png"])
|
|
async def test_display_name_is_kept(client: AsyncClient, filename: str) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
response = await upload(client, alice, chat, filename=filename)
|
|
assert response.status_code == 202, response.text
|
|
assert response.json()["filename"] == filename
|
|
|
|
|
|
async def test_content_waits_for_ready_and_serves_level_three_variants(
|
|
client: AsyncClient,
|
|
) -> None:
|
|
alice, bob, _, chat = await trio(client)
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
path = f"/api/v1/attachments/{attachment['id']}/content"
|
|
for variant in ("original", "thumbnail", "crop"):
|
|
early = await client.get(f"{path}?variant={variant}", headers=actor(bob))
|
|
assert early.status_code == 409, early.text
|
|
assert early.json()["error"]["code"] == "conflict"
|
|
await settle(client)
|
|
assert (await client.get(f"{path}?variant=original", headers=actor(alice))).status_code == 200
|
|
thumbnail = await client.get(f"{path}?variant=thumbnail", headers=actor(bob))
|
|
meta = (await client.get(path.removesuffix("/content"), headers=actor(bob))).json()
|
|
(variant,) = meta["variants"]
|
|
assert thumbnail.headers["content-type"] == "image/jpeg"
|
|
assert len(thumbnail.content) == variant["size_bytes"]
|
|
assert sha256(thumbnail.content).hexdigest() == variant["sha256"]
|
|
assert (variant["width"], variant["height"]) == (256, 160)
|
|
for absent in ("crop", "optimized"):
|
|
missing = await client.get(f"{path}?variant={absent}", headers=actor(alice))
|
|
assert missing.status_code == 404
|
|
assert (await client.get(f"{path}?variant=preview", headers=actor(alice))).status_code == 400
|
|
assert (
|
|
await client.get("/api/v1/attachments/x/content", headers=actor(alice))
|
|
).status_code == 400
|
|
unknown = await client.get(f"/api/v1/attachments/{uuid4()}", headers=actor(alice))
|
|
assert unknown.status_code == 404
|
|
|
|
|
|
async def test_storage_failure_leaves_nothing_visible(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
ready = await upload(client, alice, chat)
|
|
await settle(client)
|
|
store = await storage(client)
|
|
store.failing = True
|
|
try:
|
|
failed = await upload(client, alice, chat, filename="failed.png")
|
|
download = await client.get(
|
|
f"/api/v1/attachments/{ready.json()['id']}/content", headers=actor(alice)
|
|
)
|
|
metadata = await client.get(
|
|
f"/api/v1/attachments/{ready.json()['id']}", headers=actor(alice)
|
|
)
|
|
finally:
|
|
store.failing = False
|
|
assert failed.status_code == 503
|
|
assert failed.json()["error"]["code"] == "unavailable"
|
|
assert download.status_code == 503
|
|
assert metadata.status_code == 200
|
|
status = await scalar(
|
|
"SELECT string_agg(status, ',') FROM attachments WHERE chat_id = $1 AND filename = $2",
|
|
chat["id"],
|
|
"failed.png",
|
|
)
|
|
assert status == "pending"
|
|
|
|
|
|
async def test_lost_object_is_not_found(client: AsyncClient) -> None:
|
|
alice, _, _, chat = await trio(client)
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
await settle(client)
|
|
(await storage(client)).objects.clear()
|
|
response = await client.get(
|
|
f"/api/v1/attachments/{attachment['id']}/content", headers=actor(alice)
|
|
)
|
|
assert response.status_code == 404
|
|
assert response.json()["error"]["code"] == "not_found"
|