Files
mtusi2026lab4/student-tests/test_attachments.py
T
Alexey BodanyukandClaude Opus 5.5 a2d867bec6 lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key;
  dry-run read-only конструктивно (default_transaction_read_only + S3 без записи),
  --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо),
  перепроверка под FOR UPDATE, предохранитель от массового удаления;
- attachment_ids хранится в строке сообщения (история остаётся одним index range scan),
  message_attachments — для FK и обратного поиска;
- семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY);
- тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки,
  приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler;
- бенчмарки памяти загрузок и стоимости обхода reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 14:05:03 +03:00

269 lines
14 KiB
Python

"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits,
content checks, filename rules and messages with attachment_ids.
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
filename, text=null / missing text / attachment_ids=null in a message."""
from __future__ import annotations
import hashlib
import subprocess
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
MiB = 1024 * 1024
@pytest.fixture()
def trio(api):
"""alice + bob share a chat; eve is an outsider with her own chat."""
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
def content(api, actor, attachment_id, expected=200, query=""):
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
# ------------------------------------------------------------------ positive path
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
alice, bob, _, chat, _ = trio
jpg = imagegen.jpeg(321, 123, noise=True)
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json()
assert meta == att
got = content(api, bob, att["id"])
assert got.content == data and got.headers["content-type"] == ctype
assert got.headers["content-length"] == str(len(data))
assert got.headers["x-content-type-options"] == "nosniff"
assert got.headers["content-disposition"].startswith("attachment;")
assert "location" not in got.headers # bytes through the API, never a redirect to S3
def test_metadata_reveals_no_storage_details(api, trio):
alice, _, _, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["object_key"].startswith("o/")
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower()
# The key is random: neither the id nor the filename is part of it.
assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"]
assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket
def test_original_is_not_stored_on_api_containers(api, trio):
"""The API keeps no file of the original: no recently written file of that size anywhere in
the replica's filesystem (the bytes live only in the bucket)."""
alice, _, _, chat, _ = trio
data = imagegen.png_of_size(300_000)
api.upload(alice, chat, data)
size = len(data)
for cid in compose("ps", "-q", "api").stdout.split():
out = subprocess.run(
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
capture_output=True, text=True, timeout=60).stdout.strip()
assert out == "", out
# ------------------------------------------------------------------ access control
def test_outsider_and_unknown_get_404(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
for suffix in ("", "/content"):
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
# Upload into a foreign chat: 404 before anything about the body is looked at.
api.upload(eve, chat, SAMPLE, 404)
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
def test_attachment_ids_are_checked_per_chat(api, trio):
alice, bob, eve, chat, eve_chat = trio
other = api.chat(alice) # alice's second chat
att_other = api.upload(alice, other, SAMPLE).json()
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
# Alice is a member of both chats, but a file is bound to its chat.
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
# Eve cannot post into alice's chat even with her own file: membership first.
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
# Any member may reference an attachment of the chat.
att = api.upload(alice, chat, SAMPLE).json()
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
# ------------------------------------------------------------------ messages with attachments
def test_message_with_attachments_and_history(api, trio):
alice, bob, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
assert m2["text"] == " " and m3["attachment_ids"] == []
items, _ = api.history(bob, chat, 2)
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
def test_idempotent_replay_with_attachments(api, trio):
alice, _, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
# A failed attachment check does not burn the key: the same key works with a valid body later.
key2 = uuid.uuid4().hex
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
@pytest.mark.parametrize("raw", [
b'{"attachment_ids":["%(a)s"]}', # text missing
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
b'{"text":1,"attachment_ids":["%(a)s"]}',
b'{"text":"x","attachment_ids":null}', # attachment_ids null
b'{"text":"x","attachment_ids":"%(a)s"}',
b'{"text":"x","attachment_ids":{}}',
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
b'{"text":"x","attachment_ids":["%(a)s",1]}',
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
b'{"text":"","attachment_ids":[]}', # neither text nor files
b'{"text":" "}',
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
], ids=lambda raw: raw.decode()[:40])
def test_message_body_validation(api, trio, raw):
alice, _, _, chat, _ = trio
a = api.upload(alice, chat, SAMPLE).json()["id"]
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
def test_at_most_ten_attachments(api, trio):
alice, _, _, chat, _ = trio
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
# ------------------------------------------------------------------ limits
def test_size_limit_through_the_load_balancer(api, trio):
alice, bob, _, chat, _ = trio
exact = imagegen.png_exactly(10 * MiB)
assert len(exact) == 10 * MiB
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
assert content(api, bob, att["id"]).content == exact
started = time.monotonic()
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
assert time.monotonic() - started < 5
over = exact + bytes(1024)
def chunks(): # no Content-Length: chunked body
for i in range(0, len(over), 256 * 1024):
yield over[i:i + 256 * 1024]
error = api.upload(alice, chat, chunks(), 413).json()
assert error["error"]["code"] == "payload_too_large"
def test_pixel_limit_before_decoding(api, trio):
alice, _, _, chat, _ = trio
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
started = time.monotonic()
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
def test_exactly_40_megapixels_is_accepted(api, trio):
alice, _, _, chat, _ = trio
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
att = api.upload(alice, chat, data).json()
assert att["size_bytes"] == len(data)
# ------------------------------------------------------------------ content checks
BROKEN = {
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
"png: truncated": (SAMPLE[:-20], "image/png"),
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
"type: text/plain": (SAMPLE, "text/plain"),
"type: octet-stream": (SAMPLE, "application/octet-stream"),
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
"type: missing": (SAMPLE, None),
}
@pytest.mark.parametrize("name", list(BROKEN))
def test_bad_content_is_415(api, trio, name):
alice, _, _, chat, _ = trio
data, ctype = BROKEN[name]
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
assert error["error"]["code"] == "unsupported_media_type"
def test_empty_body_is_400(api, trio):
alice, _, _, chat, _ = trio
api.upload(alice, chat, b"", 400)
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("query", [
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
], ids=lambda q: q[:40] or "missing")
def test_bad_filename_is_400(api, trio, query):
alice, _, _, chat, _ = trio
api.upload(alice, chat, SAMPLE, 400, query=query)
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
alice, _, _, chat, _ = trio
name = 'отчёт "Q3"; ok.png'
att = api.upload(alice, chat, SAMPLE, filename=name).json()
assert att["filename"] == name
disposition = content(api, alice, att["id"]).headers["content-disposition"]
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
assert "filename*=UTF-8''" in disposition
# ------------------------------------------------------------------ variants
def test_variant_parameter(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4
content(api, alice, att["id"], 400, query="?variant=huge")
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed