forked from mtusi2026/mtusi2026lab4
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key; dry-run read-only конструктивно (default_transaction_read_only + S3 без записи), --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо), перепроверка под FOR UPDATE, предохранитель от массового удаления; - attachment_ids хранится в строке сообщения (история остаётся одним index range scan), message_attachments — для FK и обратного поиска; - семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY); - тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки, приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler; - бенчмарки памяти загрузок и стоимости обхода reconciler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
269 lines
14 KiB
Python
269 lines
14 KiB
Python
"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits,
|
|
content checks, filename rules and messages with attachment_ids.
|
|
|
|
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
|
|
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
|
|
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
|
|
filename, text=null / missing text / attachment_ids=null in a message."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import imagegen
|
|
from conftest import attachment_row, compose, s3_size
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
MiB = 1024 * 1024
|
|
|
|
|
|
@pytest.fixture()
|
|
def trio(api):
|
|
"""alice + bob share a chat; eve is an outsider with her own chat."""
|
|
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
|
|
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
|
|
|
|
|
|
def content(api, actor, attachment_id, expected=200, query=""):
|
|
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
|
|
|
|
|
|
# ------------------------------------------------------------------ positive path
|
|
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
|
|
alice, bob, _, chat, _ = trio
|
|
jpg = imagegen.jpeg(321, 123, noise=True)
|
|
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
|
|
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
|
|
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None
|
|
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
|
|
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
|
|
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
|
|
meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json()
|
|
assert meta == att
|
|
got = content(api, bob, att["id"])
|
|
assert got.content == data and got.headers["content-type"] == ctype
|
|
assert got.headers["content-length"] == str(len(data))
|
|
assert got.headers["x-content-type-options"] == "nosniff"
|
|
assert got.headers["content-disposition"].startswith("attachment;")
|
|
assert "location" not in got.headers # bytes through the API, never a redirect to S3
|
|
|
|
|
|
def test_metadata_reveals_no_storage_details(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
row = attachment_row(att["id"])
|
|
assert row["status"] == "ready" and row["object_key"].startswith("o/")
|
|
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
|
|
assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower()
|
|
# The key is random: neither the id nor the filename is part of it.
|
|
assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"]
|
|
assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket
|
|
|
|
|
|
def test_original_is_not_stored_on_api_containers(api, trio):
|
|
"""The API keeps no file of the original: no recently written file of that size anywhere in
|
|
the replica's filesystem (the bytes live only in the bucket)."""
|
|
alice, _, _, chat, _ = trio
|
|
data = imagegen.png_of_size(300_000)
|
|
api.upload(alice, chat, data)
|
|
size = len(data)
|
|
for cid in compose("ps", "-q", "api").stdout.split():
|
|
out = subprocess.run(
|
|
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
|
|
capture_output=True, text=True, timeout=60).stdout.strip()
|
|
assert out == "", out
|
|
|
|
|
|
# ------------------------------------------------------------------ access control
|
|
def test_outsider_and_unknown_get_404(api, trio):
|
|
alice, _, eve, chat, _ = trio
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
for suffix in ("", "/content"):
|
|
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
|
|
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
|
|
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
|
|
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
|
|
# Upload into a foreign chat: 404 before anything about the body is looked at.
|
|
api.upload(eve, chat, SAMPLE, 404)
|
|
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
|
|
|
|
|
|
def test_attachment_ids_are_checked_per_chat(api, trio):
|
|
alice, bob, eve, chat, eve_chat = trio
|
|
other = api.chat(alice) # alice's second chat
|
|
att_other = api.upload(alice, other, SAMPLE).json()
|
|
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
|
|
# Alice is a member of both chats, but a file is bound to its chat.
|
|
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
|
|
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
|
|
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
|
|
# Eve cannot post into alice's chat even with her own file: membership first.
|
|
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
|
|
# Any member may reference an attachment of the chat.
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
|
|
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
|
|
|
|
|
|
# ------------------------------------------------------------------ messages with attachments
|
|
def test_message_with_attachments_and_history(api, trio):
|
|
alice, bob, _, chat, _ = trio
|
|
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
|
|
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
|
|
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
|
|
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
|
|
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
|
|
assert m2["text"] == " " and m3["attachment_ids"] == []
|
|
items, _ = api.history(bob, chat, 2)
|
|
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
|
|
|
|
|
|
def test_idempotent_replay_with_attachments(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
|
|
key = uuid.uuid4().hex
|
|
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
|
|
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
|
|
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
|
|
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
|
|
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
|
|
# A failed attachment check does not burn the key: the same key works with a valid body later.
|
|
key2 = uuid.uuid4().hex
|
|
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
|
|
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
|
|
|
|
|
|
@pytest.mark.parametrize("raw", [
|
|
b'{"attachment_ids":["%(a)s"]}', # text missing
|
|
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
|
|
b'{"text":1,"attachment_ids":["%(a)s"]}',
|
|
b'{"text":"x","attachment_ids":null}', # attachment_ids null
|
|
b'{"text":"x","attachment_ids":"%(a)s"}',
|
|
b'{"text":"x","attachment_ids":{}}',
|
|
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
|
|
b'{"text":"x","attachment_ids":["%(a)s",1]}',
|
|
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
|
|
b'{"text":"","attachment_ids":[]}', # neither text nor files
|
|
b'{"text":" "}',
|
|
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
|
|
], ids=lambda raw: raw.decode()[:40])
|
|
def test_message_body_validation(api, trio, raw):
|
|
alice, _, _, chat, _ = trio
|
|
a = api.upload(alice, chat, SAMPLE).json()["id"]
|
|
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
|
|
|
|
|
|
def test_at_most_ten_attachments(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
|
|
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
|
|
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
|
|
|
|
|
|
# ------------------------------------------------------------------ limits
|
|
def test_size_limit_through_the_load_balancer(api, trio):
|
|
alice, bob, _, chat, _ = trio
|
|
exact = imagegen.png_exactly(10 * MiB)
|
|
assert len(exact) == 10 * MiB
|
|
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
|
|
assert content(api, bob, att["id"]).content == exact
|
|
started = time.monotonic()
|
|
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
|
|
assert time.monotonic() - started < 5
|
|
over = exact + bytes(1024)
|
|
|
|
def chunks(): # no Content-Length: chunked body
|
|
for i in range(0, len(over), 256 * 1024):
|
|
yield over[i:i + 256 * 1024]
|
|
error = api.upload(alice, chat, chunks(), 413).json()
|
|
assert error["error"]["code"] == "payload_too_large"
|
|
|
|
|
|
def test_pixel_limit_before_decoding(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
|
|
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
|
|
started = time.monotonic()
|
|
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
|
|
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
|
|
|
|
|
|
def test_exactly_40_megapixels_is_accepted(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
|
|
att = api.upload(alice, chat, data).json()
|
|
assert att["size_bytes"] == len(data)
|
|
|
|
|
|
# ------------------------------------------------------------------ content checks
|
|
BROKEN = {
|
|
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
|
|
"png: truncated": (SAMPLE[:-20], "image/png"),
|
|
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
|
|
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
|
|
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
|
|
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
|
|
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
|
|
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
|
|
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
|
|
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
|
|
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
|
|
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
|
|
"type: text/plain": (SAMPLE, "text/plain"),
|
|
"type: octet-stream": (SAMPLE, "application/octet-stream"),
|
|
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
|
|
"type: missing": (SAMPLE, None),
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("name", list(BROKEN))
|
|
def test_bad_content_is_415(api, trio, name):
|
|
alice, _, _, chat, _ = trio
|
|
data, ctype = BROKEN[name]
|
|
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
|
|
assert error["error"]["code"] == "unsupported_media_type"
|
|
|
|
|
|
def test_empty_body_is_400(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
api.upload(alice, chat, b"", 400)
|
|
|
|
|
|
# ------------------------------------------------------------------ filename
|
|
@pytest.mark.parametrize("query", [
|
|
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
|
|
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
|
|
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
|
|
], ids=lambda q: q[:40] or "missing")
|
|
def test_bad_filename_is_400(api, trio, query):
|
|
alice, _, _, chat, _ = trio
|
|
api.upload(alice, chat, SAMPLE, 400, query=query)
|
|
|
|
|
|
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
|
|
alice, _, _, chat, _ = trio
|
|
name = 'отчёт "Q3"; ok.png'
|
|
att = api.upload(alice, chat, SAMPLE, filename=name).json()
|
|
assert att["filename"] == name
|
|
disposition = content(api, alice, att["id"]).headers["content-disposition"]
|
|
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
|
|
assert "filename*=UTF-8''" in disposition
|
|
|
|
|
|
# ------------------------------------------------------------------ variants
|
|
def test_variant_parameter(api, trio):
|
|
alice, _, eve, chat, _ = trio
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
|
|
content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4
|
|
content(api, alice, att["id"], 400, query="?variant=huge")
|
|
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed
|