forked from mtusi2026/mtusi2026lab4
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
356 lines
16 KiB
Python
356 lines
16 KiB
Python
"""Lab4 grade 3/4: attachment upload/download, ACL, limits, MIME sniffing, filenames."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import hashlib
|
|
import http.client
|
|
import socket
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
|
|
import imagegen
|
|
from helpers import BASE_URL, call, chat_for, register, upload
|
|
|
|
MIB = 1024 * 1024
|
|
LIMIT = 10 * MIB
|
|
|
|
|
|
def _trio():
|
|
alice, bob, eve = register("att_a"), register("att_b"), register("att_e")
|
|
return alice, bob, eve, chat_for(alice, bob)
|
|
|
|
|
|
def _png(seed: int = 0) -> bytes:
|
|
return imagegen.rgb_png(16 + seed % 7, 9 + seed % 5)
|
|
|
|
|
|
def _download(token, att_id, expected=200, query=""):
|
|
return call("GET", f"/api/v1/attachments/{att_id}/content{query}", expected, token=token)
|
|
|
|
|
|
def _message(actor, chat, text, attachment_ids=None, expected=201, key=None):
|
|
body = {"text": text}
|
|
if attachment_ids is not None:
|
|
body["attachment_ids"] = attachment_ids
|
|
return call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
expected,
|
|
token=actor["token"],
|
|
body=body,
|
|
headers={"Idempotency-Key": key or uuid.uuid4().hex},
|
|
)
|
|
|
|
|
|
def test_upload_roundtrip_png_and_jpeg_variants():
|
|
alice, bob, _eve, chat = _trio()
|
|
cases = [
|
|
(imagegen.PUBLIC_PNG.read_bytes(), "image/png", "sample.png"),
|
|
(imagegen.fixture("sample.jpg"), "image/jpeg", "sample.jpg"),
|
|
(imagegen.fixture("progressive.jpg"), "image/jpeg", "progressive.jpg"),
|
|
(imagegen.fixture("gray.jpg"), "image/jpeg", "gray.jpg"),
|
|
(imagegen.interlaced_rgb_png(41, 29), "image/png", "adam7.png"),
|
|
(imagegen.palette_png(12, 7), "image/png", "palette.png"),
|
|
]
|
|
for data, ctype, name in cases:
|
|
resp = upload(alice["token"], chat["id"], data, name, ctype, 201)
|
|
att = resp["json"]
|
|
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, att
|
|
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
|
|
assert att["content_type"] == ctype and att["filename"] == name
|
|
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
|
|
assert set(att) == {"id", "chat_id", "owner_id", "filename", "content_type", "size_bytes",
|
|
"sha256", "status", "variants", "error_code", "created_at", "updated_at"}
|
|
assert resp["headers"].get("Location") == "/api/v1/attachments/" + att["id"]
|
|
meta = call("GET", "/api/v1/attachments/" + att["id"], 200, token=bob["token"])["json"]
|
|
assert meta == att
|
|
dl = _download(bob["token"], att["id"])
|
|
assert dl["bytes"] == data, name
|
|
assert dl["headers"].get_content_type() == ctype
|
|
assert dl["headers"].get("Content-Length") == str(len(data))
|
|
assert dl["headers"].get("X-Content-Type-Options") == "nosniff"
|
|
assert dl["headers"].get("ETag") == '"' + att["sha256"] + '"'
|
|
|
|
|
|
def test_outsider_and_unknown_get_404():
|
|
alice, _bob, eve, chat = _trio()
|
|
att = upload(alice["token"], chat["id"], _png(), "secret.png", "image/png", 201)["json"]
|
|
for suffix in ("", "/content", "/content?variant=original"):
|
|
resp = call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, token=eve["token"])
|
|
assert resp["json"]["error"]["code"] == "not_found"
|
|
unknown = str(uuid.uuid4())
|
|
call("GET", "/api/v1/attachments/" + unknown, 404, token=alice["token"])
|
|
call("GET", "/api/v1/attachments/" + unknown + "/content", 404, token=alice["token"])
|
|
call("GET", "/api/v1/attachments/not-a-uuid", 400, token=alice["token"])
|
|
call("GET", "/api/v1/attachments/" + att["id"], 401)
|
|
call("GET", "/api/v1/attachments/" + att["id"] + "/content", 401)
|
|
# Outsider cannot upload into the chat either.
|
|
upload(eve["token"], chat["id"], _png(), "x.png", "image/png", 404)
|
|
|
|
|
|
def test_upload_auth_and_chat_checks():
|
|
alice, _bob, _eve, chat = _trio()
|
|
upload(None, chat["id"], _png(), "x.png", "image/png", 401)
|
|
upload(alice["token"], str(uuid.uuid4()), _png(), "x.png", "image/png", 404)
|
|
upload(alice["token"], "not-a-uuid", _png(), "x.png", "image/png", 400)
|
|
|
|
|
|
def test_message_with_attachments_and_history():
|
|
alice, bob, _eve, chat = _trio()
|
|
a1 = upload(alice["token"], chat["id"], _png(1), "one.png", "image/png", 201)["json"]
|
|
a2 = upload(bob["token"], chat["id"], _png(2), "two.png", "image/png", 201)["json"]
|
|
key = uuid.uuid4().hex
|
|
msg = _message(bob, chat, "", [a2["id"], a1["id"]], key=key)["json"]
|
|
assert msg["attachment_ids"] == [a2["id"], a1["id"]] and msg["text"] == ""
|
|
replay = _message(bob, chat, "", [a2["id"], a1["id"]], key=key)["json"]
|
|
assert replay == msg
|
|
_message(bob, chat, "", [a1["id"], a2["id"]], expected=409, key=key)
|
|
plain_key = uuid.uuid4().hex
|
|
plain = _message(alice, chat, "без вложений", key=plain_key)["json"]
|
|
assert plain["attachment_ids"] == []
|
|
# Absent attachment_ids ≡ []: same key replays the same message, another set conflicts.
|
|
assert _message(alice, chat, "без вложений", [], key=plain_key)["json"] == plain
|
|
_message(alice, chat, "без вложений", [a1["id"]], expected=409, key=plain_key)
|
|
# Any member may reference any attachment of the chat, also repeatedly.
|
|
again = _message(alice, chat, "ещё раз", [a2["id"]])["json"]
|
|
assert again["attachment_ids"] == [a2["id"]]
|
|
page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=alice["token"])["json"]
|
|
by_id = {m["id"]: m for m in page["items"]}
|
|
assert by_id[msg["id"]]["attachment_ids"] == [a2["id"], a1["id"]]
|
|
assert by_id[plain["id"]]["attachment_ids"] == []
|
|
assert by_id[again["id"]]["attachment_ids"] == [a2["id"]]
|
|
|
|
|
|
def test_foreign_and_unknown_attachment_ids_404():
|
|
alice, bob, eve, chat = _trio()
|
|
other = chat_for(alice, eve, title="other chat")
|
|
foreign = upload(alice["token"], other["id"], _png(3), "other.png", "image/png", 201)["json"]
|
|
own = upload(alice["token"], chat["id"], _png(4), "own.png", "image/png", 201)["json"]
|
|
# File of another chat (even uploaded by the same user) cannot be attached.
|
|
resp = _message(alice, chat, "чужой файл", [foreign["id"]], expected=404)
|
|
assert resp["json"]["error"]["code"] == "not_found"
|
|
_message(alice, chat, "смесь", [own["id"], foreign["id"]], expected=404)
|
|
_message(bob, chat, "неизвестный", [str(uuid.uuid4())], expected=404)
|
|
# Eve is in `other` only: cannot attach chat's file there.
|
|
_message(eve, other, "украсть", [own["id"]], expected=404)
|
|
# Failed attempts did not create messages.
|
|
page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])["json"]
|
|
assert page["items"] == []
|
|
|
|
|
|
def test_attachment_ids_validation_400():
|
|
alice, _bob, _eve, chat = _trio()
|
|
att = upload(alice["token"], chat["id"], _png(), "v.png", "image/png", 201)["json"]
|
|
many = [str(uuid.uuid4()) for _ in range(11)]
|
|
for bad in (many, [att["id"], att["id"]], ["not-a-uuid"], [123], "x", {"id": att["id"]}):
|
|
_message(alice, chat, "x", bad, expected=400)
|
|
call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
400,
|
|
token=alice["token"],
|
|
body={"text": "x", "attachment_ids": None},
|
|
)
|
|
_message(alice, chat, "", [], expected=400)
|
|
_message(alice, chat, " ", None, expected=400)
|
|
_message(alice, chat, " ", [att["id"]])
|
|
|
|
|
|
def test_too_large_413():
|
|
alice, _bob, _eve, chat = _trio()
|
|
big = imagegen.large_png(LIMIT + 64 * 1024)
|
|
assert len(big) > LIMIT
|
|
resp = upload(alice["token"], chat["id"], big, "big.png", "image/png", 413)
|
|
assert resp["json"]["error"]["code"] == "payload_too_large"
|
|
# Same oversized body without Content-Length (chunked): detected while streaming.
|
|
parsed = urllib.parse.urlparse(BASE_URL)
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=60)
|
|
|
|
def body():
|
|
for i in range(0, len(big), 256 * 1024):
|
|
yield big[i:i + 256 * 1024]
|
|
|
|
conn.request(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=big.png",
|
|
body=body(),
|
|
headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"},
|
|
encode_chunked=True,
|
|
)
|
|
r = conn.getresponse()
|
|
r.read()
|
|
conn.close()
|
|
assert r.status == 413, r.status
|
|
|
|
|
|
def test_near_limit_real_png_accepted():
|
|
alice, bob, _eve, chat = _trio()
|
|
data = imagegen.large_png(LIMIT - 256 * 1024)
|
|
assert len(data) <= LIMIT
|
|
att = upload(alice["token"], chat["id"], data, "near-limit.png", "image/png", 201)["json"]
|
|
dl = _download(bob["token"], att["id"])
|
|
assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"] == hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def test_exact_limit_png_through_lb_accepted():
|
|
# 10 485 760 bytes exactly passes the LB and the API; one more byte is JSON 413 from the API.
|
|
alice, bob, _eve, chat = _trio()
|
|
data = imagegen.exact_size_png(LIMIT)
|
|
att = upload(alice["token"], chat["id"], data, "exact.png", "image/png", 201)["json"]
|
|
assert att["size_bytes"] == LIMIT
|
|
assert hashlib.sha256(_download(bob["token"], att["id"])["bytes"]).hexdigest() == hashlib.sha256(data).hexdigest()
|
|
resp = upload(alice["token"], chat["id"], imagegen.exact_size_png(LIMIT + 1), "plus1.png", "image/png", 413)
|
|
assert resp["json"]["error"]["code"] == "payload_too_large"
|
|
|
|
|
|
def test_too_many_pixels_413():
|
|
alice, _bob, _eve, chat = _trio()
|
|
bomb = imagegen.gray1_png(8000, 5001) # 40 008 000 px in a few KiB
|
|
assert len(bomb) < 64 * 1024
|
|
resp = upload(alice["token"], chat["id"], bomb, "bomb.png", "image/png", 413)
|
|
assert resp["json"]["error"]["code"] == "payload_too_large"
|
|
upload(alice["token"], chat["id"], imagegen.gray1_png(8000, 5000), "limit.png", "image/png", 201)
|
|
|
|
|
|
def test_corrupted_images_415():
|
|
alice, _bob, _eve, chat = _trio()
|
|
png = _png(5)
|
|
flipped = bytearray(png)
|
|
flipped[-20] ^= 0xFF
|
|
jpeg = imagegen.fixture("sample.jpg")
|
|
for data, ctype in (
|
|
(png[: len(png) // 2], "image/png"),
|
|
(bytes(flipped), "image/png"),
|
|
(png + b"<script>", "image/png"),
|
|
(jpeg[: len(jpeg) - 200], "image/jpeg"),
|
|
(b"\xff\xd8\xff\xe0" + bytes(range(256)) * 8, "image/jpeg"),
|
|
(b"\x89PNG\r\n\x1a\n" + b"\x00" * 64, "image/png"),
|
|
):
|
|
resp = upload(alice["token"], chat["id"], data, "broken", ctype, 415)
|
|
assert resp["json"]["error"]["code"] == "unsupported_media_type"
|
|
|
|
|
|
def test_reviewer_corrupted_images_415():
|
|
# Same inputs reviewers verify with an independent decoder; valid edge cases stay 201.
|
|
alice, bob, _eve, chat = _trio()
|
|
for name, data in imagegen.corrupted_pngs().items():
|
|
resp = upload(alice["token"], chat["id"], data, name + ".png", "image/png", 415)
|
|
assert resp["json"]["error"]["code"] == "unsupported_media_type", name
|
|
for name, data in imagegen.corrupted_jpegs().items():
|
|
resp = upload(alice["token"], chat["id"], data, name + ".jpg", "image/jpeg", 415)
|
|
assert resp["json"]["error"]["code"] == "unsupported_media_type", name
|
|
for data, ctype in ((imagegen.all_filters_png(33, 20, interlace=1), "image/png"),
|
|
(imagegen.fixture("progressive.jpg") + b"trailing", "image/jpeg")):
|
|
att = upload(alice["token"], chat["id"], data, "edge", ctype, 201)["json"]
|
|
assert _download(bob["token"], att["id"])["bytes"] == data
|
|
page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])["json"]
|
|
assert page["items"] == []
|
|
|
|
|
|
def test_mime_spoofing_415():
|
|
alice, _bob, _eve, chat = _trio()
|
|
png = _png(6)
|
|
jpeg = imagegen.fixture("sample.jpg")
|
|
cases = [
|
|
(b"GIF89a\x01\x00\x01\x00\x00\x00\x00;", "image/png"),
|
|
(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n", "image/jpeg"),
|
|
(b"<html><script>alert(1)</script></html>", "image/png"),
|
|
(b"MZ\x90\x00\x03\x00\x00\x00", "image/jpeg"),
|
|
(jpeg, "image/png"),
|
|
(png, "image/jpeg"),
|
|
(png, "text/plain"),
|
|
(png, "image/gif"),
|
|
(png, "multipart/form-data; boundary=x"),
|
|
(png, "application/octet-stream"),
|
|
]
|
|
for data, ctype in cases:
|
|
resp = upload(alice["token"], chat["id"], data, "spoof.png", ctype, 415)
|
|
assert resp["json"]["error"]["code"] == "unsupported_media_type", ctype
|
|
|
|
|
|
def test_dangerous_filenames_400_and_unicode_kept():
|
|
alice, bob, _eve, chat = _trio()
|
|
png = _png(7)
|
|
for name in ("../etc/passwd", "../x.png", "a/b.png", "a\\b.png", "x\x00.png", "x\n.png", "x\r.png",
|
|
"del\x7f.png", "c1\x85.png",
|
|
"tab\t.png", "\u202egnp.exe", "zero\u200bwidth.png", "", " ", ".", "..", "a" * 256):
|
|
resp = upload(alice["token"], chat["id"], png, name, "image/png", 400)
|
|
assert resp["json"]["error"]["code"] == "invalid_request", repr(name)
|
|
upload(alice["token"], chat["id"], png, None, "image/png", 400)
|
|
for name in ("фото отпуска 😀.png", "b" * 255, "with space & ampersand;=.png"):
|
|
att = upload(alice["token"], chat["id"], png, name, "image/png", 201)["json"]
|
|
assert att["filename"] == name
|
|
call("GET", "/api/v1/attachments/" + att["id"], 200, token=bob["token"])
|
|
|
|
|
|
def test_filename_query_decoded_strictly_400():
|
|
# Invalid UTF-8 must not be "sanitized" into U+FFFD; an ambiguous repeated filename is rejected too.
|
|
alice, _bob, _eve, chat = _trio()
|
|
png = _png(10)
|
|
path = f"/api/v1/chats/{chat['id']}/attachments"
|
|
for query in ("?filename=%FF%FE.png", "?filename=%C3.png", "?filename=%ED%A0%80.png",
|
|
"?filename=ok.png&filename=a%2Fb", "?filename=a.png&filename=b.png"):
|
|
resp = call("POST", path + query, 400, token=alice["token"], raw=png, headers={"Content-Type": "image/png"})
|
|
assert resp["json"]["error"]["code"] == "invalid_request", query
|
|
att = call("POST", path + "?filename=%D1%84%2B1+2.png", 201, token=alice["token"], raw=png,
|
|
headers={"Content-Type": "image/png"})["json"]
|
|
assert att["filename"] == "ф+1 2.png"
|
|
|
|
|
|
def test_same_filename_distinct_objects():
|
|
alice, bob, _eve, chat = _trio()
|
|
first, second = _png(8), _png(9)
|
|
a = upload(alice["token"], chat["id"], first, "same.png", "image/png", 201)["json"]
|
|
b = upload(alice["token"], chat["id"], second, "same.png", "image/png", 201)["json"]
|
|
assert a["id"] != b["id"]
|
|
assert _download(bob["token"], a["id"])["bytes"] == first
|
|
assert _download(bob["token"], b["id"])["bytes"] == second
|
|
|
|
|
|
def test_empty_body_and_variants():
|
|
alice, bob, _eve, chat = _trio()
|
|
upload(alice["token"], chat["id"], b"", "empty.png", "image/png", 400)
|
|
att = upload(alice["token"], chat["id"], _png(), "v.png", "image/png", 201)["json"]
|
|
_download(bob["token"], att["id"], 200, "?variant=original")
|
|
_download(bob["token"], att["id"], 404, "?variant=thumbnail")
|
|
_download(bob["token"], att["id"], 400, "?variant=../../etc")
|
|
|
|
|
|
def test_parallel_uploads_distinct():
|
|
alice, bob, _eve, chat = _trio()
|
|
blobs = [_png(i) + b"" for i in range(12)]
|
|
|
|
def send(i):
|
|
return upload(alice["token"], chat["id"], blobs[i], f"p{i}.png", "image/png", 201)["json"]
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
|
results = list(pool.map(send, range(12)))
|
|
assert len({r["id"] for r in results}) == 12
|
|
for i, att in enumerate(results):
|
|
assert att["sha256"] == hashlib.sha256(blobs[i]).hexdigest()
|
|
|
|
|
|
def _raw_partial_upload(token: str, chat_id: str, declared: int, payload: bytes, hold: float = 0.0) -> None:
|
|
parsed = urllib.parse.urlparse(BASE_URL)
|
|
sock = socket.create_connection((parsed.hostname, parsed.port or 80), timeout=30)
|
|
head = (
|
|
f"POST /api/v1/chats/{chat_id}/attachments?filename=cut.png HTTP/1.1\r\n"
|
|
f"Host: {parsed.hostname}\r\nAuthorization: Bearer {token}\r\n"
|
|
f"Content-Type: image/png\r\nContent-Length: {declared}\r\n\r\n"
|
|
).encode()
|
|
sock.sendall(head + payload)
|
|
if hold:
|
|
time.sleep(hold)
|
|
sock.close()
|
|
|
|
|
|
def test_interrupted_upload_does_not_break_service():
|
|
alice, bob, _eve, chat = _trio()
|
|
big = imagegen.large_png(2 * MIB)
|
|
for _ in range(3):
|
|
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 300 * 1024])
|
|
att = upload(alice["token"], chat["id"], _png(), "after.png", "image/png", 201)["json"]
|
|
_download(bob["token"], att["id"])
|