forked from mtusi2026/mtusi2026lab5
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
218 lines
13 KiB
Python
218 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Негативные и граничные сценарии вложений (из lab4; в lab5 upload -> 202, затем ready). Запуск на учебном стенде:
|
||
|
||
python3 student-tests/attachments_negative.py [--base-url http://localhost:8080] [--output evidence/attachments-negative.json]
|
||
|
||
Использует публичный tests/client.py: каждый ответ проверяется на код, X-Request-Id,
|
||
X-Instance-Id и (для ошибок) формат {"error":{code,message,request_id}}. Отдельно
|
||
проверяется, что ни один пользовательский ввод не дал 5xx. Все файлы синтетические и
|
||
генерируются на лету (большие в Git не хранятся).
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import struct
|
||
import sys
|
||
import uuid
|
||
import zlib
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "tests"))
|
||
from client import Client, Failure, check # noqa: E402
|
||
|
||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||
JPEG = (ROOT / "student-tests/fixtures/sample.jpg").read_bytes()
|
||
MAX_BYTES = 10 * 1024 * 1024
|
||
results = []
|
||
|
||
|
||
def chunk(ctype, data):
|
||
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data) & 0xFFFFFFFF)
|
||
|
||
|
||
def png(width, height, raw, color_type=2, depth=8):
|
||
ihdr = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0)
|
||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", raw) + chunk(b"IEND", b"")
|
||
|
||
|
||
def noise_png(target_bytes):
|
||
"""Несжимаемый RGB PNG размером чуть меньше target_bytes (zlib level 0)."""
|
||
width = 1024
|
||
row = 1 + width * 3
|
||
height = (target_bytes - 200) // (row + 6 * 0 + 1)
|
||
while True:
|
||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||
data = png(width, height, zlib.compress(raw, 0))
|
||
if len(data) <= target_bytes:
|
||
return data
|
||
height -= 16
|
||
|
||
|
||
class Suite:
|
||
def __init__(self, c):
|
||
self.c = c
|
||
|
||
def case(self, name, method, path, expected, actor=None, **kw):
|
||
try:
|
||
r = self.c.call(method, path, expected, actor=actor, **kw)
|
||
code = r["json"]["error"]["code"] if r["json"] and "error" in r["json"] else None
|
||
results.append({"case": name, "expected": expected, "status": r["status"], "error_code": code, "pass": True})
|
||
return r
|
||
except Failure as exc:
|
||
results.append({"case": name, "expected": expected, "error": str(exc), "pass": False})
|
||
return None
|
||
|
||
def upload(self, name, actor, chat, data, expected, ctype="image/png", filename="f.png", raw_query=None):
|
||
q = raw_query if raw_query is not None else "filename=" + quote(filename, safe="")
|
||
return self.case(name, "POST", "/api/v1/chats/%s/attachments?%s" % (chat["id"], q), expected,
|
||
actor=actor, raw=data, headers={"Content-Type": ctype},
|
||
schema="Attachment" if expected == 202 else None)
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||
ap.add_argument("--output")
|
||
args = ap.parse_args()
|
||
c = Client(args.base_url, timeout=60)
|
||
s = Suite(c)
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice,
|
||
body={"title": "второй чат", "member_ids": [alice["id"], bob["id"]]}, schema="Chat")["json"]
|
||
eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve,
|
||
body={"title": "чат Евы", "member_ids": [eve["id"]]}, schema="Chat")["json"]
|
||
|
||
# --- позитивные контрольные ---
|
||
ok_png = s.upload("valid PNG fixture", alice, chat, PNG, 202)
|
||
ok_jpg = s.upload("valid JPEG", bob, chat, JPEG, 202, ctype="image/jpeg", filename="photo.jpg")
|
||
uni = s.upload("unicode filename", alice, chat, PNG, 202, filename="фото отпуска 🙂.png")
|
||
if uni:
|
||
check(uni["json"]["filename"] == "фото отпуска 🙂.png", "unicode filename changed")
|
||
def ready(r): # lab5: upload -> 202 queued, байты доступны после ready
|
||
if r:
|
||
c.wait_ready(alice if r["json"]["owner_id"] == alice["id"] else bob, r["json"])
|
||
for r in (ok_png, ok_jpg):
|
||
ready(r)
|
||
if ok_jpg:
|
||
d = c.call("GET", "/api/v1/attachments/%s/content" % ok_jpg["json"]["id"], actor=alice)
|
||
check(d["bytes"] == JPEG and d["headers"].get_content_type() == "image/jpeg", "jpeg download mismatch")
|
||
check(d["headers"].get("X-Content-Type-Options") == "nosniff", "nosniff missing")
|
||
near = noise_png(MAX_BYTES)
|
||
big = s.upload("PNG near 10 MiB limit (%d B)" % len(near), alice, chat, near, 202, filename="big.png")
|
||
if big:
|
||
check(big["json"]["sha256"] == hashlib.sha256(near).hexdigest(), "big sha mismatch")
|
||
ready(big)
|
||
d = c.call("GET", "/api/v1/attachments/%s/content" % big["json"]["id"], actor=bob)
|
||
check(hashlib.sha256(d["bytes"]).hexdigest() == big["json"]["sha256"], "big download mismatch")
|
||
|
||
# --- размер и пиксели ---
|
||
s.upload("too large: 10 MiB + 1 B", alice, chat, near + b"\0" * (MAX_BYTES + 1 - len(near)), 413)
|
||
s.upload("pixel bomb header 100000x100000", alice, chat, png(100000, 100000, zlib.compress(b"\0" * 1000)), 413)
|
||
s.upload("PNG 7000x6000 = 42 Mpx (> 40 Mpx)", alice, chat, png(7000, 6000, zlib.compress(b"\0" * 1000)), 413)
|
||
s.upload("zlib bomb: 100x100 header, 50 MB inflated", alice, chat,
|
||
png(100, 100, zlib.compress(b"\0" * 50_000_000, 9), color_type=0), 415)
|
||
|
||
# --- повреждённые изображения ---
|
||
flipped = bytearray(PNG); flipped[5000] ^= 0xFF
|
||
s.upload("corrupted PNG (byte flipped, bad CRC)", alice, chat, bytes(flipped), 415)
|
||
s.upload("truncated PNG", alice, chat, PNG[:-4000], 415)
|
||
s.upload("PNG without IEND", alice, chat, PNG[:-12], 415)
|
||
s.upload("PNG with appended ZIP (polyglot)", alice, chat, PNG + b"PK\x03\x04" + os.urandom(64), 415)
|
||
s.upload("truncated JPEG", alice, chat, JPEG[:len(JPEG) // 2], 415, ctype="image/jpeg")
|
||
s.upload("JPEG header only", alice, chat, JPEG[:200], 415, ctype="image/jpeg")
|
||
s.upload("empty body", alice, chat, b"", 415)
|
||
|
||
# --- подмена MIME ---
|
||
s.upload("JPEG bytes declared image/png", alice, chat, JPEG, 415)
|
||
s.upload("PNG bytes declared image/jpeg", alice, chat, PNG, 415, ctype="image/jpeg")
|
||
s.upload("text declared image/png", alice, chat, b"<html><script>alert(1)</script></html>" * 4, 415)
|
||
s.upload("GIF declared image/png", alice, chat, b"GIF89a" + os.urandom(200), 415)
|
||
s.upload("PNG declared text/html", alice, chat, PNG, 415, ctype="text/html")
|
||
s.upload("PNG declared image/svg+xml", alice, chat, PNG, 415, ctype="image/svg+xml")
|
||
s.upload("multipart/form-data instead of raw", alice, chat, b"--x\r\n\r\n" + PNG + b"\r\n--x--", 415,
|
||
ctype="multipart/form-data; boundary=x")
|
||
|
||
# --- опасные filename ---
|
||
for label, query in (("../etc/passwd", "filename=" + quote("../passwd.png", safe="")),
|
||
("slash", "filename=" + quote("a/b.png", safe="")),
|
||
("backslash", "filename=" + quote("a\\b.png", safe="")),
|
||
("NUL", "filename=a%00.png"),
|
||
("newline", "filename=a%0A.png"),
|
||
("bidi override U+202E", "filename=" + quote("gnp.exe", safe="")),
|
||
("..", "filename=.."),
|
||
("empty", "filename="),
|
||
("whitespace only", "filename=%20%20"),
|
||
("256 code points", "filename=" + quote("я" * 256, safe="")),
|
||
("missing filename", ""),
|
||
("filename twice", "filename=a.png&filename=b.png"),
|
||
("invalid UTF-8", "filename=%FF%FE.png")):
|
||
s.upload("filename: " + label, alice, chat, PNG, 400, raw_query=query)
|
||
|
||
# --- доступ и чужие объекты ---
|
||
eve_att = s.upload("eve uploads to her own chat", eve, eve_chat, PNG, 202)
|
||
s.upload("non-member uploads to alice chat", eve, chat, PNG, 404)
|
||
s.case("upload without session", "POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 401,
|
||
raw=PNG, headers={"Content-Type": "image/png"})
|
||
s.upload("upload to unknown chat", alice, {"id": str(uuid.uuid4())}, PNG, 404)
|
||
if eve_att and ok_png:
|
||
eid, aid = eve_att["json"]["id"], ok_png["json"]["id"]
|
||
s.case("foreign attachment metadata", "GET", "/api/v1/attachments/" + eid, 404, actor=alice)
|
||
s.case("foreign attachment content", "GET", "/api/v1/attachments/%s/content" % eid, 404, actor=alice)
|
||
s.case("eve reads alice attachment", "GET", "/api/v1/attachments/%s/content" % aid, 404, actor=eve)
|
||
key = lambda: {"Idempotency-Key": uuid.uuid4().hex} # noqa: E731
|
||
s.case("message with foreign attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404,
|
||
actor=alice, body={"text": "x", "attachment_ids": [eid]}, headers=key())
|
||
s.case("message mixing own and foreign ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404,
|
||
actor=alice, body={"text": "x", "attachment_ids": [aid, eid]}, headers=key())
|
||
s.case("attachment of OTHER chat of the same user", "POST", "/api/v1/chats/%s/messages" % chat2["id"], 404,
|
||
actor=alice, body={"text": "x", "attachment_ids": [aid]}, headers=key())
|
||
s.case("unknown attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404, actor=alice,
|
||
body={"text": "x", "attachment_ids": [str(uuid.uuid4())]}, headers=key())
|
||
s.case("malformed attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||
body={"text": "x", "attachment_ids": ["not-a-uuid"]}, headers=key())
|
||
s.case("duplicate attachment_ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||
body={"text": "x", "attachment_ids": [aid, aid]}, headers=key())
|
||
s.case("11 attachment_ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||
body={"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, headers=key())
|
||
s.case("attachment_ids not a list", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||
body={"text": "x", "attachment_ids": aid}, headers=key())
|
||
s.case("empty text without attachments", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400,
|
||
actor=alice, body={"text": " "}, headers=key())
|
||
m = s.case("empty text with attachment (bob links alice's file)", "POST",
|
||
"/api/v1/chats/%s/messages" % chat["id"], 201, actor=bob,
|
||
body={"text": "", "attachment_ids": [aid]}, headers=key(), schema="Message")
|
||
if m:
|
||
check(m["json"]["attachment_ids"] == [aid], "attachment_ids lost")
|
||
k = key()
|
||
s.case("idempotent send with attachments", "POST", "/api/v1/chats/%s/messages" % chat["id"], 201,
|
||
actor=alice, body={"attachment_ids": [aid], "text": "повтор"}, headers=k, schema="Message")
|
||
s.case("same key, same body (key order changed)", "POST", "/api/v1/chats/%s/messages" % chat["id"], 201,
|
||
actor=alice, body={"text": "повтор", "attachment_ids": [aid]}, headers=k, schema="Message")
|
||
s.case("same key, different attachment_ids -> 409", "POST", "/api/v1/chats/%s/messages" % chat["id"], 409,
|
||
actor=alice, body={"text": "повтор", "attachment_ids": []}, headers=k)
|
||
s.case("variant=thumbnail (ready, lab5)", "GET",
|
||
"/api/v1/attachments/%s/content?variant=thumbnail" % aid, 200, actor=bob)
|
||
s.case("variant=bogus", "GET", "/api/v1/attachments/%s/content?variant=bogus" % aid, 400, actor=bob)
|
||
s.case("variant=original", "GET", "/api/v1/attachments/%s/content?variant=original" % aid, 200, actor=bob)
|
||
s.case("malformed attachment id in path", "GET", "/api/v1/attachments/zzz", 400, actor=bob)
|
||
s.case("unknown attachment id", "GET", "/api/v1/attachments/" + str(uuid.uuid4()), 404, actor=bob)
|
||
|
||
failed = [r for r in results if not r["pass"]]
|
||
server_errors = [r for r in results if r.get("status", 0) >= 500]
|
||
for r in results:
|
||
print(("PASS " if r["pass"] else "FAIL ") + "%-55s -> %s %s" % (
|
||
r["case"], r.get("status", "-"), r.get("error_code") or r.get("error", "")))
|
||
summary = {"cases": len(results), "failed": len(failed), "server_errors_5xx": len(server_errors),
|
||
"pass": not failed and not server_errors}
|
||
print(json.dumps(summary, ensure_ascii=False))
|
||
if args.output:
|
||
Path(args.output).write_text(json.dumps({"summary": summary, "results": results}, ensure_ascii=False, indent=1))
|
||
return 0 if summary["pass"] else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|