forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -29,14 +29,15 @@ def upload(args):
|
||||
response = c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
|
||||
201,
|
||||
202,
|
||||
actor=alice,
|
||||
raw=content,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
attachment = response["json"]
|
||||
check(attachment["status"] == "ready", "upload is not ready")
|
||||
accepted_by = response["headers"].get("X-Instance-Id")
|
||||
attachment = c.wait_ready(alice, response["json"], wait_seconds=60)
|
||||
check(attachment["status"] == "ready", "upload did not become ready")
|
||||
check(
|
||||
attachment["sha256"] == hashlib.sha256(content).hexdigest(),
|
||||
"upload checksum mismatch",
|
||||
@@ -45,7 +46,7 @@ def upload(args):
|
||||
"attachment_id": attachment["id"],
|
||||
"sha256": attachment["sha256"],
|
||||
"size_bytes": attachment["size_bytes"],
|
||||
"instance_id": response["headers"].get("X-Instance-Id"),
|
||||
"instance_id": accepted_by,
|
||||
"token": alice["token"],
|
||||
"chat_id": chat["id"],
|
||||
"content_length": len(content),
|
||||
|
||||
@@ -9,8 +9,10 @@ envelope, not a 500.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import struct
|
||||
import sys
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
@@ -20,6 +22,34 @@ from client import Client, check # noqa: E402
|
||||
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def png_chunk(kind, payload):
|
||||
crc = zlib.crc32(kind + payload) & 0xFFFFFFFF
|
||||
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def oversized_png(width, height):
|
||||
"""Build a valid grayscale PNG whose dimensions exceed the pixel cap.
|
||||
|
||||
The image is solid black, so the IDAT stream stays tiny and the request is
|
||||
far below the byte limit. Only the declared width*height can trigger 413.
|
||||
"""
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0)
|
||||
compressor = zlib.compressobj(9)
|
||||
idat = bytearray()
|
||||
row = b"\x00" + b"\x00" * width # filter type 0 + one 8-bit sample per pixel
|
||||
for _ in range(height):
|
||||
idat += compressor.compress(row)
|
||||
idat += compressor.flush()
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(b"IHDR", ihdr)
|
||||
+ png_chunk(b"IDAT", bytes(idat))
|
||||
+ png_chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
|
||||
def upload(client, actor, chat_id, raw, filename, content_type):
|
||||
@@ -27,7 +57,7 @@ def upload(client, actor, chat_id, raw, filename, content_type):
|
||||
return client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat_id}/attachments?filename={filename}",
|
||||
expected=(201, 400, 404, 413, 415),
|
||||
expected=(202, 400, 404, 413, 415),
|
||||
actor=actor,
|
||||
raw=raw,
|
||||
headers=headers,
|
||||
@@ -40,20 +70,31 @@ def main():
|
||||
parser.add_argument("--ca-file", default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
c = Client(args.base_url, args.ca_file, timeout=60)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
png = FIXTURE.read_bytes()
|
||||
|
||||
# Happy path first, so the negative checks can refer to a real attachment.
|
||||
good = c.attachment(alice, chat)[0]
|
||||
good = c.wait_ready(alice, c.attachment(alice, chat)[0])
|
||||
check(good["status"] == "ready", "valid upload is not ready")
|
||||
check(good["error_code"] is None, "ready attachment carries an error")
|
||||
check(
|
||||
{item["name"] for item in good["variants"]} == {"thumbnail", "crop", "optimized"},
|
||||
"ready attachment is missing a grade-4 variant",
|
||||
)
|
||||
|
||||
# 1. One byte over the contract limit must be a 413, decided before decode.
|
||||
oversized = b"\x00" * (MAX_UPLOAD_BYTES + 1)
|
||||
response = upload(c, alice, chat["id"], oversized, "big.png", "image/png")
|
||||
check(response["status"] == 413, "oversized upload did not return 413")
|
||||
|
||||
# 1b. A decompression bomb passes the byte limit but exceeds the pixel
|
||||
# cap. It must be rejected from the header before pixel allocation.
|
||||
bomb = oversized_png(7000, 6000) # 42 Mp > 40 Mp hard cap
|
||||
check(len(bomb) < MAX_UPLOAD_BYTES, "pixel-bomb fixture unexpectedly exceeds bytes")
|
||||
response = upload(c, alice, chat["id"], bomb, "bomb.png", "image/png")
|
||||
check(response["status"] == 413, "pixel-limit upload did not return 413")
|
||||
|
||||
# 2. A file with a valid PNG signature but a broken body must be a 415.
|
||||
corrupt = png[:8] + b"\x00" * 64
|
||||
response = upload(c, alice, chat["id"], corrupt, "broken.png", "image/png")
|
||||
@@ -95,7 +136,7 @@ def main():
|
||||
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
other_attachment = c.attachment(alice, other)[0]
|
||||
other_attachment = c.wait_ready(alice, c.attachment(alice, other)[0])
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
@@ -159,7 +200,7 @@ def main():
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: graded negative cases (413/415/404/400), no 5xx, "
|
||||
"PASS: graded negative cases (413 byte/pixel, 415/404/400), no 5xx, "
|
||||
"cross-member download checksum matches"
|
||||
)
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ def main():
|
||||
response = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=concurrent-{index}.png",
|
||||
201,
|
||||
202,
|
||||
actor=alice,
|
||||
raw=png,
|
||||
headers={"Content-Type": "image/png"},
|
||||
@@ -102,8 +102,11 @@ def main():
|
||||
instances = sorted({item["instance"] for item in results if item["instance"]})
|
||||
check(bool(instances), "no X-Instance-Id was observed")
|
||||
|
||||
# Read one uploaded object back and compare the exact bytes before reporting
|
||||
# success; ten 201 responses alone are not enough.
|
||||
# Lab5: a 202 only means the intent is durable. Wait for the worker, then
|
||||
# read one object back and compare the exact bytes; accepted responses
|
||||
# alone are not enough.
|
||||
for item in results:
|
||||
c.wait_ready(alice, item, wait_seconds=120)
|
||||
download = c.call(
|
||||
"GET", f"/api/v1/attachments/{results[0]['id']}/content", actor=alice
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user