feat(lab5): durable async attachment pipeline through grade 4

- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
Mikhail Verkovykh
2026-11-15 00:30:36 +03:00
parent 28d68dc30b
commit 9e43faa536
48 changed files with 4077 additions and 333 deletions
+5 -4
View File
@@ -29,14 +29,15 @@ def upload(args):
response = c.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
201,
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
attachment = response["json"]
check(attachment["status"] == "ready", "upload is not ready")
accepted_by = response["headers"].get("X-Instance-Id")
attachment = c.wait_ready(alice, response["json"], wait_seconds=60)
check(attachment["status"] == "ready", "upload did not become ready")
check(
attachment["sha256"] == hashlib.sha256(content).hexdigest(),
"upload checksum mismatch",
@@ -45,7 +46,7 @@ def upload(args):
"attachment_id": attachment["id"],
"sha256": attachment["sha256"],
"size_bytes": attachment["size_bytes"],
"instance_id": response["headers"].get("X-Instance-Id"),
"instance_id": accepted_by,
"token": alice["token"],
"chat_id": chat["id"],
"content_length": len(content),
+46 -5
View File
@@ -9,8 +9,10 @@ envelope, not a 500.
"""
import argparse
import hashlib
import struct
import sys
import uuid
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
@@ -20,6 +22,34 @@ from client import Client, check # noqa: E402
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
MAX_IMAGE_PIXELS = 40_000_000
def png_chunk(kind, payload):
crc = zlib.crc32(kind + payload) & 0xFFFFFFFF
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", crc)
def oversized_png(width, height):
"""Build a valid grayscale PNG whose dimensions exceed the pixel cap.
The image is solid black, so the IDAT stream stays tiny and the request is
far below the byte limit. Only the declared width*height can trigger 413.
"""
ihdr = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0)
compressor = zlib.compressobj(9)
idat = bytearray()
row = b"\x00" + b"\x00" * width # filter type 0 + one 8-bit sample per pixel
for _ in range(height):
idat += compressor.compress(row)
idat += compressor.flush()
return (
b"\x89PNG\r\n\x1a\n"
+ png_chunk(b"IHDR", ihdr)
+ png_chunk(b"IDAT", bytes(idat))
+ png_chunk(b"IEND", b"")
)
def upload(client, actor, chat_id, raw, filename, content_type):
@@ -27,7 +57,7 @@ def upload(client, actor, chat_id, raw, filename, content_type):
return client.call(
"POST",
f"/api/v1/chats/{chat_id}/attachments?filename={filename}",
expected=(201, 400, 404, 413, 415),
expected=(202, 400, 404, 413, 415),
actor=actor,
raw=raw,
headers=headers,
@@ -40,20 +70,31 @@ def main():
parser.add_argument("--ca-file", default=None)
args = parser.parse_args()
c = Client(args.base_url, args.ca_file)
c = Client(args.base_url, args.ca_file, timeout=60)
alice, bob, eve, chat = c.bootstrap()
png = FIXTURE.read_bytes()
# Happy path first, so the negative checks can refer to a real attachment.
good = c.attachment(alice, chat)[0]
good = c.wait_ready(alice, c.attachment(alice, chat)[0])
check(good["status"] == "ready", "valid upload is not ready")
check(good["error_code"] is None, "ready attachment carries an error")
check(
{item["name"] for item in good["variants"]} == {"thumbnail", "crop", "optimized"},
"ready attachment is missing a grade-4 variant",
)
# 1. One byte over the contract limit must be a 413, decided before decode.
oversized = b"\x00" * (MAX_UPLOAD_BYTES + 1)
response = upload(c, alice, chat["id"], oversized, "big.png", "image/png")
check(response["status"] == 413, "oversized upload did not return 413")
# 1b. A decompression bomb passes the byte limit but exceeds the pixel
# cap. It must be rejected from the header before pixel allocation.
bomb = oversized_png(7000, 6000) # 42 Mp > 40 Mp hard cap
check(len(bomb) < MAX_UPLOAD_BYTES, "pixel-bomb fixture unexpectedly exceeds bytes")
response = upload(c, alice, chat["id"], bomb, "bomb.png", "image/png")
check(response["status"] == 413, "pixel-limit upload did not return 413")
# 2. A file with a valid PNG signature but a broken body must be a 415.
corrupt = png[:8] + b"\x00" * 64
response = upload(c, alice, chat["id"], corrupt, "broken.png", "image/png")
@@ -95,7 +136,7 @@ def main():
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
other_attachment = c.attachment(alice, other)[0]
other_attachment = c.wait_ready(alice, c.attachment(alice, other)[0])
c.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
@@ -159,7 +200,7 @@ def main():
)
print(
"PASS: graded negative cases (413/415/404/400), no 5xx, "
"PASS: graded negative cases (413 byte/pixel, 415/404/400), no 5xx, "
"cross-member download checksum matches"
)
+6 -3
View File
@@ -72,7 +72,7 @@ def main():
response = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=concurrent-{index}.png",
201,
202,
actor=alice,
raw=png,
headers={"Content-Type": "image/png"},
@@ -102,8 +102,11 @@ def main():
instances = sorted({item["instance"] for item in results if item["instance"]})
check(bool(instances), "no X-Instance-Id was observed")
# Read one uploaded object back and compare the exact bytes before reporting
# success; ten 201 responses alone are not enough.
# Lab5: a 202 only means the intent is durable. Wait for the worker, then
# read one object back and compare the exact bytes; accepted responses
# alone are not enough.
for item in results:
c.wait_ready(alice, item, wait_seconds=120)
download = c.call(
"GET", f"/api/v1/attachments/{results[0]['id']}/content", actor=alice
)