Отделить обработку вложений от приёма файла.

Upload отвечает 202 после staging и записи задания, worker через RabbitMQ собирает thumbnail, crop и optimized и переживает разрыв до publish.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-11-09 01:08:24 +03:00
co-authored by Cursor
parent 9046d5206d
commit f2a3698480
24 changed files with 1579 additions and 82 deletions
+27 -8
View File
@@ -1,18 +1,21 @@
#!/usr/bin/env python3
"""Negative attachment cases. Prints statuses only, never tokens."""
import os
import base64
from urllib.parse import urlparse
import hashlib
import json
import socket
import struct
import time
import uuid
import urllib.error
import urllib.request
import zlib
from pathlib import Path
BASE = "http://127.0.0.1:8080"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
ROOT = Path(__file__).resolve().parents[1]
JPEG = base64.b64decode(
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5soooroMD/9k="
@@ -69,6 +72,19 @@ class Api:
return response.status, response.headers, parsed, payload
def wait_ready(api: Api, attachment_id: str, token: str):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
status, _, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll attachment")
if body["status"] == "failed":
raise SystemExit("FAIL attachment failed")
if body["status"] == "ready":
return body
time.sleep(0.3)
raise SystemExit("FAIL attachment did not become ready")
def expect(status, got, label):
if got == 500:
raise SystemExit("FAIL %s returned 500" % label)
@@ -93,15 +109,17 @@ def user(api: Api, label: str):
def declared_length(token, chat_id):
parsed = urlparse(BASE)
host = parsed.netloc or "127.0.0.1:8080"
request = (
"POST /api/v1/chats/%s/attachments?filename=big.png HTTP/1.1\r\n"
"Host: 127.0.0.1:8080\r\n"
"Host: %s\r\n"
"Authorization: Bearer %s\r\n"
"Content-Type: image/png\r\n"
"Content-Length: 10485761\r\n"
"Connection: close\r\n\r\n" % (chat_id, token)
"Connection: close\r\n\r\n" % (chat_id, host, token)
).encode() + b"\x89PNG"
sock = socket.create_connection(("127.0.0.1", 8080), timeout=5)
sock = socket.create_connection((parsed.hostname or "127.0.0.1", parsed.port or 80), timeout=5)
try:
sock.sendall(request)
data = b""
@@ -157,9 +175,10 @@ def main():
raw=good,
content_type="image/png",
)
expect(201, status, "png upload")
if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "ready":
expect(202, status, "png upload")
if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "queued":
raise SystemExit("FAIL png metadata")
attachment = wait_ready(api, attachment["id"], bob_token)
status, headers, _, downloaded = api.call(
"GET",
"/api/v1/attachments/%s/content" % attachment["id"],
@@ -178,7 +197,7 @@ def main():
raw=JPEG,
content_type="image/jpeg",
)
expect(201, status, "jpeg upload")
expect(202, status, "jpeg upload")
if jpeg_meta["content_type"] != "image/jpeg" or jpeg_meta["sha256"] != hashlib.sha256(JPEG).hexdigest():
raise SystemExit("FAIL jpeg metadata")
@@ -269,7 +288,7 @@ def main():
raw=good,
content_type="image/png",
)
expect(201, status, "upload into other chat")
expect(202, status, "upload into other chat")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
+2 -2
View File
@@ -180,7 +180,7 @@ def memory_sample():
check=False,
)
for line in stats.stdout.splitlines():
if "mtusi2026lab4-api-" in line:
if "mtusi2026lab5-api-" in line:
samples.append(line.strip())
time.sleep(0.4)
@@ -192,7 +192,7 @@ def memory_sample():
for thread in threads:
thread.join()
watcher.join()
if codes.count(201) != 10:
if codes.count(202) != 10:
raise SystemExit("concurrent uploads %s" % codes)
def mib(line):
+152
View File
@@ -0,0 +1,152 @@
#!/usr/bin/env python3
"""Lab5 fault checks: worker down, content 409, then the same upload reaches ready.
Prints statuses and ids only. Does not print tokens.
"""
import base64
import hashlib
import json
import os
import subprocess
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
def compose(*args: str) -> None:
command = ["docker", "compose", "--env-file", ".env"]
extra = os.environ.get("LAB5_COMPOSE_FILES", "")
if extra:
for name in extra.split(os.pathsep):
if name:
command.extend(["-f", name])
subprocess.run([*command, *args], cwd=ROOT, check=True)
class Api:
def __init__(self) -> None:
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None):
hdrs = dict(headers or {})
data = raw
if body is not None:
data = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if content_type:
hdrs["Content-Type"] = content_type
if token:
hdrs["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
try:
response = self.opener.open(request, timeout=30)
except urllib.error.HTTPError as error:
response = error
with response:
payload = response.read()
parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None
return response.status, parsed, payload
def expect(status, got, label):
if got != status:
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
print("PASS", label, got)
def user(api: Api):
username = "l5_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
status, body, _ = api.call(
"POST",
"/api/v1/users",
{"username": username, "display_name": "Lab5", "password": password},
)
expect(201, status, "register")
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
expect(201, status, "login")
return body["id"], session["token"]
def wait_ready(api: Api, attachment_id: str, token: str):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll")
if body["status"] == "failed":
raise SystemExit("FAIL attachment failed " + str(body.get("error_code")))
if body["status"] == "ready":
return body
time.sleep(0.4)
raise SystemExit("FAIL not ready")
def main() -> None:
api = Api()
alice, alice_token = user(api)
bob, bob_token = user(api)
status, chat, _ = api.call(
"POST",
"/api/v1/chats",
{"title": "lab5", "member_ids": [alice, bob]},
token=alice_token,
)
expect(201, status, "chat")
chat_id = chat["id"]
print("chat", chat_id)
compose("stop", "worker")
content = FIXTURE.read_bytes()
status, attachment, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
token=alice_token,
raw=content,
content_type="image/png",
)
expect(202, status, "upload while worker is stopped")
if attachment["status"] != "queued" or attachment["sha256"] != hashlib.sha256(content).hexdigest():
raise SystemExit("FAIL queued metadata")
print("attachment", attachment["id"])
status, _, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token)
expect(409, status, "content before ready")
status, message, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
{"text": "очередь не блокирует чат", "attachment_ids": [attachment["id"]]},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
expect(201, status, "message while queued")
if message["attachment_ids"] != [attachment["id"]]:
raise SystemExit("FAIL queued attachment was not linkable")
compose("start", "worker")
ready = wait_ready(api, attachment["id"], bob_token)
thumbs = [item for item in ready["variants"] if item["name"] == "thumbnail"]
if len(thumbs) != 1 or (thumbs[0]["width"], thumbs[0]["height"]) != (256, 160):
raise SystemExit("FAIL thumbnail metadata %s" % thumbs)
names = sorted(item["name"] for item in ready["variants"])
if names != ["crop", "optimized", "thumbnail"]:
raise SystemExit("FAIL variants %s" % names)
status, _, payload = api.call(
"GET",
"/api/v1/attachments/%s/content?variant=thumbnail" % attachment["id"],
token=bob_token,
)
expect(200, status, "thumbnail bytes")
if hashlib.sha256(payload).hexdigest() != thumbs[0]["sha256"]:
raise SystemExit("FAIL thumbnail checksum")
print("PASS lab5 worker-down upload became ready", attachment["id"])
if __name__ == "__main__":
main()
+2 -2
View File
@@ -4,7 +4,7 @@ set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
if [ -f .env ]; then
docker compose --env-file .env restart postgres redis s3 api
docker compose --env-file .env restart postgres redis s3 rabbitmq api worker
else
docker compose restart postgres redis s3 api
docker compose restart postgres redis s3 rabbitmq api worker
fi
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Stop the worker and leave it stopped. The broker, API and volumes keep running.
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
if [ -f .env ]; then
docker compose --env-file .env stop worker
else
docker compose stop worker
fi
+20 -6
View File
@@ -28,6 +28,19 @@ def expect(status, got, label):
raise SystemExit("FAIL %s returned 500" % label)
if got != status:
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
def wait_ready(api, attachment_id, token):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll attachment")
if body["status"] == "failed":
raise SystemExit("FAIL attachment failed")
if body["status"] == "ready":
return body
time.sleep(0.3)
raise SystemExit("FAIL attachment did not become ready")
print("PASS", label, got)
@@ -109,7 +122,7 @@ def peak_during(action):
check=False,
)
for line in stats.stdout.splitlines():
if "mtusi2026lab4-api-" not in line:
if "mtusi2026lab5-api-" not in line:
continue
value = line.split()[1]
number = float(value[:-3])
@@ -251,9 +264,10 @@ def negatives(api, alice_token, bob_token, eve_token, chat_id):
{"sha256": digest},
token=alice_token,
)
expect(201, status, "complete happy path")
if attachment["sha256"] != digest or attachment["status"] != "ready":
expect(202, status, "complete happy path")
if attachment["sha256"] != digest or attachment["status"] != "queued":
raise SystemExit("FAIL complete metadata")
attachment = wait_ready(api, attachment["id"], bob_token)
status, _, downloaded = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token)
expect(200, status, "member downloads direct upload")
if downloaded != small:
@@ -272,7 +286,7 @@ def negatives(api, alice_token, bob_token, eve_token, chat_id):
raw=small,
content_type="image/png",
)
expect(201, status, "proxy endpoint still works")
expect(202, status, "proxy endpoint still works")
return proxy["id"]
@@ -289,7 +303,7 @@ def compare(api, token, chat_id):
content_type="image/png",
)
)
if proxy["result"][0] != 201:
if proxy["result"][0] != 202:
raise SystemExit("FAIL proxy %s %s" % (label, proxy["result"][0]))
status, started, _ = api.call(
"POST",
@@ -309,7 +323,7 @@ def compare(api, token, chat_id):
token=token,
)
)
if direct_complete["result"][0] != 201:
if direct_complete["result"][0] != 202:
raise SystemExit("FAIL complete %s %s" % (label, direct_complete["result"][0]))
rows.append(
{