forked from mtusi2026/mtusi2026lab5
Отделить обработку вложений от приёма файла.
Upload отвечает 202 после staging и записи задания, worker через RabbitMQ собирает thumbnail, crop и optimized и переживает разрыв до publish. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,18 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative attachment cases. Prints statuses only, never tokens."""
|
||||
|
||||
import os
|
||||
import base64
|
||||
from urllib.parse import urlparse
|
||||
import hashlib
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import uuid
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
JPEG = base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5soooroMD/9k="
|
||||
@@ -69,6 +72,19 @@ class Api:
|
||||
return response.status, response.headers, parsed, payload
|
||||
|
||||
|
||||
def wait_ready(api: Api, attachment_id: str, token: str):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
status, _, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll attachment")
|
||||
if body["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed")
|
||||
if body["status"] == "ready":
|
||||
return body
|
||||
time.sleep(0.3)
|
||||
raise SystemExit("FAIL attachment did not become ready")
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got == 500:
|
||||
raise SystemExit("FAIL %s returned 500" % label)
|
||||
@@ -93,15 +109,17 @@ def user(api: Api, label: str):
|
||||
|
||||
|
||||
def declared_length(token, chat_id):
|
||||
parsed = urlparse(BASE)
|
||||
host = parsed.netloc or "127.0.0.1:8080"
|
||||
request = (
|
||||
"POST /api/v1/chats/%s/attachments?filename=big.png HTTP/1.1\r\n"
|
||||
"Host: 127.0.0.1:8080\r\n"
|
||||
"Host: %s\r\n"
|
||||
"Authorization: Bearer %s\r\n"
|
||||
"Content-Type: image/png\r\n"
|
||||
"Content-Length: 10485761\r\n"
|
||||
"Connection: close\r\n\r\n" % (chat_id, token)
|
||||
"Connection: close\r\n\r\n" % (chat_id, host, token)
|
||||
).encode() + b"\x89PNG"
|
||||
sock = socket.create_connection(("127.0.0.1", 8080), timeout=5)
|
||||
sock = socket.create_connection((parsed.hostname or "127.0.0.1", parsed.port or 80), timeout=5)
|
||||
try:
|
||||
sock.sendall(request)
|
||||
data = b""
|
||||
@@ -157,9 +175,10 @@ def main():
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(201, status, "png upload")
|
||||
if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "ready":
|
||||
expect(202, status, "png upload")
|
||||
if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "queued":
|
||||
raise SystemExit("FAIL png metadata")
|
||||
attachment = wait_ready(api, attachment["id"], bob_token)
|
||||
status, headers, _, downloaded = api.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/%s/content" % attachment["id"],
|
||||
@@ -178,7 +197,7 @@ def main():
|
||||
raw=JPEG,
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
expect(201, status, "jpeg upload")
|
||||
expect(202, status, "jpeg upload")
|
||||
if jpeg_meta["content_type"] != "image/jpeg" or jpeg_meta["sha256"] != hashlib.sha256(JPEG).hexdigest():
|
||||
raise SystemExit("FAIL jpeg metadata")
|
||||
|
||||
@@ -269,7 +288,7 @@ def main():
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(201, status, "upload into other chat")
|
||||
expect(202, status, "upload into other chat")
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
|
||||
@@ -180,7 +180,7 @@ def memory_sample():
|
||||
check=False,
|
||||
)
|
||||
for line in stats.stdout.splitlines():
|
||||
if "mtusi2026lab4-api-" in line:
|
||||
if "mtusi2026lab5-api-" in line:
|
||||
samples.append(line.strip())
|
||||
time.sleep(0.4)
|
||||
|
||||
@@ -192,7 +192,7 @@ def memory_sample():
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
watcher.join()
|
||||
if codes.count(201) != 10:
|
||||
if codes.count(202) != 10:
|
||||
raise SystemExit("concurrent uploads %s" % codes)
|
||||
|
||||
def mib(line):
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab5 fault checks: worker down, content 409, then the same upload reaches ready.
|
||||
|
||||
Prints statuses and ids only. Does not print tokens.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
|
||||
|
||||
def compose(*args: str) -> None:
|
||||
command = ["docker", "compose", "--env-file", ".env"]
|
||||
extra = os.environ.get("LAB5_COMPOSE_FILES", "")
|
||||
if extra:
|
||||
for name in extra.split(os.pathsep):
|
||||
if name:
|
||||
command.extend(["-f", name])
|
||||
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None):
|
||||
hdrs = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
hdrs["Content-Type"] = content_type
|
||||
if token:
|
||||
hdrs["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=30)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None
|
||||
return response.status, parsed, payload
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
def user(api: Api):
|
||||
username = "l5_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, body, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Lab5", "password": password},
|
||||
)
|
||||
expect(201, status, "register")
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
expect(201, status, "login")
|
||||
return body["id"], session["token"]
|
||||
|
||||
|
||||
def wait_ready(api: Api, attachment_id: str, token: str):
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll")
|
||||
if body["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed " + str(body.get("error_code")))
|
||||
if body["status"] == "ready":
|
||||
return body
|
||||
time.sleep(0.4)
|
||||
raise SystemExit("FAIL not ready")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api = Api()
|
||||
alice, alice_token = user(api)
|
||||
bob, bob_token = user(api)
|
||||
status, chat, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "lab5", "member_ids": [alice, bob]},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "chat")
|
||||
chat_id = chat["id"]
|
||||
print("chat", chat_id)
|
||||
|
||||
compose("stop", "worker")
|
||||
content = FIXTURE.read_bytes()
|
||||
status, attachment, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=content,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "upload while worker is stopped")
|
||||
if attachment["status"] != "queued" or attachment["sha256"] != hashlib.sha256(content).hexdigest():
|
||||
raise SystemExit("FAIL queued metadata")
|
||||
print("attachment", attachment["id"])
|
||||
status, _, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token)
|
||||
expect(409, status, "content before ready")
|
||||
status, message, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "очередь не блокирует чат", "attachment_ids": [attachment["id"]]},
|
||||
token=alice_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(201, status, "message while queued")
|
||||
if message["attachment_ids"] != [attachment["id"]]:
|
||||
raise SystemExit("FAIL queued attachment was not linkable")
|
||||
|
||||
compose("start", "worker")
|
||||
ready = wait_ready(api, attachment["id"], bob_token)
|
||||
thumbs = [item for item in ready["variants"] if item["name"] == "thumbnail"]
|
||||
if len(thumbs) != 1 or (thumbs[0]["width"], thumbs[0]["height"]) != (256, 160):
|
||||
raise SystemExit("FAIL thumbnail metadata %s" % thumbs)
|
||||
names = sorted(item["name"] for item in ready["variants"])
|
||||
if names != ["crop", "optimized", "thumbnail"]:
|
||||
raise SystemExit("FAIL variants %s" % names)
|
||||
status, _, payload = api.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/%s/content?variant=thumbnail" % attachment["id"],
|
||||
token=bob_token,
|
||||
)
|
||||
expect(200, status, "thumbnail bytes")
|
||||
if hashlib.sha256(payload).hexdigest() != thumbs[0]["sha256"]:
|
||||
raise SystemExit("FAIL thumbnail checksum")
|
||||
print("PASS lab5 worker-down upload became ready", attachment["id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+2
-2
@@ -4,7 +4,7 @@ set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env restart postgres redis s3 api
|
||||
docker compose --env-file .env restart postgres redis s3 rabbitmq api worker
|
||||
else
|
||||
docker compose restart postgres redis s3 api
|
||||
docker compose restart postgres redis s3 rabbitmq api worker
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Stop the worker and leave it stopped. The broker, API and volumes keep running.
|
||||
set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env stop worker
|
||||
else
|
||||
docker compose stop worker
|
||||
fi
|
||||
@@ -28,6 +28,19 @@ def expect(status, got, label):
|
||||
raise SystemExit("FAIL %s returned 500" % label)
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
|
||||
|
||||
def wait_ready(api, attachment_id, token):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll attachment")
|
||||
if body["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed")
|
||||
if body["status"] == "ready":
|
||||
return body
|
||||
time.sleep(0.3)
|
||||
raise SystemExit("FAIL attachment did not become ready")
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
@@ -109,7 +122,7 @@ def peak_during(action):
|
||||
check=False,
|
||||
)
|
||||
for line in stats.stdout.splitlines():
|
||||
if "mtusi2026lab4-api-" not in line:
|
||||
if "mtusi2026lab5-api-" not in line:
|
||||
continue
|
||||
value = line.split()[1]
|
||||
number = float(value[:-3])
|
||||
@@ -251,9 +264,10 @@ def negatives(api, alice_token, bob_token, eve_token, chat_id):
|
||||
{"sha256": digest},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "complete happy path")
|
||||
if attachment["sha256"] != digest or attachment["status"] != "ready":
|
||||
expect(202, status, "complete happy path")
|
||||
if attachment["sha256"] != digest or attachment["status"] != "queued":
|
||||
raise SystemExit("FAIL complete metadata")
|
||||
attachment = wait_ready(api, attachment["id"], bob_token)
|
||||
status, _, downloaded = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token)
|
||||
expect(200, status, "member downloads direct upload")
|
||||
if downloaded != small:
|
||||
@@ -272,7 +286,7 @@ def negatives(api, alice_token, bob_token, eve_token, chat_id):
|
||||
raw=small,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(201, status, "proxy endpoint still works")
|
||||
expect(202, status, "proxy endpoint still works")
|
||||
return proxy["id"]
|
||||
|
||||
|
||||
@@ -289,7 +303,7 @@ def compare(api, token, chat_id):
|
||||
content_type="image/png",
|
||||
)
|
||||
)
|
||||
if proxy["result"][0] != 201:
|
||||
if proxy["result"][0] != 202:
|
||||
raise SystemExit("FAIL proxy %s %s" % (label, proxy["result"][0]))
|
||||
status, started, _ = api.call(
|
||||
"POST",
|
||||
@@ -309,7 +323,7 @@ def compare(api, token, chat_id):
|
||||
token=token,
|
||||
)
|
||||
)
|
||||
if direct_complete["result"][0] != 201:
|
||||
if direct_complete["result"][0] != 202:
|
||||
raise SystemExit("FAIL complete %s %s" % (label, direct_complete["result"][0]))
|
||||
rows.append(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user