Отделить приём вложений от обработки через очередь и worker.

This commit is contained in:
samsik
2026-11-20 01:30:00 +03:00
parent 276dbdad8a
commit b92fa218ba
40 changed files with 4054 additions and 33 deletions
+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env python3
"""Сценарии уровня 4. Стенд уже запущен. Секреты и токены не печатаются."""
import base64
import hashlib
import json
import struct
import subprocess
import sys
import time
import uuid
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from image_probe import jpeg_size # noqa: E402
BASE = "http://127.0.0.1:8080"
def compose(*args, check_code=True):
proc = subprocess.run(
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args],
cwd=ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
if check_code and proc.returncode != 0:
raise Failure(proc.stdout[-2000:])
return proc
def psql(sql):
proc = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-At", "-c", sql)
return proc.stdout.strip()
def queues():
proc = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages")
return proc.stdout
def queue_depth(name):
text = queues()
for line in text.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0] == name and parts[-1].isdigit():
return int(parts[-1])
raise Failure("queue %s missing:\n%s" % (name, text))
def wait_depth(name, minimum, seconds=20):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
last = queue_depth(name)
except Failure:
last = None
if last is not None and last >= minimum:
return last
time.sleep(0.5)
raise Failure("queue %s depth %s, want >= %s" % (name, last, minimum))
def wait_status(client, actor, att, wanted, seconds=60):
deadline = time.monotonic() + seconds
last = att
while time.monotonic() < deadline:
last = client.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
if last["status"] == wanted:
return last
if wanted != "failed" and last["status"] == "failed":
raise Failure("attachment failed: " + str(last.get("error_code")))
time.sleep(0.4)
raise Failure("status %s did not become %s" % (last.get("status"), wanted))
def wait_postgres_ready(att_id, seconds=70):
deadline = time.monotonic() + seconds
last = ""
while time.monotonic() < deadline:
last = psql("SELECT status FROM attachments WHERE id = '%s'" % att_id)
if last == "ready":
return
time.sleep(0.5)
raise Failure("postgres status stayed " + last)
def register(client):
username = "u_g4_" + uuid.uuid4().hex[:12]
password = "T3st-" + uuid.uuid4().hex
user = client.call(
"POST", "/api/v1/users", 201,
body={"username": username, "display_name": "Проверка 4", "password": password},
schema="User",
)["json"]
raw = base64.b64encode((username + ":" + password).encode()).decode()
session = client.call(
"POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + raw},
schema="Session",
)["json"]
return {"id": user["id"], "token": session["token"], "user": user}
def upload(client, actor, chat_id, filename, content, content_type="image/png"):
return client.call(
"POST",
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename),
202,
actor=actor,
raw=content,
headers={"Content-Type": content_type},
schema="Attachment",
)["json"]
def bad_png():
ihdr = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
chunk = b"IHDR" + ihdr
crc = zlib.crc32(chunk) & 0xFFFFFFFF
return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc)
def webp_size(data):
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
index = 12
while index + 8 <= len(data):
kind = data[index:index + 4]
size = int.from_bytes(data[index + 4:index + 8], "little")
chunk = data[index + 8:index + 8 + size]
if kind == b"VP8X" and len(chunk) >= 10:
return (1 + int.from_bytes(chunk[4:7], "little"), 1 + int.from_bytes(chunk[7:10], "little"))
if kind == b"VP8 " and len(chunk) >= 10:
return (int.from_bytes(chunk[6:8], "little") & 0x3FFF, int.from_bytes(chunk[8:10], "little") & 0x3FFF)
if kind == b"VP8L" and len(chunk) >= 5:
bits = int.from_bytes(chunk[1:5], "little")
return ((bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1)
index += 8 + size + (size & 1)
raise Failure("WebP dimensions not found")
def variant_count(att_id):
return int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att_id))
def main():
sample = (ROOT / "tests/fixtures/sample.png").read_bytes()
client = Client(BASE, None)
client.call("GET", "/health/ready", schema="Health")
alice = register(client)
bob_name = "u_g4b_" + uuid.uuid4().hex[:12]
bob_password = "T3st-" + uuid.uuid4().hex
bob_user = client.call(
"POST", "/api/v1/users", 201,
body={"username": bob_name, "display_name": "Боб", "password": bob_password},
schema="User",
)["json"]
chat = client.call(
"POST", "/api/v1/chats", 201, actor=alice,
body={"title": "Очередь", "member_ids": [alice["id"], bob_user["id"]]},
schema="Chat",
)["json"]
notes = []
att = upload(client, alice, chat["id"], "sample.png", sample)
check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload snapshot")
ready = wait_status(client, alice, att, "ready")
names = [item["name"] for item in ready["variants"]]
check(names == ["thumbnail", "crop", "optimized"], "variant set " + ",".join(names))
thumb = next(item for item in ready["variants"] if item["name"] == "thumbnail")
crop = next(item for item in ready["variants"] if item["name"] == "crop")
optimized = next(item for item in ready["variants"] if item["name"] == "optimized")
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail meta")
check((crop["width"], crop["height"]) == (128, 128) and crop["content_type"] == "image/jpeg", "crop meta")
check(optimized["content_type"] == "image/webp" and (optimized["width"], optimized["height"]) == (320, 200), "webp meta")
original = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=alice)
check(original["bytes"] == sample, "original bytes changed")
thumb_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=alice)
crop_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=alice)
webp_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=alice)
check(jpeg_size(thumb_bytes["bytes"]) == (256, 160), "thumbnail jpeg")
check(jpeg_size(crop_bytes["bytes"]) == (128, 128), "crop jpeg")
check(webp_size(webp_bytes["bytes"]) == (320, 200), "webp size")
check(hashlib.sha256(thumb_bytes["bytes"]).hexdigest() == thumb["sha256"], "thumb sha")
replay = client.call("POST", "/api/v1/attachments/%s/replay" % att["id"], 200, actor=alice, schema="Attachment")["json"]
check(replay["status"] == "ready" and replay["variants"] == ready["variants"], "ready replay changed variants")
check(variant_count(att["id"]) == 3, "replay created another variant set")
check(psql("SELECT outcome FROM job_replays WHERE attachment_id = '%s' ORDER BY created_at DESC LIMIT 1" % att["id"]) == "ignored_ready", "replay audit")
notes.append("happy-path ready id=%s variants=3 replay=ignored_ready" % att["id"])
compose("stop", "worker")
held = upload(client, alice, chat["id"], "while-worker-down.png", sample)
client.call("GET", "/api/v1/attachments/%s/content" % held["id"], 409, actor=alice)
client.call(
"POST", "/api/v1/chats/%s/messages" % chat["id"], 201, actor=alice,
body={"text": "очередь не блокирует чат", "attachment_ids": [held["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
schema="Message",
)
before = wait_depth("attachment.process", 1)
compose("restart", "rabbitmq")
for _ in range(40):
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
if probe.returncode == 0:
break
time.sleep(1)
else:
raise Failure("rabbitmq did not come back")
after = wait_depth("attachment.process", 1, seconds=30)
notes.append("broker restart kept %s then %s process messages" % (before, after))
compose("start", "worker")
held_ready = wait_status(client, alice, held, "ready")
check(variant_count(held["id"]) == 3, "restarted job duplicated variants")
notes.append("worker-down then broker restart kept queue; ready id=%s" % held_ready["id"])
compose("stop", "worker")
compose("stop", "rabbitmq")
gap = upload(client, alice, chat["id"], "outbox-gap.png", sample)
unpublished = psql(
"SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id "
"WHERE j.attachment_id = '%s' AND o.published_at IS NULL" % gap["id"]
)
check(unpublished == "1", "outbox was published while broker was down: " + unpublished)
compose("stop", "api")
compose("start", "rabbitmq")
for _ in range(40):
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
if probe.returncode == 0:
break
time.sleep(1)
compose("start", "worker")
wait_postgres_ready(gap["id"])
ready_before_api = psql("SELECT status FROM attachments WHERE id = '%s'" % gap["id"])
check(ready_before_api == "ready", "worker did not recover unpublished outbox")
compose("start", "api")
for _ in range(40):
try:
client.call("GET", "/health/ready", schema="Health")
break
except Failure:
time.sleep(1)
else:
raise Failure("api did not return")
gap_body = client.call("GET", "/api/v1/attachments/%s" % gap["id"], actor=alice, schema="Attachment")["json"]
check(gap_body["status"] == "ready" and len(gap_body["variants"]) == 3, "gap attachment not ready via api")
notes.append("commit/publish gap recovered by worker without a new upload id=%s" % gap["id"])
broken = upload(client, alice, chat["id"], "broken.png", bad_png())
failed = wait_status(client, alice, broken, "failed")
check(failed["error_code"] == "invalid_image", "permanent code " + str(failed.get("error_code")))
check(variant_count(broken["id"]) == 0, "failed job stored variants")
client.call(
"POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
body={"text": "", "attachment_ids": [broken["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
again = client.call("POST", "/api/v1/attachments/%s/replay" % broken["id"], 202, actor=alice, schema="Attachment")["json"]
check(again["status"] == "queued", "replay of failed did not queue")
failed_again = wait_status(client, alice, broken, "failed")
check(failed_again["error_code"] == "invalid_image", "replay did not finish as permanent failure")
check(variant_count(broken["id"]) == 0, "replay of failed created variants")
dlq = queues()
check("attachment.failed" in dlq, "dlq missing")
notes.append("permanent invalid_image + replay id=%s dlq present" % broken["id"])
flaky = upload(client, alice, chat["id"], "__lab_fault_transient.png", sample)
flaky_ready = wait_status(client, alice, flaky, "ready", seconds=40)
failures = psql(
"SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND permanent = false AND error_code = 'unavailable'"
% flaky["id"]
)
check(int(failures) >= 2, "expected two transient failures, got " + failures)
check(variant_count(flaky["id"]) == 3, "transient retries duplicated variants")
attempt = psql("SELECT attempt FROM attachment_jobs WHERE attachment_id = '%s'" % flaky["id"])
notes.append("backoff retries=%s final_attempt=%s id=%s" % (failures, attempt, flaky_ready["id"]))
compose("stop", "worker")
crash = upload(client, alice, chat["id"], "crash-before-ack.png", sample)
wait_depth("attachment.process", 1)
crashed = subprocess.run(
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "run", "--rm", "--no-deps", "--name", "lab5-crash-once", "-e", "CRASH_AFTER_S3=1", "worker"],
cwd=ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=45,
)
check(crashed.returncode != 0, "crash hook did not stop the worker")
status_after_crash = psql("SELECT status FROM attachments WHERE id = '%s'" % crash["id"])
check(status_after_crash == "processing", "crash did not stop between claim and ready: " + status_after_crash)
compose("start", "worker")
crash_ready = wait_status(client, alice, crash, "ready")
check(variant_count(crash["id"]) == 3, "redelivery created a second variant set")
original_after = client.call("GET", "/api/v1/attachments/%s/content" % crash["id"], actor=alice)
check(original_after["bytes"] == sample, "redelivery changed the original")
staging_left = psql("SELECT staging_key IS NULL FROM attachments WHERE id = '%s'" % crash["id"])
check(staging_left == "t", "staging was not cleared after ready")
notes.append("crash after s3 before ack redelivery id=%s variants=3 staging cleared" % crash_ready["id"])
compose("stop", "worker")
compose("stop", "rabbitmq")
leased = upload(client, alice, chat["id"], "expired-lease.png", sample)
psql(
"UPDATE attachment_jobs SET status = 'processing', attempt = 1, "
"lease_token = gen_random_uuid(), lease_until = now() - interval '2 minutes', "
"updated_at = now() WHERE attachment_id = '%s'" % leased["id"]
)
psql("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '%s'" % leased["id"])
psql(
"UPDATE outbox SET published_at = now() WHERE job_id = "
"(SELECT id FROM attachment_jobs WHERE attachment_id = '%s')" % leased["id"]
)
compose("start", "rabbitmq")
for _ in range(40):
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
if probe.returncode == 0:
break
time.sleep(1)
compose("start", "worker")
lease_ready = wait_status(client, alice, leased, "ready")
lease_rows = psql(
"SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND error_code = 'lease_expired'"
% leased["id"]
)
check(int(lease_rows) >= 1, "expired lease was not recovered")
check(variant_count(leased["id"]) == 3, "lease recovery duplicated variants")
notes.append("expired processing lease recovered id=%s" % lease_ready["id"])
text = "\n".join(notes)
print(text)
evidence = ROOT / "evidence" / "grade4.txt"
evidence.write_text(text + "\n", encoding="utf-8")
print("PASS: grade 4 fault scenarios")
if __name__ == "__main__":
try:
main()
except (Failure, OSError, subprocess.SubprocessError) as exc:
print("FAIL:", exc)
raise SystemExit(1)
Executable → Regular
View File
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Перезапуск API, worker и всех хранилищ, включая брокер. Volumes не удаляются.
docker compose --env-file .env -f compose.yaml restart api worker postgres valkey s3 rabbitmq
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
if [ -z "${replicas}" ]; then
replicas=2
fi
docker compose --env-file .env -f compose.yaml up -d --no-deps --scale "api=${replicas}" --no-recreate api
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env -f compose.yaml start worker
Executable → Regular
View File
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
cd "$(dirname "$0")/.."
found=""
for cid in $(docker compose --env-file .env -f compose.yaml ps -q api); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$cid")
short=$(printf '%s' "$cid" | cut -c1-12)
if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then
found=$cid
break
fi
done
if [ -z "$found" ]; then
echo "API replica was not found" >&2
exit 1
fi
# docker stop оставляет реплику выключенной: restart policy unless-stopped.
docker stop -t 8 "$found" >/dev/null
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env -f compose.yaml stop worker
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ ! -f .env ]; then
echo "Copy .env.example to .env" >&2
exit 1
fi
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
if [ -z "${replicas}" ]; then
replicas=2
fi
docker compose --env-file .env -f compose.yaml up -d --build --scale "api=${replicas}" --remove-orphans