Files
mtusi2026lab5/student-tests/lab5_test_level4.py
T

513 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Level 4 для lab5: варианты, outbox, идемпотентность, DLQ, lease, retry.
Запуск:
$env:PYTHONUTF8="1"
python student-tests/lab5_test_level4.py --base-url http://localhost:8080
Требует рабочий стенд из compose.yaml (LB + 2 api + worker + outbox +
RabbitMQ + MinIO + PostgreSQL + Redis). Использует psql/mc через
docker run в compose-сети, как lab4_test_level4.py.
Сценарии:
=== variants ===
Все 4 варианта: original, thumbnail (256×160), crop (128×128),
optimized (WebP). Реальные bytes и размеры, а не JSON-обещания.
=== outbox crash between COMMIT and publish ===
Остановить outbox → upload → строка outbox есть, published_at NULL
→ запустить outbox → published_at заполнено, attachment готов.
=== duplicate delivery does not create duplicates ===
Ручная повторная публикация того же job → variants не дублируются.
=== worker stopped: 202 → content 409 → ready ===
Остановить worker → upload → 202 queued → content 409 → старт
worker → ready и thumbnail доступен.
=== broken staging → retry_exhausted → failed + DLQ ===
Удалить staging-объект → worker исчерпает retry → failed
error_code=retry_exhausted + сообщение в DLQ.
=== stuck processing (lease) → queued → ready ===
Симулировать processing с истёкшим lease → recover (через s3_cleanup
или через явный SQL) возвращает в queued → worker обрабатывает → ready.
"""
import hashlib
import io
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
from image_probe import jpeg_size # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab5_default")
PG_CONTAINER = os.environ.get("PG_CONTAINER", "mtuci-postgres")
RABBIT_CONTAINER = os.environ.get("RABBIT_CONTAINER", "mtuci-rabbitmq")
MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920")
# ---------- helpers ----------
def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str):
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
req = urllib.request.Request(
url, data=data, method="POST",
headers={
"Authorization": "Bearer " + actor["token"],
"Content-Type": ctype,
},
)
try:
with c.opener.open(req, timeout=30) as resp:
return resp.code, json.loads(resp.read()), resp.headers
except urllib.error.HTTPError as e:
body = e.read()
try:
return e.code, json.loads(body), e.headers
except Exception:
return e.code, None, e.headers
def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"):
return _raw_upload(c, actor, chat["id"], filename, data, ctype)
def _env_from_file():
env = os.environ.copy()
env_file = ROOT / ".env"
if env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
env.setdefault(k.strip(), v.strip())
return env
def _psql(sql: str) -> str:
"""psql через docker compose exec в mtuci-postgres."""
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-U", "messenger", "-d", "high_load_messenger",
"-tA", "-v", "ON_ERROR_STOP=1", "-c", sql],
cwd=str(ROOT), capture_output=True, text=True, timeout=60,
)
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
return r.stdout.strip()
def _compose(action: str, service: str):
return subprocess.run(
["docker", "compose", "--env-file", ".env", action, service],
cwd=str(ROOT), capture_output=True, text=True, timeout=90,
)
def _wait_ready(c, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
return
except Failure:
time.sleep(1)
raise Failure("API не вернулся в ready за %d секунд" % timeout)
def _mc(*args: str) -> subprocess.CompletedProcess:
"""Запуск mc через одноразовый контейнер в compose-сети."""
env = _env_from_file()
user = env.get("MINIO_ROOT_USER", "minioadmin")
password = env.get("MINIO_ROOT_PASSWORD", "minioadmin")
mc_host = "http://%s:%s@minio:9000" % (user, password)
ins = subprocess.run(
["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"],
capture_output=True, text=True, timeout=10,
)
ep = (ins.stdout or "").strip()
prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"]
return subprocess.run(
["docker", "run", "--rm", "--network", COMPOSE_NETWORK,
"-e", "MC_HOST_local=" + mc_host, MC_IMAGE]
+ prefix + list(args),
capture_output=True, text=True, timeout=60,
)
def _mc_rm(key: str) -> None:
env = _env_from_file()
bucket = env.get("S3_BUCKET", "messenger-attachments")
r = _mc("rm", "local/%s/%s" % (bucket, key))
check(r.returncode == 0, "mc rm failed: %s" % r.stderr[:300])
def _dlq_count() -> int:
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
"rabbitmqctl", "list_queues", "name", "messages"],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
for line in r.stdout.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0] == "attachment.process.v1.dlq":
return int(parts[1])
return 0
def _queue_consumers(qname: str) -> int:
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
"rabbitmqctl", "list_queues", "name", "consumers"],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
for line in r.stdout.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0] == qname:
return int(parts[1])
return 0
# ---------- 1. Все 4 варианта ----------
def test_variants(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202, got %s" % status)
check(att["status"] == "queued", "upload status must be queued, got %s" % att["status"])
ready = c.wait_ready(alice, att, wait_seconds=60)
check(ready["status"] == "ready", "must be ready")
check(ready["error_code"] is None, "ready must have error_code=null")
names = sorted(v["name"] for v in ready["variants"])
check(names == ["crop", "optimized", "thumbnail"],
"expected variants thumbnail+crop+optimized, got %s" % names)
# original
download = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)
check(download["bytes"] == data, "original bytes differ")
check(download["headers"].get_content_type() == "image/png", "original content-type")
# thumbnail: JPEG 256x160, sha256, size
v = next(x for x in ready["variants"] if x["name"] == "thumbnail")
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be image/jpeg")
check(len(thumb["bytes"]) == v["size_bytes"], "thumbnail size_bytes mismatch")
check(hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], "thumbnail sha256 mismatch")
w, h = jpeg_size(thumb["bytes"])
check((w, h) == (256, 160), "thumbnail dimensions %sx%s, expected 256x160" % (w, h))
check((v["width"], v["height"]) == (256, 160), "thumbnail metadata dimensions mismatch")
# crop: JPEG 128x128
v = next(x for x in ready["variants"] if x["name"] == "crop")
crop = c.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=bob)
check(crop["headers"].get_content_type() == "image/jpeg", "crop must be image/jpeg")
check(len(crop["bytes"]) == v["size_bytes"], "crop size_bytes mismatch")
w, h = jpeg_size(crop["bytes"])
check((w, h) == (128, 128), "crop dimensions %sx%s, expected 128x128" % (w, h))
# optimized: WebP
v = next(x for x in ready["variants"] if x["name"] == "optimized")
opt = c.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=bob)
check(opt["headers"].get_content_type() == "image/webp", "optimized must be image/webp")
check(len(opt["bytes"]) == v["size_bytes"], "optimized size_bytes mismatch")
check(hashlib.sha256(opt["bytes"]).hexdigest() == v["sha256"], "optimized sha256 mismatch")
# WebP RIFF-заголовок
check(opt["bytes"][:4] == b"RIFF" and opt["bytes"][8:12] == b"WEBP", "optimized is not WebP")
# ---------- 2. Outbox: crash между COMMIT и publish ----------
def test_outbox_crash(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
# Останавливаем outbox, чтобы job застряла в БД после COMMIT.
r = _compose("stop", "outbox")
check(r.returncode == 0, "stop outbox failed: %s" % r.stderr[:300])
try:
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202 with stopped outbox, got %s" % status)
# В БД должна быть строка outbox с published_at IS NULL.
unpublished = int(_psql(
"SELECT count(*) FROM outbox WHERE published_at IS NULL"
" AND payload->>'attachment_id' = '%s';" % att["id"]
))
check(unpublished == 1,
"expected 1 unpublished outbox row for %s, got %d" % (att["id"], unpublished))
# attachment остаётся queued.
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
check(st == "queued", "attachment must stay queued while outbox stopped, got %s" % st)
finally:
r = _compose("start", "outbox")
check(r.returncode == 0, "start outbox failed: %s" % r.stderr[:300])
# После старта outbox job должна быть опубликована и обработана.
ready = c.wait_ready(alice, att, wait_seconds=90)
check(ready["status"] == "ready", "outbox recovery: attachment must become ready")
published = int(_psql(
"SELECT count(*) FROM outbox WHERE published_at IS NOT NULL"
" AND payload->>'attachment_id' = '%s';" % att["id"]
))
check(published == 1, "outbox row must be marked published, got %d" % published)
# ---------- 3. Повторная доставка не создаёт дублей ----------
def test_duplicate_delivery_no_duplicates(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202, got %s" % status)
ready = c.wait_ready(alice, att, wait_seconds=60)
check(ready["status"] == "ready", "attachment must become ready")
# Ровно 3 non-original варианта в БД.
n_before = int(_psql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
))
check(n_before == 3, "expected exactly 3 variants, got %d" % n_before)
# Повторно публикуем тот же job через ручную публикацию в RabbitMQ
# с тем же payload (эмуляция redelivery после side effect до ack).
payload_json = _psql(
"SELECT payload::text FROM outbox WHERE payload->>'attachment_id' = '%s' LIMIT 1;" % att["id"]
)
check(bool(payload_json), "payload not found in outbox")
# Публикуем в основную очередь через pika в контейнере worker.
script = (
"import os, json, pika\n"
"url = os.environ['RABBITMQ_URL']\n"
"params = pika.URLParameters(url)\n"
"conn = pika.BlockingConnection(params)\n"
"ch = conn.channel()\n"
"ch.confirm_delivery()\n"
"ch.exchange_declare('messenger.attachments', exchange_type='direct', durable=True)\n"
"body = %r\n"
"ch.basic_publish(exchange='messenger.attachments', routing_key='attachment.process.v1',\n"
" body=body.encode('utf-8'),\n"
" properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'),\n"
" mandatory=True)\n"
"conn.close()\n"
"print('PUBLISHED')\n" % payload_json
)
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "worker",
"python", "-c", script],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
check(r.returncode == 0 and "PUBLISHED" in r.stdout,
"manual duplicate publish failed: %s" % (r.stderr[:300] or r.stdout[:300]))
# Даём worker-у обработать дубль.
time.sleep(5)
n_after = int(_psql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
))
check(n_after == n_before,
"duplicate delivery created extra variants: %d -> %d" % (n_before, n_after))
# Всё ещё ready и не failed.
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
check(st == "ready", "duplicate must not change status, got %s" % st)
# ---------- 4. Остановка worker: 202 → 409 → ready ----------
def test_worker_stopped(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
r = _compose("stop", "worker")
check(r.returncode == 0, "stop worker failed: %s" % r.stderr[:300])
try:
# Upload должен вернуть 202, attachment = queued, content — 409.
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload with stopped worker must be 202, got %s" % status)
check(att["status"] == "queued", "attachment must be queued")
# content должен вернуть 409, пока не ready.
r_content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"],
(409, 404), actor=bob)
check(r_content["status"] == 409,
"content must be 409 for queued, got %s" % r_content["status"])
# API остаётся доступен для других эндпоинтов.
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
finally:
r = _compose("start", "worker")
check(r.returncode == 0, "start worker failed: %s" % r.stderr[:300])
# Ждём consumer_started и processing.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if _queue_consumers("attachment.process.v1") >= 1:
break
time.sleep(1)
else:
raise Failure("worker не подключился к очереди за 30 секунд")
ready = c.wait_ready(alice, att, wait_seconds=90)
check(ready["status"] == "ready", "after worker start attachment must become ready")
# thumbnail доступен
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail content-type")
# ---------- 5. Broken staging → retry_exhausted → failed + DLQ ----------
def test_broken_staging_dlq(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
# Останавливаем worker, чтобы сначала зафиксировать staging_key в БД,
# затем удалить staging-объект и запустить worker.
r = _compose("stop", "worker")
check(r.returncode == 0, "stop worker failed")
try:
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202")
# Удаляем staging-объект из S3, чтобы worker не смог его прочитать.
staging = _psql(
"SELECT object_key FROM attachments WHERE id = '%s';" % att["id"]
)
check(staging.startswith("staging/"),
"queued attachment object_key must be staging/*, got %r" % staging)
_mc_rm(staging)
finally:
r = _compose("start", "worker")
check(r.returncode == 0, "start worker failed")
dlq_before = _dlq_count()
# Ждём финального failed (retry: 5+15+60 ≈ 80c + обработка).
deadline = time.monotonic() + 180
final_status = None
error_code = None
while time.monotonic() < deadline:
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
if st == "failed":
final_status = st
error_code = _psql(
"SELECT coalesce(error_code, '') FROM attachments WHERE id = '%s';" % att["id"]
)
break
if st == "ready":
raise Failure("attachment unexpectedly became ready: staging was deleted")
time.sleep(5)
check(final_status == "failed",
"expected failed after retry exhaustion, got %s" % final_status)
check(error_code == "retry_exhausted",
"expected error_code=retry_exhausted, got %r" % error_code)
# DLQ должен содержать хотя бы одно сообщение.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if _dlq_count() > dlq_before:
break
time.sleep(1)
else:
raise Failure("DLQ did not receive the message after retry exhaustion "
"(dlq_before=%d)" % dlq_before)
# ---------- 6. Stuck processing (lease) → queued → ready ----------
def test_stuck_processing_recovered(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
# Останавливаем worker, чтобы job осталась queued и мы могли её «сломать».
r = _compose("stop", "worker")
check(r.returncode == 0, "stop worker failed")
try:
status, att, _h = _upload(c, alice, chat, data)
check(status == 202, "upload must be 202, got %s" % status)
# Симулируем зависший processing: status=processing, lease в прошлом.
_psql(
"UPDATE attachments SET status='processing',"
" processing_until = now() - interval '5 minutes',"
" updated_at = now() WHERE id = '%s';" % att["id"]
)
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
check(st == "processing", "setup failed, expected processing, got %s" % st)
finally:
r = _compose("start", "worker")
check(r.returncode == 0, "start worker failed")
# Проверяем, что SQL-recover (эквивалент recover_stuck_processing)
# переводит запись в queued. Это и есть логика, которую вызывает
# s3_cleanup по интервалу. Здесь дёргаем явно, чтобы не зависеть от
# CLEANUP_INTERVAL_SECONDS.
recovered = int(_psql(
"WITH upd AS ("
" UPDATE attachments SET status='queued', processing_until=NULL, updated_at=now()"
" WHERE status='processing' AND processing_until IS NOT NULL"
" AND processing_until < now()"
" RETURNING id"
") SELECT count(*) FROM upd;"
))
check(recovered >= 1, "recover_stuck_processing did not return any row")
# worker после старта подхватит queued и доведёт до ready.
ready = c.wait_ready(alice, att, wait_seconds=90)
check(ready["status"] == "ready", "recovered attachment must become ready")
# ---------- main ----------
def main():
args = arguments("Lab5 level 4 checks")
c = Client(args.base_url, args.ca_file)
c.call("GET", "/health/ready", schema="Health")
print("=== variants (thumbnail/crop/optimized) ===")
test_variants(c)
print("PASS: original, thumbnail 256×160, crop 128×128, optimized WebP")
print("=== outbox crash between COMMIT and publish ===")
test_outbox_crash(c)
print("PASS: outbox recovers job after stop/start without manual intervention")
print("=== duplicate delivery does not create duplicates ===")
test_duplicate_delivery_no_duplicates(c)
print("PASS: re-delivery of the same job is a no-op")
print("=== worker stopped: 202 → content 409 → ready ===")
test_worker_stopped(c)
print("PASS: accepted work survives worker restart")
print("=== broken staging → retry_exhausted → failed + DLQ ===")
test_broken_staging_dlq(c)
print("PASS: bounded retry, failed with safe error_code, message in DLQ")
print("=== stuck processing (lease) → queued → ready ===")
test_stuck_processing_recovered(c)
print("PASS: expired lease is recovered, same attachment becomes ready")
print("PASS: all lab5 level-4 checks passed")
if __name__ == "__main__":
run(main)