513 lines
22 KiB
Python
513 lines
22 KiB
Python
#!/usr/bin/env python3
|
||
"""Level 4 для lab5: варианты, outbox, идемпотентность, DLQ, lease, retry.
|
||
|
||
Запуск:
|
||
$env:PYTHONUTF8="1"
|
||
python student-tests/lab5_test_level4.py --base-url http://localhost:8080
|
||
|
||
Требует рабочий стенд из compose.yaml (LB + 2 api + worker + outbox +
|
||
RabbitMQ + MinIO + PostgreSQL + Redis). Использует psql/mc через
|
||
docker run в compose-сети, как lab4_test_level4.py.
|
||
|
||
Сценарии:
|
||
=== variants ===
|
||
Все 4 варианта: original, thumbnail (256×160), crop (128×128),
|
||
optimized (WebP). Реальные bytes и размеры, а не JSON-обещания.
|
||
=== outbox crash between COMMIT and publish ===
|
||
Остановить outbox → upload → строка outbox есть, published_at NULL
|
||
→ запустить outbox → published_at заполнено, attachment готов.
|
||
=== duplicate delivery does not create duplicates ===
|
||
Ручная повторная публикация того же job → variants не дублируются.
|
||
=== worker stopped: 202 → content 409 → ready ===
|
||
Остановить worker → upload → 202 queued → content 409 → старт
|
||
worker → ready и thumbnail доступен.
|
||
=== broken staging → retry_exhausted → failed + DLQ ===
|
||
Удалить staging-объект → worker исчерпает retry → failed
|
||
error_code=retry_exhausted + сообщение в DLQ.
|
||
=== stuck processing (lease) → queued → ready ===
|
||
Симулировать processing с истёкшим lease → recover (через s3_cleanup
|
||
или через явный SQL) возвращает в queued → worker обрабатывает → ready.
|
||
"""
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||
from image_probe import jpeg_size # noqa: E402
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
|
||
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab5_default")
|
||
PG_CONTAINER = os.environ.get("PG_CONTAINER", "mtuci-postgres")
|
||
RABBIT_CONTAINER = os.environ.get("RABBIT_CONTAINER", "mtuci-rabbitmq")
|
||
MC_IMAGE = os.environ.get("MC_IMAGE", "netvark/mc:20260920")
|
||
|
||
|
||
# ---------- helpers ----------
|
||
|
||
def _raw_upload(c: Client, actor, chat_id: str, filename: str, data: bytes, ctype: str):
|
||
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
|
||
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
|
||
req = urllib.request.Request(
|
||
url, data=data, method="POST",
|
||
headers={
|
||
"Authorization": "Bearer " + actor["token"],
|
||
"Content-Type": ctype,
|
||
},
|
||
)
|
||
try:
|
||
with c.opener.open(req, timeout=30) as resp:
|
||
return resp.code, json.loads(resp.read()), resp.headers
|
||
except urllib.error.HTTPError as e:
|
||
body = e.read()
|
||
try:
|
||
return e.code, json.loads(body), e.headers
|
||
except Exception:
|
||
return e.code, None, e.headers
|
||
|
||
|
||
def _upload(c, actor, chat, data, filename="sample.png", ctype="image/png"):
|
||
return _raw_upload(c, actor, chat["id"], filename, data, ctype)
|
||
|
||
|
||
def _env_from_file():
|
||
env = os.environ.copy()
|
||
env_file = ROOT / ".env"
|
||
if env_file.is_file():
|
||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||
line = line.strip()
|
||
if not line or line.startswith("#") or "=" not in line:
|
||
continue
|
||
k, v = line.split("=", 1)
|
||
env.setdefault(k.strip(), v.strip())
|
||
return env
|
||
|
||
|
||
def _psql(sql: str) -> str:
|
||
"""psql через docker compose exec в mtuci-postgres."""
|
||
r = subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
|
||
"psql", "-U", "messenger", "-d", "high_load_messenger",
|
||
"-tA", "-v", "ON_ERROR_STOP=1", "-c", sql],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=60,
|
||
)
|
||
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
|
||
return r.stdout.strip()
|
||
|
||
|
||
def _compose(action: str, service: str):
|
||
return subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", action, service],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=90,
|
||
)
|
||
|
||
|
||
def _wait_ready(c, timeout=90):
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
c.call("GET", "/health/ready", 200)
|
||
return
|
||
except Failure:
|
||
time.sleep(1)
|
||
raise Failure("API не вернулся в ready за %d секунд" % timeout)
|
||
|
||
|
||
def _mc(*args: str) -> subprocess.CompletedProcess:
|
||
"""Запуск mc через одноразовый контейнер в compose-сети."""
|
||
env = _env_from_file()
|
||
user = env.get("MINIO_ROOT_USER", "minioadmin")
|
||
password = env.get("MINIO_ROOT_PASSWORD", "minioadmin")
|
||
mc_host = "http://%s:%s@minio:9000" % (user, password)
|
||
|
||
ins = subprocess.run(
|
||
["docker", "image", "inspect", MC_IMAGE, "--format", "{{json .Config.Entrypoint}}"],
|
||
capture_output=True, text=True, timeout=10,
|
||
)
|
||
ep = (ins.stdout or "").strip()
|
||
prefix = [] if (ep and ep != "null" and "mc" in ep) else ["mc"]
|
||
|
||
return subprocess.run(
|
||
["docker", "run", "--rm", "--network", COMPOSE_NETWORK,
|
||
"-e", "MC_HOST_local=" + mc_host, MC_IMAGE]
|
||
+ prefix + list(args),
|
||
capture_output=True, text=True, timeout=60,
|
||
)
|
||
|
||
|
||
def _mc_rm(key: str) -> None:
|
||
env = _env_from_file()
|
||
bucket = env.get("S3_BUCKET", "messenger-attachments")
|
||
r = _mc("rm", "local/%s/%s" % (bucket, key))
|
||
check(r.returncode == 0, "mc rm failed: %s" % r.stderr[:300])
|
||
|
||
|
||
def _dlq_count() -> int:
|
||
r = subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
|
||
"rabbitmqctl", "list_queues", "name", "messages"],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||
)
|
||
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
|
||
for line in r.stdout.splitlines():
|
||
parts = line.split()
|
||
if len(parts) == 2 and parts[0] == "attachment.process.v1.dlq":
|
||
return int(parts[1])
|
||
return 0
|
||
|
||
|
||
def _queue_consumers(qname: str) -> int:
|
||
r = subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
|
||
"rabbitmqctl", "list_queues", "name", "consumers"],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||
)
|
||
check(r.returncode == 0, "rabbitmqctl failed: %s" % r.stderr[:300])
|
||
for line in r.stdout.splitlines():
|
||
parts = line.split()
|
||
if len(parts) == 2 and parts[0] == qname:
|
||
return int(parts[1])
|
||
return 0
|
||
|
||
|
||
# ---------- 1. Все 4 варианта ----------
|
||
|
||
def test_variants(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
status, att, _h = _upload(c, alice, chat, data)
|
||
check(status == 202, "upload must be 202, got %s" % status)
|
||
check(att["status"] == "queued", "upload status must be queued, got %s" % att["status"])
|
||
|
||
ready = c.wait_ready(alice, att, wait_seconds=60)
|
||
check(ready["status"] == "ready", "must be ready")
|
||
check(ready["error_code"] is None, "ready must have error_code=null")
|
||
|
||
names = sorted(v["name"] for v in ready["variants"])
|
||
check(names == ["crop", "optimized", "thumbnail"],
|
||
"expected variants thumbnail+crop+optimized, got %s" % names)
|
||
|
||
# original
|
||
download = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)
|
||
check(download["bytes"] == data, "original bytes differ")
|
||
check(download["headers"].get_content_type() == "image/png", "original content-type")
|
||
|
||
# thumbnail: JPEG 256x160, sha256, size
|
||
v = next(x for x in ready["variants"] if x["name"] == "thumbnail")
|
||
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
|
||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be image/jpeg")
|
||
check(len(thumb["bytes"]) == v["size_bytes"], "thumbnail size_bytes mismatch")
|
||
check(hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], "thumbnail sha256 mismatch")
|
||
w, h = jpeg_size(thumb["bytes"])
|
||
check((w, h) == (256, 160), "thumbnail dimensions %sx%s, expected 256x160" % (w, h))
|
||
check((v["width"], v["height"]) == (256, 160), "thumbnail metadata dimensions mismatch")
|
||
|
||
# crop: JPEG 128x128
|
||
v = next(x for x in ready["variants"] if x["name"] == "crop")
|
||
crop = c.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=bob)
|
||
check(crop["headers"].get_content_type() == "image/jpeg", "crop must be image/jpeg")
|
||
check(len(crop["bytes"]) == v["size_bytes"], "crop size_bytes mismatch")
|
||
w, h = jpeg_size(crop["bytes"])
|
||
check((w, h) == (128, 128), "crop dimensions %sx%s, expected 128x128" % (w, h))
|
||
|
||
# optimized: WebP
|
||
v = next(x for x in ready["variants"] if x["name"] == "optimized")
|
||
opt = c.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=bob)
|
||
check(opt["headers"].get_content_type() == "image/webp", "optimized must be image/webp")
|
||
check(len(opt["bytes"]) == v["size_bytes"], "optimized size_bytes mismatch")
|
||
check(hashlib.sha256(opt["bytes"]).hexdigest() == v["sha256"], "optimized sha256 mismatch")
|
||
# WebP RIFF-заголовок
|
||
check(opt["bytes"][:4] == b"RIFF" and opt["bytes"][8:12] == b"WEBP", "optimized is not WebP")
|
||
|
||
|
||
# ---------- 2. Outbox: crash между COMMIT и publish ----------
|
||
|
||
def test_outbox_crash(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
|
||
# Останавливаем outbox, чтобы job застряла в БД после COMMIT.
|
||
r = _compose("stop", "outbox")
|
||
check(r.returncode == 0, "stop outbox failed: %s" % r.stderr[:300])
|
||
|
||
try:
|
||
status, att, _h = _upload(c, alice, chat, data)
|
||
check(status == 202, "upload must be 202 with stopped outbox, got %s" % status)
|
||
|
||
# В БД должна быть строка outbox с published_at IS NULL.
|
||
unpublished = int(_psql(
|
||
"SELECT count(*) FROM outbox WHERE published_at IS NULL"
|
||
" AND payload->>'attachment_id' = '%s';" % att["id"]
|
||
))
|
||
check(unpublished == 1,
|
||
"expected 1 unpublished outbox row for %s, got %d" % (att["id"], unpublished))
|
||
|
||
# attachment остаётся queued.
|
||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||
check(st == "queued", "attachment must stay queued while outbox stopped, got %s" % st)
|
||
finally:
|
||
r = _compose("start", "outbox")
|
||
check(r.returncode == 0, "start outbox failed: %s" % r.stderr[:300])
|
||
|
||
# После старта outbox job должна быть опубликована и обработана.
|
||
ready = c.wait_ready(alice, att, wait_seconds=90)
|
||
check(ready["status"] == "ready", "outbox recovery: attachment must become ready")
|
||
published = int(_psql(
|
||
"SELECT count(*) FROM outbox WHERE published_at IS NOT NULL"
|
||
" AND payload->>'attachment_id' = '%s';" % att["id"]
|
||
))
|
||
check(published == 1, "outbox row must be marked published, got %d" % published)
|
||
|
||
|
||
# ---------- 3. Повторная доставка не создаёт дублей ----------
|
||
|
||
def test_duplicate_delivery_no_duplicates(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
status, att, _h = _upload(c, alice, chat, data)
|
||
check(status == 202, "upload must be 202, got %s" % status)
|
||
ready = c.wait_ready(alice, att, wait_seconds=60)
|
||
check(ready["status"] == "ready", "attachment must become ready")
|
||
|
||
# Ровно 3 non-original варианта в БД.
|
||
n_before = int(_psql(
|
||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
|
||
))
|
||
check(n_before == 3, "expected exactly 3 variants, got %d" % n_before)
|
||
|
||
# Повторно публикуем тот же job через ручную публикацию в RabbitMQ
|
||
# с тем же payload (эмуляция redelivery после side effect до ack).
|
||
payload_json = _psql(
|
||
"SELECT payload::text FROM outbox WHERE payload->>'attachment_id' = '%s' LIMIT 1;" % att["id"]
|
||
)
|
||
check(bool(payload_json), "payload not found in outbox")
|
||
# Публикуем в основную очередь через pika в контейнере worker.
|
||
script = (
|
||
"import os, json, pika\n"
|
||
"url = os.environ['RABBITMQ_URL']\n"
|
||
"params = pika.URLParameters(url)\n"
|
||
"conn = pika.BlockingConnection(params)\n"
|
||
"ch = conn.channel()\n"
|
||
"ch.confirm_delivery()\n"
|
||
"ch.exchange_declare('messenger.attachments', exchange_type='direct', durable=True)\n"
|
||
"body = %r\n"
|
||
"ch.basic_publish(exchange='messenger.attachments', routing_key='attachment.process.v1',\n"
|
||
" body=body.encode('utf-8'),\n"
|
||
" properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'),\n"
|
||
" mandatory=True)\n"
|
||
"conn.close()\n"
|
||
"print('PUBLISHED')\n" % payload_json
|
||
)
|
||
r = subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", "exec", "-T", "worker",
|
||
"python", "-c", script],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||
)
|
||
check(r.returncode == 0 and "PUBLISHED" in r.stdout,
|
||
"manual duplicate publish failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||
|
||
# Даём worker-у обработать дубль.
|
||
time.sleep(5)
|
||
|
||
n_after = int(_psql(
|
||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='%s';" % att["id"]
|
||
))
|
||
check(n_after == n_before,
|
||
"duplicate delivery created extra variants: %d -> %d" % (n_before, n_after))
|
||
# Всё ещё ready и не failed.
|
||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||
check(st == "ready", "duplicate must not change status, got %s" % st)
|
||
|
||
|
||
# ---------- 4. Остановка worker: 202 → 409 → ready ----------
|
||
|
||
def test_worker_stopped(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
|
||
r = _compose("stop", "worker")
|
||
check(r.returncode == 0, "stop worker failed: %s" % r.stderr[:300])
|
||
try:
|
||
# Upload должен вернуть 202, attachment = queued, content — 409.
|
||
status, att, _h = _upload(c, alice, chat, data)
|
||
check(status == 202, "upload with stopped worker must be 202, got %s" % status)
|
||
check(att["status"] == "queued", "attachment must be queued")
|
||
|
||
# content должен вернуть 409, пока не ready.
|
||
r_content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"],
|
||
(409, 404), actor=bob)
|
||
check(r_content["status"] == 409,
|
||
"content must be 409 for queued, got %s" % r_content["status"])
|
||
|
||
# API остаётся доступен для других эндпоинтов.
|
||
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||
finally:
|
||
r = _compose("start", "worker")
|
||
check(r.returncode == 0, "start worker failed: %s" % r.stderr[:300])
|
||
|
||
# Ждём consumer_started и processing.
|
||
deadline = time.monotonic() + 30
|
||
while time.monotonic() < deadline:
|
||
if _queue_consumers("attachment.process.v1") >= 1:
|
||
break
|
||
time.sleep(1)
|
||
else:
|
||
raise Failure("worker не подключился к очереди за 30 секунд")
|
||
|
||
ready = c.wait_ready(alice, att, wait_seconds=90)
|
||
check(ready["status"] == "ready", "after worker start attachment must become ready")
|
||
|
||
# thumbnail доступен
|
||
thumb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)
|
||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail content-type")
|
||
|
||
|
||
# ---------- 5. Broken staging → retry_exhausted → failed + DLQ ----------
|
||
|
||
def test_broken_staging_dlq(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
|
||
# Останавливаем worker, чтобы сначала зафиксировать staging_key в БД,
|
||
# затем удалить staging-объект и запустить worker.
|
||
r = _compose("stop", "worker")
|
||
check(r.returncode == 0, "stop worker failed")
|
||
try:
|
||
status, att, _h = _upload(c, alice, chat, data)
|
||
check(status == 202, "upload must be 202")
|
||
|
||
# Удаляем staging-объект из S3, чтобы worker не смог его прочитать.
|
||
staging = _psql(
|
||
"SELECT object_key FROM attachments WHERE id = '%s';" % att["id"]
|
||
)
|
||
check(staging.startswith("staging/"),
|
||
"queued attachment object_key must be staging/*, got %r" % staging)
|
||
_mc_rm(staging)
|
||
finally:
|
||
r = _compose("start", "worker")
|
||
check(r.returncode == 0, "start worker failed")
|
||
|
||
dlq_before = _dlq_count()
|
||
|
||
# Ждём финального failed (retry: 5+15+60 ≈ 80c + обработка).
|
||
deadline = time.monotonic() + 180
|
||
final_status = None
|
||
error_code = None
|
||
while time.monotonic() < deadline:
|
||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||
if st == "failed":
|
||
final_status = st
|
||
error_code = _psql(
|
||
"SELECT coalesce(error_code, '') FROM attachments WHERE id = '%s';" % att["id"]
|
||
)
|
||
break
|
||
if st == "ready":
|
||
raise Failure("attachment unexpectedly became ready: staging was deleted")
|
||
time.sleep(5)
|
||
|
||
check(final_status == "failed",
|
||
"expected failed after retry exhaustion, got %s" % final_status)
|
||
check(error_code == "retry_exhausted",
|
||
"expected error_code=retry_exhausted, got %r" % error_code)
|
||
|
||
# DLQ должен содержать хотя бы одно сообщение.
|
||
deadline = time.monotonic() + 30
|
||
while time.monotonic() < deadline:
|
||
if _dlq_count() > dlq_before:
|
||
break
|
||
time.sleep(1)
|
||
else:
|
||
raise Failure("DLQ did not receive the message after retry exhaustion "
|
||
"(dlq_before=%d)" % dlq_before)
|
||
|
||
|
||
# ---------- 6. Stuck processing (lease) → queued → ready ----------
|
||
|
||
def test_stuck_processing_recovered(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
|
||
# Останавливаем worker, чтобы job осталась queued и мы могли её «сломать».
|
||
r = _compose("stop", "worker")
|
||
check(r.returncode == 0, "stop worker failed")
|
||
try:
|
||
status, att, _h = _upload(c, alice, chat, data)
|
||
check(status == 202, "upload must be 202, got %s" % status)
|
||
|
||
# Симулируем зависший processing: status=processing, lease в прошлом.
|
||
_psql(
|
||
"UPDATE attachments SET status='processing',"
|
||
" processing_until = now() - interval '5 minutes',"
|
||
" updated_at = now() WHERE id = '%s';" % att["id"]
|
||
)
|
||
st = _psql("SELECT status FROM attachments WHERE id = '%s';" % att["id"])
|
||
check(st == "processing", "setup failed, expected processing, got %s" % st)
|
||
finally:
|
||
r = _compose("start", "worker")
|
||
check(r.returncode == 0, "start worker failed")
|
||
|
||
# Проверяем, что SQL-recover (эквивалент recover_stuck_processing)
|
||
# переводит запись в queued. Это и есть логика, которую вызывает
|
||
# s3_cleanup по интервалу. Здесь дёргаем явно, чтобы не зависеть от
|
||
# CLEANUP_INTERVAL_SECONDS.
|
||
recovered = int(_psql(
|
||
"WITH upd AS ("
|
||
" UPDATE attachments SET status='queued', processing_until=NULL, updated_at=now()"
|
||
" WHERE status='processing' AND processing_until IS NOT NULL"
|
||
" AND processing_until < now()"
|
||
" RETURNING id"
|
||
") SELECT count(*) FROM upd;"
|
||
))
|
||
check(recovered >= 1, "recover_stuck_processing did not return any row")
|
||
|
||
# worker после старта подхватит queued и доведёт до ready.
|
||
ready = c.wait_ready(alice, att, wait_seconds=90)
|
||
check(ready["status"] == "ready", "recovered attachment must become ready")
|
||
|
||
|
||
# ---------- main ----------
|
||
|
||
def main():
|
||
args = arguments("Lab5 level 4 checks")
|
||
c = Client(args.base_url, args.ca_file)
|
||
c.call("GET", "/health/ready", schema="Health")
|
||
|
||
print("=== variants (thumbnail/crop/optimized) ===")
|
||
test_variants(c)
|
||
print("PASS: original, thumbnail 256×160, crop 128×128, optimized WebP")
|
||
|
||
print("=== outbox crash between COMMIT and publish ===")
|
||
test_outbox_crash(c)
|
||
print("PASS: outbox recovers job after stop/start without manual intervention")
|
||
|
||
print("=== duplicate delivery does not create duplicates ===")
|
||
test_duplicate_delivery_no_duplicates(c)
|
||
print("PASS: re-delivery of the same job is a no-op")
|
||
|
||
print("=== worker stopped: 202 → content 409 → ready ===")
|
||
test_worker_stopped(c)
|
||
print("PASS: accepted work survives worker restart")
|
||
|
||
print("=== broken staging → retry_exhausted → failed + DLQ ===")
|
||
test_broken_staging_dlq(c)
|
||
print("PASS: bounded retry, failed with safe error_code, message in DLQ")
|
||
|
||
print("=== stuck processing (lease) → queued → ready ===")
|
||
test_stuck_processing_recovered(c)
|
||
print("PASS: expired lease is recovered, same attachment becomes ready")
|
||
|
||
print("PASS: all lab5 level-4 checks passed")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main) |