forked from mtusi2026/mtusi2026lab5
lab5 уровень 4: jobctl (реестр failed, replay, reprocess), тесты окон отказа и demo
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> jobctl: replay только failed с живым источником (новое задание replay_of + аудит), reprocess только ready и только более новой версии; решения под той же блокировкой строки, что и уборка. test_pipeline_flow: все окна отказа attachment-states.md, retry с jitter, retry_exhausted -> replay, постоянные ошибки, лимиты памяти/времени, DLQ, зависший worker (docker pause), graceful stop, prefetch, уборка vs replay, кейсы из рецензий.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
|
||||
"""Lab 4 defense scenario (kept in lab 5; upload is now 202 queued, the download waits for ready):
|
||||
upload the fixture, send a message with attachment_ids,
|
||||
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
|
||||
request straight to S3 (from inside the storage network) is refused, the signed one works.
|
||||
|
||||
@@ -67,6 +68,16 @@ def user(c: Client, label: str) -> dict:
|
||||
return {"id": session["user"]["id"], "token": session["token"], "name": name}
|
||||
|
||||
|
||||
def wait_ready(c: Client, actor: dict, attachment_id: str, timeout: float = 60):
|
||||
import time
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{attachment_id}", token=actor["token"])
|
||||
if st != 200 or json.loads(b)["status"] in ("ready", "failed") or time.monotonic() > deadline:
|
||||
return st, h, b
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
@@ -77,7 +88,7 @@ def main() -> int:
|
||||
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
|
||||
print(f"users: alice, bob (members), eve (outsider)")
|
||||
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
|
||||
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
|
||||
print("\n1. upload -> message -> download by the second member")
|
||||
@@ -88,8 +99,8 @@ def main() -> int:
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id} (bob)", st, h, b)
|
||||
st, h, b = wait_ready(c, bob, att["id"])
|
||||
show("GET /attachments/{id} (bob) until ready", st, h, b)
|
||||
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
|
||||
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab 5 defense scenario, steps 2-3 (README): worker stopped -> upload 202 -> content 409 while the
|
||||
rest of the API works -> a message with the attachment is created -> API and broker restarted with
|
||||
their volumes -> worker started -> the same attachment becomes ready, thumbnail downloadable.
|
||||
|
||||
python scripts/demo_pipeline.py [--base-url http://127.0.0.1:8080]
|
||||
|
||||
Prints every step with HTTP codes, IDs, the outbox row and the queue depth; never prints tokens or
|
||||
keys. Creates synthetic users. Always starts the worker again at the end.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from demo_attachments import FIXTURE, ROOT, Client, show, user, wait_ready # noqa: E402
|
||||
|
||||
|
||||
def compose(*args: str, check: bool = True) -> str:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
|
||||
check=check, timeout=300).stdout
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", sql).strip()
|
||||
|
||||
|
||||
def queue_depth() -> str:
|
||||
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
|
||||
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
|
||||
_, messages, consumers = line.split("\t")
|
||||
return f"messages={messages} consumers={consumers}"
|
||||
|
||||
|
||||
def wait_api(c: Client, timeout: float = 120) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if c.call("GET", "/health/ready")[0] == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("API did not become ready")
|
||||
|
||||
|
||||
def step(text: str) -> None:
|
||||
print(f"\n[{time.strftime('%H:%M:%S')}] {text}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url)
|
||||
content = FIXTURE.read_bytes()
|
||||
alice, bob = user(c, "alice"), user(c, "bob")
|
||||
_, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
try:
|
||||
step("1. docker compose stop worker")
|
||||
compose("stop", "worker")
|
||||
print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip())
|
||||
|
||||
step("2. upload while the worker is stopped")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
raw=content, headers={"Content-Type": "image/png"})
|
||||
show("POST /chats/{id}/attachments", st, h, b)
|
||||
att = json.loads(b)
|
||||
print(f" attachment_id = {att['id']} status = {att['status']}")
|
||||
for query in ("", "?variant=thumbnail"):
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}/content{query}", token=bob["token"])
|
||||
show(f"GET /attachments/{{id}}/content{query}", st, h, b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=bob["token"],
|
||||
body={"text": "обычное сообщение работает"}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /messages (text only)", st, h, b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "фото", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /messages attachment_ids=[queued id]", st, h, b)
|
||||
message_id = json.loads(b)["id"]
|
||||
time.sleep(2)
|
||||
print(" job + outbox in PostgreSQL: " + psql(
|
||||
"SELECT j.id || ' state=' || j.state || ' attempt=' || j.attempt || ' | outbox published_at=' || "
|
||||
f"coalesce(o.published_at::text, 'NULL') FROM jobs j JOIN outbox o ON o.job_id = j.id "
|
||||
f"WHERE j.attachment_id = '{att['id']}'"))
|
||||
print(" RabbitMQ queue messenger.attachment-jobs: " + queue_depth())
|
||||
|
||||
step("3. docker compose restart api broker (volumes preserved)")
|
||||
compose("restart", "api", "broker")
|
||||
wait_api(c)
|
||||
time.sleep(3)
|
||||
print(" RabbitMQ queue after the restart: " + queue_depth())
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id}", st, h, b)
|
||||
finally:
|
||||
step("4. docker compose start worker")
|
||||
compose("start", "worker")
|
||||
st, h, b = wait_ready(c, bob, att["id"], timeout=90)
|
||||
show("GET /attachments/{id} until ready", st, h, b)
|
||||
ready = json.loads(b)
|
||||
thumb = next(v for v in ready["variants"] if v["name"] == "thumbnail")
|
||||
st, h, data = c.call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=bob["token"])
|
||||
show("GET /content?variant=thumbnail", st, h, data, binary=True)
|
||||
from PIL import Image
|
||||
size = Image.open(io.BytesIO(data)).size
|
||||
ok = (st == 200 and hashlib.sha256(data).hexdigest() == thumb["sha256"] and size == (256, 160)
|
||||
and ready["status"] == "ready")
|
||||
print(f" thumbnail: real JPEG {size[0]}x{size[1]}, sha256 matches metadata: {ok}")
|
||||
st, h, orig = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
print(f" original unchanged byte for byte: {orig == content}")
|
||||
print(f" message {message_id} still references it: "
|
||||
f"{psql(f'SELECT attachment_ids FROM messages WHERE id = {chr(39)}{message_id}{chr(39)}')}")
|
||||
step("5. job history (jobctl show)")
|
||||
shown = json.loads(compose("run", "--rm", "--no-deps", "-T", "jobctl", "show", att["id"]))
|
||||
for e in shown["events"]:
|
||||
print(f" {e['at']} {e['event']:16} attempt={e['attempt']} actor={e['actor']}")
|
||||
return 0 if ok and orig == content else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user