forked from mtusi2026/mtusi2026lab5
360 lines
20 KiB
Python
360 lines
20 KiB
Python
"""Grade 4/5 scenarios for the local stand. Prints no session tokens."""
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib import error, request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
BASE = "http://localhost:8080"
|
|
EVIDENCE = ROOT / "evidence" / "grade5-track2.json"
|
|
ALLOWED_MESSAGE_KEYS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", "traceparent", "tracestate"}
|
|
|
|
|
|
def env_value(name):
|
|
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
|
if line.startswith(name + "="):
|
|
return line.split("=", 1)[1]
|
|
raise SystemExit("missing local setting " + name)
|
|
|
|
|
|
TOKEN = env_value("OPERATOR_TOKEN")
|
|
RESULTS = []
|
|
|
|
|
|
def record(name, detail):
|
|
RESULTS.append({"scenario": name, "result": "pass", "detail": detail})
|
|
print("PASS:", name)
|
|
|
|
|
|
def fail(name, message):
|
|
RESULTS.append({"scenario": name, "result": "fail", "detail": message})
|
|
EVIDENCE.parent.mkdir(exist_ok=True)
|
|
EVIDENCE.write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
raise SystemExit(name + ": " + message)
|
|
|
|
|
|
def compose(*args):
|
|
completed = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True, capture_output=True)
|
|
if completed.returncode != 0:
|
|
raise SystemExit("compose " + " ".join(args) + " failed: " + completed.stderr[-400:])
|
|
return completed.stdout
|
|
|
|
|
|
def queue_depth():
|
|
completed = subprocess.run(
|
|
["docker", "exec", "mtusi-lab5-rabbitmq", "rabbitmqctl", "list_queues", "name", "messages"],
|
|
text=True, capture_output=True,
|
|
)
|
|
if completed.returncode != 0:
|
|
return None
|
|
for line in completed.stdout.splitlines():
|
|
parts = line.split()
|
|
if len(parts) == 2 and parts[0] == "attachment.process" and parts[1].isdigit():
|
|
return int(parts[1])
|
|
return 0
|
|
|
|
|
|
def operator(method, path, body=None, expected=200):
|
|
data = None if body is None else json.dumps(body).encode("utf-8")
|
|
headers = {"X-Operator-Token": TOKEN, "Accept": "application/json"}
|
|
if body is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
call = request.Request(BASE + path, data=data, headers=headers, method=method)
|
|
try:
|
|
response = request.urlopen(call, timeout=10)
|
|
except error.HTTPError as exc:
|
|
response = exc
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
if response.status != expected:
|
|
raise Failure(method + " " + path + " -> " + str(response.status) + " " + payload.get("error", {}).get("code", ""))
|
|
return payload
|
|
|
|
|
|
def reset_controls():
|
|
operator("POST", "/internal/control/fault", {"mode": None})
|
|
operator("POST", "/internal/control/outbox", {"paused": False})
|
|
operator("POST", "/internal/control/backpressure", {"max_queued_jobs": None})
|
|
compose("start", "worker")
|
|
|
|
|
|
def wait_health(timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure:
|
|
time.sleep(0.5)
|
|
raise SystemExit("API did not become ready")
|
|
|
|
|
|
def wait_attachment(client, actor, attachment_id, wanted, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
last = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"]
|
|
if last["status"] == wanted:
|
|
return last
|
|
if wanted != "failed" and last["status"] == "failed":
|
|
raise Failure("attachment failed with " + str(last["error_code"]))
|
|
time.sleep(0.4)
|
|
raise Failure("timed out waiting for " + wanted + ", last status " + str(last and last["status"]))
|
|
|
|
|
|
def upload_sample(client, actor, chat):
|
|
content = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
attachment = client.call(
|
|
"POST",
|
|
"/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
|
202,
|
|
actor=actor,
|
|
raw=content,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)["json"]
|
|
if attachment["status"] != "queued":
|
|
raise Failure("upload snapshot is not queued")
|
|
return attachment, content
|
|
|
|
|
|
def main():
|
|
reset_controls()
|
|
wait_health()
|
|
client = Client(BASE, timeout=10)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
|
|
attachment, content = upload_sample(client, alice, chat)
|
|
ready = wait_attachment(client, bob, attachment["id"], "ready")
|
|
names = {item["name"]: item for item in ready["variants"]}
|
|
if set(names) != {"thumbnail", "crop", "optimized"}:
|
|
fail("variants", "unexpected variant set")
|
|
if (names["thumbnail"]["width"], names["thumbnail"]["height"]) != (256, 160):
|
|
fail("variants", "thumbnail metadata")
|
|
if (names["crop"]["width"], names["crop"]["height"]) != (128, 128) or names["crop"]["content_type"] != "image/jpeg":
|
|
fail("variants", "crop contract")
|
|
if names["optimized"]["content_type"] != "image/webp" or (names["optimized"]["width"], names["optimized"]["height"]) != (320, 200):
|
|
fail("variants", "optimized contract")
|
|
original = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
|
thumb = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=bob)
|
|
optimized = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=optimized", actor=bob)
|
|
if original["bytes"] != content or thumb["headers"].get_content_type() != "image/jpeg":
|
|
fail("variants", "bytes or thumbnail type")
|
|
if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"] or optimized["bytes"][8:12] != b"WEBP":
|
|
fail("variants", "checksum or webp header")
|
|
state = operator("GET", "/internal/attachments/" + attachment["id"])
|
|
if any(set(message) - ALLOWED_MESSAGE_KEYS or any(isinstance(value, str) and len(value) > 512 for value in message.values()) for message in state["messages"]):
|
|
fail("variants", "broker payload contains unexpected data")
|
|
record("grade4 variants and broker payload", {"attachment_id": attachment["id"], "job_id": state["jobs"][0]["job_id"]})
|
|
|
|
key = "order-" + uuid.uuid4().hex
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
first = client.call("POST", path, 201, actor=alice, raw=b'{"text":"same-body","attachment_ids":[]}', headers={"Content-Type": "application/json", "Idempotency-Key": key}, schema="Message")["json"]
|
|
second = client.call("POST", path, 201, actor=alice, raw=b'{"attachment_ids":[],"text":"same-body"}', headers={"Content-Type": "application/json", "Idempotency-Key": key}, schema="Message")["json"]
|
|
if first != second:
|
|
fail("idempotency-order", "canonical replay differs")
|
|
record("idempotency key order", {"message_id": first["id"]})
|
|
|
|
client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short", "extra": 1})
|
|
client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bad.png", 415, actor=alice, raw=b"not-an-image", headers={"Content-Type": "image/png"})
|
|
import struct
|
|
import zlib
|
|
ihdr = struct.pack(">IIBBBBB", 8000, 8000, 8, 2, 0, 0, 0)
|
|
chunk = struct.pack(">I", len(ihdr)) + b"IHDR" + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr) & 0xFFFFFFFF)
|
|
bomb = b"\x89PNG\r\n\x1a\n" + chunk + struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND") & 0xFFFFFFFF)
|
|
client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bomb.png", 413, actor=alice, raw=bomb, headers={"Content-Type": "image/png"})
|
|
record("validation", {"rejected": ["extra_field", "invalid_image", "pixel_limit"]})
|
|
|
|
operator("POST", "/internal/control/backpressure", {"max_queued_jobs": 0})
|
|
try:
|
|
client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=blocked.png", 503, actor=alice, raw=content, headers={"Content-Type": "image/png"})
|
|
finally:
|
|
operator("POST", "/internal/control/backpressure", {"max_queued_jobs": None})
|
|
record("backpressure", {"max_queued_jobs": 0, "status": 503})
|
|
|
|
operator("POST", "/internal/control/outbox", {"paused": True})
|
|
try:
|
|
paused, _content = upload_sample(client, alice, chat)
|
|
time.sleep(1.2)
|
|
metrics = operator("GET", "/internal/metrics")
|
|
hidden = operator("GET", "/internal/attachments/" + paused["id"])
|
|
if metrics["outbox_unpublished"] < 1 or hidden["attachment"]["status"] != "queued" or hidden["jobs"][0]["job_id"] is None:
|
|
fail("outbox-gap", "intent was not retained while publish was paused")
|
|
if queue_depth() not in {0, None}:
|
|
fail("outbox-gap", "broker received a message before publish resumed")
|
|
job_id = hidden["jobs"][0]["job_id"]
|
|
finally:
|
|
operator("POST", "/internal/control/outbox", {"paused": False})
|
|
resumed = wait_attachment(client, alice, paused["id"], "ready")
|
|
after = operator("GET", "/internal/attachments/" + paused["id"])
|
|
if after["jobs"][0]["job_id"] != job_id or resumed["status"] != "ready":
|
|
fail("outbox-gap", "recovery created a different job")
|
|
record("outbox commit/publish gap", {"attachment_id": paused["id"], "job_id": job_id})
|
|
|
|
compose("stop", "worker")
|
|
try:
|
|
held, held_bytes = upload_sample(client, alice, chat)
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline and operator("GET", "/internal/metrics")["outbox_unpublished"] != 0:
|
|
time.sleep(0.3)
|
|
if operator("GET", "/internal/attachments/" + held["id"])["attachment"]["status"] != "queued":
|
|
fail("worker-stop", "API processed the image synchronously")
|
|
client.call("GET", "/api/v1/attachments/" + held["id"] + "/content", 409, actor=bob)
|
|
client.message(bob, chat, "чат жив без worker")
|
|
depth_before = queue_depth()
|
|
compose("restart", "api-1", "api-2", "rabbitmq")
|
|
wait_health()
|
|
depth_after = None
|
|
deadline = time.monotonic() + 40
|
|
while time.monotonic() < deadline:
|
|
depth_after = queue_depth()
|
|
if depth_after is not None:
|
|
break
|
|
time.sleep(1)
|
|
if not depth_before or not depth_after:
|
|
fail("worker-stop", "durable queue did not retain the message across broker restart")
|
|
finally:
|
|
compose("start", "worker")
|
|
held_ready = wait_attachment(client, bob, held["id"], "ready")
|
|
downloaded = client.call("GET", "/api/v1/attachments/" + held["id"] + "/content?variant=thumbnail", actor=alice)
|
|
if held_ready["id"] != held["id"] or downloaded["headers"].get_content_type() != "image/jpeg":
|
|
fail("worker-stop", "restarted worker did not finish the accepted job")
|
|
if client.call("GET", "/api/v1/attachments/" + held["id"] + "/content", actor=bob)["bytes"] != held_bytes:
|
|
fail("worker-stop", "original changed")
|
|
record("worker stop, broker restart, resume", {"attachment_id": held["id"], "queue_before": depth_before, "queue_after_restart": depth_after})
|
|
|
|
operator("POST", "/internal/control/fault", {"mode": "permanent"})
|
|
rejected, _bytes = upload_sample(client, alice, chat)
|
|
failed = wait_attachment(client, alice, rejected["id"], "failed")
|
|
if failed["error_code"] != "invalid_image":
|
|
fail("permanent-replay", "unexpected error code")
|
|
client.call("GET", "/api/v1/attachments/" + rejected["id"] + "/content", 409, actor=alice)
|
|
client.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 409, actor=alice, body={"text": "", "attachment_ids": [rejected["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
|
replay = operator("POST", "/internal/attachments/" + rejected["id"] + "/replay")
|
|
if not replay["replayed"] or replay["job_id"] != operator("GET", "/internal/attachments/" + rejected["id"])["jobs"][0]["job_id"]:
|
|
fail("permanent-replay", "replay changed the logical job")
|
|
restored = wait_attachment(client, alice, rejected["id"], "ready")
|
|
again = operator("POST", "/internal/attachments/" + rejected["id"] + "/replay")
|
|
if again["reason"] != "already_ready" or len(restored["variants"]) != 3:
|
|
fail("permanent-replay", "successful job was replayed as new work")
|
|
record("permanent failure, DLQ registry, safe replay", {"attachment_id": rejected["id"], "job_id": replay["job_id"]})
|
|
|
|
operator("POST", "/internal/control/fault", {"mode": "transient", "remaining": 1})
|
|
retried, _bytes = upload_sample(client, alice, chat)
|
|
retried_ready = wait_attachment(client, alice, retried["id"], "ready", timeout=60)
|
|
retried_state = operator("GET", "/internal/attachments/" + retried["id"])
|
|
if retried_state["jobs"][0]["attempt"] != 2 or retried_ready["status"] != "ready":
|
|
fail("backoff", "single transient failure did not retry exactly once")
|
|
record("transient retry with backoff", {"attachment_id": retried["id"], "attempt": 2})
|
|
|
|
operator("POST", "/internal/control/fault", {"mode": "transient", "remaining": 5})
|
|
exhausted, _bytes = upload_sample(client, alice, chat)
|
|
exhausted_state = wait_attachment(client, alice, exhausted["id"], "failed", timeout=60)
|
|
if exhausted_state["error_code"] != "retry_exhausted":
|
|
fail("retry-budget", "attempts were not capped")
|
|
if operator("GET", "/internal/metrics")["failed_registry"] < 1:
|
|
fail("retry-budget", "failed registry is empty")
|
|
operator("POST", "/internal/attachments/" + exhausted["id"] + "/replay")
|
|
wait_attachment(client, alice, exhausted["id"], "ready", timeout=60)
|
|
record("retry budget then explicit replay", {"attachment_id": exhausted["id"]})
|
|
|
|
operator("POST", "/internal/control/fault", {"mode": "duplicate"})
|
|
duplicated, _bytes = upload_sample(client, alice, chat)
|
|
wait_attachment(client, alice, duplicated["id"], "ready")
|
|
duplicate_state = operator("GET", "/internal/attachments/" + duplicated["id"])
|
|
visible = [item for item in duplicate_state["variants"] if item["pipeline_version"] == 1]
|
|
if len(visible) != 3 or "noop_ready" not in {item["action"] for item in duplicate_state["audits"]}:
|
|
fail("duplicate", "inline duplicate created another variant set or skipped the no-op")
|
|
before = {item["sha256"] for item in visible}
|
|
operator("POST", "/internal/jobs/" + duplicate_state["jobs"][0]["job_id"] + "/redeliver")
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline and sum(1 for item in operator("GET", "/internal/attachments/" + duplicated["id"])["audits"] if item["action"] == "noop_ready") < 2:
|
|
time.sleep(0.4)
|
|
after_state = operator("GET", "/internal/attachments/" + duplicated["id"])
|
|
after = {item["sha256"] for item in after_state["variants"] if item["pipeline_version"] == 1}
|
|
if before != after or len(after_state["variants"]) != 3:
|
|
fail("duplicate", "broker redelivery rewrote the result")
|
|
record("duplicate delivery is a no-op", {"attachment_id": duplicated["id"], "job_id": duplicate_state["jobs"][0]["job_id"]})
|
|
|
|
for mode, label in (("before_effect", "crash before side effect"), ("after_s3", "crash after object write"), ("after_commit", "crash after commit before ack")):
|
|
operator("POST", "/internal/control/fault", {"mode": mode})
|
|
crashed, _bytes = upload_sample(client, alice, chat)
|
|
wait_attachment(client, alice, crashed["id"], "ready", timeout=90)
|
|
audits = {item["action"] for item in operator("GET", "/internal/attachments/" + crashed["id"])["audits"]}
|
|
expected = {"before_effect": "fault_before_effect", "after_s3": "fault_after_s3", "after_commit": "fault_after_commit"}[mode]
|
|
if expected not in audits or len([item for item in operator("GET", "/internal/attachments/" + crashed["id"])["variants"] if item["pipeline_version"] == 1]) != 3:
|
|
fail(label, "fault was not recovered idempotently")
|
|
record(label, {"attachment_id": crashed["id"], "audit": expected})
|
|
|
|
baseline, _bytes = upload_sample(client, alice, chat)
|
|
baseline_ready = wait_attachment(client, alice, baseline["id"], "ready")
|
|
first_sha = {item["name"]: item["sha256"] for item in baseline_ready["variants"]}
|
|
reprocess = operator("POST", "/internal/attachments/" + baseline["id"] + "/reprocess", {"pipeline_version": 2})
|
|
version_two = wait_attachment(client, alice, baseline["id"], "ready")
|
|
second_sha = {item["name"]: item["sha256"] for item in version_two["variants"]}
|
|
version_state = operator("GET", "/internal/attachments/" + baseline["id"])
|
|
if version_state["attachment"]["ready_pipeline_version"] != 2 or first_sha["thumbnail"] == second_sha["thumbnail"]:
|
|
fail("pipeline-version", "version 2 did not publish a distinct result")
|
|
if len(version_state["variants"]) != 6:
|
|
fail("pipeline-version", "versioned variants were overwritten in place")
|
|
operator("POST", "/internal/jobs/" + version_state["jobs"][0]["job_id"] + "/redeliver")
|
|
time.sleep(2)
|
|
stale = operator("GET", "/internal/attachments/" + baseline["id"])
|
|
public = client.call("GET", "/api/v1/attachments/" + baseline["id"], actor=alice, schema="Attachment")["json"]
|
|
if stale["attachment"]["ready_pipeline_version"] != 2 or {item["sha256"] for item in public["variants"]} != set(second_sha.values()):
|
|
fail("pipeline-version", "stale pipeline overwrote the newer result")
|
|
record("versioned pipeline rejects stale result", {"attachment_id": baseline["id"], "new_job_id": reprocess["job_id"]})
|
|
|
|
operator("POST", "/internal/control/outbox", {"paused": True})
|
|
try:
|
|
ordered, _bytes = upload_sample(client, alice, chat)
|
|
operator("POST", "/internal/attachments/" + ordered["id"] + "/reprocess", {"pipeline_version": 2})
|
|
finally:
|
|
operator("POST", "/internal/control/outbox", {"paused": False})
|
|
ordered_ready = wait_attachment(client, alice, ordered["id"], "ready")
|
|
ordered_state = operator("GET", "/internal/attachments/" + ordered["id"])
|
|
if ordered_state["attachment"]["ready_pipeline_version"] != 2 or "superseded" not in {job["status"] for job in ordered_state["jobs"]}:
|
|
fail("reorder", "older queued job was not superseded by the newer pipeline")
|
|
if {item["name"] for item in ordered_ready["variants"]} != {"thumbnail", "crop", "optimized"}:
|
|
fail("reorder", "public result is not the complete newer pipeline")
|
|
record("reordered older job cannot win", {"attachment_id": ordered["id"]})
|
|
|
|
persisted = client.message(alice, chat, "пережить рестарт")["json"]
|
|
compose("restart", "postgres", "minio", "rabbitmq", "api-1", "api-2", "worker", "nginx")
|
|
wait_health(120)
|
|
me = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
|
|
history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
|
|
if me["id"] != alice["id"] or not any(item == persisted for item in history["items"]):
|
|
fail("persistence", "session or message did not survive restart")
|
|
if client.call("GET", "/api/v1/attachments/" + baseline["id"] + "/content", actor=bob)["bytes"] != content:
|
|
fail("persistence", "object bytes did not survive restart")
|
|
record("volume restart preserves session message and object", {"message_id": persisted["id"], "attachment_id": baseline["id"]})
|
|
|
|
observed = set()
|
|
for _ in range(20):
|
|
observed.add(client.call("GET", "/api/v1/users/me", actor=bob, schema="User")["headers"]["X-Instance-Id"])
|
|
if observed != {"api-1", "api-2"}:
|
|
fail("replicas", "load balancer did not expose both API replicas")
|
|
record("two API replicas", {"instances": sorted(observed)})
|
|
EVIDENCE.parent.mkdir(exist_ok=True)
|
|
EVIDENCE.write_text(json.dumps({"track": 2, "results": RESULTS}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print("PASS: grade 4 and track 2 scenarios")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Failure as exc:
|
|
fail("client", str(exc))
|
|
finally:
|
|
try:
|
|
reset_controls()
|
|
except Exception:
|
|
pass
|