Files
mtusi2026lab5/student_tests/run_grade.py
T

360 lines
20 KiB
Python

"""Grade 4/5 scenarios for the local stand. Prints no session tokens."""
import hashlib
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib import error, request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
BASE = "http://localhost:8080"
EVIDENCE = ROOT / "evidence" / "grade5-track2.json"
ALLOWED_MESSAGE_KEYS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", "traceparent", "tracestate"}
def env_value(name):
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
if line.startswith(name + "="):
return line.split("=", 1)[1]
raise SystemExit("missing local setting " + name)
TOKEN = env_value("OPERATOR_TOKEN")
RESULTS = []
def record(name, detail):
RESULTS.append({"scenario": name, "result": "pass", "detail": detail})
print("PASS:", name)
def fail(name, message):
RESULTS.append({"scenario": name, "result": "fail", "detail": message})
EVIDENCE.parent.mkdir(exist_ok=True)
EVIDENCE.write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2), encoding="utf-8")
raise SystemExit(name + ": " + message)
def compose(*args):
completed = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True, capture_output=True)
if completed.returncode != 0:
raise SystemExit("compose " + " ".join(args) + " failed: " + completed.stderr[-400:])
return completed.stdout
def queue_depth():
completed = subprocess.run(
["docker", "exec", "mtusi-lab5-rabbitmq", "rabbitmqctl", "list_queues", "name", "messages"],
text=True, capture_output=True,
)
if completed.returncode != 0:
return None
for line in completed.stdout.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0] == "attachment.process" and parts[1].isdigit():
return int(parts[1])
return 0
def operator(method, path, body=None, expected=200):
data = None if body is None else json.dumps(body).encode("utf-8")
headers = {"X-Operator-Token": TOKEN, "Accept": "application/json"}
if body is not None:
headers["Content-Type"] = "application/json"
call = request.Request(BASE + path, data=data, headers=headers, method=method)
try:
response = request.urlopen(call, timeout=10)
except error.HTTPError as exc:
response = exc
payload = json.loads(response.read().decode("utf-8"))
if response.status != expected:
raise Failure(method + " " + path + " -> " + str(response.status) + " " + payload.get("error", {}).get("code", ""))
return payload
def reset_controls():
operator("POST", "/internal/control/fault", {"mode": None})
operator("POST", "/internal/control/outbox", {"paused": False})
operator("POST", "/internal/control/backpressure", {"max_queued_jobs": None})
compose("start", "worker")
def wait_health(timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
return
except Failure:
time.sleep(0.5)
raise SystemExit("API did not become ready")
def wait_attachment(client, actor, attachment_id, wanted, timeout=90):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
last = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"]
if last["status"] == wanted:
return last
if wanted != "failed" and last["status"] == "failed":
raise Failure("attachment failed with " + str(last["error_code"]))
time.sleep(0.4)
raise Failure("timed out waiting for " + wanted + ", last status " + str(last and last["status"]))
def upload_sample(client, actor, chat):
content = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
attachment = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202,
actor=actor,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
if attachment["status"] != "queued":
raise Failure("upload snapshot is not queued")
return attachment, content
def main():
reset_controls()
wait_health()
client = Client(BASE, timeout=10)
alice, bob, _eve, chat = client.bootstrap()
attachment, content = upload_sample(client, alice, chat)
ready = wait_attachment(client, bob, attachment["id"], "ready")
names = {item["name"]: item for item in ready["variants"]}
if set(names) != {"thumbnail", "crop", "optimized"}:
fail("variants", "unexpected variant set")
if (names["thumbnail"]["width"], names["thumbnail"]["height"]) != (256, 160):
fail("variants", "thumbnail metadata")
if (names["crop"]["width"], names["crop"]["height"]) != (128, 128) or names["crop"]["content_type"] != "image/jpeg":
fail("variants", "crop contract")
if names["optimized"]["content_type"] != "image/webp" or (names["optimized"]["width"], names["optimized"]["height"]) != (320, 200):
fail("variants", "optimized contract")
original = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
thumb = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=bob)
optimized = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=optimized", actor=bob)
if original["bytes"] != content or thumb["headers"].get_content_type() != "image/jpeg":
fail("variants", "bytes or thumbnail type")
if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"] or optimized["bytes"][8:12] != b"WEBP":
fail("variants", "checksum or webp header")
state = operator("GET", "/internal/attachments/" + attachment["id"])
if any(set(message) - ALLOWED_MESSAGE_KEYS or any(isinstance(value, str) and len(value) > 512 for value in message.values()) for message in state["messages"]):
fail("variants", "broker payload contains unexpected data")
record("grade4 variants and broker payload", {"attachment_id": attachment["id"], "job_id": state["jobs"][0]["job_id"]})
key = "order-" + uuid.uuid4().hex
path = "/api/v1/chats/" + chat["id"] + "/messages"
first = client.call("POST", path, 201, actor=alice, raw=b'{"text":"same-body","attachment_ids":[]}', headers={"Content-Type": "application/json", "Idempotency-Key": key}, schema="Message")["json"]
second = client.call("POST", path, 201, actor=alice, raw=b'{"attachment_ids":[],"text":"same-body"}', headers={"Content-Type": "application/json", "Idempotency-Key": key}, schema="Message")["json"]
if first != second:
fail("idempotency-order", "canonical replay differs")
record("idempotency key order", {"message_id": first["id"]})
client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short", "extra": 1})
client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bad.png", 415, actor=alice, raw=b"not-an-image", headers={"Content-Type": "image/png"})
import struct
import zlib
ihdr = struct.pack(">IIBBBBB", 8000, 8000, 8, 2, 0, 0, 0)
chunk = struct.pack(">I", len(ihdr)) + b"IHDR" + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr) & 0xFFFFFFFF)
bomb = b"\x89PNG\r\n\x1a\n" + chunk + struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND") & 0xFFFFFFFF)
client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bomb.png", 413, actor=alice, raw=bomb, headers={"Content-Type": "image/png"})
record("validation", {"rejected": ["extra_field", "invalid_image", "pixel_limit"]})
operator("POST", "/internal/control/backpressure", {"max_queued_jobs": 0})
try:
client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=blocked.png", 503, actor=alice, raw=content, headers={"Content-Type": "image/png"})
finally:
operator("POST", "/internal/control/backpressure", {"max_queued_jobs": None})
record("backpressure", {"max_queued_jobs": 0, "status": 503})
operator("POST", "/internal/control/outbox", {"paused": True})
try:
paused, _content = upload_sample(client, alice, chat)
time.sleep(1.2)
metrics = operator("GET", "/internal/metrics")
hidden = operator("GET", "/internal/attachments/" + paused["id"])
if metrics["outbox_unpublished"] < 1 or hidden["attachment"]["status"] != "queued" or hidden["jobs"][0]["job_id"] is None:
fail("outbox-gap", "intent was not retained while publish was paused")
if queue_depth() not in {0, None}:
fail("outbox-gap", "broker received a message before publish resumed")
job_id = hidden["jobs"][0]["job_id"]
finally:
operator("POST", "/internal/control/outbox", {"paused": False})
resumed = wait_attachment(client, alice, paused["id"], "ready")
after = operator("GET", "/internal/attachments/" + paused["id"])
if after["jobs"][0]["job_id"] != job_id or resumed["status"] != "ready":
fail("outbox-gap", "recovery created a different job")
record("outbox commit/publish gap", {"attachment_id": paused["id"], "job_id": job_id})
compose("stop", "worker")
try:
held, held_bytes = upload_sample(client, alice, chat)
deadline = time.monotonic() + 20
while time.monotonic() < deadline and operator("GET", "/internal/metrics")["outbox_unpublished"] != 0:
time.sleep(0.3)
if operator("GET", "/internal/attachments/" + held["id"])["attachment"]["status"] != "queued":
fail("worker-stop", "API processed the image synchronously")
client.call("GET", "/api/v1/attachments/" + held["id"] + "/content", 409, actor=bob)
client.message(bob, chat, "чат жив без worker")
depth_before = queue_depth()
compose("restart", "api-1", "api-2", "rabbitmq")
wait_health()
depth_after = None
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
depth_after = queue_depth()
if depth_after is not None:
break
time.sleep(1)
if not depth_before or not depth_after:
fail("worker-stop", "durable queue did not retain the message across broker restart")
finally:
compose("start", "worker")
held_ready = wait_attachment(client, bob, held["id"], "ready")
downloaded = client.call("GET", "/api/v1/attachments/" + held["id"] + "/content?variant=thumbnail", actor=alice)
if held_ready["id"] != held["id"] or downloaded["headers"].get_content_type() != "image/jpeg":
fail("worker-stop", "restarted worker did not finish the accepted job")
if client.call("GET", "/api/v1/attachments/" + held["id"] + "/content", actor=bob)["bytes"] != held_bytes:
fail("worker-stop", "original changed")
record("worker stop, broker restart, resume", {"attachment_id": held["id"], "queue_before": depth_before, "queue_after_restart": depth_after})
operator("POST", "/internal/control/fault", {"mode": "permanent"})
rejected, _bytes = upload_sample(client, alice, chat)
failed = wait_attachment(client, alice, rejected["id"], "failed")
if failed["error_code"] != "invalid_image":
fail("permanent-replay", "unexpected error code")
client.call("GET", "/api/v1/attachments/" + rejected["id"] + "/content", 409, actor=alice)
client.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 409, actor=alice, body={"text": "", "attachment_ids": [rejected["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
replay = operator("POST", "/internal/attachments/" + rejected["id"] + "/replay")
if not replay["replayed"] or replay["job_id"] != operator("GET", "/internal/attachments/" + rejected["id"])["jobs"][0]["job_id"]:
fail("permanent-replay", "replay changed the logical job")
restored = wait_attachment(client, alice, rejected["id"], "ready")
again = operator("POST", "/internal/attachments/" + rejected["id"] + "/replay")
if again["reason"] != "already_ready" or len(restored["variants"]) != 3:
fail("permanent-replay", "successful job was replayed as new work")
record("permanent failure, DLQ registry, safe replay", {"attachment_id": rejected["id"], "job_id": replay["job_id"]})
operator("POST", "/internal/control/fault", {"mode": "transient", "remaining": 1})
retried, _bytes = upload_sample(client, alice, chat)
retried_ready = wait_attachment(client, alice, retried["id"], "ready", timeout=60)
retried_state = operator("GET", "/internal/attachments/" + retried["id"])
if retried_state["jobs"][0]["attempt"] != 2 or retried_ready["status"] != "ready":
fail("backoff", "single transient failure did not retry exactly once")
record("transient retry with backoff", {"attachment_id": retried["id"], "attempt": 2})
operator("POST", "/internal/control/fault", {"mode": "transient", "remaining": 5})
exhausted, _bytes = upload_sample(client, alice, chat)
exhausted_state = wait_attachment(client, alice, exhausted["id"], "failed", timeout=60)
if exhausted_state["error_code"] != "retry_exhausted":
fail("retry-budget", "attempts were not capped")
if operator("GET", "/internal/metrics")["failed_registry"] < 1:
fail("retry-budget", "failed registry is empty")
operator("POST", "/internal/attachments/" + exhausted["id"] + "/replay")
wait_attachment(client, alice, exhausted["id"], "ready", timeout=60)
record("retry budget then explicit replay", {"attachment_id": exhausted["id"]})
operator("POST", "/internal/control/fault", {"mode": "duplicate"})
duplicated, _bytes = upload_sample(client, alice, chat)
wait_attachment(client, alice, duplicated["id"], "ready")
duplicate_state = operator("GET", "/internal/attachments/" + duplicated["id"])
visible = [item for item in duplicate_state["variants"] if item["pipeline_version"] == 1]
if len(visible) != 3 or "noop_ready" not in {item["action"] for item in duplicate_state["audits"]}:
fail("duplicate", "inline duplicate created another variant set or skipped the no-op")
before = {item["sha256"] for item in visible}
operator("POST", "/internal/jobs/" + duplicate_state["jobs"][0]["job_id"] + "/redeliver")
deadline = time.monotonic() + 20
while time.monotonic() < deadline and sum(1 for item in operator("GET", "/internal/attachments/" + duplicated["id"])["audits"] if item["action"] == "noop_ready") < 2:
time.sleep(0.4)
after_state = operator("GET", "/internal/attachments/" + duplicated["id"])
after = {item["sha256"] for item in after_state["variants"] if item["pipeline_version"] == 1}
if before != after or len(after_state["variants"]) != 3:
fail("duplicate", "broker redelivery rewrote the result")
record("duplicate delivery is a no-op", {"attachment_id": duplicated["id"], "job_id": duplicate_state["jobs"][0]["job_id"]})
for mode, label in (("before_effect", "crash before side effect"), ("after_s3", "crash after object write"), ("after_commit", "crash after commit before ack")):
operator("POST", "/internal/control/fault", {"mode": mode})
crashed, _bytes = upload_sample(client, alice, chat)
wait_attachment(client, alice, crashed["id"], "ready", timeout=90)
audits = {item["action"] for item in operator("GET", "/internal/attachments/" + crashed["id"])["audits"]}
expected = {"before_effect": "fault_before_effect", "after_s3": "fault_after_s3", "after_commit": "fault_after_commit"}[mode]
if expected not in audits or len([item for item in operator("GET", "/internal/attachments/" + crashed["id"])["variants"] if item["pipeline_version"] == 1]) != 3:
fail(label, "fault was not recovered idempotently")
record(label, {"attachment_id": crashed["id"], "audit": expected})
baseline, _bytes = upload_sample(client, alice, chat)
baseline_ready = wait_attachment(client, alice, baseline["id"], "ready")
first_sha = {item["name"]: item["sha256"] for item in baseline_ready["variants"]}
reprocess = operator("POST", "/internal/attachments/" + baseline["id"] + "/reprocess", {"pipeline_version": 2})
version_two = wait_attachment(client, alice, baseline["id"], "ready")
second_sha = {item["name"]: item["sha256"] for item in version_two["variants"]}
version_state = operator("GET", "/internal/attachments/" + baseline["id"])
if version_state["attachment"]["ready_pipeline_version"] != 2 or first_sha["thumbnail"] == second_sha["thumbnail"]:
fail("pipeline-version", "version 2 did not publish a distinct result")
if len(version_state["variants"]) != 6:
fail("pipeline-version", "versioned variants were overwritten in place")
operator("POST", "/internal/jobs/" + version_state["jobs"][0]["job_id"] + "/redeliver")
time.sleep(2)
stale = operator("GET", "/internal/attachments/" + baseline["id"])
public = client.call("GET", "/api/v1/attachments/" + baseline["id"], actor=alice, schema="Attachment")["json"]
if stale["attachment"]["ready_pipeline_version"] != 2 or {item["sha256"] for item in public["variants"]} != set(second_sha.values()):
fail("pipeline-version", "stale pipeline overwrote the newer result")
record("versioned pipeline rejects stale result", {"attachment_id": baseline["id"], "new_job_id": reprocess["job_id"]})
operator("POST", "/internal/control/outbox", {"paused": True})
try:
ordered, _bytes = upload_sample(client, alice, chat)
operator("POST", "/internal/attachments/" + ordered["id"] + "/reprocess", {"pipeline_version": 2})
finally:
operator("POST", "/internal/control/outbox", {"paused": False})
ordered_ready = wait_attachment(client, alice, ordered["id"], "ready")
ordered_state = operator("GET", "/internal/attachments/" + ordered["id"])
if ordered_state["attachment"]["ready_pipeline_version"] != 2 or "superseded" not in {job["status"] for job in ordered_state["jobs"]}:
fail("reorder", "older queued job was not superseded by the newer pipeline")
if {item["name"] for item in ordered_ready["variants"]} != {"thumbnail", "crop", "optimized"}:
fail("reorder", "public result is not the complete newer pipeline")
record("reordered older job cannot win", {"attachment_id": ordered["id"]})
persisted = client.message(alice, chat, "пережить рестарт")["json"]
compose("restart", "postgres", "minio", "rabbitmq", "api-1", "api-2", "worker", "nginx")
wait_health(120)
me = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
if me["id"] != alice["id"] or not any(item == persisted for item in history["items"]):
fail("persistence", "session or message did not survive restart")
if client.call("GET", "/api/v1/attachments/" + baseline["id"] + "/content", actor=bob)["bytes"] != content:
fail("persistence", "object bytes did not survive restart")
record("volume restart preserves session message and object", {"message_id": persisted["id"], "attachment_id": baseline["id"]})
observed = set()
for _ in range(20):
observed.add(client.call("GET", "/api/v1/users/me", actor=bob, schema="User")["headers"]["X-Instance-Id"])
if observed != {"api-1", "api-2"}:
fail("replicas", "load balancer did not expose both API replicas")
record("two API replicas", {"instances": sorted(observed)})
EVIDENCE.parent.mkdir(exist_ok=True)
EVIDENCE.write_text(json.dumps({"track": 2, "results": RESULTS}, ensure_ascii=False, indent=2), encoding="utf-8")
print("PASS: grade 4 and track 2 scenarios")
if __name__ == "__main__":
try:
main()
except Failure as exc:
fail("client", str(exc))
finally:
try:
reset_controls()
except Exception:
pass