Files
mtusi2026lab5/student-tests/grade4.py

409 lines
16 KiB
Python

#!/usr/bin/env python3
"""Сценарии оценки 4: варианты, остановка worker, outbox, crash до ack, lease, replay.
Стенд уже поднят. Скрипт останавливает worker, relay, rabbitmq и API и снова их запускает.
Токены и пароли не печатаются.
"""
import hashlib
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check, run
from image_probe import jpeg_size
EVIDENCE = ROOT / "evidence"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
FAULTS = ROOT / "infra" / "faults.json"
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
SAFE = {
"inject_transient": False,
"crash_after_s3": False,
"relay_exit_before_publish": False,
"stall_seconds": 0,
"timeout_seconds": 0,
}
def compose(*args, timeout=180):
proc = subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=timeout,
encoding="utf-8",
errors="replace",
)
if proc.returncode != 0:
raise Failure("docker compose %s failed" % args[0])
return proc.stdout
def psql(sql):
proc = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
"-At",
"-c",
sql,
],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=60,
encoding="utf-8",
errors="replace",
)
if proc.returncode != 0:
raise Failure("database query failed")
return proc.stdout.strip()
def logs(service):
proc = subprocess.run(
["docker", "compose", "--env-file", ".env", "logs", "--since", "3m", "--no-log-prefix", service],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=60,
encoding="utf-8",
errors="replace",
)
return proc.stdout
def write_faults(data):
payload = dict(SAFE)
payload.update(data)
temporary = FAULTS.with_suffix(".tmp")
temporary.write_text(json.dumps(payload), encoding="utf-8")
temporary.replace(FAULTS)
def job_id_for(attachment_id):
uuid.UUID(attachment_id)
return psql("SELECT job_id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment_id)
def variant_counts(attachment_id):
uuid.UUID(attachment_id)
text = psql(
"SELECT name || ':' || count(*) FROM attachment_variants WHERE attachment_id = '%s' GROUP BY name ORDER BY name"
% attachment_id
)
counts = {}
if text:
for line in text.splitlines():
name, amount = line.split(":", 1)
counts[name] = int(amount)
return counts
def assert_single_variants(attachment_id):
counts = variant_counts(attachment_id)
check(counts == {"crop": 1, "optimized": 1, "thumbnail": 1}, "variant rows are not unique")
return counts
def wait_status(client, actor, attachment, status, seconds=90):
deadline = time.monotonic() + seconds
last = attachment
while time.monotonic() < deadline:
last = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=actor, schema="Attachment")["json"]
if last["status"] == status:
return last
if status != "failed" and last["status"] == "failed":
raise Failure("attachment failed while waiting for " + status)
time.sleep(0.4)
raise Failure("attachment stayed " + str(last.get("status")) + " while waiting for " + status)
def replay(job):
uuid.UUID(job)
proc = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"run",
"--rm",
"--no-deps",
"--entrypoint",
"python",
"worker",
"-m",
"app.replay",
job,
],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=90,
encoding="utf-8",
errors="replace",
)
if proc.returncode != 0:
raise Failure("replay command failed")
line = [item for item in proc.stdout.splitlines() if item.startswith("{")]
check(line, "replay did not print a result")
return json.loads(line[-1])
def check_ready_image(client, actor, ready, content):
variants = {item["name"]: item for item in ready["variants"]}
check(set(variants) == {"thumbnail", "crop", "optimized"}, "ready variants mismatch")
thumb = variants["thumbnail"]
crop = variants["crop"]
optimized = variants["optimized"]
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail metadata mismatch")
check(thumb["content_type"] == "image/jpeg", "thumbnail type mismatch")
check((crop["width"], crop["height"]) == (128, 128), "crop metadata mismatch")
check(crop["content_type"] == "image/jpeg", "crop type mismatch")
check((optimized["width"], optimized["height"]) == (320, 200), "optimized metadata mismatch")
check(optimized["content_type"] == "image/webp", "optimized type mismatch")
original = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor)
check(original["bytes"] == content, "original bytes changed")
for name, meta, magic in (
("thumbnail", thumb, b"\xff\xd8"),
("crop", crop, b"\xff\xd8"),
("optimized", optimized, b"RIFF"),
):
body = client.call(
"GET",
"/api/v1/attachments/" + ready["id"] + "/content?variant=" + name,
actor=actor,
)
check(body["headers"].get_content_type() == meta["content_type"], name + " content type mismatch")
check(len(body["bytes"]) == meta["size_bytes"], name + " size mismatch")
check(hashlib.sha256(body["bytes"]).hexdigest() == meta["sha256"], name + " checksum mismatch")
check(body["bytes"][:4] == magic or body["bytes"][:2] == magic, name + " magic mismatch")
if name != "optimized":
check(jpeg_size(body["bytes"]) == (meta["width"], meta["height"]), name + " jpeg size mismatch")
check(optimized and body["bytes"][8:12] == b"WEBP", "optimized is not webp")
def main():
write_faults({})
evidence = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
client = Client(BASE, timeout=30)
alice, bob, _eve, chat = client.bootstrap()
try:
att, content = client.attachment(alice, chat)
check(att["status"] == "queued" and att["variants"] == [], "upload snapshot is not queued")
ready = client.wait_ready(bob, att, wait_seconds=90)
check_ready_image(client, bob, ready, content)
assert_single_variants(att["id"])
evidence["happy_path"] = {"attachment_id": att["id"], "job_id": job_id_for(att["id"]), "status": "ready"}
print("PASS: thumbnail, crop and optimized")
compose("stop", "worker")
queued, queued_bytes = client.attachment(alice, chat)
blocked = client.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", 409, actor=bob)
check(blocked["json"]["error"]["code"] == "conflict", "content of a queued file is not 409")
linked = client.message(alice, chat, "", attachments=[queued["id"]])["json"]
check(linked["attachment_ids"] == [queued["id"]], "queued attachment was rejected")
client.message(bob, chat, "сообщение без вложения")
compose("restart", "api")
compose("restart", "rabbitmq")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
client.call("GET", "/health/ready", schema="Health")
break
except Failure:
time.sleep(1)
else:
raise Failure("api did not recover after restart")
still = client.call("GET", "/api/v1/attachments/" + queued["id"], actor=alice, schema="Attachment")["json"]
check(still["status"] in {"queued", "processing", "ready"}, "accepted job disappeared")
compose("start", "worker")
recovered = wait_status(client, bob, queued, "ready", 90)
downloaded = client.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", actor=bob)
check(downloaded["bytes"] == queued_bytes, "recovered original changed")
assert_single_variants(queued["id"])
evidence["worker_stopped"] = {
"attachment_id": queued["id"],
"job_id": job_id_for(queued["id"]),
"content_before_worker": 409,
"status_after_recovery": recovered["status"],
"rabbit_restarted": True,
"api_restarted": True,
}
print("PASS: stopped worker, broker restart, same job reached ready")
compose("stop", "relay")
held, _held_bytes = client.attachment(alice, chat)
held_job = job_id_for(held["id"])
unpublished = psql(
"SELECT count(*) FROM job_outbox WHERE job_id = '%s' AND published_at IS NULL" % held_job
)
check(unpublished == "1", "outbox row was published while relay was stopped")
compose("start", "relay")
held_ready = wait_status(client, alice, held, "ready", 90)
evidence["outbox_gap"] = {
"attachment_id": held["id"],
"job_id": held_job,
"unpublished_before_restart": int(unpublished),
"status": held_ready["status"],
}
print("PASS: commit without publish recovered from the outbox")
write_faults({"relay_exit_before_publish": True})
time.sleep(0.8)
crashed, _bytes = client.attachment(alice, chat)
crash_job = job_id_for(crashed["id"])
seen = False
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
if "injected exit after outbox commit before broker publish" in logs("relay"):
seen = True
break
time.sleep(0.5)
check(seen, "relay did not exit between commit and publish")
crash_ready = wait_status(client, alice, crashed, "ready", 90)
gap = psql("SELECT gap_injected FROM job_outbox WHERE job_id = '%s' ORDER BY created_at LIMIT 1" % crash_job)
check(gap == "t", "gap latch was not stored")
jobs = psql("SELECT count(*) FROM attachment_jobs WHERE attachment_id = '%s'" % crashed["id"])
check(jobs == "1", "crash recovery created another job")
evidence["relay_crash"] = {
"attachment_id": crashed["id"],
"job_id": crash_job,
"gap_injected": True,
"log_seen": True,
"status": crash_ready["status"],
}
write_faults({})
print("PASS: relay crash before publish recovered the same job")
write_faults({"crash_after_s3": True})
time.sleep(0.8)
side, side_bytes = client.attachment(alice, chat)
side_job = job_id_for(side["id"])
seen = False
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
if "injected crash after object write before ack job_id=%s" % side_job in logs("worker"):
seen = True
break
time.sleep(0.5)
check(seen, "worker did not exit after the object write")
side_ready = wait_status(client, bob, side, "ready", 90)
side_download = client.call("GET", "/api/v1/attachments/" + side["id"] + "/content", actor=bob)
check(side_download["bytes"] == side_bytes, "redelivery changed the original")
assert_single_variants(side["id"])
evidence["crash_before_ack"] = {
"attachment_id": side["id"],
"job_id": side_job,
"log_seen": True,
"status": side_ready["status"],
"variant_rows": assert_single_variants(side["id"]),
}
write_faults({})
print("PASS: crash after object write redelivered the same result")
compose("stop", "worker", "relay")
leased, _leased_bytes = client.attachment(alice, chat)
leased_job = job_id_for(leased["id"])
psql(
"UPDATE attachment_jobs SET status = 'processing', attempt = 1, lease_owner = 'dead', "
"lease_until = now() - interval '2 minutes' WHERE job_id = '%s'" % leased_job
)
psql("UPDATE job_outbox SET published_at = now() WHERE job_id = '%s' AND published_at IS NULL" % leased_job)
psql("UPDATE attachments SET status = 'processing', error_code = NULL WHERE id = '%s'" % leased["id"])
compose("start", "relay", "worker")
lease_ready = wait_status(client, alice, leased, "ready", 90)
evidence["expired_lease"] = {
"attachment_id": leased["id"],
"job_id": leased_job,
"status": lease_ready["status"],
}
print("PASS: expired processing lease was recovered")
write_faults({"inject_transient": True})
time.sleep(0.8)
failed, _failed_bytes = client.attachment(alice, chat)
failed_job = job_id_for(failed["id"])
failed_meta = wait_status(client, alice, failed, "failed", 90)
check(failed_meta["error_code"] == "retry_exhausted", "exhausted job has no safe error code")
denied = client.call("GET", "/api/v1/attachments/" + failed["id"] + "/content", 409, actor=alice)
check(denied["json"]["error"]["code"] == "conflict", "failed content is not 409")
registry = psql("SELECT count(*) FROM job_failures WHERE job_id = '%s'" % failed_job)
check(int(registry) >= 1, "failure registry is empty")
dlq = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages")
check("attachment.jobs.dlq" in dlq, "dlq is missing")
write_faults({})
time.sleep(0.5)
first = replay(failed_job)
check(first["outcome"] == "enqueued", "failed job was not replayed")
replayed = wait_status(client, alice, failed, "ready", 90)
second = replay(failed_job)
check(second["outcome"] == "already_ready", "replay of a ready job was not a no-op")
assert_single_variants(failed["id"])
evidence["retry_and_replay"] = {
"attachment_id": failed["id"],
"job_id": failed_job,
"error_code": failed_meta["error_code"],
"failure_rows": int(registry),
"first_replay": first["outcome"],
"second_replay": second["outcome"],
"status": replayed["status"],
}
print("PASS: backoff exhausted to failed, replay is safe")
write_faults({"stall_seconds": 5, "timeout_seconds": 1})
time.sleep(0.8)
stalled, _stalled_bytes = client.attachment(alice, chat)
client.message(alice, chat, "API отвечает, пока worker ограничен по времени")
client.call("GET", "/health/ready", schema="Health")
stalled_meta = wait_status(client, alice, stalled, "failed", 90)
check(stalled_meta["error_code"] == "retry_exhausted", "timed out job was not exhausted")
write_faults({})
time.sleep(0.5)
stalled_job = job_id_for(stalled["id"])
outcome = replay(stalled_job)
check(outcome["outcome"] == "enqueued", "timed out job was not replayed")
stalled_ready = wait_status(client, alice, stalled, "ready", 90)
evidence["timeout"] = {
"attachment_id": stalled["id"],
"job_id": stalled_job,
"error_code": stalled_meta["error_code"],
"status": stalled_ready["status"],
"api_ready_during_stall": True,
}
print("PASS: worker time limit does not block the API")
finally:
write_faults({})
try:
compose("start", "worker", "relay", "api", "rabbitmq")
except Failure:
pass
EVIDENCE.mkdir(exist_ok=True)
(EVIDENCE / "grade4.json").write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("PASS: grade 4 failure scenarios")
if __name__ == "__main__":
run(main)