forked from mtusi2026/mtusi2026lab5
409 lines
16 KiB
Python
409 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Сценарии оценки 4: варианты, остановка worker, outbox, crash до ack, lease, replay.
|
|
|
|
Стенд уже поднят. Скрипт останавливает worker, relay, rabbitmq и API и снова их запускает.
|
|
Токены и пароли не печатаются.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check, run
|
|
from image_probe import jpeg_size
|
|
|
|
EVIDENCE = ROOT / "evidence"
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
FAULTS = ROOT / "infra" / "faults.json"
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
SAFE = {
|
|
"inject_transient": False,
|
|
"crash_after_s3": False,
|
|
"relay_exit_before_publish": False,
|
|
"stall_seconds": 0,
|
|
"timeout_seconds": 0,
|
|
}
|
|
|
|
|
|
def compose(*args, timeout=180):
|
|
proc = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", *args],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("docker compose %s failed" % args[0])
|
|
return proc.stdout
|
|
|
|
|
|
def psql(sql):
|
|
proc = subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--env-file",
|
|
".env",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-At",
|
|
"-c",
|
|
sql,
|
|
],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("database query failed")
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def logs(service):
|
|
proc = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "logs", "--since", "3m", "--no-log-prefix", service],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
return proc.stdout
|
|
|
|
|
|
def write_faults(data):
|
|
payload = dict(SAFE)
|
|
payload.update(data)
|
|
temporary = FAULTS.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(payload), encoding="utf-8")
|
|
temporary.replace(FAULTS)
|
|
|
|
|
|
def job_id_for(attachment_id):
|
|
uuid.UUID(attachment_id)
|
|
return psql("SELECT job_id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment_id)
|
|
|
|
|
|
def variant_counts(attachment_id):
|
|
uuid.UUID(attachment_id)
|
|
text = psql(
|
|
"SELECT name || ':' || count(*) FROM attachment_variants WHERE attachment_id = '%s' GROUP BY name ORDER BY name"
|
|
% attachment_id
|
|
)
|
|
counts = {}
|
|
if text:
|
|
for line in text.splitlines():
|
|
name, amount = line.split(":", 1)
|
|
counts[name] = int(amount)
|
|
return counts
|
|
|
|
|
|
def assert_single_variants(attachment_id):
|
|
counts = variant_counts(attachment_id)
|
|
check(counts == {"crop": 1, "optimized": 1, "thumbnail": 1}, "variant rows are not unique")
|
|
return counts
|
|
|
|
|
|
def wait_status(client, actor, attachment, status, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
last = attachment
|
|
while time.monotonic() < deadline:
|
|
last = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=actor, schema="Attachment")["json"]
|
|
if last["status"] == status:
|
|
return last
|
|
if status != "failed" and last["status"] == "failed":
|
|
raise Failure("attachment failed while waiting for " + status)
|
|
time.sleep(0.4)
|
|
raise Failure("attachment stayed " + str(last.get("status")) + " while waiting for " + status)
|
|
|
|
|
|
def replay(job):
|
|
uuid.UUID(job)
|
|
proc = subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--env-file",
|
|
".env",
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"--entrypoint",
|
|
"python",
|
|
"worker",
|
|
"-m",
|
|
"app.replay",
|
|
job,
|
|
],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=90,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("replay command failed")
|
|
line = [item for item in proc.stdout.splitlines() if item.startswith("{")]
|
|
check(line, "replay did not print a result")
|
|
return json.loads(line[-1])
|
|
|
|
|
|
def check_ready_image(client, actor, ready, content):
|
|
variants = {item["name"]: item for item in ready["variants"]}
|
|
check(set(variants) == {"thumbnail", "crop", "optimized"}, "ready variants mismatch")
|
|
thumb = variants["thumbnail"]
|
|
crop = variants["crop"]
|
|
optimized = variants["optimized"]
|
|
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail metadata mismatch")
|
|
check(thumb["content_type"] == "image/jpeg", "thumbnail type mismatch")
|
|
check((crop["width"], crop["height"]) == (128, 128), "crop metadata mismatch")
|
|
check(crop["content_type"] == "image/jpeg", "crop type mismatch")
|
|
check((optimized["width"], optimized["height"]) == (320, 200), "optimized metadata mismatch")
|
|
check(optimized["content_type"] == "image/webp", "optimized type mismatch")
|
|
original = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor)
|
|
check(original["bytes"] == content, "original bytes changed")
|
|
for name, meta, magic in (
|
|
("thumbnail", thumb, b"\xff\xd8"),
|
|
("crop", crop, b"\xff\xd8"),
|
|
("optimized", optimized, b"RIFF"),
|
|
):
|
|
body = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + ready["id"] + "/content?variant=" + name,
|
|
actor=actor,
|
|
)
|
|
check(body["headers"].get_content_type() == meta["content_type"], name + " content type mismatch")
|
|
check(len(body["bytes"]) == meta["size_bytes"], name + " size mismatch")
|
|
check(hashlib.sha256(body["bytes"]).hexdigest() == meta["sha256"], name + " checksum mismatch")
|
|
check(body["bytes"][:4] == magic or body["bytes"][:2] == magic, name + " magic mismatch")
|
|
if name != "optimized":
|
|
check(jpeg_size(body["bytes"]) == (meta["width"], meta["height"]), name + " jpeg size mismatch")
|
|
check(optimized and body["bytes"][8:12] == b"WEBP", "optimized is not webp")
|
|
|
|
|
|
def main():
|
|
write_faults({})
|
|
evidence = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
|
client = Client(BASE, timeout=30)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
try:
|
|
att, content = client.attachment(alice, chat)
|
|
check(att["status"] == "queued" and att["variants"] == [], "upload snapshot is not queued")
|
|
ready = client.wait_ready(bob, att, wait_seconds=90)
|
|
check_ready_image(client, bob, ready, content)
|
|
assert_single_variants(att["id"])
|
|
evidence["happy_path"] = {"attachment_id": att["id"], "job_id": job_id_for(att["id"]), "status": "ready"}
|
|
print("PASS: thumbnail, crop and optimized")
|
|
|
|
compose("stop", "worker")
|
|
queued, queued_bytes = client.attachment(alice, chat)
|
|
blocked = client.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", 409, actor=bob)
|
|
check(blocked["json"]["error"]["code"] == "conflict", "content of a queued file is not 409")
|
|
linked = client.message(alice, chat, "", attachments=[queued["id"]])["json"]
|
|
check(linked["attachment_ids"] == [queued["id"]], "queued attachment was rejected")
|
|
client.message(bob, chat, "сообщение без вложения")
|
|
compose("restart", "api")
|
|
compose("restart", "rabbitmq")
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
break
|
|
except Failure:
|
|
time.sleep(1)
|
|
else:
|
|
raise Failure("api did not recover after restart")
|
|
still = client.call("GET", "/api/v1/attachments/" + queued["id"], actor=alice, schema="Attachment")["json"]
|
|
check(still["status"] in {"queued", "processing", "ready"}, "accepted job disappeared")
|
|
compose("start", "worker")
|
|
recovered = wait_status(client, bob, queued, "ready", 90)
|
|
downloaded = client.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", actor=bob)
|
|
check(downloaded["bytes"] == queued_bytes, "recovered original changed")
|
|
assert_single_variants(queued["id"])
|
|
evidence["worker_stopped"] = {
|
|
"attachment_id": queued["id"],
|
|
"job_id": job_id_for(queued["id"]),
|
|
"content_before_worker": 409,
|
|
"status_after_recovery": recovered["status"],
|
|
"rabbit_restarted": True,
|
|
"api_restarted": True,
|
|
}
|
|
print("PASS: stopped worker, broker restart, same job reached ready")
|
|
|
|
compose("stop", "relay")
|
|
held, _held_bytes = client.attachment(alice, chat)
|
|
held_job = job_id_for(held["id"])
|
|
unpublished = psql(
|
|
"SELECT count(*) FROM job_outbox WHERE job_id = '%s' AND published_at IS NULL" % held_job
|
|
)
|
|
check(unpublished == "1", "outbox row was published while relay was stopped")
|
|
compose("start", "relay")
|
|
held_ready = wait_status(client, alice, held, "ready", 90)
|
|
evidence["outbox_gap"] = {
|
|
"attachment_id": held["id"],
|
|
"job_id": held_job,
|
|
"unpublished_before_restart": int(unpublished),
|
|
"status": held_ready["status"],
|
|
}
|
|
print("PASS: commit without publish recovered from the outbox")
|
|
|
|
write_faults({"relay_exit_before_publish": True})
|
|
time.sleep(0.8)
|
|
crashed, _bytes = client.attachment(alice, chat)
|
|
crash_job = job_id_for(crashed["id"])
|
|
seen = False
|
|
deadline = time.monotonic() + 40
|
|
while time.monotonic() < deadline:
|
|
if "injected exit after outbox commit before broker publish" in logs("relay"):
|
|
seen = True
|
|
break
|
|
time.sleep(0.5)
|
|
check(seen, "relay did not exit between commit and publish")
|
|
crash_ready = wait_status(client, alice, crashed, "ready", 90)
|
|
gap = psql("SELECT gap_injected FROM job_outbox WHERE job_id = '%s' ORDER BY created_at LIMIT 1" % crash_job)
|
|
check(gap == "t", "gap latch was not stored")
|
|
jobs = psql("SELECT count(*) FROM attachment_jobs WHERE attachment_id = '%s'" % crashed["id"])
|
|
check(jobs == "1", "crash recovery created another job")
|
|
evidence["relay_crash"] = {
|
|
"attachment_id": crashed["id"],
|
|
"job_id": crash_job,
|
|
"gap_injected": True,
|
|
"log_seen": True,
|
|
"status": crash_ready["status"],
|
|
}
|
|
write_faults({})
|
|
print("PASS: relay crash before publish recovered the same job")
|
|
|
|
write_faults({"crash_after_s3": True})
|
|
time.sleep(0.8)
|
|
side, side_bytes = client.attachment(alice, chat)
|
|
side_job = job_id_for(side["id"])
|
|
seen = False
|
|
deadline = time.monotonic() + 40
|
|
while time.monotonic() < deadline:
|
|
if "injected crash after object write before ack job_id=%s" % side_job in logs("worker"):
|
|
seen = True
|
|
break
|
|
time.sleep(0.5)
|
|
check(seen, "worker did not exit after the object write")
|
|
side_ready = wait_status(client, bob, side, "ready", 90)
|
|
side_download = client.call("GET", "/api/v1/attachments/" + side["id"] + "/content", actor=bob)
|
|
check(side_download["bytes"] == side_bytes, "redelivery changed the original")
|
|
assert_single_variants(side["id"])
|
|
evidence["crash_before_ack"] = {
|
|
"attachment_id": side["id"],
|
|
"job_id": side_job,
|
|
"log_seen": True,
|
|
"status": side_ready["status"],
|
|
"variant_rows": assert_single_variants(side["id"]),
|
|
}
|
|
write_faults({})
|
|
print("PASS: crash after object write redelivered the same result")
|
|
|
|
compose("stop", "worker", "relay")
|
|
leased, _leased_bytes = client.attachment(alice, chat)
|
|
leased_job = job_id_for(leased["id"])
|
|
psql(
|
|
"UPDATE attachment_jobs SET status = 'processing', attempt = 1, lease_owner = 'dead', "
|
|
"lease_until = now() - interval '2 minutes' WHERE job_id = '%s'" % leased_job
|
|
)
|
|
psql("UPDATE job_outbox SET published_at = now() WHERE job_id = '%s' AND published_at IS NULL" % leased_job)
|
|
psql("UPDATE attachments SET status = 'processing', error_code = NULL WHERE id = '%s'" % leased["id"])
|
|
compose("start", "relay", "worker")
|
|
lease_ready = wait_status(client, alice, leased, "ready", 90)
|
|
evidence["expired_lease"] = {
|
|
"attachment_id": leased["id"],
|
|
"job_id": leased_job,
|
|
"status": lease_ready["status"],
|
|
}
|
|
print("PASS: expired processing lease was recovered")
|
|
|
|
write_faults({"inject_transient": True})
|
|
time.sleep(0.8)
|
|
failed, _failed_bytes = client.attachment(alice, chat)
|
|
failed_job = job_id_for(failed["id"])
|
|
failed_meta = wait_status(client, alice, failed, "failed", 90)
|
|
check(failed_meta["error_code"] == "retry_exhausted", "exhausted job has no safe error code")
|
|
denied = client.call("GET", "/api/v1/attachments/" + failed["id"] + "/content", 409, actor=alice)
|
|
check(denied["json"]["error"]["code"] == "conflict", "failed content is not 409")
|
|
registry = psql("SELECT count(*) FROM job_failures WHERE job_id = '%s'" % failed_job)
|
|
check(int(registry) >= 1, "failure registry is empty")
|
|
dlq = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages")
|
|
check("attachment.jobs.dlq" in dlq, "dlq is missing")
|
|
write_faults({})
|
|
time.sleep(0.5)
|
|
first = replay(failed_job)
|
|
check(first["outcome"] == "enqueued", "failed job was not replayed")
|
|
replayed = wait_status(client, alice, failed, "ready", 90)
|
|
second = replay(failed_job)
|
|
check(second["outcome"] == "already_ready", "replay of a ready job was not a no-op")
|
|
assert_single_variants(failed["id"])
|
|
evidence["retry_and_replay"] = {
|
|
"attachment_id": failed["id"],
|
|
"job_id": failed_job,
|
|
"error_code": failed_meta["error_code"],
|
|
"failure_rows": int(registry),
|
|
"first_replay": first["outcome"],
|
|
"second_replay": second["outcome"],
|
|
"status": replayed["status"],
|
|
}
|
|
print("PASS: backoff exhausted to failed, replay is safe")
|
|
|
|
write_faults({"stall_seconds": 5, "timeout_seconds": 1})
|
|
time.sleep(0.8)
|
|
stalled, _stalled_bytes = client.attachment(alice, chat)
|
|
client.message(alice, chat, "API отвечает, пока worker ограничен по времени")
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
stalled_meta = wait_status(client, alice, stalled, "failed", 90)
|
|
check(stalled_meta["error_code"] == "retry_exhausted", "timed out job was not exhausted")
|
|
write_faults({})
|
|
time.sleep(0.5)
|
|
stalled_job = job_id_for(stalled["id"])
|
|
outcome = replay(stalled_job)
|
|
check(outcome["outcome"] == "enqueued", "timed out job was not replayed")
|
|
stalled_ready = wait_status(client, alice, stalled, "ready", 90)
|
|
evidence["timeout"] = {
|
|
"attachment_id": stalled["id"],
|
|
"job_id": stalled_job,
|
|
"error_code": stalled_meta["error_code"],
|
|
"status": stalled_ready["status"],
|
|
"api_ready_during_stall": True,
|
|
}
|
|
print("PASS: worker time limit does not block the API")
|
|
finally:
|
|
write_faults({})
|
|
try:
|
|
compose("start", "worker", "relay", "api", "rabbitmq")
|
|
except Failure:
|
|
pass
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
(EVIDENCE / "grade4.json").write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS: grade 4 failure scenarios")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|