Files
mtusi2026lab5-fork/scripts/check_outbox_recovery.py
T

148 lines
5.1 KiB
Python

"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, check
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(
COMPOSE + list(arguments),
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
)
def require_compose(*arguments: str) -> str:
result = compose(*arguments)
check(result.returncode == 0, "Compose command failed; output hidden")
return result.stdout
def main() -> None:
parser = argparse.ArgumentParser(
description="Отказ publisher после COMMIT перед RabbitMQ publish"
)
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(
require_compose("ps", "--status", "running", "--services").splitlines()
)
check(
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
"Start the application stack before this scenario",
)
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "messenger-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
"json"
]
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
expected=409,
actor=alice,
)
killed = compose(
"run",
"--rm",
"--no-deps",
"--name",
fault_container,
"publisher",
"python",
"-m",
"app.core.outbox_publisher",
"--crash-before-publish",
attachment["id"],
)
marker = json.dumps(
{"fault": "before_publish", "attachment_id": attachment["id"]}
)
check(
killed.returncode == 137 and marker in killed.stdout,
"Expected targeted SIGKILL before publish was not observed",
)
pending = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
actor=alice,
schema="Attachment",
)["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
try:
subprocess.run(
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check(
{variant["name"] for variant in ready["variants"]}
== {"thumbnail", "crop", "optimized"},
"Missing required variants",
)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"Original changed after recovery",
)
for variant in ready["variants"]:
downloaded = client.call(
"GET", path + "?variant=" + variant["name"], actor=alice
)["bytes"]
check(
len(downloaded) == variant["size_bytes"]
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
"Recovered variant checksum mismatch",
)
print(
json.dumps(
{
"scenario": "commit_before_publish",
"attachment_id": ready["id"],
"message_id": message["id"],
"publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"],
"status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}
)
)
if __name__ == "__main__":
main()