created skeleton for lab 6

This commit is contained in:
2026-11-14 03:51:10 +03:00
parent 470d9d6f1e
commit 498ce1e0d2
144 changed files with 11428 additions and 21 deletions
+1
View File
@@ -0,0 +1 @@
"""Команды управления учебным стендом."""
+429
View File
@@ -0,0 +1,429 @@
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
import argparse
from datetime import datetime
import hashlib
from io import BytesIO
import json
from pathlib import Path
import subprocess
import time
from uuid import UUID, uuid4
from PIL import Image
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = """
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.models.job import AttachmentJob
from app.models.attachment import Attachment
from app.storage.s3 import create_s3_storage
async def main():
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
objects=[]
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
for obj in entries:
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
try:
async for chunk in stream: checksum.update(chunk)
finally: await stream.aclose()
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
"""
def probe(attachment_id):
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
result = subprocess.run(
COMPOSE + ["exec", "-T", "api", "python", "-"],
input=code,
text=True,
capture_output=True,
cwd=ROOT,
timeout=30,
)
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose(
"run",
"--rm",
"--no-deps",
"worker",
"python",
"-m",
"app.core.job_admin",
*arguments
)
check(result.returncode == expected, "Unexpected operator command exit status")
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready["id"] == attachment["id"], "Attachment identity changed")
path = "/api/v1/attachments/" + ready["id"] + "/content"
check(
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
)
check(
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
"Incomplete pipeline",
)
for variant in ready["variants"]:
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
"bytes"
]
check(
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
)
with Image.open(BytesIO(data)) as image:
check(
image.size == (variant["width"], variant["height"]),
"Metadata dimensions mismatch",
)
check(
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
"Wrong format",
)
check(
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp"),
"Metadata leaked",
)
if variant["name"] == "crop":
check(image.size == (128, 128), "Wrong crop size")
snapshot = probe(ready["id"])
check(
len(snapshot["objects"]) == 4,
"Expected exactly one original and three variants",
)
return snapshot
def crash(client, actor, chat, mode):
name = "messenger-worker-fault-" + uuid4().hex
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
result = compose(
"run",
"--rm",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
phase = (
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
)
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
check(
result.returncode == 137 and marker in result.stdout,
"Targeted crash not observed",
)
before = probe(attachment["id"])
check(
before["status"]
== ("processing" if mode == "crash-after-object" else "ready"),
"Unexpected crash state",
)
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
finally:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
if mode == "crash-before-ack":
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after = probe(attachment["id"])
check(after == before, "Ready duplicate rewrote result")
else:
check(
after["attempt"] == before["attempt"] + 1,
"Lease did not recover the interrupted attempt",
)
check(
[(x["name"], x["sha256"]) for x in before["objects"]]
== [(x["name"], x["sha256"]) for x in after["objects"]],
"Replay changed object content",
)
return dict(
scenario=mode,
attachment_id=attachment["id"],
exit_code=137,
before=before,
after=after,
)
def retry_replay(client, actor, chat, permanent):
name = "messenger-retry-fault-" + uuid4().hex
samples = []
configuration = ComposeProject().config["services"]["worker"]["environment"]
retry_base = float(configuration["JOB_RETRY_SECONDS"])
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
mode = "fail-permanent" if permanent else "fail-transient"
require_compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
sample = probe(attachment["id"])
if (
sample["attempt"]
and sample["status"] in {"queued", "failed"}
and (not samples or samples[-1]["attempt"] != sample["attempt"])
):
samples.append(sample)
if sample["status"] == "failed":
break
time.sleep(0.5)
check(sample["status"] == "failed", "Retry never reached failed")
check(
sample["attempt"] == (1 if permanent else sample["limit"]),
"Incorrect attempt budget",
)
check(sample["permanent"] == permanent, "Incorrect error classification")
if not permanent:
check(len(samples) == sample["limit"], "Missed retry samples")
for entry in samples:
delay = (
datetime.fromisoformat(entry["next_attempt_at"])
- datetime.fromisoformat(entry["updated_at"])
).total_seconds()
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
check(
ceiling / 2 <= delay <= ceiling,
"Retry delay outside configured jitter bounds",
)
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
job_id = sample["job_id"]
request_id = str(uuid4())
listing = admin("list-failed")
check(
any(row["job_id"] == job_id for row in listing),
"Job missing from failed registry",
)
subprocess.run(
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
)
first = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(first["status"] == "queued", "Replay did not queue job")
finally:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
repeat = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(repeat["already_applied"], "Replay request id not idempotent")
refused = admin(
"replay",
job_id,
"--request-id",
str(uuid4()),
"--actor",
"verification",
"--reason",
"manual_review",
expected=2,
)
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
return dict(
scenario=mode,
attachment_id=attachment["id"],
samples=samples,
after=after,
replay_idempotent=True,
)
def queue_state():
output = require_compose(
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_queues",
"--formatter=json",
"name",
"messages_ready",
"messages_unacknowledged",
"consumers",
"durable",
)
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
def wait_queue_empty():
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
state = queue_state()
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
return state
time.sleep(0.5)
check(False, "RabbitMQ did not acknowledge all completed deliveries")
def persistence(client, actor, chat):
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
client.message(actor, chat, "", attachments=[attachment["id"]])
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
time.sleep(2)
queued_before = queue_state()
check(
queued_before["messages_ready"] >= 1 and queued_before["durable"],
"No durable queued message before restart",
)
result = subprocess.run(
[str(ROOT / "scripts/restart.sh")],
cwd=ROOT,
capture_output=True,
timeout=180,
)
check(result.returncode == 0, "Restart hook failed")
check(
"worker"
not in require_compose(
"ps", "--status", "running", "--services"
).splitlines(),
"Restart started deliberately stopped worker",
)
queued_after = queue_state()
check(
queued_after["messages_ready"] >= 1 and queued_after["durable"],
"Broker lost queued delivery across restart",
)
client.message(actor, chat, "session and chat survived broker/API restart")
finally:
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
return dict(
scenario="worker_stopped_restart_all",
attachment_id=attachment["id"],
queue_before=queued_before,
queue_after=queued_after,
after=after,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
parser.add_argument(
"--scenario",
choices=[
"all",
"persistence",
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
],
default="all",
)
parser.add_argument("--output", default="evidence/jobs.json")
args = parser.parse_args()
client = Client(args.base_url, args.ca_file, timeout=40)
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
check(
client.call("GET", "/health/live", schema="Health")["headers"].get(
"X-Instance-Id"
)
in aliases,
"Wrong Compose project for BASE_URL",
)
actor, _, _, chat = client.bootstrap()
results = []
operations = {
"persistence": persistence,
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
"fail-transient": lambda *x: retry_replay(*x, False),
"fail-permanent": lambda *x: retry_replay(*x, True),
}
selected = (
operations.values() if args.scenario == "all" else [operations[args.scenario]]
)
for operation in selected:
result = operation(client, actor, chat)
results.append(result)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
)
print("PASS: " + result["scenario"], flush=True)
Path(args.output).write_text(
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
)
if __name__ == "__main__":
main()
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения."""
from concurrent.futures import ThreadPoolExecutor
import gzip
import json
from pathlib import Path
import subprocess
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
IMAGE = "nginx:1.28.0-alpine"
PAYLOAD = json.dumps({"text": "message " * 256}).encode()
def check(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
class Stand:
def __init__(self, directory: str) -> None:
self.directory = Path(directory)
self.project = "messenger-nginx-check-" + uuid4().hex[:12]
self.base_url = ""
backend = self.directory / "backend.conf"
backend.write_text(
"worker_processes 1;\n"
"events { worker_connections 1024; }\n"
"http {\n"
" log_format backend 'BACKEND $request_method $uri';\n"
" access_log /dev/stdout backend;\n"
" server {\n"
" listen 8080;\n"
" default_type application/json;\n"
" add_header X-Instance-Id $hostname always;\n"
" add_header X-Request-Id $request_id always;\n"
" add_header Retry-After 7 always;\n"
' location /health/ { return 200 \'{"status":"ok"}\'; }\n'
" location = /api/limited {\n"
' return 429 \'{"error":{"code":"rate_limited",'
'"message":"upstream limit","request_id":"$request_id"}}\';\n'
" }\n"
" location = /api/post-failure {\n"
' return 503 \'{"error":{"code":"unavailable",'
'"message":"upstream failure","request_id":"$request_id"}}\';\n'
" }\n"
" location / { return 200 '" + PAYLOAD.decode() + "'; }\n"
" }\n"
"}\n"
)
self.compose_file = self.directory / "compose.json"
self.compose_file.write_text(
json.dumps(
{
"services": {
"api": {
"image": IMAGE,
"volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"],
},
"nginx": {
"image": IMAGE,
"volumes": [
f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro"
],
"ports": ["127.0.0.1::8080"],
},
}
}
)
)
def compose(self, *args: str) -> str:
# Явный файл, уникальный проект и пустой env-file исключают основной стенд.
result = subprocess.run(
[
"docker",
"compose",
"--env-file",
"/dev/null",
"-f",
str(self.compose_file),
"-p",
self.project,
*args,
],
cwd=self.directory,
capture_output=True,
text=True,
timeout=45,
)
if result.returncode:
raise RuntimeError(result.stdout + result.stderr)
return result.stdout + result.stderr
def call(self, path: str, *, method: str = "GET", encoding: str = "identity"):
request = Request(
self.base_url + path,
method=method,
headers={"Accept-Encoding": encoding},
data=b"{}" if method == "POST" else None,
)
try:
response = urlopen(request, timeout=3)
except HTTPError as error:
response = error
except (URLError, TimeoutError):
return 0, {}, b""
with response:
return (
response.status,
{key.lower(): value for key, value in response.headers.items()},
response.read(),
)
def wait_instances(self, count: int) -> set[str]:
seen: set[str] = set()
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
status, headers, _ = self.call("/health/live")
if status == 200:
seen.add(headers["x-instance-id"])
if len(seen) == count:
return seen
time.sleep(0.05)
raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}")
def check_error(response, status: int, code: str) -> None:
actual_status, headers, body = response
check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}")
check(
headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON"
)
error = json.loads(body)["error"]
check(error["code"] == code, "Неверный код ошибки")
check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает")
def run_checks(stand: Stand) -> None:
stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx")
address = stand.compose("port", "nginx", "8080").strip()
stand.base_url = "http://" + address
stand.wait_instances(2)
print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip())
stand.compose("exec", "-T", "nginx", "nginx", "-t")
print("PASS: nginx -t и балансировка двух реплик")
status, headers, body = stand.call("/api/large", encoding="gzip")
check(
status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON"
)
check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое")
check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ")
check(
"accept-encoding" in headers.get("vary", "").lower(),
"Нет Vary: Accept-Encoding",
)
check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён")
small = stand.call("/health/live", encoding="gzip")
check(
small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат"
)
print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено")
limited = stand.call("/api/limited")
check_error(limited, 429, "rate_limited")
check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения")
check("x-instance-id" in limited[1], "Потерян id backend на ошибке API")
print("PASS: JSON и заголовки ошибки API передаются без подмены")
failure = stand.call("/api/post-failure", method="POST")
check_error(failure, 503, "unavailable")
backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api")
check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился")
print("PASS: отправленный POST не повторяется на другой реплике")
with ThreadPoolExecutor(max_workers=32) as pool:
responses = list(pool.map(lambda _: stand.call("/api/large"), range(500)))
rejected = [response for response in responses if response[0] == 429]
check(bool(rejected), "Не удалось вызвать ограничение частоты")
check(
all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке"
)
for response in rejected:
check_error(response, 429, "rate_limited")
check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика")
check("x-instance-id" not in response[1], "LB выдал себя за API-реплику")
with ThreadPoolExecutor(max_workers=16) as pool:
health = list(pool.map(lambda _: stand.call("/health/ready"), range(100)))
check(all(response[0] == 200 for response in health), "Лимит затронул health")
check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна")
print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен")
stand.compose(
"up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api"
)
instances = stand.wait_instances(3)
print("PASS: третья реплика обнаружена через DNS без reload nginx")
target = sorted(instances)[0]
containers = stand.compose("ps", "-q", "api").splitlines()
matching = [container for container in containers if container.startswith(target)]
check(len(matching) == 1, "Не найдена тестовая реплика для остановки")
# Все id взяты только из уникального временного Compose-проекта.
started = time.monotonic()
subprocess.run(
["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True
)
recovery = None
for _ in range(20):
status, headers, _ = stand.call("/health/live")
if status == 200:
check(headers["x-instance-id"] != target, "Остановленная реплика отвечает")
if recovery is None:
recovery = time.monotonic() - started
check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с")
print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с")
stand.compose("stop", "--timeout", "1", "api")
unavailable = stand.call("/health/ready")
check_error(unavailable, 503, "unavailable")
check(
unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API"
)
check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики")
check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB")
print("PASS: недоступность всех backend возвращает JSON 503")
logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx")
entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
check(bool(entries), "Нет JSON access log")
check(
all("request_id" in entry and "upstream_time" in entry for entry in entries),
"Неполный лог",
)
check(
any(entry["limit_status"] == "REJECTED" for entry in entries),
"Лимит не отражён в логе",
)
print("PASS: JSON access log содержит время backend и результат лимита")
def main() -> None:
with tempfile.TemporaryDirectory(prefix="messenger-nginx-check-") as directory:
stand = Stand(directory)
try:
run_checks(stand)
except Exception:
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
raise
finally:
stand.compose("down", "--volumes", "--remove-orphans", "--timeout", "1")
if __name__ == "__main__":
main()
+147
View File
@@ -0,0 +1,147 @@
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, check
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(
COMPOSE + list(arguments),
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
)
def require_compose(*arguments: str) -> str:
result = compose(*arguments)
check(result.returncode == 0, "Compose command failed; output hidden")
return result.stdout
def main() -> None:
parser = argparse.ArgumentParser(
description="Отказ publisher после COMMIT перед RabbitMQ publish"
)
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(
require_compose("ps", "--status", "running", "--services").splitlines()
)
check(
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
"Start the application stack before this scenario",
)
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "messenger-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
"json"
]
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
expected=409,
actor=alice,
)
killed = compose(
"run",
"--rm",
"--no-deps",
"--name",
fault_container,
"publisher",
"python",
"-m",
"app.core.outbox_publisher",
"--crash-before-publish",
attachment["id"],
)
marker = json.dumps(
{"fault": "before_publish", "attachment_id": attachment["id"]}
)
check(
killed.returncode == 137 and marker in killed.stdout,
"Expected targeted SIGKILL before publish was not observed",
)
pending = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
actor=alice,
schema="Attachment",
)["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
try:
subprocess.run(
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check(
{variant["name"] for variant in ready["variants"]}
== {"thumbnail", "crop", "optimized"},
"Missing required variants",
)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"Original changed after recovery",
)
for variant in ready["variants"]:
downloaded = client.call(
"GET", path + "?variant=" + variant["name"], actor=alice
)["bytes"]
check(
len(downloaded) == variant["size_bytes"]
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
"Recovered variant checksum mismatch",
)
print(
json.dumps(
{
"scenario": "commit_before_publish",
"attachment_id": ready["id"],
"message_id": message["id"],
"publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"],
"status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}
)
)
if __name__ == "__main__":
main()
+152
View File
@@ -0,0 +1,152 @@
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
import json
import subprocess
import time
from scripts.deployment import ComposeProject
from tests.client import Client, arguments, check, run
PROBE = r"""
import asyncio, json, sys
from pathlib import Path
from uuid import UUID
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.storage.s3 import create_s3_storage
from app.models.attachment import Attachment
from minio.error import S3Error
async def main():
settings = Settings()
engine = create_db_engine(settings)
storage = create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
key = record.object_key
results = {}
for method in ('GET', 'PUT'):
request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
method=method, data=b'x' if method == 'PUT' else None)
try:
with urlopen(request, timeout=3) as response:
results[method] = response.status
except HTTPError as error:
results[method] = error.code
error.close()
try:
storage._client.get_bucket_policy(settings.s3.bucket)
results['admin_denied'] = False
except S3Error as error:
results['admin_denied'] = error.code == 'AccessDenied'
try:
buckets = storage._client.list_buckets()
results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
except S3Error as error:
results['other_buckets_denied'] = error.code == 'AccessDenied'
try:
with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
results['console_unreachable'] = False
except HTTPError as error:
results['console_unreachable'] = False
error.close()
except URLError:
results['console_unreachable'] = True
results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
for name in ('minio_root_user', 'minio_root_password'))
print(json.dumps(results))
finally:
await storage.aclose()
await engine.dispose()
asyncio.run(main())
"""
def main():
def configure(parser):
parser.add_argument(
"--with-outage",
action="store_true",
help="временно остановить MinIO выбранного проекта",
)
args = arguments("MinIO private bucket and failure checks", configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
attachment, original = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
for _ in range(2):
project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
replicas = project.replicas()
check(bool(replicas), "API not running")
result = subprocess.run(
["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
input=json.dumps({"attachment_id": attachment["id"]}),
text=True,
capture_output=True,
timeout=20,
)
check(result.returncode == 0, "private bucket probe failed (details hidden)")
statuses = json.loads(result.stdout)
check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
check(
statuses["admin_denied"]
and statuses["other_buckets_denied"]
and statuses["root_files_absent"],
"API has excessive access",
)
check(statuses["console_unreachable"], "console reachable from API network")
check(statuses["temp_originals_absent"], "API retained temporary originals")
check(
not project.config["services"]["minio"].get("ports"),
"MinIO publishes host ports",
)
env = project.config["services"]["minio"]["environment"]
check(env["MINIO_BROWSER"] == "off", "browser enabled")
check(
"127.0.0.1:9001" in project.config["services"]["minio"]["command"],
"console is not bound to loopback",
)
path = "/api/v1/attachments/" + attachment["id"] + "/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"bootstrap changed original",
)
if args.with_outage:
try:
project.run("stop", "minio")
client.call("GET", path, expected=503, actor=alice)
finally:
project.run("start", "minio")
deadline = time.monotonic() + 60
while True:
try:
client.call("GET", "/health/ready", schema="Health")
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(0.5)
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"original lost after MinIO restart",
)
print(
"PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
+ (", MinIO outage/recovery" if args.with_outage else "")
)
if __name__ == "__main__":
run(main)
+73
View File
@@ -0,0 +1,73 @@
"""Общие операции с репликами текущего Compose-проекта."""
from dataclasses import dataclass
import json
import os
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
@dataclass(frozen=True)
class Replica:
"""Состояние контейнера API."""
id: str
image: str
instance_id: str
running: bool
health: str | None
class ComposeProject:
"""Работает только с выбранным Compose-проектом."""
def __init__(self) -> None:
self.command = ["docker", "compose", "--env-file", ".env"]
self.config = json.loads(self.run("config", "--format", "json"))
self.name = self.config["name"]
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
"""Выполняет Docker-команду с ограничением времени."""
result = subprocess.run(
["docker", *args],
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
return result.stdout.strip()
def run(self, *args: str, timeout: int = 90, env=None) -> str:
"""Выполняет команду Compose с текущими overrides."""
return self.docker(*self.command[1:], *args, timeout=timeout, env=env)
def replicas(self) -> list[Replica]:
"""Читает состояние API без окружения контейнеров."""
ids = self.run("ps", "--all", "--quiet", "api").splitlines()
if not ids:
return []
template = (
'{"id":{{json .Id}},"image":{{json .Image}},'
'"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},'
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
)
rows = self.docker("inspect", "--format", template, *ids).splitlines()
configured = str(
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
or ""
).strip()
return [
Replica(
row["id"],
row["image"],
configured or row["hostname"],
row["running"],
row["health"],
)
for row in map(json.loads, rows)
]
+38
View File
@@ -0,0 +1,38 @@
"""Создаёт только отсутствующие локальные настройки и файловые секреты."""
import os
from pathlib import Path
import secrets
def main() -> None:
root = Path(__file__).resolve().parents[1]
destination = root / ".env"
if not destination.exists():
with destination.open("x") as target:
target.write((root / ".env.example").read_text())
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in (
"db_password",
"redis_password",
"broker_password",
"minio_root_user",
"minio_root_password",
"s3_access_key_id",
"s3_secret_access_key",
):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
except FileExistsError:
continue
with os.fdopen(descriptor, "w") as target:
target.write(secrets.token_urlsafe(24))
print("Создан файл " + str(path.relative_to(root)))
print("Настройки готовы; существующие значения сохранены")
if __name__ == "__main__":
main()
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
compose() {
docker compose --env-file .env "$@"
}
wait_healthy() {
for service in "$@"; do
attempts=0
until states=$(compose ps --format '{{.Health}}' "$service") \
&& [ -n "$states" ] \
&& [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do
attempts=$((attempts + 1))
if [ "$attempts" -ge 60 ]; then
echo "$service не стал healthy за 60 секунд" >&2
exit 1
fi
sleep 1
done
done
}
worker_running=$(compose ps --status running --quiet worker)
compose stop worker publisher cleanup
compose restart postgres redis minio broker
wait_healthy postgres redis minio broker
compose restart api
wait_healthy api
compose restart nginx
wait_healthy nginx
compose start publisher cleanup
# Сценарий с заранее остановленным worker сохраняет его остановленным.
if [ -n "$worker_running" ]; then
compose start worker
fi
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!a-zA-Z0-9_.:-]*|'') exit 1 ;;
esac
target=""
for container in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \
| sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then
if [ -n "$target" ]; then
echo 'Instance id matches multiple API containers' >&2
exit 1
fi
target="$container"
fi
done
if [ -z "$target" ]; then
echo 'Target API instance not found in this Compose project' >&2
exit 1
fi
echo "Stopping API container $target" >&2
docker stop --timeout 8 "$target" >/dev/null