- carry the lab 2 solution over and plan lab 3 (PLAN.md, OpenSpec change instance-id-idempotency) - rename the compose project to mtusi-lab3 and the image to :lab3 - add X-Instance-Id to every response and instance_id to every log record ([app] instance_id, hostname by default) - require Idempotency-Key on POST messages: keys table in Postgres, key and message in one transaction, replay returns the same 201, another body gets 409, TTL from [idempotency] ttl_seconds - send the key from student-tests, k6 scenarios and the skill driver; add idempotency, X-Instance-Id, key race and fault tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
279 lines
9.8 KiB
Python
Executable File
279 lines
9.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Hook стенда отказов для проверок `fault` (`make failure_checks`).
|
||
|
||
Тесты вызывают его как `[FAULT_HOOK, действие, аргументы…]` без shell —
|
||
так же `tests/resilience.py` вызывает `ACTION_SCRIPT`. Сами тесты docker
|
||
не трогают. Hook работает только с проектом compose из `FAULT_PROJECT`,
|
||
его задаёт `student-tests/run_failures.py`. Проект рабочего стенда hook
|
||
не трогает. Решение 3 design change `failure-checks`.
|
||
|
||
Действия:
|
||
stop|start|pause|unpause postgres|valkey
|
||
flush-cache FLUSHALL
|
||
cache-hits keyspace_hits из INFO stats
|
||
cache-has-chat <chat_id> EXISTS ключа карточки: 0 или 1
|
||
fail-members <title> вставка участников чата с этим названием
|
||
падает (триггер в базе стенда отказов)
|
||
clear-fail-members убрать триггер
|
||
fail-messages <text> вставка сообщения с этим текстом падает
|
||
(триггер; ключ идемпотентности к этому
|
||
моменту уже вставлен в той же транзакции)
|
||
clear-fail-messages убрать триггер
|
||
count-chats <title> число чатов с этим названием
|
||
|
||
Пароли не печатает: valkey-cli берёт пароль из VALKEYCLI_AUTH
|
||
контейнера, psql работает от POSTGRES_USER контейнера. Только stdlib.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "student-tests" / "failures.compose.yml",
|
||
)
|
||
PROJECT_SUFFIX = "-failures"
|
||
SERVICES = ("postgres", "valkey")
|
||
# та же проверка, что в healthcheck сервиса в docker-compose.yml
|
||
PROBES = {
|
||
"postgres": 'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" '
|
||
'-d "$POSTGRES_DB"',
|
||
"valkey": 'test "$(valkey-cli -p "$VALKEY_PORT" ping)" = PONG',
|
||
}
|
||
PROBE_TIMEOUT_S = 30
|
||
# начало ключа карточки — копия CACHE_KEY_PREFIX из src/utils/constants.py:
|
||
# hook на stdlib и код приложения не импортирует. Меняется формат ключа —
|
||
# менять и здесь, иначе упадут test_flush_cache и test_valkey_back
|
||
CACHE_KEY_PREFIX = "messenger:v1"
|
||
|
||
FAIL_MEMBERS_SQL = """\
|
||
CREATE OR REPLACE FUNCTION fault_fail_members() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF (SELECT title FROM chats WHERE id = NEW.chat_id) = TG_ARGV[0] THEN
|
||
RAISE EXCEPTION 'fault injected: chat members insert';
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_fail_members ON chat_members;
|
||
CREATE TRIGGER fault_fail_members BEFORE INSERT ON chat_members
|
||
FOR EACH ROW EXECUTE FUNCTION fault_fail_members(:'title');
|
||
"""
|
||
CLEAR_FAIL_MEMBERS_SQL = """\
|
||
DROP TRIGGER IF EXISTS fault_fail_members ON chat_members;
|
||
DROP FUNCTION IF EXISTS fault_fail_members();
|
||
"""
|
||
FAIL_MESSAGES_SQL = """\
|
||
CREATE OR REPLACE FUNCTION fault_fail_messages() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF NEW.text = TG_ARGV[0] THEN
|
||
RAISE EXCEPTION 'fault injected: message insert';
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_fail_messages ON messages;
|
||
CREATE TRIGGER fault_fail_messages BEFORE INSERT ON messages
|
||
FOR EACH ROW EXECUTE FUNCTION fault_fail_messages(:'title');
|
||
"""
|
||
CLEAR_FAIL_MESSAGES_SQL = """\
|
||
DROP TRIGGER IF EXISTS fault_fail_messages ON messages;
|
||
DROP FUNCTION IF EXISTS fault_fail_messages();
|
||
"""
|
||
COUNT_CHATS_SQL = "SELECT count(*) FROM chats WHERE title = :'title';\n"
|
||
|
||
|
||
class HookError(Exception):
|
||
"""Неверный вызов или проект: код 2 и сообщение в stderr."""
|
||
|
||
|
||
def working_project() -> str:
|
||
"""Имя проекта рабочего стенда — из docker-compose.yml без `-p`."""
|
||
result = subprocess.run(
|
||
["docker", "compose", "config", "--format", "json"],
|
||
cwd=ROOT,
|
||
text=True,
|
||
capture_output=True,
|
||
check=True,
|
||
)
|
||
return json.loads(result.stdout)["name"]
|
||
|
||
|
||
def fault_project() -> str:
|
||
project = os.environ.get("FAULT_PROJECT", "")
|
||
if not project.endswith(PROJECT_SUFFIX):
|
||
raise HookError(
|
||
f"FAULT_PROJECT must end with {PROJECT_SUFFIX!r}: "
|
||
"run make failure_checks",
|
||
)
|
||
if project == working_project():
|
||
raise HookError("FAULT_PROJECT is the working stand, refusing")
|
||
return project
|
||
|
||
|
||
def compose_run(
|
||
project: str,
|
||
*args: str,
|
||
stdin: str | None = None,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return subprocess.run(
|
||
["docker", "compose", "-p", project, *files, *args],
|
||
cwd=ROOT,
|
||
input=stdin,
|
||
text=True,
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
|
||
def compose(project: str, *args: str, stdin: str | None = None) -> str:
|
||
result = compose_run(project, *args, stdin=stdin)
|
||
if result.returncode != 0:
|
||
sys.stderr.write(result.stderr)
|
||
raise SystemExit(result.returncode)
|
||
return result.stdout.strip()
|
||
|
||
|
||
def wait_answers(project: str, name: str) -> str:
|
||
"""Ждать, пока сервис ответит своей проверкой из healthcheck.
|
||
|
||
После `unpause` Docker до следующего запуска healthcheck (до 5 с)
|
||
показывает `unhealthy`, и `up --wait` сразу выходит с ошибкой, хотя
|
||
сервис уже отвечает. Поэтому проверка — напрямую.
|
||
"""
|
||
deadline = time.monotonic() + PROBE_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
probe = compose_run(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
name,
|
||
"sh",
|
||
"-c",
|
||
PROBES[name],
|
||
)
|
||
if probe.returncode == 0:
|
||
return ""
|
||
time.sleep(0.2)
|
||
raise HookError(f"{name} does not answer in {PROBE_TIMEOUT_S}s")
|
||
|
||
|
||
def valkey(project: str, *args: str) -> str:
|
||
# аргументы — позиционные параметры sh, без подстановки в строку
|
||
return compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"valkey",
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" "$@"',
|
||
"sh",
|
||
*args,
|
||
)
|
||
|
||
|
||
def psql(project: str, sql: str, *, title: str = "") -> str:
|
||
return compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"sh",
|
||
"-c",
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq '
|
||
'-v ON_ERROR_STOP=1 -v title="$1"',
|
||
"sh",
|
||
title,
|
||
stdin=sql,
|
||
)
|
||
|
||
|
||
def service(name: str) -> str:
|
||
if name not in SERVICES:
|
||
raise HookError(f"service must be one of {SERVICES}")
|
||
return name
|
||
|
||
|
||
def wait_healthy(project: str, name: str) -> str:
|
||
# тот же контейнер, без пересоздания; --wait ждёт его healthcheck
|
||
return compose(project, "up", "-d", "--wait", "--no-recreate", name)
|
||
|
||
|
||
def cache_hits(project: str) -> str:
|
||
for line in valkey(project, "INFO", "stats").splitlines():
|
||
if line.startswith("keyspace_hits:"):
|
||
return line.split(":", 1)[1].strip()
|
||
raise HookError("keyspace_hits not found in INFO stats")
|
||
|
||
|
||
def cache_has_chat(project: str, chat_id: str) -> str:
|
||
database = compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"printenv",
|
||
"POSTGRES_DB",
|
||
)
|
||
key = f"{CACHE_KEY_PREFIX}:{database}:chat:{uuid.UUID(chat_id)}"
|
||
return valkey(project, "EXISTS", key)
|
||
|
||
|
||
def run(project: str, action: str, args: list[str]) -> str:
|
||
"""Выполнить действие; синтаксис без match: hook запускают и 3.9."""
|
||
if action in ("stop", "pause") and len(args) == 1:
|
||
return compose(project, action, service(args[0]))
|
||
if action == "start" and len(args) == 1:
|
||
return wait_healthy(project, service(args[0]))
|
||
if action == "unpause" and len(args) == 1:
|
||
compose(project, "unpause", service(args[0]))
|
||
return wait_answers(project, args[0])
|
||
if action == "flush-cache" and not args:
|
||
return valkey(project, "FLUSHALL")
|
||
if action == "cache-hits" and not args:
|
||
return cache_hits(project)
|
||
if action == "cache-has-chat" and len(args) == 1:
|
||
return cache_has_chat(project, args[0])
|
||
if action == "fail-members" and len(args) == 1:
|
||
return psql(project, FAIL_MEMBERS_SQL, title=args[0])
|
||
if action == "clear-fail-members" and not args:
|
||
return psql(project, CLEAR_FAIL_MEMBERS_SQL)
|
||
if action == "fail-messages" and len(args) == 1:
|
||
# аргумент триггера передаётся той же переменной psql, что title
|
||
return psql(project, FAIL_MESSAGES_SQL, title=args[0])
|
||
if action == "clear-fail-messages" and not args:
|
||
return psql(project, CLEAR_FAIL_MESSAGES_SQL)
|
||
if action == "count-chats" and len(args) == 1:
|
||
return psql(project, COUNT_CHATS_SQL, title=args[0])
|
||
raise HookError(f"unknown action or arguments: {action} {args}")
|
||
|
||
|
||
def main() -> int:
|
||
if len(sys.argv) < 2:
|
||
print(__doc__, file=sys.stderr)
|
||
return 2
|
||
try:
|
||
output = run(fault_project(), sys.argv[1], sys.argv[2:])
|
||
except (HookError, ValueError) as e:
|
||
print(f"fault_hook: {e}", file=sys.stderr)
|
||
return 2
|
||
if output:
|
||
print(output)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|