Files
mtusi2026lab4/student-tests/fault_hook.py
T

504 lines
21 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Hook стенда отказов для проверок `fault` (`make failure-checks`).
Тесты вызывают его как `[FAULT_HOOK, действие, аргументы…]` без shell —
так же `tests/resilience.py` вызывает `ACTION_SCRIPT`. Сами тесты docker
не трогают. Hook работает только с проектом compose из `FAULT_PROJECT`
(его и пароли стенда передаёт `student-tests/stand.py` через окружение) и
отказывается работать с проектом рабочего стенда.
Действия:
stop|start|pause|unpause postgres|valkey|s3
flush-cache FLUSHALL
cache-hits keyspace_hits из INFO stats
cache-has-chat <chat_id> EXISTS ключа карточки: 0 или 1
fail-members <title> вставка участников чата с этим названием
падает (триггер в базе стенда отказов)
clear-fail-members убрать триггер
count-chats <title> число чатов с этим названием
fail-messages <text> вставка сообщения с этим текстом падает —
уже после вставки ключа идемпотентности
clear-fail-messages убрать триггер
count-keys <chat_id> число ключей идемпотентности чата
Вложения и S3 (лаба 4):
attachment <id> строка: «status lost size object_key» или none
count-attachments <chat_id> «pending ready» строк чата
pending-ids <chat_id> id строк pending чата, по строке на id
prefix префикс объектов вложений базы стенда
object <key> объект в bucket: «size sha256» или none
count-objects объектов под префиксом вложений
put-object <key> записать объект мимо API (тело — stdin)
delete-object <key> удалить объект мимо API
fail-ready <filename> перевод в ready строки с этим filename
падает как отказ PostgreSQL (SQLSTATE 08006)
stall-ready <filename> <s> перевод в ready ждёт <s> секунд
clear-fail-ready убрать оба триггера
delete-row <id> удалить строку вложения мимо API
set-size <id> <bytes> другой size_bytes у строки
age-row <id> <seconds> сдвинуть created_at и updated_at в прошлое
set-key <id> <key> другой object_key у строки
kill-api | start-api docker kill / start обычного API стенда
stop-cleaner | start-cleaner уборщик стенда
reconcile [аргументы] разовая сверка в контейнере уборщика
(grace стенда); вывод и строка exit_code=N
Пароли и ключи не печатает: valkey-cli берёт пароль из VALKEYCLI_AUTH
контейнера, psql работает от POSTGRES_USER контейнера, запросы к S3
делает контейнер обычного API своими ключами из окружения. Только stdlib,
Python 3.9+.
"""
from __future__ import annotations
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COMPOSE_FILES = (ROOT / "compose.yaml", ROOT / "student-tests" / "stand.compose.yml")
ENV_FILE = ROOT / "student-tests" / "stand.env"
PROJECT_SUFFIX = "-failures"
SERVICES = ("postgres", "valkey", "s3")
# та же проверка, что healthcheck сервиса в compose.yaml
PROBES = {
"postgres": 'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"',
"valkey": 'test "$(valkey-cli ping)" = PONG',
"s3": (
"wget -q -O /dev/null http://127.0.0.1:9333/cluster/healthz && "
"wget -q -O - http://127.0.0.1:9333/dir/status | grep -qF "
"'\"DataNodes\":[{' && wget -q -O /dev/null http://127.0.0.1:8888/healthz "
"&& wget -q -O /dev/null http://127.0.0.1:8333/healthz"
),
}
PROBE_TIMEOUT = 30
# срок остановки S3 на стенде отказов: плавная остановка SeaweedFS — ~15 с,
# проверке отказа она не нужна
S3_STOP_TIMEOUT = "2"
# начало ключа карточки — копия CACHE_KEY_PREFIX из src/constants.py: hook
# на stdlib и код приложения не импортирует. Меняется формат ключа —
# менять и здесь, иначе упадут test_flush_cache и test_valkey_back
CACHE_KEY_PREFIX = "messenger:v1"
# часть ключа объекта — копия S3_KEY_ATTACHMENTS из src/constants.py
S3_KEY_ATTACHMENTS = "attachments"
# grace уборщика стенда — CLEANER_ENV в stand.py
CLEANER_GRACE = os.environ.get("FAULT_CLEANER_GRACE", "25")
FAIL_MEMBERS_SQL = """\
CREATE OR REPLACE FUNCTION fault_fail_members() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
IF (SELECT title FROM chats WHERE id = NEW.chat_id) = TG_ARGV[0] THEN
RAISE EXCEPTION 'fault injected: chat members insert';
END IF;
RETURN NEW;
END
$$;
DROP TRIGGER IF EXISTS fault_fail_members ON chat_members;
CREATE TRIGGER fault_fail_members BEFORE INSERT ON chat_members
FOR EACH ROW EXECUTE FUNCTION fault_fail_members(:'arg');
"""
CLEAR_FAIL_MEMBERS_SQL = """\
DROP TRIGGER IF EXISTS fault_fail_members ON chat_members;
DROP FUNCTION IF EXISTS fault_fail_members();
"""
COUNT_CHATS_SQL = "SELECT count(*) FROM chats WHERE title = :'arg';\n"
# сбой вставки сообщения: ключ идемпотентности к этому моменту уже вставлен
# той же транзакцией (src/storage/postgres.py) — откат должен убрать и его
FAIL_MESSAGES_SQL = """\
CREATE OR REPLACE FUNCTION fault_fail_messages() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
IF NEW.text = TG_ARGV[0] THEN
RAISE EXCEPTION 'fault injected: message insert';
END IF;
RETURN NEW;
END
$$;
DROP TRIGGER IF EXISTS fault_fail_messages ON messages;
CREATE TRIGGER fault_fail_messages BEFORE INSERT ON messages
FOR EACH ROW EXECUTE FUNCTION fault_fail_messages(:'arg');
"""
CLEAR_FAIL_MESSAGES_SQL = """\
DROP TRIGGER IF EXISTS fault_fail_messages ON messages;
DROP FUNCTION IF EXISTS fault_fail_messages();
"""
COUNT_KEYS_SQL = "SELECT count(*) FROM idempotency_keys WHERE chat_id = :'arg';\n"
# перевод pending → ready строки с этим filename падает как обрыв связи с
# PostgreSQL (класс 08 — «база недоступна»): API отвечает 503, объект уже
# записан — компенсация должна его удалить
FAIL_READY_SQL = """\
CREATE OR REPLACE FUNCTION fault_fail_ready() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
IF NEW.status = 'ready' AND OLD.status = 'pending'
AND NEW.filename = TG_ARGV[0] THEN
RAISE EXCEPTION USING ERRCODE = '08006',
MESSAGE = 'fault injected: attachment ready';
END IF;
RETURN NEW;
END
$$;
DROP TRIGGER IF EXISTS fault_fail_ready ON attachments;
CREATE TRIGGER fault_fail_ready BEFORE UPDATE ON attachments
FOR EACH ROW EXECUTE FUNCTION fault_fail_ready(:'arg');
"""
# перевод в ready ждёт: окно «объект записан, ready ещё нет» для kill реплики
STALL_READY_SQL = """\
CREATE OR REPLACE FUNCTION fault_stall_ready() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
IF NEW.status = 'ready' AND OLD.status = 'pending'
AND NEW.filename = TG_ARGV[0] THEN
PERFORM pg_sleep(TG_ARGV[1]::float);
END IF;
RETURN NEW;
END
$$;
DROP TRIGGER IF EXISTS fault_stall_ready ON attachments;
CREATE TRIGGER fault_stall_ready BEFORE UPDATE ON attachments
FOR EACH ROW EXECUTE FUNCTION fault_stall_ready(:'arg', :'arg2');
"""
CLEAR_FAIL_READY_SQL = """\
DROP TRIGGER IF EXISTS fault_fail_ready ON attachments;
DROP FUNCTION IF EXISTS fault_fail_ready();
DROP TRIGGER IF EXISTS fault_stall_ready ON attachments;
DROP FUNCTION IF EXISTS fault_stall_ready();
"""
ATTACHMENT_SQL = (
"SELECT status, (lost_at IS NOT NULL)::int, size_bytes, object_key "
"FROM attachments WHERE id = :'arg';\n"
)
COUNT_ATTACHMENTS_SQL = (
"SELECT count(*) FILTER (WHERE status = 'pending'), "
"count(*) FILTER (WHERE status = 'ready') "
"FROM attachments WHERE chat_id = :'arg';\n"
)
PENDING_IDS_SQL = (
"SELECT id FROM attachments WHERE chat_id = :'arg' AND status = 'pending' "
"ORDER BY created_at;\n"
)
DELETE_ROW_SQL = "DELETE FROM attachments WHERE id = :'arg';\n"
SET_SIZE_SQL = "UPDATE attachments SET size_bytes = :'arg2'::int WHERE id = :'arg';\n"
AGE_ROW_SQL = (
"UPDATE attachments SET created_at = created_at - make_interval(secs => "
":'arg2'::float), updated_at = updated_at - make_interval(secs => "
":'arg2'::float) WHERE id = :'arg';\n"
)
SET_KEY_SQL = "UPDATE attachments SET object_key = :'arg2' WHERE id = :'arg';\n"
# запросы к S3 — из контейнера обычного API, его ключами (окружение
# контейнера): у пользователя API есть чтение, запись и удаление объектов
S3_SCRIPT = """\
import hashlib, os, sys
import botocore.session
from botocore.config import Config
client = botocore.session.get_session().create_client(
"s3",
endpoint_url=os.environ["S3_ENDPOINT"],
region_name=os.environ.get("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
config=Config(s3={"addressing_style": "path"}, retries={"total_max_attempts": 1}),
)
bucket = os.environ["S3_BUCKET"]
action, key = sys.argv[1], sys.argv[2]
if action == "get":
try:
body = client.get_object(Bucket=bucket, Key=key)["Body"].read()
except client.exceptions.NoSuchKey:
print("none")
else:
print(len(body), hashlib.sha256(body).hexdigest())
elif action == "put":
client.put_object(Bucket=bucket, Key=key, Body=sys.stdin.buffer.read())
elif action == "delete":
client.delete_object(Bucket=bucket, Key=key)
"""
class HookError(Exception):
"""Неверный вызов или проект: код 2 и сообщение в stderr."""
def fault_project() -> str:
project = os.environ.get("FAULT_PROJECT", "")
if not project.endswith(PROJECT_SUFFIX) or project == PROJECT_SUFFIX:
raise HookError(
f"FAULT_PROJECT must end with {PROJECT_SUFFIX!r}: run make failure-checks",
)
return project
def compose_run(project: str, *args: str, stdin: str | None = None):
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
return subprocess.run(
[
"docker",
"compose",
"-p",
project,
"--env-file",
str(ENV_FILE),
*files,
*args,
],
cwd=ROOT,
input=stdin,
text=True,
capture_output=True,
check=False,
)
def compose(project: str, *args: str, stdin: str | None = None) -> str:
result = compose_run(project, *args, stdin=stdin)
if result.returncode != 0:
sys.stderr.write(result.stderr)
raise SystemExit(result.returncode)
return result.stdout.strip()
def docker(*args: str, stdin: bytes | None = None) -> str:
result = subprocess.run(
["docker", *args], cwd=ROOT, input=stdin, capture_output=True, check=False
)
if result.returncode != 0:
sys.stderr.write(result.stderr.decode(errors="replace"))
raise SystemExit(result.returncode)
return result.stdout.decode().strip()
def service(name: str) -> str:
if name not in SERVICES:
raise HookError(f"service must be one of {SERVICES}")
return name
def wait_answers(project: str, name: str) -> str:
"""Ждать, пока сервис ответит проверкой из своего healthcheck.
После `unpause` Docker до следующего запуска healthcheck показывает
старое состояние, и `up --wait` выходит с ошибкой, хотя сервис уже
отвечает. Поэтому проверка — напрямую.
"""
deadline = time.monotonic() + PROBE_TIMEOUT
while time.monotonic() < deadline:
probe = compose_run(project, "exec", "-T", name, "sh", "-c", PROBES[name])
if probe.returncode == 0:
return ""
time.sleep(0.2)
raise HookError(f"{name} does not answer in {PROBE_TIMEOUT}s")
def valkey(project: str, *args: str) -> str:
# аргументы — позиционные параметры sh, без подстановки в строку
return compose(
project,
"exec",
"-T",
"valkey",
"sh",
"-c",
'exec valkey-cli "$@"',
"sh",
*args,
)
def psql(project: str, sql: str, *, arg: str = "", arg2: str = "") -> str:
# аргументы — переменные psql :'arg' и :'arg2' (литерал экранирует psql)
return compose(
project,
"exec",
"-T",
"postgres",
"sh",
"-c",
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq '
'-v ON_ERROR_STOP=1 -v arg="$1" -v arg2="$2"',
"sh",
arg,
arg2,
stdin=sql,
)
def database(project: str) -> str:
return compose(project, "exec", "-T", "postgres", "printenv", "POSTGRES_DB")
def cache_hits(project: str) -> str:
for line in valkey(project, "INFO", "stats").splitlines():
if line.startswith("keyspace_hits:"):
return line.split(":", 1)[1].strip()
raise HookError("keyspace_hits not found in INFO stats")
def cache_has_chat(project: str, chat_id: str) -> str:
key = f"{CACHE_KEY_PREFIX}:{database(project)}:chat:{uuid.UUID(chat_id)}"
return valkey(project, "EXISTS", key)
def prefix(project: str) -> str:
return f"{database(project)}/{S3_KEY_ATTACHMENTS}/"
def own_key(project: str, key: str) -> str:
"""Hook трогает только объекты под префиксом вложений базы стенда."""
if not key.startswith(prefix(project)) or ".." in key:
raise HookError("object key must be under the attachments prefix")
return key
def s3_object(project: str, action: str, key: str, body: bytes | None = None) -> str:
return docker(
"exec",
"-i",
f"{project}-api",
"python",
"-c",
S3_SCRIPT,
action,
own_key(project, key),
stdin=body,
)
def count_objects(project: str) -> str:
"""Файлы `original` под префиксом вложений — `fs.tree` filer SeaweedFS."""
bucket = docker("exec", f"{project}-api", "printenv", "S3_BUCKET")
path = f"/buckets/{bucket}/{prefix(project).rstrip('/')}"
out = compose(
project,
"exec",
"-T",
"s3",
"sh",
"-c",
'printf "fs.tree %s\\n" "$1" | weed shell -master=127.0.0.1:9333 2>/dev/null',
"sh",
path,
)
return str(sum(1 for line in out.splitlines() if line.strip().endswith("original")))
def reconcile(project: str, args: list[str]) -> str:
result = compose_run(
project,
"run",
"--rm",
"--no-deps",
"-e",
f"CLEANER_GRACE={CLEANER_GRACE}",
"-e",
"RECONCILE_INTERVAL=0",
"cleaner",
"reconcile",
*args,
)
return f"{result.stdout}{result.stderr}\nexit_code={result.returncode}"
def run(project: str, action: str, args: list[str]) -> str:
"""Выполнить действие; без match: hook запускается и на Python 3.9."""
if action == "stop" and len(args) == 1 and args[0] == "s3":
return compose(project, "stop", "-t", S3_STOP_TIMEOUT, "s3")
if action in ("stop", "pause") and len(args) == 1:
return compose(project, action, service(args[0]))
if action == "start" and len(args) == 1:
# тот же контейнер, без пересоздания; --wait ждёт его healthcheck
return compose(project, "up", "-d", "--wait", "--no-recreate", service(args[0]))
if action == "unpause" and len(args) == 1:
compose(project, "unpause", service(args[0]))
return wait_answers(project, args[0])
if action == "flush-cache" and not args:
return valkey(project, "FLUSHALL")
if action == "cache-hits" and not args:
return cache_hits(project)
if action == "cache-has-chat" and len(args) == 1:
return cache_has_chat(project, args[0])
if action == "fail-members" and len(args) == 1:
return psql(project, FAIL_MEMBERS_SQL, arg=args[0])
if action == "clear-fail-members" and not args:
return psql(project, CLEAR_FAIL_MEMBERS_SQL)
if action == "count-chats" and len(args) == 1:
return psql(project, COUNT_CHATS_SQL, arg=args[0])
if action == "fail-messages" and len(args) == 1:
return psql(project, FAIL_MESSAGES_SQL, arg=args[0])
if action == "clear-fail-messages" and not args:
return psql(project, CLEAR_FAIL_MESSAGES_SQL)
if action == "count-keys" and len(args) == 1:
return psql(project, COUNT_KEYS_SQL, arg=str(uuid.UUID(args[0])))
return run_attachments(project, action, args)
def run_attachments(project: str, action: str, args: list[str]) -> str:
"""Действия лабы 4: строки вложений, объекты S3, уборщик, сверка."""
if action == "attachment" and len(args) == 1:
row = psql(project, ATTACHMENT_SQL, arg=str(uuid.UUID(args[0])))
return row.replace("|", " ") or "none"
if action == "count-attachments" and len(args) == 1:
row = psql(project, COUNT_ATTACHMENTS_SQL, arg=str(uuid.UUID(args[0])))
return row.replace("|", " ")
if action == "pending-ids" and len(args) == 1:
return psql(project, PENDING_IDS_SQL, arg=str(uuid.UUID(args[0])))
if action == "prefix" and not args:
return prefix(project)
if action == "object" and len(args) == 1:
return s3_object(project, "get", args[0])
if action == "count-objects" and not args:
return count_objects(project)
if action == "put-object" and len(args) == 1:
return s3_object(project, "put", args[0], sys.stdin.buffer.read())
if action == "delete-object" and len(args) == 1:
return s3_object(project, "delete", args[0])
if action == "fail-ready" and len(args) == 1:
return psql(project, FAIL_READY_SQL, arg=args[0])
if action == "stall-ready" and len(args) == 2:
return psql(project, STALL_READY_SQL, arg=args[0], arg2=str(float(args[1])))
if action == "clear-fail-ready" and not args:
return psql(project, CLEAR_FAIL_READY_SQL)
if action == "delete-row" and len(args) == 1:
return psql(project, DELETE_ROW_SQL, arg=str(uuid.UUID(args[0])))
if action == "set-size" and len(args) == 2:
return psql(
project, SET_SIZE_SQL, arg=str(uuid.UUID(args[0])), arg2=str(int(args[1]))
)
if action == "age-row" and len(args) == 2:
return psql(
project, AGE_ROW_SQL, arg=str(uuid.UUID(args[0])), arg2=str(float(args[1]))
)
if action == "set-key" and len(args) == 2:
return psql(project, SET_KEY_SQL, arg=str(uuid.UUID(args[0])), arg2=args[1])
if action == "kill-api" and not args:
return docker("kill", f"{project}-api")
if action == "start-api" and not args:
return docker("start", f"{project}-api")
if action == "stop-cleaner" and not args:
return docker("stop", "-t", "10", f"{project}-cleaner")
if action == "start-cleaner" and not args:
return docker("start", f"{project}-cleaner")
if action == "reconcile":
return reconcile(project, args)
raise HookError(f"unknown action or arguments: {action} {args}")
def main() -> int:
if len(sys.argv) < 2:
print(__doc__, file=sys.stderr)
return 2
try:
output = run(fault_project(), sys.argv[1], sys.argv[2:])
except (HookError, ValueError) as exc:
print(f"fault_hook: {exc}", file=sys.stderr)
return 2
if output:
print(output)
return 0
if __name__ == "__main__":
sys.exit(main())