Close the source image after converting to RGBA so PNGs at the pixel limit fit the render memory limit. Strengthen the fencing test so that it fails without the attempt check, and measure worker and relay stops under load. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1910 lines
79 KiB
Python
Executable File
1910 lines
79 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Hook стенда отказов для проверок `fault` (`make failure_checks`).
|
||
|
||
Тесты вызывают его как `[FAULT_HOOK, действие, аргументы…]` без shell —
|
||
так же `tests/resilience.py` вызывает `ACTION_SCRIPT`. Сами тесты docker
|
||
не трогают. Hook работает только с проектом compose из `FAULT_PROJECT`,
|
||
его задаёт `student-tests/run_failures.py`. Проект рабочего стенда hook
|
||
не трогает. Решение 3 design change `failure-checks`.
|
||
|
||
Действия:
|
||
stop|start|pause|unpause postgres|valkey|s3
|
||
flush-cache FLUSHALL
|
||
cache-hits keyspace_hits из INFO stats
|
||
cache-has-chat <chat_id> EXISTS ключа карточки: 0 или 1
|
||
fail-members <title> вставка участников чата с этим названием
|
||
падает (триггер в базе стенда отказов)
|
||
clear-fail-members убрать триггер
|
||
fail-messages <text> вставка сообщения с этим текстом падает
|
||
(триггер; ключ идемпотентности к этому
|
||
моменту уже вставлен в той же транзакции)
|
||
clear-fail-messages убрать триггер
|
||
count-chats <title> число чатов с этим названием
|
||
fill-connections <секунд> занять все слоты max_connections сессиями
|
||
psql (`SELECT pg_sleep`) и оборвать
|
||
соединения API: их пулы переподключаются
|
||
и получают «too many clients» (53300)
|
||
clear-connections освободить слоты и дождаться подключения
|
||
db-sessions pg_stat_database.sessions базы стенда:
|
||
сколько сессий PostgreSQL приняла за всё
|
||
время (учитывает и сессию самого вызова)
|
||
attachment-ids строки `<id> <status>` всех вложений базы
|
||
стенда
|
||
last-pending `id` самого нового вложения в статусе
|
||
pending или пусто
|
||
object-ids `id` вложений, у которых в bucket стенда
|
||
есть объект: файл `original` под
|
||
`<база>/attachments/<id>/` (filer через
|
||
`weed shell fs.tree` на loopback контейнера
|
||
s3). Каталог `<id>` без файла не считается:
|
||
после DELETE SeaweedFS убирает его лишь
|
||
через ~2 мин
|
||
staging-ids то же под `<база>/staging/<id>/`
|
||
variant-object-count <id> сколько у вложения объектов вариантов
|
||
(`<база>/attachments/<id>/v<N>-<имя>`)
|
||
job <attachment_id> строка задания вложения: JSON с id, state,
|
||
attempt, attempt_base, last_error_code,
|
||
next_attempt_at, lease_expires_at,
|
||
published_at, updated_at; нет задания — пусто
|
||
fail-attachments ready-commit-stall|relay-stall <filename>
|
||
окна без точек сбоя в коде (design change
|
||
`pipeline-reliability`, решение 14):
|
||
ready-commit-stall — COMMIT фиксации ready ждёт
|
||
pg_sleep(3) в отложенном триггере; relay-stall
|
||
— отметка published_at ждёт pg_sleep(3), брокер
|
||
сообщение уже подтвердил
|
||
stop|start|kill|pause|unpause relay
|
||
relay стенда; start — docker start контейнера
|
||
freeze worker docker kill -s STOP: контейнер остаётся running
|
||
(не paused), поэтому его можно убить посреди
|
||
COMMIT
|
||
pause|unpause worker заморозка контейнера (зависший worker)
|
||
run-worker K=V… второй worker `<FAULT_PROJECT>-worker-extra` с
|
||
этим окружением
|
||
run-worker-second K=V… третий worker `<FAULT_PROJECT>-worker-second-extra`:
|
||
вторая попытка рядом с замороженной первой
|
||
run-relay K=V… второй relay `<FAULT_PROJECT>-relay-extra`
|
||
stop-extra убрать дополнительные контейнеры
|
||
extra-log worker|worker-second|relay
|
||
журнал дополнительного контейнера
|
||
kill-extra|pause-extra|unpause-extra worker|worker-second
|
||
то же с дополнительным worker
|
||
expire-lease <id> срок аренды задания вложения — в прошлом: окно,
|
||
где таймеры прежней попытки ещё не сработали
|
||
attachment-row <id> JSON: status, staging_key, error_code
|
||
attachment-free <id> `id` строки, если её не держит чужая транзакция;
|
||
занята — пусто
|
||
cleaner-log журнал контейнера уборщика, все запуски
|
||
relay-log журнал контейнера relay, все запуски
|
||
relay-state JSON: running, restart_count, exit_code,
|
||
started_at
|
||
kill-render [extra] SIGKILL процессу обработки в контейнере worker;
|
||
печатает число убитых
|
||
render-env [extra] имена переменных окружения процесса обработки;
|
||
значения не печатаются
|
||
render-count [extra] сколько процессов обработки в контейнере
|
||
wait-commit-stall ждать сессию PostgreSQL в PgSleep на COMMIT
|
||
object-sha256 <ключ> SHA-256 объекта процессом в контейнере API; нет
|
||
объекта — пусто
|
||
broker-memory-limit <значение>
|
||
rabbitmqctl set_vm_memory_high_watermark
|
||
absolute <значение>, например 1MiB
|
||
set-sha256 <id> <hex> изменить sha256 строки
|
||
set-staging-key <id> <ключ> изменить staging_key строки (только БД)
|
||
set-next-attempt <id> <секунд>
|
||
срок следующей попытки задания: сейчас + секунд
|
||
delete-variant <id> <имя> строка и объект варианта готового вложения
|
||
cancel-to-pending <id> как отмена приёма прежней версии: задание
|
||
удалено, вложение queued → pending с
|
||
updated_at позже created_at
|
||
replay [аргументы] `docker compose run --rm --no-deps -T relay
|
||
replay …`: записи процесса и строка
|
||
`exit_code=<n>`; код hook — 0
|
||
replays <id> записи аудита задания вложения, JSON в строку
|
||
fail-attachments insert|queued|stall|commit|outlive|undeletable <filename>
|
||
триггеры на вложения с этим filename (сбой —
|
||
SQLSTATE 08006, для API это отказ БД, 503):
|
||
insert — вставка строки падает; queued —
|
||
перевод в queued падает до COMMIT; stall —
|
||
перевод в queued ждёт pg_sleep(4); commit —
|
||
перевод в queued падает на COMMIT (исход для
|
||
API неизвестен); outlive — строка
|
||
вставляется на час старше (уборщик берёт её
|
||
сразу), а перевод в queued ждёт до 4,5 с, пока
|
||
строка pending не исчезнет; undeletable — как
|
||
outlive, и удаление строки падает
|
||
fail-attachments ready-fail|ready-stall <filename>
|
||
фиксация worker: перевод в ready падает
|
||
(08006) или ждёт pg_sleep(4)
|
||
stop|start|pause|unpause|kill|recreate rabbitmq
|
||
брокер стенда; recreate — up --force-recreate
|
||
на том же volume; после start, recreate и
|
||
unpause — ожидание ответа брокера
|
||
stop|start|kill worker worker стенда; start — docker start контейнера
|
||
worker-state JSON: running, restart_count, oom_killed,
|
||
exit_code
|
||
worker-log журнал контейнера worker (stdout), все
|
||
запуски подряд
|
||
queue-state JSON очереди: messages, messages_ready,
|
||
messages_unacknowledged, consumers, durable;
|
||
очереди нет — null
|
||
peek-job тело одного сообщения очереди, прочитанного
|
||
без подтверждения (возвращается в очередь);
|
||
JSON с полями body (JSON-строка), delivery_mode;
|
||
сообщения нет — пусто. Python в одноразовом
|
||
контейнере relay его настройками: пароль в hook
|
||
не попадает
|
||
publish-job сообщение с телом из stdin в очередь,
|
||
persistent; тот же контейнер
|
||
delete-queue удалить очередь (rabbitmqctl в контейнере
|
||
брокера); отсутствие — не ошибка
|
||
fail-attachments mark <filename>
|
||
пометка потерянного оригинала (UPDATE
|
||
lost_at) вложения с этим filename падает
|
||
(08006); свой триггер BEFORE UPDATE OF
|
||
lost_at, прежние режимы колонку не трогают
|
||
clear-fail-attachments убрать триггеры
|
||
kill-api docker kill обычного API стенда
|
||
(`<FAULT_PROJECT>-api`)
|
||
start-api docker start того же контейнера и ожидание
|
||
healthy его healthcheck
|
||
put-object <ключ> записать объект с телом из stdin в bucket
|
||
стенда мимо API: botocore в контейнере
|
||
обычного API его ключами из окружения
|
||
контейнера. Ключ — только под
|
||
`<FAULT_DATABASE>/attachments/` или
|
||
`<FAULT_DATABASE>/staging/`, символы
|
||
[A-Za-z0-9._/-]
|
||
delete-object <ключ> удалить объект мимо шлюза S3: `weed shell
|
||
fs.rm /buckets/<bucket>/<ключ>`; ключ — с той
|
||
же проверкой
|
||
delete-row <id> удалить строку вложения с его заданием,
|
||
записями аудита и вариантами (неиспользуемого:
|
||
ссылку сообщения не даст удалить FK)
|
||
set-size <id> <байт> изменить size_bytes строки
|
||
age-row <id> <секунд> сдвинуть created_at и updated_at строки
|
||
назад на одно и то же время: CHECK и
|
||
непомеченность pending сохраняются
|
||
set-key <id> <ключ> изменить object_key строки (только БД,
|
||
bucket не трогается): ключ — под
|
||
`<FAULT_DATABASE>/`, в том числе вне
|
||
`attachments/`, символы [A-Za-z0-9._/-]
|
||
lost-ids `id` вложений с потерянным оригиналом
|
||
(lost_at задан)
|
||
reconcile dry-run|apply [<предел>] [--strategy batch|head] [--deep]
|
||
[--max-objects-per-second R] [--staging-retention S] [--grace S]
|
||
разовая сверка (`docker compose run --rm
|
||
--no-deps -T cleaner reconcile [--apply]
|
||
[--max-changes N] …`): стратегия обхода,
|
||
глубокая сверка и предел скорости (R —
|
||
неотрицательное число, 0 — без предела)
|
||
проверяются по списку и передаются команде
|
||
как есть (design change
|
||
`track2-reconcile-cost`, решение 14);
|
||
--staging-retention и --grace — окружение
|
||
процесса CLEANER__STAGING_RETENTION_SECONDS и
|
||
CLEANER__PENDING_GRACE_SECONDS;
|
||
записи процесса и строка `exit_code=<n>`;
|
||
код hook — 0
|
||
stop|start cleaner сервис уборщика стенда
|
||
run-cleaner <секунд> отдельный уборщик
|
||
`<FAULT_PROJECT>-cleaner-scheduled` с
|
||
плановой сверкой раз в 1 с
|
||
(RECONCILER__INTERVAL_SECONDS=1) до конца
|
||
первого прохода, не дольше этого срока, затем
|
||
docker stop, его журнал и docker rm
|
||
|
||
Bucket и имя базы для object-ids и ключей объектов — FAULT_S3_BUCKET и
|
||
FAULT_DATABASE, их задаёт run_failures.py. Пароли и ключи не получает и
|
||
не печатает: valkey-cli берёт пароль из VALKEYCLI_AUTH контейнера, psql
|
||
работает от POSTGRES_USER контейнера, weed shell — с ключом подписи из
|
||
окружения контейнера s3, put-object и object-sha256 — ключами API из окружения
|
||
контейнера API, peek-job и publish-job — настройками брокера в одноразовом
|
||
контейнере relay: у API их больше нет. Только stdlib. Действия вложений — design
|
||
change `upload-consistency`, решение 10; расхождения хранилищ и сверка — design
|
||
change `track2-reconciler`, решение 11; брокер, worker и задания — design change
|
||
`async-attachments`, решение 15; relay, окна сбоя и replay — design change
|
||
`pipeline-reliability`, решение 14.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "student-tests" / "failures.compose.yml",
|
||
)
|
||
PROJECT_SUFFIX = "-failures"
|
||
SERVICES = ("postgres", "valkey", "s3", "rabbitmq")
|
||
# та же проверка, что в healthcheck сервиса в docker-compose.yml. У
|
||
# PostgreSQL стенда отказов healthcheck переопределён на базу postgres
|
||
# (failures.compose.yml), а проба после start / unpause ходит в базу
|
||
# приложения и добавляет в её pg_stat_database одну сессию: тест «Соединения
|
||
# пула не переоткрываются» start в своём окне не вызывает
|
||
PROBES = {
|
||
"postgres": 'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" '
|
||
'-d "$POSTGRES_DB"',
|
||
"valkey": 'test "$(valkey-cli -p "$VALKEY_PORT" ping)" = PONG',
|
||
# master выбран, том зарегистрирован у master, filer и шлюз S3
|
||
# отвечают — как healthcheck сервиса s3
|
||
"s3": "wget -q -O /dev/null http://127.0.0.1:9333/cluster/healthz && "
|
||
"wget -q -O - http://127.0.0.1:9333/dir/status | "
|
||
"grep -qF '\"DataNodes\":[{' && "
|
||
"wget -q -O /dev/null http://127.0.0.1:8888/healthz && "
|
||
'wget -q -O /dev/null "http://127.0.0.1:$S3_PORT/healthz"',
|
||
"rabbitmq": "rabbitmq-diagnostics -q check_port_connectivity",
|
||
}
|
||
PROBE_TIMEOUT_S = 30
|
||
# начало ключа карточки — копия CACHE_KEY_PREFIX из src/utils/constants.py:
|
||
# hook на stdlib и код приложения не импортирует. Меняется формат ключа —
|
||
# менять и здесь, иначе упадут test_flush_cache и test_valkey_back
|
||
CACHE_KEY_PREFIX = "messenger:v1"
|
||
|
||
FAIL_MEMBERS_SQL = """\
|
||
CREATE OR REPLACE FUNCTION fault_fail_members() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF (SELECT title FROM chats WHERE id = NEW.chat_id) = TG_ARGV[0] THEN
|
||
RAISE EXCEPTION 'fault injected: chat members insert';
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_fail_members ON chat_members;
|
||
CREATE TRIGGER fault_fail_members BEFORE INSERT ON chat_members
|
||
FOR EACH ROW EXECUTE FUNCTION fault_fail_members(:'title');
|
||
"""
|
||
CLEAR_FAIL_MEMBERS_SQL = """\
|
||
DROP TRIGGER IF EXISTS fault_fail_members ON chat_members;
|
||
DROP FUNCTION IF EXISTS fault_fail_members();
|
||
"""
|
||
FAIL_MESSAGES_SQL = """\
|
||
CREATE OR REPLACE FUNCTION fault_fail_messages() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF NEW.text = TG_ARGV[0] THEN
|
||
RAISE EXCEPTION 'fault injected: message insert';
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_fail_messages ON messages;
|
||
CREATE TRIGGER fault_fail_messages BEFORE INSERT ON messages
|
||
FOR EACH ROW EXECUTE FUNCTION fault_fail_messages(:'title');
|
||
"""
|
||
CLEAR_FAIL_MESSAGES_SQL = """\
|
||
DROP TRIGGER IF EXISTS fault_fail_messages ON messages;
|
||
DROP FUNCTION IF EXISTS fault_fail_messages();
|
||
"""
|
||
COUNT_CHATS_SQL = "SELECT count(*) FROM chats WHERE title = :'title';\n"
|
||
# Сессия psql учитывает себя в sessions, только когда сбросит свои
|
||
# счётчики: обычно на первом простое до запроса, но без ожидания
|
||
# блокировки, и при занятой — позже. Принудительный сброс в отдельной
|
||
# транзакции делает это детерминированным: значение всегда включает
|
||
# сессию самого вызова
|
||
DB_SESSIONS_SQL = """\
|
||
SELECT pg_stat_force_next_flush();
|
||
SELECT sessions FROM pg_stat_database WHERE datname = current_database();
|
||
"""
|
||
|
||
# Заполнение слотов: отвязанный sh в контейнере Postgres. Сначала столько
|
||
# попыток, сколько слотов, затем новая попытка раз в 0,05 с до конца
|
||
# срока: слот, освободившийся после обрыва соединения API, сразу занимает
|
||
# снова psql, а не переподключение пула. Лишние попытки получают отказ и
|
||
# выходят. Сессии — через сокет ([local]), пользователь контейнера.
|
||
# После CLEAR_FILL_SH фоновые psql — сироты пространства PID: их подбирает
|
||
# init стенда отказов (init: true у postgres в failures.compose.yml), а не
|
||
# postmaster, для которого выход чужого потомка с кодом 2 — авария
|
||
FILL_PID = "/tmp/fault-fill.pid"
|
||
FILL_SH = f"""\
|
||
echo $$ > {FILL_PID}
|
||
hold() {{
|
||
PGAPPNAME=fault_fill psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq \\
|
||
-c "SELECT pg_sleep($1)" >/dev/null 2>&1 &
|
||
}}
|
||
end=$(( $(date +%s) + $1 ))
|
||
slots=$(psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq \\
|
||
-c 'SHOW max_connections')
|
||
i=0
|
||
while [ "$i" -lt "$slots" ]; do hold "$1"; i=$((i + 1)); done
|
||
while [ "$(date +%s)" -lt "$end" ]; do hold "$1"; sleep 0.05; done
|
||
"""
|
||
# Обрыв соединений по TCP (это пулы API) сигналом SIGTERM — то же, что
|
||
# pg_terminate_backend, но без своего слота. Заголовок процесса backend —
|
||
# «postgres: <user> <db> <адрес>(<порт>) <состояние>»; сокет ([local]) и
|
||
# проверку готовности с 127.0.0.1 не трогаем. Печатает число оборванных.
|
||
KILL_TCP_SH = """\
|
||
n=0
|
||
for d in /proc/[0-9]*; do
|
||
t=$(tr '\\0' ' ' < "$d/cmdline" 2>/dev/null) || continue
|
||
case "$t" in
|
||
"postgres: "*" 127.0.0.1("*) ;;
|
||
"postgres: "*"("*")"*) kill -TERM "${d#/proc/}" 2>/dev/null \\
|
||
&& n=$((n + 1)) ;;
|
||
esac
|
||
done
|
||
echo "$n"
|
||
"""
|
||
CLEAR_FILL_SH = f"""\
|
||
[ -f {FILL_PID} ] && kill "$(cat {FILL_PID})" 2>/dev/null
|
||
rm -f {FILL_PID}
|
||
for d in /proc/[0-9]*; do
|
||
t=$(tr '\\0' ' ' < "$d/cmdline" 2>/dev/null) || continue
|
||
case "$t" in
|
||
"postgres: "*" [local] "*) kill -TERM "${{d#/proc/}}" 2>/dev/null ;;
|
||
esac
|
||
done
|
||
"""
|
||
ATTACHMENT_IDS_SQL = """\
|
||
SELECT id || ' ' || status FROM attachments ORDER BY created_at, id;
|
||
"""
|
||
LAST_PENDING_SQL = """\
|
||
SELECT id FROM attachments WHERE status = 'pending'
|
||
ORDER BY created_at DESC, id DESC LIMIT 1;
|
||
"""
|
||
# режим и filename — аргументы триггеров (TG_ARGV). SQLSTATE 08006
|
||
# (connection_failure) приложение считает отказом БД: 503, а не 500
|
||
# (design change upload-consistency, пробы). Три триггера, каждый
|
||
# срабатывает только в своих режимах:
|
||
# - fault_fail_attachments (строка, BEFORE): insert, queued, stall; в outlive
|
||
# и undeletable вставка «на час старше» — created_at и updated_at вместе,
|
||
# строка остаётся непомеченной (mark_queued требует updated_at =
|
||
# created_at); в undeletable удаление строки падает; ready-fail и
|
||
# ready-stall — перевод worker в ready;
|
||
# - fault_fail_attachments_commit (отложенный триггер ограничения, AFTER
|
||
# UPDATE): commit — перевод в queued падает на COMMIT, исход для API
|
||
# неизвестен;
|
||
# - fault_wait_attachments (оператор, BEFORE UPDATE OF status): outlive и
|
||
# undeletable — перевод в queued ждёт, пока строка pending с этим
|
||
# filename не исчезнет, но не дольше WAIT. На уровне оператора строка ещё
|
||
# не заблокирована, уборщик её берёт. UPDATE OF status: пометку уборщика
|
||
# (меняет только updated_at) ожидание не задерживает.
|
||
# stall и ожидание короче statement_timeout (5 с): завершаются сами
|
||
FAIL_ATTACHMENTS_SQL = """\
|
||
CREATE OR REPLACE FUNCTION fault_fail_attachments() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF TG_OP = 'DELETE' THEN
|
||
IF OLD.filename = TG_ARGV[1] AND TG_ARGV[0] = 'undeletable' THEN
|
||
RAISE EXCEPTION 'fault injected: attachment delete'
|
||
USING ERRCODE = '08006';
|
||
END IF;
|
||
RETURN OLD;
|
||
END IF;
|
||
IF NEW.filename <> TG_ARGV[1] THEN
|
||
RETURN NEW;
|
||
END IF;
|
||
IF TG_OP = 'INSERT' THEN
|
||
IF TG_ARGV[0] = 'insert' THEN
|
||
RAISE EXCEPTION 'fault injected: attachment insert'
|
||
USING ERRCODE = '08006';
|
||
END IF;
|
||
IF TG_ARGV[0] IN ('outlive', 'undeletable') THEN
|
||
NEW.created_at := NEW.created_at - interval '1 hour';
|
||
NEW.updated_at := NEW.updated_at - interval '1 hour';
|
||
END IF;
|
||
RETURN NEW;
|
||
END IF;
|
||
IF OLD.status = 'pending' AND NEW.status = 'queued' THEN
|
||
IF TG_ARGV[0] = 'queued' THEN
|
||
RAISE EXCEPTION 'fault injected: attachment queued'
|
||
USING ERRCODE = '08006';
|
||
END IF;
|
||
IF TG_ARGV[0] = 'stall' THEN
|
||
PERFORM pg_sleep(4);
|
||
END IF;
|
||
END IF;
|
||
IF OLD.status <> 'ready' AND NEW.status = 'ready' THEN
|
||
IF TG_ARGV[0] = 'ready-fail' THEN
|
||
RAISE EXCEPTION 'fault injected: attachment ready'
|
||
USING ERRCODE = '08006';
|
||
END IF;
|
||
IF TG_ARGV[0] = 'ready-stall' THEN
|
||
PERFORM pg_sleep(4);
|
||
END IF;
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
CREATE OR REPLACE FUNCTION fault_fail_attachments_commit() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF TG_ARGV[0] = 'commit' AND NEW.filename = TG_ARGV[1]
|
||
AND OLD.status = 'pending' AND NEW.status = 'queued' THEN
|
||
RAISE EXCEPTION 'fault injected: attachment queued commit'
|
||
USING ERRCODE = '08006';
|
||
END IF;
|
||
IF TG_ARGV[0] = 'ready-commit-stall' AND NEW.filename = TG_ARGV[1]
|
||
AND OLD.status <> 'ready' AND NEW.status = 'ready' THEN
|
||
PERFORM pg_sleep(3);
|
||
END IF;
|
||
RETURN NULL;
|
||
END
|
||
$$;
|
||
CREATE OR REPLACE FUNCTION fault_stall_published() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF TG_ARGV[0] = 'relay-stall' AND NEW.published_at IS NOT NULL
|
||
AND (SELECT filename FROM attachments
|
||
WHERE id = NEW.attachment_id) = TG_ARGV[1] THEN
|
||
PERFORM pg_sleep(3);
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_stall_published ON attachment_jobs;
|
||
CREATE TRIGGER fault_stall_published
|
||
BEFORE UPDATE OF published_at ON attachment_jobs
|
||
FOR EACH ROW EXECUTE FUNCTION fault_stall_published(:'mode', :'title');
|
||
CREATE OR REPLACE FUNCTION fault_wait_attachments() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
DECLARE
|
||
deadline timestamptz := clock_timestamp() + interval '4.5 seconds';
|
||
BEGIN
|
||
IF TG_ARGV[0] NOT IN ('outlive', 'undeletable') THEN
|
||
RETURN NULL;
|
||
END IF;
|
||
WHILE clock_timestamp() < deadline AND EXISTS (
|
||
SELECT 1 FROM attachments
|
||
WHERE filename = TG_ARGV[1] AND status = 'pending'
|
||
) LOOP
|
||
PERFORM pg_sleep(0.1);
|
||
END LOOP;
|
||
RETURN NULL;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_fail_attachments ON attachments;
|
||
CREATE TRIGGER fault_fail_attachments
|
||
BEFORE INSERT OR UPDATE OR DELETE ON attachments
|
||
FOR EACH ROW EXECUTE FUNCTION fault_fail_attachments(:'mode', :'title');
|
||
DROP TRIGGER IF EXISTS fault_fail_attachments_commit ON attachments;
|
||
CREATE CONSTRAINT TRIGGER fault_fail_attachments_commit
|
||
AFTER UPDATE ON attachments DEFERRABLE INITIALLY DEFERRED
|
||
FOR EACH ROW
|
||
EXECUTE FUNCTION fault_fail_attachments_commit(:'mode', :'title');
|
||
DROP TRIGGER IF EXISTS fault_wait_attachments ON attachments;
|
||
CREATE TRIGGER fault_wait_attachments
|
||
BEFORE UPDATE OF status ON attachments
|
||
FOR EACH STATEMENT
|
||
EXECUTE FUNCTION fault_wait_attachments(:'mode', :'title');
|
||
"""
|
||
# пометка потерянного оригинала падает: отдельный триггер на колонку
|
||
# lost_at (design change track2-reconciler, решение 11) — прежние режимы и
|
||
# их функция от новой колонки не зависят
|
||
FAIL_ATTACHMENTS_MARK_SQL = """\
|
||
CREATE OR REPLACE FUNCTION fault_fail_attachments_mark() RETURNS trigger
|
||
LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF NEW.filename = TG_ARGV[0] AND OLD.lost_at IS NULL
|
||
AND NEW.lost_at IS NOT NULL THEN
|
||
RAISE EXCEPTION 'fault injected: attachment mark lost'
|
||
USING ERRCODE = '08006';
|
||
END IF;
|
||
RETURN NEW;
|
||
END
|
||
$$;
|
||
DROP TRIGGER IF EXISTS fault_fail_attachments_mark ON attachments;
|
||
CREATE TRIGGER fault_fail_attachments_mark
|
||
BEFORE UPDATE OF lost_at ON attachments
|
||
FOR EACH ROW EXECUTE FUNCTION fault_fail_attachments_mark(:'title');
|
||
"""
|
||
CLEAR_FAIL_ATTACHMENTS_SQL = """\
|
||
DROP TRIGGER IF EXISTS fault_fail_attachments ON attachments;
|
||
DROP TRIGGER IF EXISTS fault_fail_attachments_commit ON attachments;
|
||
DROP TRIGGER IF EXISTS fault_wait_attachments ON attachments;
|
||
DROP TRIGGER IF EXISTS fault_fail_attachments_mark ON attachments;
|
||
DROP TRIGGER IF EXISTS fault_stall_published ON attachment_jobs;
|
||
DROP FUNCTION IF EXISTS fault_fail_attachments();
|
||
DROP FUNCTION IF EXISTS fault_fail_attachments_commit();
|
||
DROP FUNCTION IF EXISTS fault_stall_published();
|
||
DROP FUNCTION IF EXISTS fault_wait_attachments();
|
||
DROP FUNCTION IF EXISTS fault_fail_attachments_mark();
|
||
"""
|
||
# расхождения хранилищ (design change track2-reconciler, решение 11):
|
||
# значения — переменные psql (:'title' — id, :'mode' — число)
|
||
# задание и варианты ссылаются на строку без каскада: у готового вложения
|
||
# метаданные удаляются вместе с ними
|
||
DELETE_ROW_SQL = """\
|
||
BEGIN;
|
||
DELETE FROM attachment_variants WHERE attachment_id = :'title'::uuid;
|
||
DELETE FROM attachment_job_replays WHERE job_id IN (
|
||
SELECT id FROM attachment_jobs WHERE attachment_id = :'title'::uuid
|
||
);
|
||
DELETE FROM attachment_jobs WHERE attachment_id = :'title'::uuid;
|
||
DELETE FROM attachments WHERE id = :'title'::uuid RETURNING id;
|
||
COMMIT;
|
||
"""
|
||
SET_SIZE_SQL = """\
|
||
UPDATE attachments SET size_bytes = :'mode'::integer
|
||
WHERE id = :'title'::uuid RETURNING id;
|
||
"""
|
||
AGE_ROW_SQL = """\
|
||
UPDATE attachments
|
||
SET created_at = created_at - make_interval(secs => :'mode'::float8),
|
||
updated_at = updated_at - make_interval(secs => :'mode'::float8)
|
||
WHERE id = :'title'::uuid RETURNING id;
|
||
"""
|
||
SET_KEY_SQL = """\
|
||
UPDATE attachments SET object_key = :'mode'
|
||
WHERE id = :'title'::uuid RETURNING id;
|
||
"""
|
||
LOST_IDS_SQL = """\
|
||
SELECT id FROM attachments WHERE lost_at IS NOT NULL ORDER BY id;
|
||
"""
|
||
JOB_SQL = """\
|
||
SELECT row_to_json(j) FROM (
|
||
SELECT id, state, attempt, attempt_base, last_error_code, next_attempt_at,
|
||
lease_expires_at, published_at, updated_at
|
||
FROM attachment_jobs WHERE attachment_id = :'title'::uuid
|
||
) j;
|
||
"""
|
||
SET_SHA256_SQL = """\
|
||
UPDATE attachments SET sha256 = decode(:'mode', 'hex')
|
||
WHERE id = :'title'::uuid RETURNING id;
|
||
"""
|
||
SET_STAGING_KEY_SQL = """\
|
||
UPDATE attachments SET staging_key = :'mode'
|
||
WHERE id = :'title'::uuid RETURNING id;
|
||
"""
|
||
SET_NEXT_ATTEMPT_SQL = """\
|
||
UPDATE attachment_jobs
|
||
SET next_attempt_at = clock_timestamp() + make_interval(secs => :'mode'::float8)
|
||
WHERE attachment_id = :'title'::uuid RETURNING id;
|
||
"""
|
||
DELETE_VARIANT_SQL = """\
|
||
DELETE FROM attachment_variants
|
||
WHERE attachment_id = :'title'::uuid AND variant = :'mode'
|
||
RETURNING object_key;
|
||
"""
|
||
# как условная отмена прежней версии: задание удалено, строка pending с
|
||
# updated_at позже created_at (design change async-attachments, решение 3)
|
||
CANCEL_TO_PENDING_SQL = """\
|
||
BEGIN;
|
||
DELETE FROM attachment_jobs WHERE attachment_id = :'title'::uuid;
|
||
UPDATE attachments
|
||
SET status = 'pending', updated_at = updated_at + interval '1 microsecond'
|
||
WHERE id = :'title'::uuid AND status = 'queued' RETURNING id;
|
||
COMMIT;
|
||
"""
|
||
EXPIRE_LEASE_SQL = """\
|
||
UPDATE attachment_jobs SET lease_expires_at = clock_timestamp() - interval '1s'
|
||
WHERE attachment_id = :'title'::uuid AND state = 'processing' RETURNING id;
|
||
"""
|
||
ATTACHMENT_ROW_SQL = """\
|
||
SELECT row_to_json(a) FROM (
|
||
SELECT status, staging_key, error_code FROM attachments
|
||
WHERE id = :'title'::uuid
|
||
) a;
|
||
"""
|
||
# `FOR SHARE` конфликтует и с `FOR UPDATE`, и с `FOR NO KEY UPDATE`: занятая
|
||
# строка не вернётся. Проба держит блокировку только на время одного выражения
|
||
ATTACHMENT_FREE_SQL = """\
|
||
SELECT id FROM attachments
|
||
WHERE id = :'title'::uuid
|
||
FOR SHARE SKIP LOCKED;
|
||
"""
|
||
REPLAYS_SQL = """\
|
||
SELECT row_to_json(r) FROM (
|
||
SELECT r.id, r.job_id, r.operator, r.error_code, r.reason, r.attempt,
|
||
r.created_at
|
||
FROM attachment_job_replays r
|
||
JOIN attachment_jobs j ON j.id = r.job_id
|
||
WHERE j.attachment_id = :'title'::uuid
|
||
ORDER BY r.created_at, r.id
|
||
) r;
|
||
"""
|
||
COMMIT_STALL_SQL = """\
|
||
SELECT count(*) FROM pg_stat_activity
|
||
WHERE wait_event = 'PgSleep' AND state = 'active' AND query ILIKE 'COMMIT%';
|
||
"""
|
||
# брокер — настройками процесса в одноразовом контейнере relay: пароль в hook не
|
||
# попадает, у контейнера API их больше нет (design change pipeline-reliability,
|
||
# решение 1). Соединение прямо на сообщение, без подтверждения (peek): закрытие
|
||
# соединения возвращает его в очередь. Очередь — из имени базы процесса
|
||
PEEK_JOB_PY = """\
|
||
import asyncio
|
||
import json
|
||
import aio_pika
|
||
from src.config import settings
|
||
from src.infra.broker import queue_name
|
||
async def main():
|
||
broker = settings.broker
|
||
connection = await aio_pika.connect(
|
||
host=broker.host, port=broker.port, login=broker.user,
|
||
password=broker.password.get_secret_value(), timeout=5,
|
||
)
|
||
async with connection:
|
||
channel = await connection.channel()
|
||
queue = await channel.declare_queue(
|
||
queue_name(settings.database.name), durable=True,
|
||
)
|
||
message = await queue.get(no_ack=False, fail=False, timeout=5)
|
||
if message is None:
|
||
return
|
||
print(json.dumps({
|
||
"body": message.body.decode(),
|
||
"delivery_mode": message.delivery_mode,
|
||
}))
|
||
await message.nack(requeue=True)
|
||
asyncio.run(main())
|
||
"""
|
||
PUBLISH_JOB_PY = """\
|
||
import asyncio
|
||
import sys
|
||
import aio_pika
|
||
from src.config import settings
|
||
from src.infra.broker import queue_name
|
||
async def main():
|
||
broker = settings.broker
|
||
connection = await aio_pika.connect(
|
||
host=broker.host, port=broker.port, login=broker.user,
|
||
password=broker.password.get_secret_value(), timeout=5,
|
||
)
|
||
async with connection:
|
||
channel = await connection.channel(publisher_confirms=True)
|
||
name = queue_name(settings.database.name)
|
||
await channel.declare_queue(name, durable=True)
|
||
await channel.default_exchange.publish(
|
||
aio_pika.Message(
|
||
sys.stdin.buffer.read(),
|
||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||
),
|
||
routing_key=name,
|
||
)
|
||
asyncio.run(main())
|
||
"""
|
||
# поля list_queues; очередь — точное имя из FAULT_DATABASE
|
||
QUEUE_COLUMNS = (
|
||
"name",
|
||
"messages",
|
||
"messages_ready",
|
||
"messages_unacknowledged",
|
||
"consumers",
|
||
"durable",
|
||
)
|
||
# копия ATTACHMENT_QUEUE_SUFFIX из src/utils/constants.py
|
||
QUEUE_SUFFIX = "attachment.process.v1"
|
||
# объект мимо API: botocore образа (зависимость aiobotocore) с адресом,
|
||
# регионом и ключами из настроек процесса в контейнере API — ключи в hook
|
||
# не попадают. Ключ — argv[1], тело — stdin
|
||
PUT_OBJECT_PY = """\
|
||
import sys
|
||
import botocore.session
|
||
from botocore.config import Config
|
||
from src.config import settings
|
||
s3 = settings.s3
|
||
client = botocore.session.get_session().create_client(
|
||
"s3",
|
||
endpoint_url=s3.endpoint_url,
|
||
region_name=s3.region,
|
||
aws_access_key_id=s3.access_key_id.get_secret_value(),
|
||
aws_secret_access_key=s3.secret_access_key.get_secret_value(),
|
||
config=Config(s3={"addressing_style": "path"},
|
||
retries={"total_max_attempts": 1}),
|
||
)
|
||
body = sys.stdin.buffer.read()
|
||
client.put_object(Bucket=s3.bucket, Key=sys.argv[1], Body=body)
|
||
"""
|
||
# SHA-256 объекта тем же способом; нет объекта — пустой вывод
|
||
OBJECT_SHA256_PY = """\
|
||
import hashlib
|
||
import sys
|
||
import botocore.session
|
||
from botocore.config import Config
|
||
from botocore.exceptions import ClientError
|
||
from src.config import settings
|
||
s3 = settings.s3
|
||
client = botocore.session.get_session().create_client(
|
||
"s3",
|
||
endpoint_url=s3.endpoint_url,
|
||
region_name=s3.region,
|
||
aws_access_key_id=s3.access_key_id.get_secret_value(),
|
||
aws_secret_access_key=s3.secret_access_key.get_secret_value(),
|
||
config=Config(s3={"addressing_style": "path"},
|
||
retries={"total_max_attempts": 1}),
|
||
)
|
||
try:
|
||
body = client.get_object(Bucket=s3.bucket, Key=sys.argv[1])["Body"].read()
|
||
except ClientError:
|
||
sys.exit(0)
|
||
print(hashlib.sha256(body).hexdigest())
|
||
"""
|
||
# процессы обработки в контейнере: только stdlib образа, тот же пользователь.
|
||
# Значения окружения не печатаются
|
||
RENDER_TOOL_PY = """\
|
||
import os
|
||
import signal
|
||
import sys
|
||
def pids():
|
||
found = []
|
||
for name in os.listdir("/proc"):
|
||
if not name.isdigit():
|
||
continue
|
||
try:
|
||
with open(f"/proc/{name}/cmdline", "rb") as cmdline:
|
||
# точный аргумент: текст этой пробы в её argv тоже содержит имя
|
||
if b"src.worker.render" in cmdline.read().split(b"\\0"):
|
||
found.append(int(name))
|
||
except OSError:
|
||
pass
|
||
return found
|
||
action = sys.argv[1]
|
||
found = pids()
|
||
if action == "count":
|
||
print(len(found))
|
||
elif action == "kill":
|
||
for pid in found:
|
||
os.kill(pid, signal.SIGKILL)
|
||
print(len(found))
|
||
elif found:
|
||
with open(f"/proc/{found[0]}/environ", "rb") as environ:
|
||
items = environ.read().split(b"\\0")
|
||
print(" ".join(sorted(i.split(b"=")[0].decode() for i in items if i)))
|
||
"""
|
||
# окружение дополнительного процесса: только имена настроек своих секций, чтобы
|
||
# через hook нельзя было подменить пароли и адреса
|
||
EXTRA_ENV_RE = re.compile(
|
||
r"^((WORKER|RELAY)__[A-Z_]+|S3__TIMEOUT_SECONDS)=[A-Za-z0-9._-]+$",
|
||
)
|
||
BROKER_LIMIT_RE = re.compile(r"^[0-9]+(MiB|GiB|MB|GB)?$")
|
||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||
# варианты контракта: имя в DELETE_VARIANT_SQL
|
||
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
|
||
SECOND_WORKER = "worker-second"
|
||
EXTRA_SERVICES = ("worker", SECOND_WORKER, "relay")
|
||
COMMIT_STALL_TIMEOUT_S = 15
|
||
# аргументы replay: значения проверяет сама команда, а эти — чтобы в compose
|
||
# run не ушло лишнее
|
||
REPLAY_OPTION_RE = re.compile(r"^[A-Za-z0-9_:.-]+$")
|
||
# символы ключа объекта, который hook пишет или удаляет мимо API
|
||
OBJECT_KEY_RE = re.compile(r"^[A-Za-z0-9._/-]+$")
|
||
RECONCILE_MODES = {"dry-run": (), "apply": ("--apply",)}
|
||
# стратегии обхода и предел скорости сверки (change track2-reconcile-cost):
|
||
# неотрицательное число, целое или с дробной частью
|
||
RECONCILE_STRATEGIES = ("batch", "head")
|
||
RATE_PATTERN = re.compile(r"[0-9]+(\.[0-9]+)?")
|
||
# плановая сверка отдельного уборщика: раз в секунду
|
||
SCHEDULED_CLEANER_ENV = "RECONCILER__INTERVAL_SECONDS=1"
|
||
SCHEDULED_CLEANER_MAX_S = 60
|
||
SCHEDULED_PASS_DONE = '"event": "Finished reconcile pass"'
|
||
FAIL_ATTACHMENTS_MODES = (
|
||
"insert",
|
||
"queued",
|
||
"stall",
|
||
"commit",
|
||
"outlive",
|
||
"undeletable",
|
||
"ready-fail",
|
||
"ready-stall",
|
||
"ready-commit-stall",
|
||
"relay-stall",
|
||
)
|
||
# как S3_BUCKET_NAME_PATTERN в src/utils/constants.py; имя базы — как в
|
||
# run_isolated.py: путь filer собирается только из проверенных имён
|
||
BUCKET_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")
|
||
DB_NAME_RE = re.compile(r"^[a-z0-9_]+$")
|
||
# каталог вложения в filer — его id; копия ATTACHMENT_KEY_PREFIX и
|
||
# ATTACHMENT_ORIGINAL_SUFFIX из src/utils/constants.py
|
||
ATTACHMENT_DIR_RE = re.compile(
|
||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
|
||
)
|
||
ATTACHMENT_KEY_PREFIX = "attachments"
|
||
ATTACHMENT_STAGING_PREFIX = "staging"
|
||
ATTACHMENT_ORIGINAL = "original"
|
||
# объект варианта в каталоге вложения: v<версия>-<имя>
|
||
VARIANT_FILE_RE = re.compile(r"^v[1-9][0-9]*-(thumbnail|crop|optimized)$")
|
||
# строка fs.tree: отступ по 4 символа на уровень («│ » или « »), затем
|
||
# «├──» или «└──» и имя
|
||
TREE_LINE_RE = re.compile(r"^((?:[│ ] {3})*)[├└]──\s*(.*?)\s*$")
|
||
API_HEALTHY_TIMEOUT_S = 60
|
||
PROBE_SQL_SH = (
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -c "SELECT 1"'
|
||
)
|
||
TOO_MANY = "too many clients"
|
||
FILL_TIMEOUT_S = 20
|
||
|
||
|
||
class HookError(Exception):
|
||
"""Неверный вызов или проект: код 2 и сообщение в stderr."""
|
||
|
||
|
||
def working_project() -> str:
|
||
"""Имя проекта рабочего стенда — из docker-compose.yml без `-p`."""
|
||
result = subprocess.run(
|
||
["docker", "compose", "config", "--format", "json"],
|
||
cwd=ROOT,
|
||
text=True,
|
||
capture_output=True,
|
||
check=True,
|
||
)
|
||
return json.loads(result.stdout)["name"]
|
||
|
||
|
||
def fault_project() -> str:
|
||
project = os.environ.get("FAULT_PROJECT", "")
|
||
if not project.endswith(PROJECT_SUFFIX):
|
||
raise HookError(
|
||
f"FAULT_PROJECT must end with {PROJECT_SUFFIX!r}: "
|
||
"run make failure_checks",
|
||
)
|
||
if project == working_project():
|
||
raise HookError("FAULT_PROJECT is the working stand, refusing")
|
||
return project
|
||
|
||
|
||
def compose_run(
|
||
project: str,
|
||
*args: str,
|
||
stdin: str | None = None,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return subprocess.run(
|
||
["docker", "compose", "-p", project, *files, *args],
|
||
cwd=ROOT,
|
||
input=stdin,
|
||
text=True,
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
|
||
def compose(project: str, *args: str, stdin: str | None = None) -> str:
|
||
result = compose_run(project, *args, stdin=stdin)
|
||
if result.returncode != 0:
|
||
sys.stderr.write(result.stderr)
|
||
raise SystemExit(result.returncode)
|
||
return result.stdout.strip()
|
||
|
||
|
||
def wait_answers(project: str, name: str) -> str:
|
||
"""Ждать, пока сервис ответит своей проверкой из healthcheck.
|
||
|
||
После `unpause` Docker до следующего запуска healthcheck (до 5 с)
|
||
показывает `unhealthy`, и `up --wait` сразу выходит с ошибкой, хотя
|
||
сервис уже отвечает. Поэтому проверка — напрямую.
|
||
"""
|
||
deadline = time.monotonic() + PROBE_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
probe = compose_run(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
name,
|
||
"sh",
|
||
"-c",
|
||
PROBES[name],
|
||
)
|
||
if probe.returncode == 0:
|
||
return ""
|
||
time.sleep(0.2)
|
||
raise HookError(f"{name} does not answer in {PROBE_TIMEOUT_S}s")
|
||
|
||
|
||
def valkey(project: str, *args: str) -> str:
|
||
# аргументы — позиционные параметры sh, без подстановки в строку
|
||
return compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"valkey",
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" "$@"',
|
||
"sh",
|
||
*args,
|
||
)
|
||
|
||
|
||
def psql(project: str, sql: str, *, title: str = "", mode: str = "") -> str:
|
||
# значения — переменные psql (:'title', :'mode'), без подстановки в SQL
|
||
return compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"sh",
|
||
"-c",
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq '
|
||
'-v ON_ERROR_STOP=1 -v title="$1" -v mode="$2"',
|
||
"sh",
|
||
title,
|
||
mode,
|
||
stdin=sql,
|
||
)
|
||
|
||
|
||
def service(name: str) -> str:
|
||
if name not in SERVICES:
|
||
raise HookError(f"service must be one of {SERVICES}")
|
||
return name
|
||
|
||
|
||
def wait_healthy(project: str, name: str) -> str:
|
||
# тот же контейнер, без пересоздания; --wait ждёт его healthcheck
|
||
return compose(project, "up", "-d", "--wait", "--no-recreate", name)
|
||
|
||
|
||
def cache_hits(project: str) -> str:
|
||
for line in valkey(project, "INFO", "stats").splitlines():
|
||
if line.startswith("keyspace_hits:"):
|
||
return line.split(":", 1)[1].strip()
|
||
raise HookError("keyspace_hits not found in INFO stats")
|
||
|
||
|
||
def cache_has_chat(project: str, chat_id: str) -> str:
|
||
database = compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"printenv",
|
||
"POSTGRES_DB",
|
||
)
|
||
key = f"{CACHE_KEY_PREFIX}:{database}:chat:{uuid.UUID(chat_id)}"
|
||
return valkey(project, "EXISTS", key)
|
||
|
||
|
||
def postgres_sh(project: str, script: str, *args: str) -> str:
|
||
return compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"sh",
|
||
"-c",
|
||
script,
|
||
"sh",
|
||
*args,
|
||
)
|
||
|
||
|
||
def probe_connect(project: str) -> subprocess.CompletedProcess[str]:
|
||
"""Новое подключение psql и `SELECT 1`."""
|
||
return compose_run(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"sh",
|
||
"-c",
|
||
PROBE_SQL_SH,
|
||
)
|
||
|
||
|
||
def slots_full(project: str) -> bool:
|
||
"""Новое подключение получает «too many clients»."""
|
||
probe = probe_connect(project)
|
||
return probe.returncode != 0 and TOO_MANY in probe.stderr
|
||
|
||
|
||
def fill_connections(project: str, seconds: str) -> str:
|
||
"""Все слоты заняты psql, соединений API не осталось.
|
||
|
||
Готово, когда новое подключение получает отказ, а обрывать уже
|
||
некого: пул API, переподключаясь, получит 53300.
|
||
"""
|
||
if int(seconds) <= 0:
|
||
raise HookError("seconds must be positive")
|
||
compose(
|
||
project,
|
||
"exec",
|
||
"-d",
|
||
"postgres",
|
||
"sh",
|
||
"-c",
|
||
FILL_SH,
|
||
"sh",
|
||
str(int(seconds)),
|
||
)
|
||
deadline = time.monotonic() + FILL_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
if slots_full(project) and postgres_sh(project, KILL_TCP_SH) == "0":
|
||
return ""
|
||
time.sleep(0.1)
|
||
raise HookError(f"connections are not filled in {FILL_TIMEOUT_S}s")
|
||
|
||
|
||
def clear_connections(project: str) -> str:
|
||
deadline = time.monotonic() + FILL_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
postgres_sh(project, CLEAR_FILL_SH)
|
||
if probe_connect(project).returncode == 0:
|
||
return ""
|
||
time.sleep(0.2)
|
||
raise HookError(f"connections are not freed in {FILL_TIMEOUT_S}s")
|
||
|
||
|
||
def ids_with_original(tree: str) -> list[str]:
|
||
"""`id` каталогов первого уровня, в которых есть файл `original`.
|
||
|
||
Вывод `fs.tree` каталога `attachments`: первая строка — сам каталог,
|
||
затем `├──<id>` и под ним `│ └──original`. Каталог `<id>` без файла
|
||
— остаток после DELETE, объекта в нём нет.
|
||
"""
|
||
found: list[str] = []
|
||
current = ""
|
||
for line in tree.splitlines():
|
||
match = TREE_LINE_RE.match(line)
|
||
if match is None:
|
||
continue
|
||
depth = len(match.group(1)) // 4 + 1
|
||
name = match.group(2)
|
||
if depth == 1:
|
||
current = name if ATTACHMENT_DIR_RE.fullmatch(name) else ""
|
||
elif depth == 2 and name == ATTACHMENT_ORIGINAL and current:
|
||
found.append(current)
|
||
return found
|
||
|
||
|
||
def variant_files(tree: str, attachment_id: str) -> list[str]:
|
||
"""Файлы вариантов в каталоге `attachment_id` вывода `fs.tree`."""
|
||
found: list[str] = []
|
||
current = ""
|
||
for line in tree.splitlines():
|
||
match = TREE_LINE_RE.match(line)
|
||
if match is None:
|
||
continue
|
||
depth = len(match.group(1)) // 4 + 1
|
||
name = match.group(2)
|
||
if depth == 1:
|
||
current = name
|
||
elif (
|
||
depth == 2
|
||
and current == attachment_id
|
||
and VARIANT_FILE_RE.fullmatch(name)
|
||
):
|
||
found.append(name)
|
||
return found
|
||
|
||
|
||
def tree_of(project: str, prefix: str) -> str:
|
||
"""`fs.tree` каталога `<база>/<prefix>` bucket стенда; нет — пусто."""
|
||
bucket, database = storage_names()
|
||
return compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"s3",
|
||
"sh",
|
||
"-c",
|
||
"exec weed shell -master=127.0.0.1:9333 2>&1",
|
||
stdin=f"fs.tree /buckets/{bucket}/{database}/{prefix}\n",
|
||
)
|
||
|
||
|
||
def object_ids(project: str, prefix: str = ATTACHMENT_KEY_PREFIX) -> str:
|
||
"""`id` вложений с объектом `original` под префиксом bucket стенда,
|
||
строка на `id`. Нет префикса — пусто."""
|
||
return "\n".join(ids_with_original(tree_of(project, prefix)))
|
||
|
||
|
||
def variant_object_count(project: str, attachment_id: str) -> str:
|
||
wanted = str(uuid.UUID(attachment_id))
|
||
tree = tree_of(project, ATTACHMENT_KEY_PREFIX)
|
||
return str(len(variant_files(tree, wanted)))
|
||
|
||
|
||
def queue() -> str:
|
||
_, database = storage_names()
|
||
return f"{database}.{QUEUE_SUFFIX}"
|
||
|
||
|
||
def queue_state(project: str) -> str:
|
||
"""Состояние очереди стенда JSON-ом или `null`: `rabbitmqctl` в контейнере
|
||
брокера, без management API."""
|
||
output = compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"rabbitmq",
|
||
"rabbitmqctl",
|
||
"-q",
|
||
"list_queues",
|
||
*QUEUE_COLUMNS,
|
||
"--formatter",
|
||
"json",
|
||
)
|
||
for item in json.loads(output or "[]"):
|
||
if item["name"] == queue():
|
||
return json.dumps(item)
|
||
return "null"
|
||
|
||
|
||
def delete_queue(project: str) -> str:
|
||
"""Удалить очередь стенда; её отсутствие — не ошибка."""
|
||
result = compose_run(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"rabbitmq",
|
||
"rabbitmqctl",
|
||
"-q",
|
||
"delete_queue",
|
||
queue(),
|
||
)
|
||
if result.returncode != 0 and "No such queue" not in (
|
||
result.stdout + result.stderr
|
||
):
|
||
sys.stderr.write(result.stderr)
|
||
raise SystemExit(result.returncode)
|
||
return ""
|
||
|
||
|
||
def relay_python(project: str, code: str, stdin: bytes | None = None) -> str:
|
||
"""Код в одноразовом контейнере relay: настройки процесса, а не hook, знают
|
||
пароль брокера. У реплик API его нет."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
result = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
project,
|
||
*files,
|
||
"run",
|
||
"--rm",
|
||
"--no-deps",
|
||
"-T",
|
||
"--entrypoint",
|
||
"python",
|
||
"relay",
|
||
"-c",
|
||
code,
|
||
],
|
||
cwd=ROOT,
|
||
input=b"" if stdin is None else stdin,
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if result.returncode != 0:
|
||
tail = result.stderr.decode(errors="replace").strip().splitlines()[-1:]
|
||
raise HookError(f"relay python failed: {tail}")
|
||
# лишние записи настройки логгера в stdout: строка результата — последняя
|
||
lines = result.stdout.decode(errors="replace").strip().splitlines()
|
||
return lines[-1] if lines else ""
|
||
|
||
|
||
def service_container(project: str, name: str) -> str:
|
||
"""Контейнер сервиса, без одноразовых: `compose ps -a` назвал бы и второй
|
||
процесс hook `run-worker`."""
|
||
container = docker(
|
||
"ps",
|
||
"-a",
|
||
"-q",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={project}",
|
||
"--filter",
|
||
f"label=com.docker.compose.service={name}",
|
||
"--filter",
|
||
"label=com.docker.compose.oneoff=False",
|
||
)
|
||
if not container:
|
||
raise HookError(f"{name} container of the stand not found")
|
||
return container
|
||
|
||
|
||
def relay_action(project: str, action: str) -> str:
|
||
"""`start` — тот же контейнер, как у worker: одноразовых зависимостей
|
||
compose start не найдёт."""
|
||
if action in ("stop", "pause", "unpause"):
|
||
return compose(project, action, "relay")
|
||
docker(action, service_container(project, "relay"))
|
||
return ""
|
||
|
||
|
||
def container_state(container: str) -> str:
|
||
fields = docker(
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}} {{.RestartCount}} {{.State.ExitCode}} "
|
||
"{{.State.StartedAt}}",
|
||
container,
|
||
).split()
|
||
return json.dumps(
|
||
{
|
||
"running": fields[0] == "true",
|
||
"restart_count": int(fields[1]),
|
||
"exit_code": int(fields[2]),
|
||
"started_at": fields[3],
|
||
},
|
||
)
|
||
|
||
|
||
def extra_name(project: str, name: str) -> str:
|
||
if name not in EXTRA_SERVICES:
|
||
raise HookError(f"extra service must be one of {EXTRA_SERVICES}")
|
||
return f"{project}-{name}-extra"
|
||
|
||
|
||
def run_extra(
|
||
project: str,
|
||
name: str,
|
||
pairs: list[str],
|
||
*,
|
||
service: str | None = None,
|
||
) -> str:
|
||
"""Ещё один процесс сервиса с этим окружением; прежний такой убирается.
|
||
`service` — сервис compose, если он не совпадает с именем процесса."""
|
||
for pair in pairs:
|
||
if not EXTRA_ENV_RE.fullmatch(pair):
|
||
raise HookError(f"bad environment pair: {pair!r}")
|
||
container = extra_name(project, name)
|
||
subprocess.run(
|
||
["docker", "rm", "-f", container],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
options = [arg for pair in pairs for arg in ("-e", pair)]
|
||
compose(
|
||
project,
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
container,
|
||
*options,
|
||
service or name,
|
||
)
|
||
return ""
|
||
|
||
|
||
def stop_extra(project: str) -> str:
|
||
for name in EXTRA_SERVICES:
|
||
subprocess.run(
|
||
["docker", "rm", "-f", extra_name(project, name)],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
return ""
|
||
|
||
|
||
def render_tool(project: str, action: str, args: list[str]) -> str:
|
||
if args not in ([], ["extra"]):
|
||
raise HookError("render action takes only an optional extra")
|
||
container = (
|
||
extra_name(project, "worker") if args else worker_container(project)
|
||
)
|
||
return docker(
|
||
"exec",
|
||
container,
|
||
"python",
|
||
"-I",
|
||
"-c",
|
||
RENDER_TOOL_PY,
|
||
action,
|
||
)
|
||
|
||
|
||
def wait_commit_stall(project: str) -> str:
|
||
"""Фиксация дошла до COMMIT и ждёт в отложенном триггере."""
|
||
deadline = time.monotonic() + COMMIT_STALL_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
if psql(project, COMMIT_STALL_SQL).strip() not in ("", "0"):
|
||
return ""
|
||
time.sleep(0.05)
|
||
raise HookError(f"no COMMIT in pg_sleep in {COMMIT_STALL_TIMEOUT_S}s")
|
||
|
||
|
||
def object_sha256(project: str, key: str) -> str:
|
||
result = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
api_container(project),
|
||
"python",
|
||
"-c",
|
||
OBJECT_SHA256_PY,
|
||
object_key(key),
|
||
],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if result.returncode != 0:
|
||
tail = result.stderr.decode(errors="replace").strip().splitlines()[-1:]
|
||
raise HookError(f"object-sha256 failed: {tail}")
|
||
lines = result.stdout.decode(errors="replace").strip().splitlines()
|
||
return lines[-1] if lines else ""
|
||
|
||
|
||
def broker_memory_limit(project: str, value: str) -> str:
|
||
if not BROKER_LIMIT_RE.fullmatch(value):
|
||
raise HookError(f"bad memory limit: {value!r}")
|
||
compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"rabbitmq",
|
||
"rabbitmqctl",
|
||
"-q",
|
||
"set_vm_memory_high_watermark",
|
||
"absolute",
|
||
value,
|
||
)
|
||
return ""
|
||
|
||
|
||
def delete_variant(project: str, attachment_id: str, name: str) -> str:
|
||
"""Вариант готового вложения: строка, затем объект (filer)."""
|
||
if name not in VARIANT_NAMES:
|
||
raise HookError(f"variant must be one of {VARIANT_NAMES}")
|
||
key = psql(
|
||
project,
|
||
DELETE_VARIANT_SQL,
|
||
title=str(uuid.UUID(attachment_id)),
|
||
mode=name,
|
||
).strip()
|
||
if not key:
|
||
raise HookError(f"variant {name} of {attachment_id} not found")
|
||
return delete_object(project, key.splitlines()[0])
|
||
|
||
|
||
def replay(project: str, args: list[str]) -> str:
|
||
"""Команда реестра в одноразовом контейнере relay: записи процесса и
|
||
строка `exit_code=<n>`."""
|
||
for arg in args:
|
||
if not REPLAY_OPTION_RE.fullmatch(arg):
|
||
raise HookError(f"bad replay argument: {arg!r}")
|
||
result = compose_run(
|
||
project,
|
||
"run",
|
||
"--rm",
|
||
"--no-deps",
|
||
"-T",
|
||
"relay",
|
||
"replay",
|
||
*args,
|
||
)
|
||
lines = [*result.stdout.splitlines(), *result.stderr.splitlines()]
|
||
return "\n".join([*lines, f"exit_code={result.returncode}"])
|
||
|
||
|
||
def worker_container(project: str) -> str:
|
||
return service_container(project, "worker")
|
||
|
||
|
||
def worker_action(project: str, action: str) -> str:
|
||
"""`start` — тот же контейнер, `docker start`, как у уборщика: у compose
|
||
start нет контейнера одноразовых зависимостей."""
|
||
if action == "stop":
|
||
return compose(project, "stop", "worker")
|
||
if action in ("pause", "unpause"):
|
||
return compose(project, action, "worker")
|
||
if action == "freeze":
|
||
docker("kill", "-s", "STOP", worker_container(project))
|
||
return ""
|
||
docker(action, worker_container(project))
|
||
return ""
|
||
|
||
|
||
def worker_state(project: str) -> str:
|
||
fields = docker(
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}} {{.RestartCount}} {{.State.OOMKilled}} "
|
||
"{{.State.ExitCode}}",
|
||
worker_container(project),
|
||
).split()
|
||
return json.dumps(
|
||
{
|
||
"running": fields[0] == "true",
|
||
"restart_count": int(fields[1]),
|
||
"oom_killed": fields[2] == "true",
|
||
"exit_code": int(fields[3]),
|
||
},
|
||
)
|
||
|
||
|
||
def broker_action(project: str, action: str) -> str:
|
||
"""Действия брокера; ждёт ответа после возврата."""
|
||
if action in ("stop", "pause"):
|
||
compose(project, action, "rabbitmq")
|
||
return ""
|
||
if action == "kill":
|
||
docker("kill", compose(project, "ps", "-q", "rabbitmq"))
|
||
return ""
|
||
if action == "start":
|
||
docker("start", compose(project, "ps", "-a", "-q", "rabbitmq"))
|
||
elif action == "unpause":
|
||
compose(project, "unpause", "rabbitmq")
|
||
else:
|
||
compose(
|
||
project,
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--force-recreate",
|
||
"rabbitmq",
|
||
)
|
||
return wait_answers(project, "rabbitmq")
|
||
|
||
|
||
def fail_attachments(project: str, mode: str, filename: str) -> str:
|
||
if mode == "mark":
|
||
return psql(project, FAIL_ATTACHMENTS_MARK_SQL, title=filename)
|
||
if mode not in FAIL_ATTACHMENTS_MODES:
|
||
raise HookError(
|
||
f"mode must be one of {(*FAIL_ATTACHMENTS_MODES, 'mark')}",
|
||
)
|
||
return psql(project, FAIL_ATTACHMENTS_SQL, title=filename, mode=mode)
|
||
|
||
|
||
def storage_names() -> tuple[str, str]:
|
||
"""Проверенные bucket и имя базы стенда (FAULT_S3_BUCKET,
|
||
FAULT_DATABASE): пути filer и ключи собираются только из них."""
|
||
bucket = os.environ.get("FAULT_S3_BUCKET", "")
|
||
database = os.environ.get("FAULT_DATABASE", "")
|
||
if not BUCKET_NAME_RE.fullmatch(bucket):
|
||
raise HookError(f"FAULT_S3_BUCKET is not a bucket name: {bucket!r}")
|
||
if not DB_NAME_RE.fullmatch(database):
|
||
raise HookError(f"FAULT_DATABASE is not a database name: {database!r}")
|
||
return bucket, database
|
||
|
||
|
||
def object_key(key: str) -> str:
|
||
"""Ключ объекта вложений или staging базы стенда; иначе — отказ (код
|
||
2)."""
|
||
_, database = storage_names()
|
||
prefixes = (
|
||
f"{database}/{ATTACHMENT_KEY_PREFIX}/",
|
||
f"{database}/{ATTACHMENT_STAGING_PREFIX}/",
|
||
)
|
||
if (
|
||
not any(
|
||
key.startswith(prefix) and len(key) > len(prefix)
|
||
for prefix in prefixes
|
||
)
|
||
or not OBJECT_KEY_RE.fullmatch(key)
|
||
or ".." in key.split("/")
|
||
):
|
||
raise HookError(f"object key must be under {prefixes}: {key!r}")
|
||
return key
|
||
|
||
|
||
def row_key(key: str) -> str:
|
||
"""Ключ для `set-key`: под `<FAULT_DATABASE>/` (в том числе вне
|
||
`attachments/` — ключ «вне префикса вложений»), иначе — отказ (код 2).
|
||
Меняется только строка БД стенда, bucket hook не трогает."""
|
||
_, database = storage_names()
|
||
prefix = f"{database}/"
|
||
if (
|
||
not key.startswith(prefix)
|
||
or len(key) == len(prefix)
|
||
or not OBJECT_KEY_RE.fullmatch(key)
|
||
or ".." in key.split("/")
|
||
):
|
||
raise HookError(f"row key must be under {prefix}: {key!r}")
|
||
return key
|
||
|
||
|
||
def put_object(project: str, key: str) -> str:
|
||
"""Объект с телом из stdin мимо API — процессом в контейнере API."""
|
||
result = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
"-i",
|
||
api_container(project),
|
||
"python",
|
||
"-c",
|
||
PUT_OBJECT_PY,
|
||
object_key(key),
|
||
],
|
||
input=sys.stdin.buffer.read(),
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if result.returncode != 0:
|
||
sys.stderr.write(result.stderr.decode(errors="replace"))
|
||
raise SystemExit(result.returncode)
|
||
return ""
|
||
|
||
|
||
def delete_object(project: str, key: str) -> str:
|
||
"""Удалить объект в filer, мимо шлюза S3. `weed shell` выходит с кодом
|
||
0 и при ошибке команды: ошибка — по её выводу."""
|
||
bucket, _ = storage_names()
|
||
output = compose(
|
||
project,
|
||
"exec",
|
||
"-T",
|
||
"s3",
|
||
"sh",
|
||
"-c",
|
||
"exec weed shell -master=127.0.0.1:9333 2>&1",
|
||
stdin=f"fs.rm /buckets/{bucket}/{object_key(key)}\n",
|
||
)
|
||
failed = [
|
||
line
|
||
for line in output.splitlines()
|
||
if "error" in line.lower() or "not found" in line.lower()
|
||
]
|
||
if failed:
|
||
raise HookError(f"fs.rm failed: {failed[0]}")
|
||
return ""
|
||
|
||
|
||
def row_change(project: str, sql: str, attachment_id: str, value: str) -> str:
|
||
"""Изменение одной строки вложения; строки нет — отказ (код 2)."""
|
||
changed = psql(
|
||
project,
|
||
sql,
|
||
title=str(uuid.UUID(attachment_id)),
|
||
mode=value,
|
||
)
|
||
if not changed:
|
||
raise HookError(f"attachment {attachment_id} not found")
|
||
return ""
|
||
|
||
|
||
def reconcile_options(args: list[str]) -> tuple[list[str], list[str]]:
|
||
"""Аргументы сверки после режима: необязательный предел изменений
|
||
первым, затем `--strategy batch|head`, `--deep`,
|
||
`--max-objects-per-second R`, `--staging-retention S`, `--grace S`. Каждый — по списку
|
||
разрешённых, иначе код 2 hook. Возвращает аргументы команды и аргументы
|
||
`docker compose run` (окружение)."""
|
||
usage = (
|
||
"reconcile dry-run|apply [<max changes>] [--strategy batch|head] "
|
||
"[--deep] [--max-objects-per-second R] [--staging-retention S] "
|
||
"[--grace S]"
|
||
)
|
||
options: list[str] = []
|
||
run_options: list[str] = []
|
||
rest = list(args)
|
||
if rest and not rest[0].startswith("--"):
|
||
if not rest[0].isdigit() or int(rest[0]) < 1:
|
||
raise HookError(usage)
|
||
options += ["--max-changes", str(int(rest.pop(0)))]
|
||
seen: set[str] = set()
|
||
while rest:
|
||
name = rest.pop(0)
|
||
if name in seen:
|
||
raise HookError(usage)
|
||
seen.add(name)
|
||
if name == "--deep":
|
||
options.append(name)
|
||
elif name == "--strategy" and rest and rest[0] in RECONCILE_STRATEGIES:
|
||
options += [name, rest.pop(0)]
|
||
elif name == "--max-objects-per-second" and rest:
|
||
value = rest.pop(0)
|
||
if not RATE_PATTERN.fullmatch(value):
|
||
raise HookError(usage)
|
||
options += [name, value]
|
||
elif name == "--staging-retention" and rest and rest[0].isdigit():
|
||
run_options += [
|
||
"-e",
|
||
f"CLEANER__STAGING_RETENTION_SECONDS={int(rest.pop(0))}",
|
||
]
|
||
elif name == "--grace" and rest and rest[0].isdigit():
|
||
run_options += [
|
||
"-e",
|
||
f"CLEANER__PENDING_GRACE_SECONDS={int(rest.pop(0))}",
|
||
]
|
||
else:
|
||
raise HookError(usage)
|
||
return options, run_options
|
||
|
||
|
||
def reconcile(project: str, mode: str, args: list[str]) -> str:
|
||
"""Разовая сверка в одноразовом контейнере уборщика стенда: записи
|
||
процесса и строка `exit_code=<n>`."""
|
||
if mode not in RECONCILE_MODES:
|
||
raise HookError("reconcile dry-run|apply [<max changes>] [options]")
|
||
options, run_options = reconcile_options(args)
|
||
result = compose_run(
|
||
project,
|
||
"run",
|
||
"--rm",
|
||
"--no-deps",
|
||
"-T",
|
||
*run_options,
|
||
"cleaner",
|
||
"reconcile",
|
||
*RECONCILE_MODES[mode],
|
||
*options,
|
||
)
|
||
lines = [*result.stdout.splitlines(), *result.stderr.splitlines()]
|
||
return "\n".join([*lines, f"exit_code={result.returncode}"])
|
||
|
||
|
||
def cleaner_action(project: str, action: str) -> str:
|
||
"""`stop` или `start` сервиса уборщика стенда — тот же контейнер.
|
||
|
||
`docker compose start` проверял бы зависимости сервиса, а одноразовый
|
||
`migrate` стенд запускал через `run --rm`, и контейнера его нет
|
||
(«missing dependency migrate»): поэтому `docker start` контейнера.
|
||
"""
|
||
if action == "stop":
|
||
return compose(project, "stop", "cleaner")
|
||
docker("start", service_container(project, "cleaner"))
|
||
return ""
|
||
|
||
|
||
def run_cleaner(project: str, seconds: str) -> str:
|
||
"""Отдельный уборщик с плановой сверкой раз в 1 с на `seconds`, затем
|
||
остановка; его журнал."""
|
||
duration = float(seconds)
|
||
if not 0 < duration <= SCHEDULED_CLEANER_MAX_S:
|
||
raise HookError(f"seconds must be in (0, {SCHEDULED_CLEANER_MAX_S}]")
|
||
name = f"{project}-cleaner-scheduled"
|
||
subprocess.run(
|
||
["docker", "rm", "-f", name],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
compose(
|
||
project,
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
name,
|
||
"-e",
|
||
SCHEDULED_CLEANER_ENV,
|
||
"cleaner",
|
||
)
|
||
try:
|
||
# проход на большом bucket при 0,25 CPU дольше старта: остановка раньше
|
||
# его конца прервала бы его сигналом
|
||
deadline = time.monotonic() + duration
|
||
while (
|
||
SCHEDULED_PASS_DONE not in docker("logs", name)
|
||
and time.monotonic() < deadline
|
||
):
|
||
time.sleep(0.5)
|
||
docker("stop", name)
|
||
return docker("logs", name)
|
||
finally:
|
||
subprocess.run(
|
||
["docker", "rm", "-f", name],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
|
||
def api_container(project: str) -> str:
|
||
# обычный API стенда отказов: имя задаёт run_failures.py
|
||
return f"{project}-api"
|
||
|
||
|
||
def start_api(project: str) -> str:
|
||
"""Тот же контейнер API; готов — `healthy` по healthcheck образа."""
|
||
name = api_container(project)
|
||
docker("start", name)
|
||
deadline = time.monotonic() + API_HEALTHY_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
status = docker(
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Health.Status}}",
|
||
name,
|
||
)
|
||
if status == "healthy":
|
||
return ""
|
||
time.sleep(0.2)
|
||
raise HookError(f"{name} is not healthy in {API_HEALTHY_TIMEOUT_S}s")
|
||
|
||
|
||
def docker(*args: str) -> str:
|
||
result = subprocess.run(
|
||
["docker", *args],
|
||
text=True,
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if result.returncode != 0:
|
||
sys.stderr.write(result.stderr)
|
||
raise SystemExit(result.returncode)
|
||
return result.stdout.strip()
|
||
|
||
|
||
def run(project: str, action: str, args: list[str]) -> str:
|
||
"""Выполнить действие; синтаксис без match: hook запускают и 3.9."""
|
||
if action in ("stop", "start") and args == ["cleaner"]:
|
||
return cleaner_action(project, action)
|
||
if action in (
|
||
"stop",
|
||
"start",
|
||
"kill",
|
||
"pause",
|
||
"unpause",
|
||
"freeze",
|
||
) and args == ["worker"]:
|
||
return worker_action(project, action)
|
||
if action in ("stop", "start", "kill", "pause", "unpause") and args == [
|
||
"relay",
|
||
]:
|
||
return relay_action(project, action)
|
||
if action == "relay-log" and not args:
|
||
return docker("logs", service_container(project, "relay"))
|
||
if action == "relay-state" and not args:
|
||
return container_state(service_container(project, "relay"))
|
||
if action == "run-worker":
|
||
return run_extra(project, "worker", args)
|
||
if action == "run-worker-second":
|
||
return run_extra(project, SECOND_WORKER, args, service="worker")
|
||
if action == "run-relay":
|
||
return run_extra(project, "relay", args)
|
||
if action == "stop-extra" and not args:
|
||
return stop_extra(project)
|
||
if action == "extra-log" and len(args) == 1:
|
||
return docker("logs", extra_name(project, args[0]))
|
||
if (
|
||
action in ("kill-extra", "pause-extra", "unpause-extra")
|
||
and len(args) == 1
|
||
and args[0] in ("worker", SECOND_WORKER)
|
||
):
|
||
docker(action.split("-")[0], extra_name(project, args[0]))
|
||
return ""
|
||
if action == "expire-lease" and len(args) == 1:
|
||
return row_change(project, EXPIRE_LEASE_SQL, args[0], "")
|
||
if action == "attachment-row" and len(args) == 1:
|
||
return psql(project, ATTACHMENT_ROW_SQL, title=str(uuid.UUID(args[0])))
|
||
if action == "attachment-free" and len(args) == 1:
|
||
return psql(
|
||
project,
|
||
ATTACHMENT_FREE_SQL,
|
||
title=str(uuid.UUID(args[0])),
|
||
)
|
||
if action == "cleaner-log" and not args:
|
||
return docker("logs", service_container(project, "cleaner"))
|
||
if action == "kill-render":
|
||
return render_tool(project, "kill", args)
|
||
if action == "render-env":
|
||
return render_tool(project, "env", args)
|
||
if action == "render-count":
|
||
return render_tool(project, "count", args)
|
||
if action == "wait-commit-stall" and not args:
|
||
return wait_commit_stall(project)
|
||
if action == "object-sha256" and len(args) == 1:
|
||
return object_sha256(project, args[0])
|
||
if action == "broker-memory-limit" and len(args) == 1:
|
||
return broker_memory_limit(project, args[0])
|
||
if action == "set-sha256" and len(args) == 2:
|
||
if not SHA256_RE.fullmatch(args[1]):
|
||
raise HookError("sha256 must be 64 hex digits")
|
||
return row_change(project, SET_SHA256_SQL, args[0], args[1])
|
||
if action == "set-staging-key" and len(args) == 2:
|
||
return row_change(
|
||
project,
|
||
SET_STAGING_KEY_SQL,
|
||
args[0],
|
||
row_key(args[1]),
|
||
)
|
||
if action == "set-next-attempt" and len(args) == 2:
|
||
return row_change(
|
||
project,
|
||
SET_NEXT_ATTEMPT_SQL,
|
||
args[0],
|
||
str(float(args[1])),
|
||
)
|
||
if action == "delete-variant" and len(args) == 2:
|
||
return delete_variant(project, args[0], args[1])
|
||
if action == "cancel-to-pending" and len(args) == 1:
|
||
return row_change(project, CANCEL_TO_PENDING_SQL, args[0], "")
|
||
if action == "replay":
|
||
return replay(project, args)
|
||
if action == "replays" and len(args) == 1:
|
||
return psql(project, REPLAYS_SQL, title=str(uuid.UUID(args[0])))
|
||
if action in (
|
||
"stop",
|
||
"start",
|
||
"pause",
|
||
"unpause",
|
||
"kill",
|
||
"recreate",
|
||
) and args == ["rabbitmq"]:
|
||
return broker_action(project, action)
|
||
if action == "worker-state" and not args:
|
||
return worker_state(project)
|
||
if action == "worker-log" and not args:
|
||
return docker("logs", worker_container(project))
|
||
if action == "queue-state" and not args:
|
||
return queue_state(project)
|
||
if action == "delete-queue" and not args:
|
||
return delete_queue(project)
|
||
if action == "peek-job" and not args:
|
||
return relay_python(project, PEEK_JOB_PY)
|
||
if action == "publish-job" and not args:
|
||
return relay_python(project, PUBLISH_JOB_PY, sys.stdin.buffer.read())
|
||
if action == "job" and len(args) == 1:
|
||
return psql(project, JOB_SQL, title=str(uuid.UUID(args[0])))
|
||
if action == "staging-ids" and not args:
|
||
return object_ids(project, ATTACHMENT_STAGING_PREFIX)
|
||
if action == "variant-object-count" and len(args) == 1:
|
||
return variant_object_count(project, args[0])
|
||
if action in ("stop", "pause") and len(args) == 1:
|
||
return compose(project, action, service(args[0]))
|
||
if action == "start" and len(args) == 1:
|
||
return wait_healthy(project, service(args[0]))
|
||
if action == "unpause" and len(args) == 1:
|
||
compose(project, "unpause", service(args[0]))
|
||
return wait_answers(project, args[0])
|
||
if action == "flush-cache" and not args:
|
||
return valkey(project, "FLUSHALL")
|
||
if action == "cache-hits" and not args:
|
||
return cache_hits(project)
|
||
if action == "cache-has-chat" and len(args) == 1:
|
||
return cache_has_chat(project, args[0])
|
||
if action == "fail-members" and len(args) == 1:
|
||
return psql(project, FAIL_MEMBERS_SQL, title=args[0])
|
||
if action == "clear-fail-members" and not args:
|
||
return psql(project, CLEAR_FAIL_MEMBERS_SQL)
|
||
if action == "fail-messages" and len(args) == 1:
|
||
# аргумент триггера передаётся той же переменной psql, что title
|
||
return psql(project, FAIL_MESSAGES_SQL, title=args[0])
|
||
if action == "clear-fail-messages" and not args:
|
||
return psql(project, CLEAR_FAIL_MESSAGES_SQL)
|
||
if action == "count-chats" and len(args) == 1:
|
||
return psql(project, COUNT_CHATS_SQL, title=args[0])
|
||
if action == "fill-connections" and len(args) == 1:
|
||
return fill_connections(project, args[0])
|
||
if action == "clear-connections" and not args:
|
||
return clear_connections(project)
|
||
if action == "db-sessions" and not args:
|
||
return psql(project, DB_SESSIONS_SQL)
|
||
if action == "attachment-ids" and not args:
|
||
return psql(project, ATTACHMENT_IDS_SQL)
|
||
if action == "last-pending" and not args:
|
||
return psql(project, LAST_PENDING_SQL)
|
||
if action == "object-ids" and not args:
|
||
return object_ids(project)
|
||
if action == "fail-attachments" and len(args) == 2:
|
||
return fail_attachments(project, args[0], args[1])
|
||
if action == "clear-fail-attachments" and not args:
|
||
return psql(project, CLEAR_FAIL_ATTACHMENTS_SQL)
|
||
if action == "kill-api" and not args:
|
||
return docker("kill", api_container(project)) and ""
|
||
if action == "start-api" and not args:
|
||
return start_api(project)
|
||
if action == "put-object" and len(args) == 1:
|
||
return put_object(project, args[0])
|
||
if action == "delete-object" and len(args) == 1:
|
||
return delete_object(project, args[0])
|
||
if action == "delete-row" and len(args) == 1:
|
||
return row_change(project, DELETE_ROW_SQL, args[0], "")
|
||
if action == "set-size" and len(args) == 2:
|
||
return row_change(project, SET_SIZE_SQL, args[0], str(int(args[1])))
|
||
if action == "age-row" and len(args) == 2:
|
||
return row_change(
|
||
project,
|
||
AGE_ROW_SQL,
|
||
args[0],
|
||
str(float(args[1])),
|
||
)
|
||
if action == "set-key" and len(args) == 2:
|
||
return row_change(project, SET_KEY_SQL, args[0], row_key(args[1]))
|
||
if action == "lost-ids" and not args:
|
||
return psql(project, LOST_IDS_SQL)
|
||
if action == "reconcile" and args:
|
||
return reconcile(project, args[0], args[1:])
|
||
if action == "run-cleaner" and len(args) == 1:
|
||
return run_cleaner(project, args[0])
|
||
raise HookError(f"unknown action or arguments: {action} {args}")
|
||
|
||
|
||
def main() -> int:
|
||
if len(sys.argv) < 2:
|
||
print(__doc__, file=sys.stderr)
|
||
return 2
|
||
try:
|
||
output = run(fault_project(), sys.argv[1], sys.argv[2:])
|
||
except (HookError, ValueError) as e:
|
||
print(f"fault_hook: {e}", file=sys.stderr)
|
||
return 2
|
||
if output:
|
||
print(output)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|