Files
mtusi2026lab6/student-tests/test_failures.py
T
ricoromaandClaude Opus 5.5 9cc6c0c7a6 chore: port lab 5 solution into the lab 6 skeleton
Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e.
Contracts, public tests and evidence of lab 5 are not carried over.
The compose project, image and CI prefix are renamed to lab6; the first
broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the
OpenSpec changes of lab 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 04:46:56 +03:00

6474 lines
245 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
Valkey и S3, уборщик незавершённых загрузок (grace и интервал —
`FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API
(`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении),
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
соединения), «узкий по загрузкам» (`FAULT_UPLOADS_BASE_URL`: одно место
загрузки, одна в очереди, ожидание 1 с, срок тела 3 с — change
`upload-memory`) и «с коротким сроком S3» (`FAULT_S3_BASE_URL`: срок вызова
S3 1 с — change `fix-upload-put-flake`), брокер и worker (4 попытки, пауза
5 с, срок попытки 10 с, срок остановки 2 с — `FAULT_WORKER_*`, change
`async-attachments`). На хранилища и API тесты действуют только через
фикстуру `stand` (hook `student-tests/fault_hook.py`) и всегда возвращают
их в рабочее состояние. Решение 4 design change `failure-checks`, решение
10 design change `upload-consistency`, решение 11 design change
`track2-reconciler` (сверка bucket и БД — в конце модуля: её тесты
оставляют помеченные потерянными вложения и объекты staging без строки,
прежние тесты таких не ждут), решение 3 design change
`fix-upload-put-flake`, решение 15 design change `async-attachments`.
"""
import hashlib
import json
import math
import os
import random
import statistics
import threading
import time
import uuid
from collections.abc import Callable, Collection, Iterator, Sequence
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
import httpx
import pytest
from jsonschema import Draft202012Validator, FormatChecker
import images
from api_client import API, Actor, Api, new_key
from contract import (
BEARER_CHALLENGE,
assert_accepted,
assert_content,
assert_error,
assert_ok,
assert_unauthorized,
assert_variant_content,
assert_variant_image,
describe,
)
pytestmark = pytest.mark.fault
type Stand = Callable[..., str]
RESTORE = {"stop": "start", "pause": "unpause", "kill": "start"}
# запас к границам времени: планировщик Docker Desktop
SLACK_S = 0.3
# [cache] timeout_seconds и retry_after_seconds по умолчанию
CACHE_TIMEOUT_S = 0.2
CACHE_RETRY_AFTER_S = 5
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
SESSION_TTL_S = 2
IDEMPOTENCY_TTL_S = 2
POOL_TIMEOUT_S = 1
# срок проверки Postgres в /health/ready
READINESS_TIMEOUT_S = 2
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
OBJECT_STORAGE_HEALTH_S = 1
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
S3_TIMEOUT_S = 5
# сколько секунд hook держит слоты max_connections; clear-connections
# освобождает их раньше
FILL_HOLD_S = 30
ME = f"{API}/users/me"
@contextmanager
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
"""`stop`, `pause` или `kill` сервиса; в `finally` — обратно."""
stand(action, service)
try:
yield
finally:
stand(RESTORE[action], service)
def timed(
call: Callable[[], httpx.Response],
) -> tuple[httpx.Response, float]:
started = time.monotonic()
resp = call()
return resp, time.monotonic() - started
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
def wait_cache_ready(
api: Api,
stand: Stand,
actor: Actor,
chat: dict[str, Any],
) -> None:
"""Дождаться, пока чтение снова пишет карточку в кэш.
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
"""
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
stand("flush-cache")
while time.monotonic() < deadline:
assert_ok(read_chat(api, actor, chat), 200, "Chat")
if stand("cache-has-chat", chat["id"]) == "1":
return
time.sleep(0.5)
raise AssertionError("cache did not come back")
def chat_titles(api: Api, actor: Actor) -> set[str]:
return {
chat["title"]
for page in api.chat_pages(actor)
for chat in page["items"]
}
# ---------------------------------------------------------------------------
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
# ---------------------------------------------------------------------------
@pytest.mark.spec("sessions", "Истёкший токен")
def test_expired_token(tight_api: Api) -> None:
actor = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
time.sleep(SESSION_TTL_S + 1)
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "Срок не продлевается")
def test_use_does_not_extend_session(tight_api: Api) -> None:
actor = tight_api.register()
logged_in = time.monotonic()
successes = 0
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
successes += 1
time.sleep(0.5)
assert successes >= 3, "too few requests before expiry"
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
actor = api.register()
with outage(stand, "stop", "postgres"):
resp = api.request("GET", ME, actor=actor)
assert_error(resp, 503)
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
# ---------------------------------------------------------------------------
# Хранение: откат, остановленная БД, исчерпанный пул
# ---------------------------------------------------------------------------
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
def test_members_failure_rolls_back_chat(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
) -> None:
title = f"fault-{uuid.uuid4().hex}"
stand("fail-members", title)
try:
resp = api.request(
"POST",
f"{API}/chats",
actor=alice,
json={"title": title, "member_ids": [alice.id, bob.id]},
)
assert_error(resp, 500)
finally:
stand("clear-fail-members")
for actor in (alice, bob):
assert title not in chat_titles(api, actor)
assert stand("count-chats", title) == "0"
@pytest.mark.spec("data-storage", "БД остановлена")
@pytest.mark.spec("data-storage", "БД вернулась")
def test_database_stopped_and_back(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
error = assert_error(resp, 503)
assert took < 2, f"503 took {took:.2f}s"
for word in ("postgres", "sql", "connect"):
assert word not in error["message"].lower()
message = api.send_message(alice, chat["id"], "после возврата")
# запись во время отказа не зафиксирована, новая — видна
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
holder = Api(str(tight_api.client.base_url), timeout=15)
first: list[int] = []
def hold_connection() -> None:
first.append(holder.request("GET", ME, actor=actor).status_code)
with outage(stand, "pause", "postgres"):
thread = threading.Thread(target=hold_connection)
thread.start()
time.sleep(SLACK_S)
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
assert_error(resp, 503)
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
thread.join()
holder.close()
assert first == [503]
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
# hook оборвал соединение пула: при выдаче pool_pre_ping
# переподключается и получает «too many clients» (SQLSTATE 53300)
stand("fill-connections", str(FILL_HOLD_S))
try:
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
error = assert_error(resp, 503)
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
for word in ("postgres", "sql", "client", "connect"):
assert word not in error["message"].lower()
finally:
stand("clear-connections")
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
# сессии 2 с за время заполнения истёк — новый вход
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
# пачка одновременных запросов больше пула обычного API (10)
BURST_THREADS = 48
BURST_AFTER_BARRIER = 4
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
STATS_FLUSH_S = 11
def burst(api: Api, actor: Actor) -> None:
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
одновременно. Все ответы — 200."""
barrier = threading.Barrier(BURST_THREADS)
clients = [
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
]
responses: list[httpx.Response] = []
errors: list[BaseException] = []
lock = threading.Lock()
def worker(client: Api) -> None:
try:
got = [client.request("GET", ME, actor=actor)]
barrier.wait()
got += [
client.request("GET", ME, actor=actor)
for _ in range(BURST_AFTER_BARRIER)
]
except BaseException as e: # noqa: BLE001 — поднимется в тесте
barrier.abort()
with lock:
errors.append(e)
return
with lock:
responses.extend(got)
threads = [
threading.Thread(target=worker, args=(client,)) for client in clients
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
for client in clients:
client.close()
assert not errors, f"burst failed: {errors[0]!r}"
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
for resp in responses:
assert_ok(resp, 200, "User")
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
def test_pool_keeps_connections(
api: Api,
tight_api: Api,
stand: Stand,
) -> None:
actor = api.register()
# соединение «тесного» API могли оборвать прежние тесты
# (fill-connections): его переподключение не должно попасть в окно
tight = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
# прогрев: пул обычного API набирает свой размер
for _ in range(2):
burst(api, actor)
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
# в общую статистику без ожидания блокировки и при занятой блокировке
# откладываются. Простаивающий backend сбрасывает их принудительно по
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
# попадали в окно по одному
time.sleep(STATS_FLUSH_S)
# калибровка: второй вызов видит сессию первого вызова hook, и только её
s1 = int(stand("db-sessions"))
s2 = int(stand("db-sessions"))
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
for _ in range(3):
burst(api, actor)
s3 = int(stand("db-sessions"))
opened = s3 - s2 - 1
assert opened == 0, (
f"PostgreSQL accepted {opened} new sessions during "
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
)
# ---------------------------------------------------------------------------
# Readiness
# ---------------------------------------------------------------------------
@pytest.mark.spec("health", "Хранилище не готово")
def test_readiness_without_database(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(lambda: api.request("GET", "/health/ready"))
assert_error(resp, 503)
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
live = api.request("GET", "/health/live")
assert live.status_code == 200
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Кэш недоступен")
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "valkey"):
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Хранилище объектов недоступно")
@pytest.mark.parametrize("action", ["stop", "pause"])
def test_readiness_without_object_storage(
api: Api,
stand: Stand,
action: str,
) -> None:
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
with outage(stand, action, "s3"):
for path in ("/health/ready", "/health/live"):
resp, took = timed(lambda path=path: api.request("GET", path))
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
# ---------------------------------------------------------------------------
# Кэш карточки чата
# ---------------------------------------------------------------------------
@pytest.mark.spec("chat-cache", "Очистка кэша")
def test_flush_cache(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
stand("flush-cache")
assert stand("cache-has-chat", chat["id"]) == "0"
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert stand("cache-has-chat", chat["id"]) == "1"
@pytest.mark.spec("chat-cache", "Valkey остановлен")
def test_valkey_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
sent = [api.send_message(alice, chat["id"], "до отказа")]
with outage(stand, "stop", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
sent.append(assert_ok(resp, 201, "Message"))
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
assert api.all_messages(bob, chat["id"]) == sent
@pytest.mark.spec("chat-cache", "Valkey завис")
def test_valkey_paused(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
baseline = statistics.median(
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
)
with outage(stand, "pause", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
f"read took {took:.2f}s, baseline {baseline:.3f}s"
)
# окно пропуска: следующий запрос таймаута не ждёт
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
@pytest.mark.spec("chat-cache", "Valkey вернулся")
def test_valkey_back(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "valkey"):
assert_ok(read_chat(api, bob, chat), 200, "Chat")
# API не перезапускался; кэш снова в работе после окна пропуска
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
assert_ok(read_chat(api, bob, chat), 200, "Chat")
assert stand("cache-has-chat", chat["id"]) == "1"
hits = int(stand("cache-hits"))
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert int(stand("cache-hits")) > hits
# ---------------------------------------------------------------------------
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
# ---------------------------------------------------------------------------
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
def test_message_failure_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
# ключ вставляется первым: триггер роняет транзакцию уже после него
text = f"fault-{uuid.uuid4().hex}"
key = new_key()
stand("fail-messages", text)
try:
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 500)
finally:
stand("clear-fail-messages")
# ключ не занят: повтор выполняется заново, а не 500 и не 409
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
def test_database_outage_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
text = "при отказе БД"
key = new_key()
with outage(stand, "stop", "postgres"):
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 503)
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Ключ после срока")
def test_key_free_after_ttl(tight_api: Api) -> None:
alice = tight_api.register()
chat = tight_api.create_chat(alice)
key = new_key()
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
# в пределах срока ключ занят: другое тело — 409
resp = tight_api.post_message(
alice,
chat["id"],
key=key,
json={"text": "после срока"},
)
assert_error(resp, 409)
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
alice = tight_api.login(alice.user, alice.password)
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
assert second["id"] != first["id"]
history = tight_api.all_messages(alice, chat["id"])
assert [item["id"] for item in history] == [first["id"], second["id"]]
# ---------------------------------------------------------------------------
# Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки
# и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает
# строки прежних тестов параллельно (design change upload-consistency,
# решение 10)
# ---------------------------------------------------------------------------
# grace и интервал уборщика стенда отказов (run_failures.py,
# failures.compose.yml)
CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20"))
CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1"))
# запас поверх grace и интервала: проход, вызов hook, планировщик Docker
# Desktop
CLEANUP_SLACK_S = 5
# задержка перевода в queued триггером stall и в ready триггером ready-stall
# (fault_hook.py)
STALL_S = 4
# срок клиента загрузок в потоке
THREAD_CLIENT_TIMEOUT_S = 30
# имя базы стенда: ключи объектов — `<база>/attachments/<id>/original` и
# `<база>/staging/<id>/original`
FAULT_DATABASE = os.environ.get("FAULT_DATABASE", "messenger")
def attachment_rows(stand: Stand) -> dict[str, str]:
"""Вложения стенда: `id` → статус."""
return dict(line.split() for line in stand("attachment-ids").splitlines())
def stored_ids(stand: Stand) -> set[str]:
"""`id` вложений с объектом в конечном префиксе bucket стенда."""
return set(stand("object-ids").splitlines())
def staged_ids(stand: Stand) -> set[str]:
"""`id` вложений с объектом в staging."""
return set(stand("staging-ids").splitlines())
def object_ids(stand: Stand) -> set[str]:
"""`id` вложений с объектом в любом из двух префиксов."""
return stored_ids(stand) | staged_ids(stand)
def orphan_objects(stand: Stand) -> set[str]:
"""Объекты вложений без строки. Сначала строки, потом объекты: уборщик
удаляет объект раньше строки, и убранная между вызовами строка не
даёт ложного «объекта без строки»."""
rows = attachment_rows(stand)
return object_ids(stand) - set(rows)
def accepted_without_staging(stand: Stand) -> set[str]:
"""Принятые вложения без своего объекта: в работе — staging, готовые —
конечного оригинала (staging у них убирает уборщик). У `failed` его может не
быть по самой причине отказа."""
rows = attachment_rows(stand)
working = {
item
for item, status in rows.items()
if status in ("queued", "processing")
}
return (working - staged_ids(stand)) | (
ready_ids(rows) - stored_ids(stand)
)
def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]:
return set(after) - set(before)
def objects_and_rows(stand: Stand) -> tuple[dict[str, str], set[str]]:
"""Снимок: строки и объекты вложений обоих префиксов."""
return attachment_rows(stand), object_ids(stand)
def new_objects(
before: tuple[dict[str, str], set[str]],
stand: Stand,
) -> set[str]:
"""Объекты, которых не было в снимке, у `id`, которых в нём тоже не было.
Worker параллельно доводит задания прежних тестов: объект конечного ключа
у прежнего `id` — не след новой загрузки."""
return object_ids(stand) - before[1] - set(before[0])
def job_row(stand: Stand, attachment_id: str) -> dict[str, Any] | None:
"""Задание вложения; нет — `None`. Два задания дали бы две строки и
ошибку разбора: задание у вложения одно."""
output = stand("job", attachment_id)
return json.loads(output) if output else None
def ready_ids(rows: dict[str, str]) -> set[str]:
return {item for item, status in rows.items() if status == "ready"}
def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None:
"""Строка и объект вложения убраны не позже grace + интервал + запас
от `since` (момент до загрузки: `created_at` строки не раньше)."""
deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
while True:
rows = attachment_rows(stand)
stored = object_ids(stand)
if attachment_id not in rows and attachment_id not in stored:
return
if time.monotonic() > deadline:
raise AssertionError(
f"attachment {attachment_id} is not cleaned "
f"{time.monotonic() - since:.1f}s after upload: "
f"row {rows.get(attachment_id)}, "
f"object {attachment_id in stored}",
)
time.sleep(0.5)
def sleep_until(moment: float) -> None:
time.sleep(max(0.0, moment - time.monotonic()))
def fault_filename() -> str:
"""Уникальное имя загрузки, на которое срабатывает триггер hook."""
return f"fault-{uuid.uuid4().hex}.png"
def upload_raw(
api: Api,
actor: Actor,
chat: dict[str, Any],
data: Any,
**kwargs: Any,
) -> httpx.Response:
return api.post_attachment(actor, chat["id"], data, **kwargs)
def assert_invisible(
api: Api,
actor: Actor,
chat: dict[str, Any],
attachment_id: str,
) -> None:
"""Незавершённая загрузка: метаданные, байты и ссылка в сообщении — 404
с тем же `message`, что у неизвестного `id`."""
def calls(item: str) -> tuple[httpx.Response, ...]:
return (
api.attachment_meta(actor, item),
api.attachment_content(actor, item),
api.post_message(
actor,
chat["id"],
json={"text": "ссылка", "attachment_ids": [item]},
),
)
unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))]
hidden = [assert_error(resp, 404) for resp in calls(attachment_id)]
assert [error["message"] for error in hidden] == [
error["message"] for error in unknown
]
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
@pytest.mark.spec("attachments", "Незавершённая загрузка не видна")
def test_upload_with_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
# хотя бы одно готовое вложение: контроль ниже не пуст случайно
ready = api.upload_ready(alice, chat["id"], data)
before = attachment_rows(stand)
assert ready["id"] in ready_ids(before)
with outage(stand, "stop", "s3"):
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
assert_error(resp, 503)
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
after = attachment_rows(stand)
# строка-намерение осталась; её уберёт уборщик
# (test_cleaner_survives_outages)
(pending,) = new_ids(before, after)
assert after[pending] == "pending"
# ready не откатывается; число готовых может расти: worker
# параллельно доводит задания прежних тестов
assert ready_ids(before) <= ready_ids(after)
# заданий и принятых не прибавилось: новая строка одна и pending
assert job_row(stand, pending) is None
# замечание S6 verify attachments: id последней pending — от hook
assert stand("last-pending") == pending
assert_invisible(api, alice, chat, pending)
api.send_message(alice, chat["id"], "текст без S3")
api.upload(alice, chat["id"], data)
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
def test_rejected_uploads_leave_nothing(
api: Api,
stand: Stand,
alice: Actor,
eve: Actor,
chat: dict[str, Any],
) -> None:
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
def chunked() -> Iterator[bytes]:
for offset in range(0, len(big), 1024 * 1024):
yield big[offset : offset + 1024 * 1024]
# хотя бы одно готовое вложение: списки hook не пусты случайно
ready = api.upload_ready(alice, chat["id"], images.sample_png())
before = objects_and_rows(stand)
assert before[0].get(ready["id"]) == "ready", before[0]
assert ready["id"] in before[1], before[1]
rejected = (
(404, upload_raw(api, eve, chat, images.sample_png())),
(413, upload_raw(api, alice, chat, big)),
(413, upload_raw(api, alice, chat, chunked())),
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
(
415,
upload_raw(
api,
alice,
chat,
images.sample_png(),
content_type="text/plain",
),
),
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
)
for status, resp in rejected:
assert_error(resp, status)
assert not new_ids(before[0], attachment_rows(stand))
assert not new_objects(before, stand)
@pytest.mark.spec("attachments", "Сбой записи метаданных")
def test_metadata_insert_failure(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
before = objects_and_rows(stand)
stand("fail-attachments", "insert", filename)
try:
resp = upload_raw(
api,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
# строки нет — объект не записывался
assert not new_ids(before[0], attachment_rows(stand))
assert not new_objects(before, stand)
class BodyAborted(Exception):
"""Клиент обрывает передачу тела загрузки."""
@pytest.mark.spec("attachments", "Обрыв передачи тела")
def test_interrupted_body(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = objects_and_rows(stand)
def aborted_body() -> Iterator[bytes]:
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
raise BodyAborted
# свой клиент: его соединение обрывается вместе с телом
with (
Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own,
pytest.raises((BodyAborted, httpx.TransportError)),
):
upload_raw(own, alice, chat, aborted_body())
# API замечает обрыв не мгновенно; строки до конца тела нет в любом
# случае
time.sleep(1)
assert not new_ids(before[0], attachment_rows(stand))
assert not new_objects(before, stand)
api.upload(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachments", "Фиксация задания не удалась")
@pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута")
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
def test_queued_failure_compensated(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "queued", filename)
try:
started = time.monotonic()
resp = upload_raw(
api,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
after = attachment_rows(stand)
(pending,) = new_ids(before, after)
assert after[pending] == "pending"
assert stand("last-pending") == pending
# компенсация: объект удалён до ответа 503, задания нет
assert pending not in object_ids(stand)
assert job_row(stand, pending) is None
assert_invisible(api, alice, chat, pending)
# моложе grace строку уборщик не трогает: к половине grace он сделал
# около G / 2 / I проходов
sleep_until(started + CLEANER_GRACE_S / 2)
assert attachment_rows(stand).get(pending) == "pending"
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
@pytest.mark.spec("storage-consistency", "Объект без фиксации убран")
def test_killed_between_put_and_ready(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
before = attachment_rows(stand)
outcome: dict[str, Any] = {}
client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
def upload() -> None:
try:
outcome["status"] = upload_raw(
client,
alice,
chat,
images.sample_png(),
filename=filename,
).status_code
except httpx.TransportError as e:
outcome["error"] = e
stand("fail-attachments", "stall", filename)
killed = False
try:
started = time.monotonic()
thread = threading.Thread(target=upload)
thread.start()
# строка pending и объект staging — до конца задержки перевода в
# queued
deadline = started + STALL_S
pending = ""
while time.monotonic() < deadline:
pending = stand("last-pending")
if (
pending
and pending not in before
and pending in staged_ids(stand)
):
break
time.sleep(0.05)
else:
raise AssertionError("no pending row with object during stall")
stand("kill-api")
killed = True
thread.join(THREAD_CLIENT_TIMEOUT_S)
assert not thread.is_alive()
# 202 до клиента не дошёл: перевод в queued ждал в триггере
assert outcome.get("status") != 202, outcome
assert attachment_rows(stand).get(pending) == "pending"
assert pending in staged_ids(stand)
assert job_row(stand, pending) is None
finally:
client.close()
stand("clear-fail-attachments")
if killed:
stand("start-api")
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
api.upload(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachments", "Исход фиксации неизвестен")
def test_queued_commit_outcome_unknown(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Перевод в queued падает на самом COMMIT (отложенный триггер): API не
знает, зафиксировано ли вложение, и объект не трогает (verify W2)."""
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
started = time.monotonic()
resp = upload_raw(
api,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
after = attachment_rows(stand)
(pending,) = new_ids(before, after)
assert after[pending] == "pending"
# компенсации нет: вложение могло стать queued
assert pending in staged_ids(stand)
assert_invisible(api, alice, chat, pending)
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
@pytest.mark.spec("storage-consistency", "Загрузка дольше grace")
def test_upload_outlives_grace(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Строка вставлена «на час старше», перевод в queued ждёт в триггере,
пока её не уберёт уборщик стенда: загрузка, пережившая grace, без
заморозки процесса (verify W3)."""
filename = fault_filename()
before = objects_and_rows(stand)
stand("fail-attachments", "outlive", filename)
# ожидание в триггере до 4,5 с: срок клиента больше
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
try:
with slow:
resp = upload_raw(
slow,
alice,
chat,
images.sample_png(),
filename=filename,
)
# не 202 (принято без объекта) и не 500 (дефект ветки)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
rows = attachment_rows(stand)
assert not new_ids(before[0], rows), rows
assert not new_objects(before, stand)
assert not orphan_objects(stand)
@pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта")
def test_cleaner_failure_after_object_delete(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Уборщик удалил объект, а удаление строки упало (08006), пока загрузка
ждёт перевода в queued: строка помечена до удаления объекта и queued уже
не станет (verify W1)."""
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "undeletable", filename)
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
try:
with slow:
resp = upload_raw(
slow,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
(pending,) = new_ids(before, attachment_rows(stand))
assert attachment_rows(stand).get(pending) == "pending"
assert job_row(stand, pending) is None
assert pending not in staged_ids(stand)
assert not accepted_without_staging(stand)
assert_invisible(api, alice, chat, pending)
finally:
stand("clear-fail-attachments")
# строка старше grace: следующий проход доводит уборку
returned = time.monotonic()
wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S)
assert not orphan_objects(stand)
assert not accepted_without_staging(stand)
@pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки")
@pytest.mark.spec("storage-consistency", "БД недоступна во время уборки")
@pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты")
def test_cleaner_survives_outages(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
ready = api.upload_ready(alice, chat["id"], data)
with outage(stand, "stop", "s3"):
before = attachment_rows(stand)
started = time.monotonic()
assert_error(upload_raw(api, alice, chat, data), 503)
(pending,) = new_ids(before, attachment_rows(stand))
# PostgreSQL недоступен, пока загрузка моложе grace
with outage(stand, "stop", "postgres"):
time.sleep(3)
# grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте
sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S)
assert attachment_rows(stand).get(pending) == "pending"
# S3 вернулся: уборщик без перезапуска убирает строку за проход
returned = time.monotonic()
deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
while pending in attachment_rows(stand):
assert time.monotonic() < deadline, (
f"pending {pending} is still there "
f"{time.monotonic() - returned:.1f}s after S3 came back"
)
time.sleep(0.5)
assert pending not in object_ids(stand)
assert not orphan_objects(stand)
# готовое вложение старше grace на месте
meta = assert_ok(
api.attachment_meta(alice, ready["id"]),
200,
"Attachment",
)
assert meta == ready
assert meta["status"] == "ready"
assert_content(
api.attachment_content(alice, ready["id"]),
data,
"image/png",
)
@pytest.mark.spec("attachments", "S3 остановлен")
def test_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
attachment = api.upload_ready(alice, chat["id"], data)
with outage(stand, "stop", "s3"):
resp, elapsed = timed(
lambda: api.attachment_content(bob, attachment["id"]),
)
assert_error(resp, 503)
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
meta = assert_ok(
api.attachment_meta(bob, attachment["id"]),
200,
"Attachment",
)
assert meta == attachment
message = api.send_message(
bob,
chat["id"],
"",
attachment_ids=[attachment["id"]],
)
# история с attachment_ids — из PostgreSQL, S3 ей не нужен
history = api.all_messages(alice, chat["id"])
assert [item["id"] for item in history] == [message["id"]]
assert history[0]["attachment_ids"] == [attachment["id"]]
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
assert_content(
api.attachment_content(bob, attachment["id"]),
data,
"image/png",
)
def test_environment_does_not_change_limits(
api: Api,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Спек `service-configuration`, «Окружение не меняет лимит» (вне
сверки покрытия — сценарий в докстринге): обычный API стенда
отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1`
(`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт,
64 000 пикселей) всё равно принимается: лимиты — константы."""
limits = dict(
item.split("=", 1)
for item in os.environ.get("FAULT_LIMITS_ENV", "").split()
)
# без этих переменных на стенде тест ничего не доказывает
assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, (
f"stand started without limit variables: {limits}"
)
api.upload(alice, chat["id"], images.sample_png(), filename="sample.png")
@pytest.mark.spec("attachments", "S3 завис")
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
def test_object_storage_paused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
attachment = api.upload_ready(alice, chat["id"], data)
before = attachment_rows(stand)
started = time.monotonic()
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
with slow, outage(stand, "pause", "s3"):
for call in (
lambda: upload_raw(slow, alice, chat, data),
lambda: slow.attachment_content(alice, attachment["id"]),
):
resp, elapsed = timed(call)
assert_error(resp, 503)
# срок вызова S3 и немного на шаги с БД, а не зависание
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
f"503 after {elapsed:.2f}s"
)
assert_content(
api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)
# загрузка во время заморозки оставила строку pending; объект после
# разморозки не дописался, строку уборщик убирает после grace
(pending,) = new_ids(before, attachment_rows(stand))
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
# ---------------------------------------------------------------------------
# Места загрузки и срок тела (change upload-memory, design, решение 11):
# «узкий по загрузкам» API — одно место, одна загрузка в очереди, ожидание
# места и срок тела из FAULT_UPLOADS_ENV (run_failures.py, UPLOADS_ENV).
# Место держит загрузка, тело которой стоит на threading.Event теста
# ---------------------------------------------------------------------------
UPLOADS_ENV = dict(
item.split("=", 1)
for item in os.environ.get("FAULT_UPLOADS_ENV", "").split()
)
# окружение, с которым тесты ниже ждут «узкий» API
EXPECTED_UPLOADS_ENV = {
"UPLOADS__MAX_CONCURRENT": "1",
"UPLOADS__MAX_WAITING": "1",
"UPLOADS__WAIT_TIMEOUT_SECONDS": "1",
"UPLOADS__BODY_TIMEOUT_SECONDS": "3",
}
WAIT_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__WAIT_TIMEOUT_SECONDS", "1"))
BODY_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__BODY_TIMEOUT_SECONDS", "3"))
# отказ без ожидания места: вход, членство и ответ
IMMEDIATE_S = 0.5
# пауза, за которую запрос в соседнем потоке доходит до места или очереди
SETTLE_S = 0.2
# сколько держатель места ждёт отпускания, если тест упал раньше
HOLD_MAX_S = 10
def assert_uploads_stand() -> None:
"""Без этого окружения «узкого» API тесты мест ничего не доказывают."""
assert UPLOADS_ENV == EXPECTED_UPLOADS_ENV, (
f"uploads stand started with {UPLOADS_ENV}, "
f"expected {EXPECTED_UPLOADS_ENV}"
)
class Upload:
"""Загрузка в своём потоке и своём соединении; ответ — `result()`."""
def __init__(self, api: Api, actor: Actor, chat: dict[str, Any]) -> None:
self._base_url = str(api.client.base_url)
self._actor = actor
self._chat = chat
self._outcome: dict[str, Any] = {}
self._thread: threading.Thread | None = None
def start(self, data: Any, **kwargs: Any) -> "Upload":
def run() -> None:
with Api(self._base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
started = time.monotonic()
try:
self._outcome["resp"] = upload_raw(
own,
self._actor,
self._chat,
data,
**kwargs,
)
except (BodyAborted, httpx.TransportError) as e:
self._outcome["error"] = e
self._outcome["elapsed"] = time.monotonic() - started
self._outcome["ended"] = time.monotonic()
self._thread = threading.Thread(target=run)
self._thread.start()
return self
def result(self) -> tuple[httpx.Response, float, float]:
"""Ответ, длительность и момент ответа (`time.monotonic`)."""
assert self._thread is not None
self._thread.join(THREAD_CLIENT_TIMEOUT_S)
assert not self._thread.is_alive(), "upload thread hangs"
assert "resp" in self._outcome, self._outcome
return (
self._outcome["resp"],
self._outcome["elapsed"],
self._outcome["ended"],
)
@contextmanager
def held_slot(
api: Api,
actor: Actor,
chat: dict[str, Any],
data: bytes,
) -> Iterator[Callable[[], httpx.Response]]:
"""Загрузка A держит место: отдаёт половину тела и ждёт отпускания.
Внутри — после того, как A заняла место; вызов отдаваемой функции
отпускает A и возвращает её ответ. В `finally` A отпускается всегда.
"""
sent = threading.Event()
release = threading.Event()
def body() -> Iterator[bytes]:
yield data[: len(data) // 2]
sent.set()
if not release.wait(HOLD_MAX_S):
raise BodyAborted
yield data[len(data) // 2 :]
holder = Upload(api, actor, chat).start(body())
try:
assert sent.wait(HOLD_MAX_S), "holder did not start its body"
# вход и членство — короткие шаги БД, затем место
time.sleep(SETTLE_S)
def finish() -> httpx.Response:
release.set()
return holder.result()[0]
yield finish
finally:
release.set()
def slow_chunks(data: bytes, *, parts: int, seconds: float) -> Iterator[bytes]:
"""Тело `parts` частями, растянутое на `seconds`."""
size = -(-len(data) // parts)
for offset in range(0, len(data), size):
if offset:
time.sleep(seconds / (parts - 1))
yield data[offset : offset + size]
@pytest.mark.spec("attachments", "Все места заняты")
@pytest.mark.spec("attachments", "Очередь ожидания полна")
@pytest.mark.spec("api-conventions", "Лимит одновременных загрузок")
def test_upload_slots_busy(
uploads_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
data = images.sample_png()
before = objects_and_rows(stand)
with held_slot(uploads_api, alice, chat, data) as finish_holder:
# B ждёт место в очереди, C застаёт полную очередь
waiting = Upload(uploads_api, alice, chat).start(data)
time.sleep(SETTLE_S)
busy, busy_s = timed(
lambda: upload_raw(uploads_api, alice, chat, data),
)
waited, waited_s, _ = waiting.result()
# A отпускается сразу: её тело ограничено сроком тела стенда
holder = finish_holder()
assert_error(busy, 429)
assert busy_s < IMMEDIATE_S, f"429 of a full queue after {busy_s:.2f}s"
assert_error(waited, 429)
assert waited_s >= WAIT_TIMEOUT_S, f"429 after {waited_s:.2f}s"
for resp in (busy, waited):
assert int(resp.headers["Retry-After"]) >= 1
attachment = assert_accepted(holder)
# из трёх загрузок осталась только принятая: у 429 ни строки, ни объекта
assert new_ids(before[0], attachment_rows(stand)) == {attachment["id"]}
assert new_objects(before, stand) == {attachment["id"]}
@pytest.mark.spec("attachments", "Отказ раньше очереди")
def test_rejections_before_queue(
uploads_api: Api,
alice: Actor,
eve: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
data = images.sample_png()
# 30 MiB, которые начинаются как PNG: Content-Length 31 457 280
big = images.PNG_SIGNATURE + bytes(
3 * images.MAX_BYTES - len(images.PNG_SIGNATURE),
)
with held_slot(uploads_api, alice, chat, data) as finish_holder:
waiting = Upload(uploads_api, alice, chat).start(data)
time.sleep(SETTLE_S)
early = [
(status, *timed(call))
for status, call in (
(404, lambda: upload_raw(uploads_api, eve, chat, data)),
(413, lambda: upload_raw(uploads_api, alice, chat, big)),
(
415,
lambda: upload_raw(
uploads_api,
alice,
chat,
data,
content_type="text/plain",
),
),
)
]
early_done = time.monotonic()
waited, _, waited_at = waiting.result()
holder = finish_holder()
# очередь была полна всё время ранних отказов
assert_error(waited, 429)
assert early_done < waited_at
for status, resp, elapsed in early:
assert_error(resp, status)
# без ожидания места: ожидающий получил бы его не раньше срока
assert elapsed < WAIT_TIMEOUT_S, f"{status} after {elapsed:.2f}s"
assert early[1][1].request.headers["Content-Length"] == "31457280"
assert_accepted(holder)
@pytest.mark.spec("attachments", "Ожидание места")
@pytest.mark.spec(
"storage-consistency",
"Ожидание места не входит в возраст загрузки",
)
def test_upload_waits_for_slot(
uploads_api: Api,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
first = images.sample_png()
second = images.png(80, 60)
with held_slot(uploads_api, alice, chat, first) as finish_holder:
waiting = Upload(uploads_api, bob, chat).start(second)
time.sleep(SETTLE_S)
# A заканчивается раньше срока ожидания B
holder = finish_holder()
held = assert_accepted(holder)
waited_resp, waited_s, _ = waiting.result()
waited = assert_accepted(waited_resp)
assert waited_s < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, waited_s
# метаданные ожидавшей записаны после перевода в queued державшей место:
# ожидание прошло до строки pending (граница B уборщика прежняя)
assert waited["created_at"] > held["updated_at"], (held, waited)
for attachment, data in ((held, first), (waited, second)):
uploads_api.wait_attachment(alice, attachment["id"])
assert_content(
uploads_api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)
@pytest.mark.spec("attachments", "Место освобождается")
def test_slot_released_after_failures(
uploads_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
over = images.png_of_size(images.MAX_BYTES + 1)
def aborted_body() -> Iterator[bytes]:
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
raise BodyAborted
def next_upload_gets_slot(after: str) -> None:
resp, elapsed = timed(
lambda: upload_raw(uploads_api, alice, chat, images.sample_png()),
)
assert_accepted(resp)
assert elapsed < WAIT_TIMEOUT_S, (
f"upload after {after} waited {elapsed:.2f}s for a slot"
)
assert_error(
upload_raw(
uploads_api,
alice,
chat,
slow_chunks(over, parts=10, seconds=0),
),
413,
)
next_upload_gets_slot("413 by counter")
assert_error(upload_raw(uploads_api, alice, chat, b"not an image"), 415)
next_upload_gets_slot("415 by content")
slow = slow_chunks(
images.sample_png(),
parts=4,
seconds=BODY_TIMEOUT_S + 1,
)
assert_error(upload_raw(uploads_api, alice, chat, slow), 400)
next_upload_gets_slot("400 by body timeout")
with (
Api(
str(uploads_api.client.base_url),
timeout=THREAD_CLIENT_TIMEOUT_S,
) as own,
pytest.raises((BodyAborted, httpx.TransportError)),
):
upload_raw(own, alice, chat, aborted_body())
# API замечает обрыв не мгновенно
time.sleep(SETTLE_S)
next_upload_gets_slot("interrupted body")
with outage(stand, "stop", "s3"):
assert_error(
upload_raw(uploads_api, alice, chat, images.sample_png()),
503,
)
next_upload_gets_slot("503 with S3 stopped")
@pytest.mark.spec("attachments", "Медленный клиент")
@pytest.mark.spec("attachments", "Медленная передача в срок")
def test_slow_body(
uploads_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
data = images.sample_png()
before = objects_and_rows(stand)
late = upload_raw(
uploads_api,
alice,
chat,
slow_chunks(data, parts=6, seconds=BODY_TIMEOUT_S + 1),
)
error = assert_error(late, 400)
assert error["message"] == "Upload body was not received in time"
assert not new_ids(before[0], attachment_rows(stand))
assert not new_objects(before, stand)
# мелкие части с паузами, но в срок: тело принято как есть
in_time = slow_chunks(data, parts=20, seconds=BODY_TIMEOUT_S / 2)
resp, elapsed = timed(
lambda: upload_raw(uploads_api, alice, chat, in_time),
)
attachment = assert_accepted(resp)
assert elapsed < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, elapsed
assert attachment["size_bytes"] == len(data)
assert attachment["sha256"] == hashlib.sha256(data).hexdigest()
uploads_api.wait_attachment(alice, attachment["id"])
assert_content(
uploads_api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)
# ---------------------------------------------------------------------------
# Простой S3 после скачиваний (change fix-upload-put-flake, design, решение
# 3): API «с коротким сроком S3» — срок вызова S3 из FAULT_S3_ENV
# (run_failures.py, S3_ENV), других тестов на нём нет
# ---------------------------------------------------------------------------
S3_ENV = dict(
item.split("=", 1) for item in os.environ.get("FAULT_S3_ENV", "").split()
)
# окружение, с которым тест ниже ждёт API с коротким сроком S3
EXPECTED_S3_ENV = {"S3__TIMEOUT_SECONDS": "1"}
SHORT_S3_TIMEOUT_S = float(S3_ENV.get("S3__TIMEOUT_SECONDS", "1"))
# раунды: скачивания, простой дольше срока вызова S3, загрузки
IDLE_ROUNDS = 8
IDLE_DOWNLOADS = 10
IDLE_UPLOADS = 12
IDLE_EXTRA_S = 0.5
# медленный клиент: пауза после каждого блока 64 KiB
IDLE_READ_PAUSE_S = 0.002
def assert_s3_stand() -> None:
"""Без этого окружения API с коротким сроком S3 тест ничего не
доказывает: простой 1,5 с был бы короче срока вызова."""
assert S3_ENV == EXPECTED_S3_ENV, (
f"short S3 timeout stand started with {S3_ENV}, "
f"expected {EXPECTED_S3_ENV}"
)
@pytest.mark.spec("attachments", "Загрузка после простоя S3")
def test_upload_after_idle_s3(
s3_api: Api,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Скачивания большого оригинала, простой S3 дольше срока вызова,
затем загрузки — каждая 202, ни одной 503.
Раунд: 10 одновременных медленных скачиваний (своё соединение у
каждого клиента, барьер, 2 мс на блок 64 KiB) PNG ~4 MiB, простой
`S3__TIMEOUT_SECONDS + 0,5` с без запросов к этому API, 12 загрузок
`sample.png` по одной: пул клиента S3 отдаёт соединения по порядку
возврата, и 12 вызовов проходят через все ≤ 10 его соединений.
До правки клиента (срок чтения aiohttp, переживший ответ) соединение
портится в раунде с p ≈ 0,28 (design, «Как подтверждено», п. 8):
промах раунда — 0,72^10 ≈ 0,037, восьми раундов — ≈ 4e-12. После
правки 8 раундов — ~16 с.
"""
assert_s3_stand()
base_url = str(s3_api.client.base_url)
original = images.png_noise(1024, 1365, seed=4)
attachment = s3_api.upload_ready(
alice,
chat["id"],
original,
filename="idle-original.png",
)
sample = images.sample_png()
def download(barrier: threading.Barrier) -> httpx.Response:
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
# соединение открыто до барьера: после него сразу скачивание
own.request("GET", "/health/live")
barrier.wait()
return own.attachment_content_slowly(
alice,
attachment["id"],
pause_s=IDLE_READ_PAUSE_S,
)
for round_number in range(1, IDLE_ROUNDS + 1):
barrier = threading.Barrier(IDLE_DOWNLOADS)
with ThreadPoolExecutor(max_workers=IDLE_DOWNLOADS) as pool:
downloads = list(
pool.map(download, [barrier] * IDLE_DOWNLOADS),
)
for resp in downloads:
assert_content(resp, original, "image/png")
time.sleep(SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S)
for upload_number in range(1, IDLE_UPLOADS + 1):
resp = upload_raw(s3_api, alice, chat, sample)
assert resp.status_code == 202, describe(
resp,
f"round {round_number}, upload {upload_number} after "
f"{SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S:.1f}s of S3 idle",
)
assert_accepted(resp)
# ---------------------------------------------------------------------------
# Фоновая обработка, брокер и worker (change async-attachments, design, решение
# 15). Worker стенда: 4 попытки, пауза 5 с, срок попытки 10 с, срок остановки
# 2 с (FAULT_WORKER_*). Перед остановкой worker очередь доводится до пустой:
# задания прежних тестов не считаются в «одно сообщение». Worker и триггеры
# возвращаются в finally, упавшая проверка не ломает следующие
# ---------------------------------------------------------------------------
WORKER_MAX_ATTEMPTS = int(os.environ.get("FAULT_WORKER_MAX_ATTEMPTS", "4"))
WORKER_RETRY_BASE_S = float(os.environ.get("FAULT_WORKER_RETRY_BASE_S", "4"))
WORKER_RETRY_MAX_S = float(os.environ.get("FAULT_WORKER_RETRY_MAX_S", "16"))
WORKER_JOB_TIMEOUT_S = float(
os.environ.get("FAULT_WORKER_JOB_TIMEOUT_S", "10"),
)
WORKER_LEASE_MARGIN_S = float(
os.environ.get("FAULT_WORKER_LEASE_MARGIN_S", "2"),
)
WORKER_LEASE_S = WORKER_JOB_TIMEOUT_S + WORKER_LEASE_MARGIN_S
RELAY_REPUBLISH_S = float(os.environ.get("FAULT_RELAY_REPUBLISH_S", "5"))
STAGING_RETENTION_S = float(
os.environ.get("FAULT_CLEANER_STAGING_RETENTION_S", "60"),
)
# [worker] concurrency по умолчанию: стенд отказов её не меняет
WORKER_CONCURRENCY = 2
# ответ на загрузку не ждёт брокера и relay (спек job-outbox)
ACCEPT_MAX_S = 1
# отказ брокера не держит health: быстрее секунды, как у S3
BROKER_HEALTH_S = 1
# срок ожидания итога задания: все попытки и паузы с запасом
JOB_WAIT_S = 60
# relay публикует намерение за проход; срок ожидания сообщения с отказом брокера
PUBLISH_WAIT_S = 15
JOBS_SCHEMA_PATH = (
Path(__file__).resolve().parents[1] / "contracts" / "jobs.schema.json"
)
# срок «Stopped worker» по контракту остановки контейнера
WORKER_STOP_MAX_S = 10
ALL_VARIANTS = (None, "original", "thumbnail", "crop", "optimized")
VARIANT_NAMES = {"thumbnail", "crop", "optimized"}
# предел памяти брокера в конфигурации стенда: его возвращает проба сдерживания
BROKER_MEMORY_LIMIT = "400MiB"
def queue_state(stand: Stand) -> dict[str, Any]:
state = json.loads(stand("queue-state"))
assert state is not None, "job queue does not exist"
return state
def wait_queue_messages(
stand: Stand,
count: int,
*,
timeout: float = PUBLISH_WAIT_S,
) -> dict[str, Any]:
"""В очереди столько сообщений: relay публикует намерения за проход."""
deadline = time.monotonic() + timeout
while True:
state = json.loads(stand("queue-state"))
if state is not None and state["messages"] == count:
return state
assert time.monotonic() < deadline, (
f"queue has not {count} messages: {state}"
)
time.sleep(0.2)
@contextmanager
def relay_stopped(stand: Stand) -> Iterator[None]:
"""Relay остановлен; в `finally` запускается."""
with outage(stand, "stop", "relay"):
yield
def relay_records(stand: Stand) -> list[dict[str, Any]]:
return [
json.loads(line)
for line in stand("relay-log").splitlines()
if line.startswith("{")
]
def wait_relay_record(
stand: Stand,
event: str,
*,
timeout: float = 30,
skip: int = 0,
**fields: Any,
) -> dict[str, Any]:
"""Запись журнала relay с этими полями после первых `skip` записей."""
deadline = time.monotonic() + timeout
while True:
for record in relay_records(stand)[skip:]:
if record.get("event") == event and all(
record.get(name) == value for name, value in fields.items()
):
return record
assert time.monotonic() < deadline, (
f"no relay record {event!r} with {fields} in {timeout} s"
)
time.sleep(0.3)
def extra_records(stand: Stand, name: str = "worker") -> list[dict[str, Any]]:
"""Записи журнала дополнительного процесса hook `run-worker`."""
return [
json.loads(line)
for line in stand("extra-log", name).splitlines()
if line.startswith("{")
]
def wait_extra_record(
stand: Stand,
event: str,
*,
timeout: float = 30,
extra: str = "worker",
**fields: Any,
) -> dict[str, Any]:
deadline = time.monotonic() + timeout
while True:
for record in extra_records(stand, extra):
if record.get("event") == event and all(
record.get(name) == value for name, value in fields.items()
):
return record
assert time.monotonic() < deadline, (
f"no {extra} record {event!r} with {fields} in {timeout} s"
)
time.sleep(0.3)
def wait_queue_empty(stand: Stand, *, timeout: float = 90) -> None:
"""В очереди не осталось сообщений, в том числе неподтверждённых."""
deadline = time.monotonic() + timeout
while True:
state = queue_state(stand)
if state["messages"] == 0:
return
assert time.monotonic() < deadline, f"queue is not empty: {state}"
time.sleep(0.5)
@contextmanager
def worker_stopped(stand: Stand) -> Iterator[None]:
"""Worker остановлен при пустой очереди; в `finally` запускается."""
wait_queue_empty(stand)
with outage(stand, "stop", "worker"):
yield
def worker_records(stand: Stand) -> list[dict[str, Any]]:
"""Записи журнала worker за все его запуски."""
return [
json.loads(line)
for line in stand("worker-log").splitlines()
if line.startswith("{")
]
def wait_worker_record(
stand: Stand,
event: str,
*,
timeout: float = 30,
skip: int = 0,
**fields: Any,
) -> dict[str, Any]:
"""Запись журнала worker с этими полями после первых `skip` записей."""
deadline = time.monotonic() + timeout
while True:
for record in worker_records(stand)[skip:]:
if record.get("event") == event and all(
record.get(name) == value for name, value in fields.items()
):
return record
assert time.monotonic() < deadline, (
f"no worker record {event!r} with {fields} in {timeout} s"
)
time.sleep(0.3)
def job_body(
job_id: str,
attachment_id: str,
*,
attempt: int = 1,
) -> bytes:
"""Сообщение задания по `contracts/jobs.schema.json`."""
return json.dumps(
{
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": attachment_id,
"pipeline_version": 1,
"attempt": attempt,
"created_at": "2026-10-31T12:00:00.000000Z",
},
).encode()
def wait_job(
stand: Stand,
attachment_id: str,
*,
state: str,
attempt: int,
timeout: float = 20,
) -> dict[str, Any]:
"""Задание в этом состоянии с этим или большим числом попыток."""
deadline = time.monotonic() + timeout
while True:
job = job_row(stand, attachment_id)
if job and job["state"] == state and job["attempt"] >= attempt:
return job
assert time.monotonic() < deadline, (
f"job of {attachment_id} is not {state} with attempt >= "
f"{attempt}: {job}"
)
time.sleep(0.2)
def assert_variants(
api: Api,
stand: Stand,
actor: Actor,
meta: dict[str, Any],
) -> dict[str, dict[str, Any]]:
"""По одному варианту каждого имени в метаданных и по объекту на каждый, с
его SHA-256; настоящие размеры и тип. Возвращает варианты по имени."""
assert meta["status"] == "ready", meta
variants = {item["name"]: item for item in meta["variants"]}
assert len(meta["variants"]) == len(VARIANT_NAMES), meta["variants"]
assert set(variants) == VARIANT_NAMES, variants
assert stand("variant-object-count", meta["id"]) == str(len(VARIANT_NAMES))
for name, variant in variants.items():
data = assert_variant_content(
api.attachment_content(actor, meta["id"], variant=name),
variant,
)
assert_variant_image(data, variant)
return variants
def assert_not_ready_content(
api: Api,
actor: Actor,
attachment_id: str,
) -> None:
"""Любое скачивание вложения не в `ready` — 409."""
for variant in ALL_VARIANTS:
assert_error(
api.attachment_content(actor, attachment_id, variant=variant),
409,
)
def staging_key(attachment_id: str) -> str:
return f"{FAULT_DATABASE}/staging/{attachment_id}/original"
def wait_staging_gone(stand: Stand, attachment_id: str) -> None:
"""Объект staging и ключ в метаданных убраны: не позже следующего прохода
уборщика с запасом."""
deadline = time.monotonic() + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
while True:
row = json.loads(stand("attachment-row", attachment_id))
if row["staging_key"] is None and attachment_id not in staged_ids(
stand,
):
return
assert time.monotonic() < deadline, (attachment_id, row)
time.sleep(0.3)
@pytest.mark.spec("attachment-processing", "Принятое вложение в очереди")
def test_queued_while_worker_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
time.sleep(3)
meta = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert meta == accepted
api.wait_attachment(alice, accepted["id"])
@pytest.mark.spec("attachment-processing", "Обработка видна в API")
def test_processing_visible(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
filename = fault_filename()
stand("fail-attachments", "ready-stall", filename)
try:
accepted = api.upload(alice, chat["id"], data, filename=filename)
api.wait_attachment(
alice,
accepted["id"],
status="processing",
timeout=JOB_WAIT_S,
)
assert_error(api.attachment_content(alice, accepted["id"]), 409)
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
finally:
stand("clear-fail-attachments")
assert_content(
api.attachment_content(alice, accepted["id"]),
data,
"image/png",
)
@pytest.mark.spec("attachment-processing", "Готовое вложение не откатывается")
@pytest.mark.spec("attachment-processing", "Повторная доставка готового")
def test_redelivery_of_ready_is_noop(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
meta = api.upload_ready(alice, chat["id"], images.sample_png())
job = job_row(stand, meta["id"])
assert job is not None
wait_queue_empty(stand)
seen = len(worker_records(stand))
stand("publish-job", stdin=job_body(job["id"], meta["id"]))
wait_worker_record(
stand,
"Skipped job message",
skip=seen,
job_id=job["id"],
reason="finished",
)
wait_queue_empty(stand)
assert (
assert_ok(api.attachment_meta(alice, meta["id"]), 200, "Attachment")
== meta
)
assert job_row(stand, meta["id"]) == job
assert_variants(api, stand, alice, meta)
@pytest.mark.spec("attachment-processing", "Что сохранено к ответу 202")
def test_durable_state_at_202(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
with relay_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
attachment_id = accepted["id"]
assert attachment_id in staged_ids(stand)
assert attachment_id not in stored_ids(stand)
assert attachment_rows(stand)[attachment_id] == "queued"
job = job_row(stand, attachment_id)
assert job is not None
assert (job["state"], job["attempt"]) == ("queued", 0), job
assert job["published_at"] is None, job
assert queue_state(stand)["messages"] == 0
wait_queue_messages(stand, 1)
published = job_row(stand, attachment_id)
assert published is not None
assert published["published_at"] is not None, published
api.wait_attachment(alice, attachment_id)
@pytest.mark.spec("attachment-processing", "Временная ошибка")
@pytest.mark.spec("attachment-processing", "Повторы под одним заданием")
def test_transient_error_retried(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Сначала S3 остановлен и возвращается за ~8 с: попыток больше одной.
Затем S3 заморожен на первую попытку и размораживается в паузе между
попытками: ровно две, задание то же."""
data = images.sample_png()
with worker_stopped(stand):
stopped = api.upload(alice, chat["id"], data)
first = job_row(stand, stopped["id"])
with outage(stand, "stop", "s3"):
stand("start", "worker")
wait_job(stand, stopped["id"], state="queued", attempt=1)
meta = api.wait_attachment(alice, stopped["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
job = job_row(stand, stopped["id"])
assert job is not None and first is not None
assert job["id"] == first["id"], (job, first)
assert job["attempt"] > 1, job
assert job["state"] == "succeeded", job
with worker_stopped(stand):
paused = api.upload(alice, chat["id"], data)
wait_queue_messages(stand, 1)
first = job_row(stand, paused["id"])
with relay_stopped(stand), outage(stand, "pause", "s3"):
stand("start", "worker")
# срок вызова S3 5 с: первая попытка заканчивается отказом S3.
# Срок следующей отодвинут, пока S3 размораживается: случайная
# пауза могла бы её начать раньше, relay стоит до этого
wait_job(stand, paused["id"], state="queued", attempt=1)
stand("set-next-attempt", paused["id"], "6")
meta = api.wait_attachment(alice, paused["id"], timeout=JOB_WAIT_S)
job = job_row(stand, paused["id"])
assert job is not None and first is not None
assert job["id"] == first["id"], (job, first)
assert job["attempt"] == 2, job
assert_variants(api, stand, alice, meta)
@pytest.mark.spec("attachment-processing", "Сообщение по схеме")
def test_job_message_matches_schema(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
schema = json.loads(JOBS_SCHEMA_PATH.read_text(encoding="utf-8"))
filename = fault_filename()
with worker_stopped(stand):
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
wait_queue_messages(stand, 1)
peeked = json.loads(stand("peek-job"))
text = peeked["body"]
body = json.loads(text)
errors = list(
Draft202012Validator(
schema,
format_checker=FormatChecker(),
).iter_errors(body),
)
assert not errors, [error.message for error in errors]
datetime.fromisoformat(body["created_at"])
job = job_row(stand, accepted["id"])
assert job is not None
assert body["attachment_id"] == accepted["id"]
assert body["job_id"] == job["id"]
assert body["attempt"] == 1
# ни имени файла, ни ключей, ни адресов, ни учётных данных
lowered = text.lower()
for word in (
filename.lower(),
os.environ.get("FAULT_S3_BUCKET", "attachments").lower(),
f"{FAULT_DATABASE}/",
"staging",
"original",
"http",
"password",
"secret",
):
assert word not in lowered, f"{word!r} in the job message"
api.wait_attachment(alice, accepted["id"])
@pytest.mark.spec("message-broker", "Очередь и сообщения устойчивы")
def test_queue_durable(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
state = wait_queue_messages(stand, 1)
assert state["durable"] is True, state
peeked = json.loads(stand("peek-job"))
assert peeked["delivery_mode"] == 2, peeked
# прочитанное без подтверждения вернулось в очередь
state = queue_state(stand)
assert (state["messages"], state["messages_ready"]) == (1, 1), state
api.wait_attachment(alice, accepted["id"])
@pytest.mark.spec("attachment-processing", "Сообщение не по схеме")
def test_malformed_message_dropped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
marker = uuid.uuid4().hex
wait_queue_empty(stand)
seen = len(worker_records(stand))
stand(
"publish-job",
stdin=json.dumps({"schema_version": 1, "note": marker}).encode(),
)
record = wait_worker_record(
stand,
"Dropped malformed job message",
skip=seen,
)
assert record["level"] == "error", record
# содержимое сообщения в журнал не попало
assert marker not in stand("worker-log")
wait_queue_empty(stand)
api.upload_ready(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachment-processing", "Неизвестное задание")
def test_unknown_job_acked(stand: Stand) -> None:
job_id, attachment_id = str(uuid.uuid4()), str(uuid.uuid4())
wait_queue_empty(stand)
before = objects_and_rows(stand)
seen = len(worker_records(stand))
stand("publish-job", stdin=job_body(job_id, attachment_id))
wait_worker_record(
stand,
"Skipped job message",
skip=seen,
job_id=job_id,
reason="unknown",
)
wait_queue_empty(stand)
assert not new_ids(before[0], attachment_rows(stand))
assert not new_objects(before, stand)
@pytest.mark.spec("attachment-processing", "Worker убит посреди обработки")
def test_worker_killed_during_processing(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
wait_queue_empty(stand)
stand("fail-attachments", "ready-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
api.wait_attachment(
alice,
accepted["id"],
status="processing",
timeout=JOB_WAIT_S,
)
stand("kill", "worker")
finally:
stand("clear-fail-attachments")
stand("start", "worker")
# сообщение вернулось, но аренда жива: новую попытку начнёт её истечение
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 2, job
wait_queue_empty(stand)
@pytest.mark.spec("attachment-processing", "Дубль во время обработки")
def test_duplicate_during_processing(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Две ветки дубля. S3 заморожен короче срока вызова, попытка ждёт его без
блокировок БД: дубли видят живую аренду («busy») и подтверждаются, а после
разморозки та же попытка доходит до `ready`. Фиксация ждёт в триггере и
держит строку задания: дубль дожидается её и видит итог («finished»)."""
with worker_stopped(stand):
live = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
job = job_row(stand, live["id"])
assert job is not None
stand("publish-job", stdin=job_body(job["id"], live["id"]))
stand("publish-job", stdin=job_body(job["id"], live["id"]))
wait_queue_messages(stand, 3)
seen = len(worker_records(stand))
with outage(stand, "pause", "s3"):
stand("start", "worker")
deadline = time.monotonic() + S3_TIMEOUT_S
while True:
busy = [
item
for item in worker_records(stand)[seen:]
if item.get("event") == "Skipped job message"
and item.get("job_id") == job["id"]
and item.get("reason") == "busy"
]
if len(busy) == 2:
break
assert time.monotonic() < deadline, busy
time.sleep(0.1)
meta = api.wait_attachment(alice, live["id"], timeout=JOB_WAIT_S)
wait_queue_empty(stand)
assert_variants(api, stand, alice, meta)
done = job_row(stand, live["id"])
assert done is not None
assert done["attempt"] == 1, done
filename = fault_filename()
seen = len(worker_records(stand))
stand("fail-attachments", "ready-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
api.wait_attachment(
alice,
accepted["id"],
status="processing",
timeout=JOB_WAIT_S,
)
job = job_row(stand, accepted["id"])
assert job is not None
stand("publish-job", stdin=job_body(job["id"], accepted["id"]))
wait_worker_record(
stand,
"Skipped job message",
skip=seen,
job_id=job["id"],
reason="finished",
)
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
finally:
stand("clear-fail-attachments")
wait_queue_empty(stand)
assert_variants(api, stand, alice, meta)
done = job_row(stand, accepted["id"])
assert done is not None
assert done["attempt"] == 1, done
@pytest.mark.spec("attachment-processing", "Оригинал неизменен")
def test_original_published_unchanged(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.png_noise(64, 48, seed=23)
meta = api.upload_ready(alice, chat["id"], data)
assert_content(
api.attachment_content(alice, meta["id"]),
data,
"image/png",
)
assert meta["sha256"] == hashlib.sha256(data).hexdigest()
assert meta["id"] in stored_ids(stand)
wait_staging_gone(stand, meta["id"])
@pytest.mark.spec("attachment-processing", "Испорченный staging")
def test_staging_corrupted(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Те же размер и другие байты — и другой размер: обе ветки сверки."""
data = images.png_noise(64, 48, seed=29)
with worker_stopped(stand):
same_size = api.upload(alice, chat["id"], data)
other_size = api.upload(alice, chat["id"], data)
put_object(
stand,
staging_key(same_size["id"]),
same_size_bytes(data, seed=29),
)
put_object(stand, staging_key(other_size["id"]), data + b"\0")
for accepted in (same_size, other_size):
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "processing_failed", meta
assert meta["variants"] == [], meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 1, job
assert job["last_error_code"] == "source_corrupted", job
assert accepted["id"] not in stored_ids(stand)
@pytest.mark.spec("attachment-processing", "Нет объекта staging")
def test_staging_missing(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
stand("delete-object", staging_key(accepted["id"]))
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "processing_failed", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 1, job
assert job["last_error_code"] == "source_missing", job
@pytest.mark.spec("attachment-processing", "Изображение в лимите пикселей")
def test_max_pixels_fit_worker_memory(
base_url: str,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Столько PNG RGBA 8000 × 5000, сколько заданий worker обрабатывает
одновременно: они берутся разом, и контейнер не завершён из-за памяти."""
data = images.png_rgba_large()
before = json.loads(stand("worker-state"))
# срок клиента: тело на 40 млн пикселей проверяется дольше 5 с по умолчанию
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
with worker_stopped(stand):
accepted = [
slow.upload(alice, chat["id"], data)
for _ in range(WORKER_CONCURRENCY)
]
sizes = {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (1280, 800),
}
for item in accepted:
meta = slow.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
variants = assert_variants(slow, stand, alice, meta)
for name, size in sizes.items():
variant = variants[name]
assert (variant["width"], variant["height"]) == size, variant
state = json.loads(stand("worker-state"))
assert state["running"] is True, state
assert state["oom_killed"] is False, state
assert state["restart_count"] == before["restart_count"], state
@pytest.mark.spec("attachment-processing", "Повтор после записи объектов")
@pytest.mark.spec("attachments", "Фиксация ready не удалась")
def test_ready_failure_retried(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Объекты записаны, фиксация падает отказом БД (08006): сообщение уходит в
очередь; пока триггер стоит, вложение не готово. Аренда жива: следующую
попытку начинает её истечение, а триггер снимается раньше."""
filename = fault_filename()
wait_queue_empty(stand)
seen = len(worker_records(stand))
stand("fail-attachments", "ready-fail", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
job = job_row(stand, accepted["id"])
assert job is not None
wait_worker_record(
stand,
"Job waits for database",
skip=seen,
job_id=job["id"],
)
assert stand("variant-object-count", accepted["id"]) == str(
len(VARIANT_NAMES),
)
assert accepted["id"] in stored_ids(stand)
meta = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert meta["status"] in ("queued", "processing"), meta
assert_not_ready_content(api, alice, accepted["id"])
finally:
stand("clear-fail-attachments")
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
done = job_row(stand, accepted["id"])
assert done is not None
assert done["id"] == job["id"], done
assert done["attempt"] >= 2, done
@pytest.mark.spec("attachment-processing", "Состояние между попытками")
def test_queued_between_attempts(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
with outage(stand, "stop", "s3"):
stand("start", "worker")
job = wait_job(stand, accepted["id"], state="queued", attempt=1)
assert job["last_error_code"] == "storage_unavailable", job
assert job["next_attempt_at"] is not None, job
meta = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert meta["status"] == "queued", meta
assert meta["error_code"] is None, meta
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("attachment-processing", "Попытки исчерпаны")
def test_attempts_exhausted(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
with outage(stand, "stop", "s3"):
stand("start", "worker")
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "retry_exhausted", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == WORKER_MAX_ATTEMPTS, job
assert job["state"] == "failed", job
wait_queue_empty(stand)
# S3 вернулся: итог окончателен
time.sleep(WORKER_RETRY_BASE_S)
after = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert after == meta
assert job_row(stand, accepted["id"]) == job
@pytest.mark.spec(
"attachment-processing",
"PostgreSQL недоступен при обработке",
)
def test_database_outage_during_processing(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = json.loads(stand("worker-state"))
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
job = job_row(stand, accepted["id"])
assert job is not None
seen = len(worker_records(stand))
with outage(stand, "stop", "postgres"):
stand("start", "worker")
# журнал называет отказ, задание ждёт в очереди
wait_worker_record(stand, "Job waits for database", skip=seen)
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
wait_queue_empty(stand)
done = job_row(stand, accepted["id"])
assert done is not None
# попытки на отказе БД не считаются
assert done["attempt"] == 1, done
state = json.loads(stand("worker-state"))
assert state["running"] is True, state
assert state["restart_count"] == before["restart_count"], state
@pytest.mark.spec("attachment-processing", "Изображение не декодируется")
def test_undecodable_becomes_failed(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
accepted = api.upload(
alice,
chat["id"],
images.jpeg_precision_12(),
content_type="image/jpeg",
filename="image.jpg",
)
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "invalid_image", meta
assert meta["variants"] == [], meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 1, job
assert job["last_error_code"] == "undecodable", job
assert_error(api.attachment_content(alice, accepted["id"]), 409)
@pytest.mark.spec("attachment-processing", "Worker недоступен")
def test_worker_down_api_serves(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
assert accepted["status"] == "queued"
message = api.send_message(alice, chat["id"], "текст без worker")
history = api.all_messages(bob, chat["id"])
assert [item["id"] for item in history] == [message["id"]]
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
api.wait_attachment(alice, accepted["id"])
@pytest.mark.spec("attachment-processing", "Ограниченная конкуренция")
def test_concurrency_bounded(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Фиксация каждого задания ждёт 4 с в триггере: заданий больше, чем
мест, и видно, сколько worker держит разом."""
filename = fault_filename()
total = WORKER_CONCURRENCY + 3
stand("fail-attachments", "ready-stall", filename)
try:
with worker_stopped(stand):
ids = [
api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)["id"]
for _ in range(total)
]
api.wait_attachment(
alice,
ids[0],
status="processing",
timeout=JOB_WAIT_S,
)
most = 0
for _ in range(4):
rows = attachment_rows(stand)
statuses = [rows[item] for item in ids]
busy = statuses.count("processing")
most = max(most, busy)
assert busy <= WORKER_CONCURRENCY, statuses
assert (
statuses.count("queued")
+ busy
+ statuses.count(
"ready",
)
== total
), statuses
state = queue_state(stand)
assert state["messages_unacknowledged"] <= WORKER_CONCURRENCY, (
state
)
time.sleep(0.3)
assert most == WORKER_CONCURRENCY, most
for item in ids:
api.wait_attachment(alice, item, timeout=JOB_WAIT_S)
finally:
stand("clear-fail-attachments")
wait_queue_empty(stand)
@pytest.mark.spec("attachment-processing", "Реплика API убита после ответа")
def test_api_killed_after_202(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
accepted = api.upload(alice, chat["id"], images.sample_png())
stand("kill-api")
try:
# реплики нет, а работа идёт: итог — от worker
wait_job(
stand,
accepted["id"],
state="succeeded",
attempt=1,
timeout=JOB_WAIT_S,
)
finally:
stand("start-api")
meta = api.wait_attachment(alice, accepted["id"])
assert_variants(api, stand, alice, meta)
@pytest.mark.spec("attachment-processing", "Рестарт брокера с volume")
def test_broker_restart_keeps_job(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
with outage(stand, "stop", "rabbitmq"):
pass
assert queue_state(stand)["messages"] == 1
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("attachment-processing", "Принудительное завершение брокера")
@pytest.mark.spec("message-broker", "Подтверждённые сообщения после KILL")
def test_broker_kill_keeps_confirmed(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
confirmed = 5
with worker_stopped(stand):
accepted = [
api.upload(alice, chat["id"], images.sample_png())
for _ in range(confirmed)
]
wait_queue_messages(stand, confirmed)
with outage(stand, "kill", "rabbitmq"):
pass
assert queue_state(stand)["messages"] == confirmed
for item in accepted:
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("attachment-processing", "Сценарий защиты")
def test_defense_scenario(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], data)
attachment_id = accepted["id"]
message = api.send_message(
alice,
chat["id"],
"",
attachment_ids=[attachment_id],
)
assert_error(api.attachment_content(alice, attachment_id), 409)
text = api.send_message(alice, chat["id"], "текст при очереди")
wait_queue_messages(stand, 1)
with outage(stand, "stop", "rabbitmq"):
stand("kill-api")
stand("start-api")
meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S)
assert meta["id"] == attachment_id
assert_content(
api.attachment_content(alice, attachment_id),
data,
"image/png",
)
assert_variants(api, stand, alice, meta)
history = {
item["id"]: item for item in api.all_messages(alice, chat["id"])
}
assert history[message["id"]]["attachment_ids"] == [attachment_id]
assert text["id"] in history
@pytest.mark.spec("message-broker", "Брокер остановлен при загрузке")
@pytest.mark.spec("message-broker", "Брокер вернулся")
@pytest.mark.spec("attachments", "Брокер не подтвердил задание")
@pytest.mark.spec("job-outbox", "Загрузка при остановленном брокере")
def test_upload_with_broker_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
ready = api.upload_ready(alice, chat["id"], data)
with outage(stand, "stop", "rabbitmq"):
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
accepted = assert_accepted(resp)
assert elapsed < ACCEPT_MAX_S, elapsed
meta = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert meta["status"] == "queued", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["published_at"] is None, job
assert accepted["id"] in staged_ids(stand)
# обязательный путь без брокера работает
api.send_message(alice, chat["id"], "текст без брокера")
assert (
assert_ok(
api.attachment_meta(alice, ready["id"]),
200,
"Attachment",
)
== ready
)
assert_content(
api.attachment_content(alice, ready["id"]),
data,
"image/png",
)
# relay и worker не перезапускались: принятое без брокера доходит до ready,
# и следующая загрузка тоже
done = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, done)
api.upload_ready(alice, chat["id"], data)
@pytest.mark.spec("message-broker", "Брокер завис")
@pytest.mark.spec("job-outbox", "Загрузка при замороженном брокере")
def test_upload_with_broker_paused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Соединение издателя живо: публикация relay уходит в заморозку и кончается
сроком, исход неизвестен. Приём этого не видит, а дубль после разморозки
worker подтверждает без эффекта."""
data = images.sample_png()
api.upload_ready(alice, chat["id"], data)
wait_queue_empty(stand)
with outage(stand, "pause", "rabbitmq"):
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
accepted = assert_accepted(resp)
assert elapsed < ACCEPT_MAX_S, elapsed
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
wait_queue_empty(stand)
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 1, job
@pytest.mark.spec("message-broker", "Очередь удалена")
def test_queue_deleted(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
# соединение издателя живо: его канал очередь больше не найдёт, а новое
# соединение объявит её заново
api.upload_ready(alice, chat["id"], data)
with worker_stopped(stand):
stand("delete-queue")
assert json.loads(stand("queue-state")) is None
accepted = [api.upload(alice, chat["id"], data) for _ in range(2)]
state = wait_queue_messages(stand, 2)
assert state["durable"] is True, state
for item in accepted:
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("message-broker", "Пересоздание контейнера")
def test_broker_recreated(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
confirmed = 2
with worker_stopped(stand):
accepted = [
api.upload(alice, chat["id"], images.sample_png())
for _ in range(confirmed)
]
wait_queue_messages(stand, confirmed)
stand("recreate", "rabbitmq")
state = queue_state(stand)
assert (state["durable"], state["messages"]) == (True, confirmed), (
state
)
for item in accepted:
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("message-broker", "Старт без брокера")
def test_start_without_broker(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with (
worker_stopped(stand),
relay_stopped(stand),
outage(stand, "stop", "rabbitmq"),
):
seen = len(worker_records(stand))
seen_relay = len(relay_records(stand))
stand("kill-api")
stand("start-api")
stand("start", "worker")
stand("start", "relay")
# реплика готова и принимает файл
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
api.send_message(alice, chat["id"], "текст при старте без брокера")
accepted = api.upload(alice, chat["id"], images.sample_png())
# worker и relay называют отказ и не завершаются
wait_worker_record(stand, "Broker is unavailable", skip=seen)
wait_relay_record(stand, "Broker is unavailable", skip=seen_relay)
state = json.loads(stand("worker-state"))
assert state["running"] is True, state
state = json.loads(stand("relay-state"))
assert state["running"] is True, state
# брокер запущен, ни реплика, ни worker не перезапускались
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("message-broker", "Рестарт брокера при работающем worker")
def test_worker_survives_broker_restart(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = json.loads(stand("worker-state"))
assert before["running"] is True, before
with outage(stand, "stop", "rabbitmq"):
pass
api.upload_ready(alice, chat["id"], images.sample_png())
state = json.loads(stand("worker-state"))
assert state["running"] is True, state
assert state["restart_count"] == before["restart_count"], state
@pytest.mark.spec("attachments", "Скачивание до готовности")
def test_content_before_ready(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Каждое скачивание вложения в `queued`, `processing` и `failed` — 409."""
filename = fault_filename()
stand("fail-attachments", "ready-stall", filename)
try:
with worker_stopped(stand):
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
assert_not_ready_content(api, alice, accepted["id"])
api.wait_attachment(
alice,
accepted["id"],
status="processing",
timeout=JOB_WAIT_S,
)
assert_not_ready_content(api, alice, accepted["id"])
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
finally:
stand("clear-fail-attachments")
failed = api.upload(
alice,
chat["id"],
images.jpeg_precision_12(),
content_type="image/jpeg",
filename="image.jpg",
)
api.wait_attachment(
alice,
failed["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert_not_ready_content(api, alice, failed["id"])
@pytest.mark.spec("messages", "Вложение в обработке")
def test_message_with_queued_attachment(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
message = api.send_message(
alice,
chat["id"],
"с вложением в очереди",
attachment_ids=[accepted["id"]],
)
meta = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert meta["status"] == "queued", meta
assert api.all_messages(bob, chat["id"]) == [message]
api.wait_attachment(alice, accepted["id"])
@pytest.mark.spec("messages", "Вложение стало failed после отправки")
def test_attachment_failed_after_message(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(
alice,
chat["id"],
images.jpeg_precision_12(),
content_type="image/jpeg",
filename="image.jpg",
)
message = api.send_message(
alice,
chat["id"],
"",
attachment_ids=[accepted["id"]],
)
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "invalid_image", meta
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Повтор после failed вложения")
def test_replay_after_attachment_failed(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
key = new_key()
with worker_stopped(stand):
accepted = api.upload(
alice,
chat["id"],
images.jpeg_precision_12(),
content_type="image/jpeg",
filename="image.jpg",
)
body = {"text": "повтор", "attachment_ids": [accepted["id"]]}
message = assert_ok(
api.post_message(alice, chat["id"], key=key, json=body),
201,
"Message",
)
api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
replay = assert_ok(
api.post_message(alice, chat["id"], key=key, json=body),
201,
"Message",
)
assert replay == message
assert_error(api.post_message(alice, chat["id"], json=body), 409)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("health", "Брокер и worker недоступны")
@pytest.mark.parametrize("action", ["stop", "pause"])
def test_readiness_without_broker_and_worker(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
action: str,
) -> None:
with (
worker_stopped(stand),
relay_stopped(stand),
outage(stand, action, "rabbitmq"),
):
for path in ("/health/ready", "/health/live"):
resp, took = timed(lambda path=path: api.request("GET", path))
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
assert took < BROKER_HEALTH_S, f"{path} took {took:.2f}s"
# загрузка от них не зависит
accepted = api.upload(alice, chat["id"], images.sample_png())
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
def test_worker_stop_during_processing(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Спек `graceful-shutdown`, «docker stop во время обработки» (вне сверки
покрытия — сценарий в докстринге): фиксация ждёт 4 с, а срок остановки
worker стенда 2 с. `stop` быстрее 10 с, код 0, сообщение снова в очереди,
а после запуска worker вложение становится `ready`."""
filename = fault_filename()
wait_queue_empty(stand)
stand("fail-attachments", "ready-stall", filename)
stopped = False
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
api.wait_attachment(
alice,
accepted["id"],
status="processing",
timeout=JOB_WAIT_S,
)
seen = len(worker_records(stand))
started = time.monotonic()
stand("stop", "worker")
stopped = True
assert time.monotonic() - started < WORKER_STOP_MAX_S
state = json.loads(stand("worker-state"))
assert (state["running"], state["exit_code"]) == (False, 0), state
assert queue_state(stand)["messages"] == 1
records = worker_records(stand)[seen:]
assert any(item.get("event") == "Stopped worker" for item in records)
finally:
stand("clear-fail-attachments")
if not stopped:
stand("stop", "worker")
stand("start", "worker")
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
wait_queue_empty(stand)
def test_worker_stop_during_render(
base_url: str,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Спек `graceful-shutdown`, «docker stop во время декодирования» (вне сверки
покрытия — сценарий в докстринге): дорогое изображение декодируется дольше
срока остановки worker стенда (2 с). `stop` быстрее 10 с, код 0, процессов
обработки нет, а задание получает следующую попытку по аренде."""
wait_queue_empty(stand)
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
accepted = slow.upload(
alice,
chat["id"],
images.jpeg_scan_bomb(),
content_type="image/jpeg",
filename="bomb.jpg",
)
wait_render_process(stand)
started = time.monotonic()
stand("stop", "worker")
try:
assert time.monotonic() - started < WORKER_STOP_MAX_S
state = json.loads(stand("worker-state"))
assert (state["running"], state["exit_code"]) == (False, 0), state
finally:
stand("start", "worker")
# контейнер снова запущен: процессов обработки прежней попытки нет,
# попытка 2 дойдёт до лимита CPU
meta = slow.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 2, job
assert meta["error_code"] == "invalid_image", meta
assert job["last_error_code"] == "limit_cpu", job
def wait_render_process(stand: Stand, *args: str, timeout: float = 15) -> None:
"""В контейнере worker идёт процесс обработки."""
deadline = time.monotonic() + timeout
while stand("render-count", *args) == "0":
assert time.monotonic() < deadline, "no render process in time"
time.sleep(0.05)
def wait_variant_objects(stand: Stand, attachment_id: str) -> None:
"""Объекты всех вариантов записаны: пока фиксация ждёт в триггере."""
deadline = time.monotonic() + STALL_S
while stand("variant-object-count", attachment_id) != str(
len(VARIANT_NAMES),
):
assert time.monotonic() < deadline, "no variant objects in time"
time.sleep(0.1)
@contextmanager
def current_work(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> Iterator[dict[str, dict[str, Any]]]:
"""Worker убит посреди задания и лежит, relay остановлен: иначе он вернул бы
P по истечении аренды. Вложения F в `failed` (конечный объект у него есть) —
старше grace, но моложе срока хранения staging, P в `processing` (объекты
записаны, фиксация не дошла) и Q в `queued` — старше grace и срока хранения
по строке. Выход — worker и relay запущены, P и Q становятся `ready`."""
wait_queue_empty(stand)
failed = api.upload(
alice,
chat["id"],
images.jpeg_precision_12(),
content_type="image/jpeg",
filename="image.jpg",
)
api.wait_attachment(
alice,
failed["id"],
status="failed",
timeout=JOB_WAIT_S,
)
filename = fault_filename()
stand("fail-attachments", "ready-stall", filename)
work: dict[str, dict[str, Any]] = {"failed": failed}
try:
processing = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
api.wait_attachment(
alice,
processing["id"],
status="processing",
timeout=JOB_WAIT_S,
)
work["processing"] = processing
# конечный объект записан до фиксации; ждём и варианты
wait_variant_objects(stand, processing["id"])
stand("kill", "worker")
stand("stop", "relay")
stand("clear-fail-attachments")
work["queued"] = api.upload(alice, chat["id"], images.sample_png())
for name, item in work.items():
age = FAILED_AGED_S if name == "failed" else AGED_S
stand("age-row", item["id"], str(age))
yield work
finally:
stand("clear-fail-attachments")
stand("start", "relay")
stand("start", "worker")
for name in ("processing", "queued"):
api.wait_attachment(alice, work[name]["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("storage-consistency", "Текущая работа не тронута")
def test_cleaner_keeps_current_work(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with current_work(api, stand, alice, chat) as work:
# проходов уборщика за это время — несколько
time.sleep(3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S)
rows = attachment_rows(stand)
staged = staged_ids(stand)
for name, item in work.items():
assert rows.get(item["id"]) == name, (name, rows.get(item["id"]))
assert item["id"] in staged, name
# ---------------------------------------------------------------------------
# Сверка bucket и БД (change track2-reconciler, design, решение 11). Hook
# вносит расхождения с обеих сторон и запускает разовую сверку; плановая
# сверка уборщика стенда выключена. Записи сверки фильтруются по своим id и
# ключам: остатки прежних тестов (строки pending) могут попасть в отчёт, и
# код выхода сверяется с записями того же прохода, а не с числом. Объекты
# состарить нельзя (LastModified ставит S3): тесты объектов без строки ждут
# G + 1 с, строки состаривает hook (age-row)
# ---------------------------------------------------------------------------
# на сколько hook состаривает строку: заведомо больше grace
AGED_S = CLEANER_GRACE_S + 60
# строка failed: старше grace, а срок хранения staging (60 с) тест не перейдёт
FAILED_AGED_S = CLEANER_GRACE_S + 5
# объекты без строки старше grace: grace и секунда точности LastModified
OBJECT_AGE_WAIT_S = CLEANER_GRACE_S + 1
# grace разового прохода, когда он сам дольше grace стенда; меньше AGED_S
FRESH_GRACE_S = int(AGED_S) - 20
FOUND = "Found storage discrepancy"
FIXED = "Fixed storage discrepancy"
FINISHED = "Finished reconcile pass"
LIMIT_EXCEEDED = "Reconcile changes exceed limit"
# сверка под нагрузкой: потоки загрузок, период сверки, пауза потока
LOAD_THREADS = 4
LOAD_EXTRA_S = 15
LOAD_RECONCILE_PERIOD_S = 3
LOAD_PAUSE_S = 0.1
# плановая сверка отдельного уборщика: столько секунд он работает
SCHEDULED_RUN_S = 60
@dataclass(frozen=True)
class Reconcile:
"""Один разовый проход сверки: записи журнала процесса и код выхода."""
records: list[dict[str, Any]]
exit_code: int
def events(self, event: str) -> list[dict[str, Any]]:
return [item for item in self.records if item.get("event") == event]
@property
def finished(self) -> dict[str, Any]:
(record,) = self.events(FINISHED)
return record
def about(
self,
event: str,
*,
ids: Collection[str] = (),
keys: Collection[str] = (),
) -> list[dict[str, Any]]:
"""Записи `event` о своих `id` вложений и ключах объектов."""
return [
item
for item in self.events(event)
if item.get("attachment_id") in ids
or item.get("object_key") in keys
]
def kinds(
self,
event: str,
*,
ids: Collection[str] = (),
keys: Collection[str] = (),
) -> dict[str, str]:
"""`id` или ключ → класс расхождения в записях `event`."""
found: dict[str, str] = {}
for item in self.about(event, ids=ids, keys=keys):
name = (
item["attachment_id"]
if item.get("attachment_id") in ids
else item["object_key"]
)
found[name] = item["kind"]
return found
def assert_code_matches(self) -> None:
"""Код выхода соответствует записям прохода: 3 — остались
расхождения (запись `Found`) или отказ по пределу, иначе 0."""
finished = self.finished
assert finished["interrupted_by"] is None, finished
expected = 3 if self.events(FOUND) or finished["refused"] else 0
assert self.exit_code == expected, (self.exit_code, finished)
def run_reconcile(
stand: Stand,
mode: str,
limit: int | None = None,
*,
options: Sequence[str] = (),
) -> Reconcile:
"""Разовая сверка hook: `dry-run` или `apply`, с пределом изменений;
`options` — стратегия, глубина и предел скорости (change
`track2-reconcile-cost`) как аргументы команды."""
args = [mode] if limit is None else [mode, str(limit)]
lines = stand("reconcile", *args, *options).splitlines()
assert lines and lines[-1].startswith("exit_code="), lines[-5:]
records = []
for line in lines[:-1]:
if line.startswith("{"):
records.append(json.loads(line))
return Reconcile(records=records, exit_code=int(lines[-1].split("=")[1]))
def object_key(attachment_id: str) -> str:
return f"{FAULT_DATABASE}/attachments/{attachment_id}/original"
def variant_key(attachment_id: str, name: str = "thumbnail") -> str:
return f"{FAULT_DATABASE}/attachments/{attachment_id}/v1-{name}"
def lost_ids(stand: Stand) -> set[str]:
return set(stand("lost-ids").splitlines())
def put_object(stand: Stand, key: str, data: bytes) -> None:
stand("put-object", key, stdin=data)
def lose_original(stand: Stand, attachment: dict[str, Any]) -> None:
"""Готовое вложение старше grace без объекта: строка состарена, объект
удалён мимо API."""
stand("age-row", attachment["id"], str(AGED_S))
stand("delete-object", object_key(attachment["id"]))
def assert_lost_content(
api: Api,
actor: Actor,
attachment: dict[str, Any],
*,
aged: bool = False,
) -> None:
"""Помеченное вложение: байты — 404, метаданные — как при загрузке.
`aged` — строку состарил hook (`age-row`): её `created_at` и
`updated_at` сдвинуты им, а не сверкой, и сравниваются остальные поля.
"""
assert_error(api.attachment_content(actor, attachment["id"]), 404)
meta = assert_ok(
api.attachment_meta(actor, attachment["id"]),
200,
"Attachment",
)
if aged:
times = ("created_at", "updated_at")
meta = {key: value for key, value in meta.items() if key not in times}
attachment = {
key: value for key, value in attachment.items() if key not in times
}
assert meta == attachment
@pytest.mark.spec("storage-consistency", "Расхождения с обеих сторон найдены")
@pytest.mark.spec("storage-consistency", "Исправление расхождений")
@pytest.mark.spec("storage-consistency", "Повторная сверка")
@pytest.mark.spec("attachments", "Потерянный оригинал")
def test_reconcile_finds_and_fixes(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
with outage(stand, "stop", "cleaner"):
a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3))
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
orphan = str(uuid.uuid4())
keys = {
"o": object_key(orphan),
"b": object_key(b["id"]),
"bv": variant_key(b["id"]),
}
put_object(stand, keys["o"], data)
stand("delete-object", object_key(a["id"]))
stand("delete-row", b["id"])
stand("set-size", c["id"], str(len(data) + 1))
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
resp = upload_raw(api, alice, chat, data, filename=filename)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
assert pending in staged_ids(stand)
# последний объект записан только что: после G + 1 всё старше grace
time.sleep(OBJECT_AGE_WAIT_S)
ids = {a["id"], c["id"], pending}
snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, ids=ids, keys=keys.values()) == {
keys["o"]: "orphan_object",
keys["b"]: "orphan_object",
keys["bv"]: "orphan_object",
a["id"]: "missing_object",
c["id"]: "size_mismatch",
pending: "stale_pending",
}
for record in dry.about(FOUND, ids=ids, keys=keys.values()):
assert record["age_seconds"] >= CLEANER_GRACE_S, record
assert dry.exit_code == 3
# dry-run ничего не меняет: ни строк, ни объектов, ни пометок
after = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
assert after == snapshot
assert_error(api.attachment_content(alice, a["id"]), 500)
assert_ok(api.attachment_meta(alice, a["id"]), 200, "Attachment")
fixed = run_reconcile(stand, "apply")
actions = {
record.get("attachment_id")
if record.get("attachment_id") in ids
else record["object_key"]: record["action"]
for record in fixed.about(FIXED, ids=ids, keys=keys.values())
}
assert actions == {
keys["o"]: "deleted_object",
keys["b"]: "deleted_object",
keys["bv"]: "deleted_object",
a["id"]: "marked_lost",
c["id"]: "marked_lost",
pending: "removed_pending",
}
assert not fixed.about(FOUND, ids=ids, keys=keys.values())
fixed.assert_code_matches()
rows, stored = attachment_rows(stand), stored_ids(stand)
assert not {orphan, b["id"], pending} & stored
assert stand("variant-object-count", b["id"]) == "0"
assert pending not in staged_ids(stand)
assert pending not in rows
assert {a["id"], c["id"]} <= lost_ids(stand)
assert rows[a["id"]] == rows[c["id"]] == "ready"
assert_lost_content(api, alice, a)
# у C размер в метаданных изменил hook (set-size), а не сверка
assert_lost_content(api, alice, c | {"size_bytes": len(data) + 1})
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids=ids, keys=keys.values())
assert not again.about(FIXED, ids=ids, keys=keys.values())
again.assert_code_matches()
@pytest.mark.spec("storage-consistency", "Свежие не тронуты")
@pytest.mark.spec("storage-consistency", "Используемое вложение не удаляется")
@pytest.mark.spec("storage-consistency", "Чужой объект не удаляется")
def test_reconcile_keeps_fresh_used_and_unknown(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
used = api.upload_ready(alice, chat["id"], data)
message = api.send_message(
alice,
chat["id"],
"",
attachment_ids=[used["id"]],
)
lose_original(stand, used)
# ключ, которого API не пишет: под префиксом вложений, но не <id>/original
unknown = f"{FAULT_DATABASE}/attachments/x-{uuid.uuid4()}"
put_object(stand, unknown, data)
time.sleep(OBJECT_AGE_WAIT_S)
# свежие: готовое вложение без объекта и объект без строки
fresh = api.upload_ready(alice, chat["id"], data)
stand("delete-object", object_key(fresh["id"]))
fresh_object = str(uuid.uuid4())
put_object(stand, object_key(fresh_object), data)
ready_before = ready_ids(attachment_rows(stand))
try:
run = run_reconcile(stand, "apply")
keys = {unknown, object_key(fresh_object)}
ids = {used["id"], fresh["id"], fresh_object}
assert run.kinds(FOUND, ids=ids, keys=keys) == {
unknown: "unknown_object",
}
assert run.kinds(FIXED, ids=ids, keys=keys) == {
used["id"]: "missing_object",
}
assert run.exit_code == 3
assert run.finished["fresh_objects"] >= 1, run.finished
assert run.finished["fresh_rows"] >= 1, run.finished
# свежий объект на месте, свежее вложение не помечено
assert fresh_object in stored_ids(stand)
lost = lost_ids(stand)
assert fresh["id"] not in lost
# используемое: помечено, метаданные и ссылка в истории на месте
assert used["id"] in lost
assert_lost_content(api, alice, used, aged=True)
history = api.all_messages(alice, chat["id"])
(sent,) = (item for item in history if item["id"] == message["id"])
assert sent["attachment_ids"] == [used["id"]]
assert ready_before <= ready_ids(attachment_rows(stand))
# чужой объект не тронут: следующий проход снова его видит
again = run_reconcile(stand, "dry-run")
assert again.kinds(FOUND, keys={unknown}) == {
unknown: "unknown_object",
}
finally:
# стенд — без чужого и лишнего объекта, у свежего — его байты
stand("delete-object", unknown)
stand("delete-object", object_key(fresh_object))
put_object(stand, object_key(fresh["id"]), data)
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
def test_reconcile_restored_original(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.png_noise(64, 48, seed=7)
restored = api.upload_ready(alice, chat["id"], data)
lose_original(stand, restored)
marked = run_reconcile(stand, "apply")
assert marked.kinds(FIXED, ids={restored["id"]}) == {
restored["id"]: "missing_object",
}
assert restored["id"] in lost_ids(stand)
assert_lost_content(api, alice, restored, aged=True)
# те же байты снова под ключом вложения
put_object(stand, object_key(restored["id"]), data)
cleared = run_reconcile(stand, "apply")
(record,) = cleared.about(FIXED, ids={restored["id"]})
assert (record["kind"], record["action"]) == (
"restored_object",
"cleared_lost",
)
assert restored["id"] not in lost_ids(stand)
assert_content(
api.attachment_content(alice, restored["id"]),
data,
"image/png",
)
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids={restored["id"]})
@pytest.mark.spec("storage-consistency", "Предел изменений")
def test_reconcile_change_limit(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
limit = 2
batch = [
api.upload_ready(alice, chat["id"], images.sample_png())
for _ in range(3)
]
ids = {item["id"] for item in batch}
for item in batch:
lose_original(stand, item)
refused = run_reconcile(stand, "apply", limit)
assert refused.exit_code == 3
(error,) = refused.events(LIMIT_EXCEEDED)
assert error["level"] == "error"
assert error["changes"] >= len(batch), error
assert error["max_changes"] == limit
assert refused.finished["refused"] is True
assert refused.finished["level"] == "warning"
assert not refused.about(FIXED, ids=ids)
assert not ids & lost_ids(stand)
assert all(
record["kind"] == "missing_object"
for record in refused.about(FOUND, ids=ids)
)
# предел не меньше числа изменений: исправлено всё
applied = run_reconcile(stand, "apply", 100)
assert set(applied.kinds(FIXED, ids=ids)) == ids
assert ids <= lost_ids(stand)
@pytest.mark.spec(
"storage-consistency",
"Объект, на который ссылается строка, не удаляется",
)
def test_reconcile_keeps_referenced_objects(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W1 verify: объект `<X>/original`, у строки которого другой
ключ, и объект `<Z>/original`, на который ссылается строка Y другого
`id`, — чужие, а не объекты без метаданных."""
data = images.sample_png()
x, y = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
z = str(uuid.uuid4())
keys = {
"x": object_key(x["id"]),
"y": object_key(y["id"]),
"z": object_key(z),
# ключ X: под префиксом вложений, объекта по нему нет
"other": f"{FAULT_DATABASE}/attachments/x-other-{uuid.uuid4()}",
}
put_object(stand, keys["z"], data)
try:
stand("set-key", x["id"], keys["other"])
stand("set-key", y["id"], keys["z"])
for item in (x, y):
stand("age-row", item["id"], str(AGED_S))
# объект Z записан только что: после G + 1 все объекты старше grace
time.sleep(OBJECT_AGE_WAIT_S)
unknown = {keys[name]: "unknown_object" for name in ("x", "y", "z")}
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, keys=keys.values()) == unknown | {
keys["other"]: "missing_object",
}
assert dry.exit_code == 3
applied = run_reconcile(stand, "apply")
assert applied.kinds(FIXED, keys=keys.values()) == {
keys["other"]: "missing_object",
}
assert applied.kinds(FOUND, keys=keys.values()) == unknown
assert applied.exit_code == 3
# ни один из трёх объектов не удалён, строки на месте
assert {x["id"], y["id"], z} <= stored_ids(stand)
rows = attachment_rows(stand)
assert rows[x["id"]] == rows[y["id"]] == "ready"
# X без объекта по своему ключу помечено, у Y объект по ключу есть
lost = lost_ids(stand)
assert x["id"] in lost
assert y["id"] not in lost
finally:
# прежние ключи; объект Z, на который больше никто не ссылается, —
# прочь, чтобы следующие проходы не удаляли его как свой
stand("set-key", x["id"], keys["x"])
stand("set-key", y["id"], keys["y"])
stand("delete-object", keys["z"])
@pytest.mark.spec(
"storage-consistency",
"Предел не останавливает уборку зависших",
)
def test_reconcile_limit_keeps_cleanup(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W1 verify: отказ по пределу не мешает уборке зависшей
загрузки — её уборщик делает и без сверки."""
data = images.sample_png()
limit = 2
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
with outage(stand, "stop", "cleaner"):
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
assert_error(
upload_raw(api, alice, chat, data, filename=filename),
503,
)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
assert pending in staged_ids(stand)
stand("age-row", pending, str(AGED_S))
batch = [api.upload_ready(alice, chat["id"], data) for _ in range(3)]
ids = {item["id"] for item in batch}
for item in batch:
lose_original(stand, item)
refused = run_reconcile(stand, "apply", limit)
assert refused.exit_code == 3
(error,) = refused.events(LIMIT_EXCEEDED)
assert error["changes"] >= len(batch), error
assert refused.finished["refused"] is True
assert refused.kinds(FIXED, ids={pending}) == {
pending: "stale_pending",
}
(record,) = refused.about(FIXED, ids={pending})
assert record["action"] == "removed_pending"
assert pending not in attachment_rows(stand)
assert pending not in staged_ids(stand)
# изменений под пределом нет: ни одной пометки
assert not refused.about(FIXED, ids=ids)
assert not ids & lost_ids(stand)
# стенд — без внесённых потерь для следующих тестов
applied = run_reconcile(stand, "apply", 100)
assert set(applied.kinds(FIXED, ids=ids)) == ids
@pytest.mark.spec(
"storage-consistency",
"Ключ вне префикса вложений не трогается",
)
def test_reconcile_foreign_key_untouched(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W2 verify: строка с ключом вне `<база>/attachments/` —
готовая и зависшая загрузка — только в отчёте как чужой объект: сверка
не делает HEAD чужого префикса, не помечает и не убирает строку."""
data = images.sample_png()
# уборщик стенда стоит: иначе зависшую загрузку убрал бы он
with outage(stand, "stop", "cleaner"):
ready = api.upload_ready(alice, chat["id"], data)
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
assert_error(
upload_raw(api, alice, chat, data, filename=filename),
503,
)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
ids = {ready["id"], pending}
try:
for item in ids:
# префикс своей базы, но не вложений: список его не видит
stand("set-key", item, f"{FAULT_DATABASE}/elsewhere/{item}")
stand("age-row", item, str(AGED_S))
unknown = dict.fromkeys(ids, "unknown_object")
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, ids=ids) == unknown
assert dry.exit_code == 3
applied = run_reconcile(stand, "apply")
assert applied.kinds(FOUND, ids=ids) == unknown
assert not applied.about(FIXED, ids=ids)
assert applied.exit_code == 3
# ни пометки, ни уборки: строки и объекты под префиксом на месте
rows = attachment_rows(stand)
assert (rows.get(ready["id"]), rows.get(pending)) == (
"ready",
"pending",
)
assert ready["id"] not in lost_ids(stand)
assert ready["id"] in stored_ids(stand)
assert pending in staged_ids(stand)
finally:
# прежние ключи у тех строк, что остались: зависшую загрузку
# после теста уберёт уборщик стенда вместе с её объектом
rows = attachment_rows(stand)
for item in ids & set(rows):
stand("set-key", item, object_key(item))
@pytest.mark.spec("storage-consistency", "S3 недоступен во время сверки")
def test_reconcile_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
missing = api.upload_ready(alice, chat["id"], images.sample_png())
lose_original(stand, missing)
with outage(stand, "stop", "s3"):
interrupted = run_reconcile(stand, "apply")
assert interrupted.exit_code == 1
assert interrupted.finished["interrupted_by"] == "object_storage"
assert not interrupted.events(FIXED)
assert missing["id"] not in lost_ids(stand)
applied = run_reconcile(stand, "apply")
assert applied.kinds(FIXED, ids={missing["id"]}) == {
missing["id"]: "missing_object",
}
assert missing["id"] in lost_ids(stand)
@pytest.mark.spec(
"storage-consistency",
"Прерванная сверка доводится повтором",
)
def test_reconcile_interrupted_by_database(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
failing = api.upload_ready(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
other = api.upload_ready(alice, chat["id"], images.sample_png())
ids = {failing["id"], other["id"]}
for item in (failing, other):
lose_original(stand, item)
stand("fail-attachments", "mark", filename)
try:
interrupted = run_reconcile(stand, "apply")
assert interrupted.exit_code == 1
assert interrupted.finished["interrupted_by"] == "database"
assert failing["id"] not in lost_ids(stand)
finally:
stand("clear-fail-attachments")
applied = run_reconcile(stand, "apply")
assert ids <= lost_ids(stand)
assert not applied.about(FOUND, ids=ids)
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids=ids)
@pytest.mark.spec("storage-consistency", "Сверка под нагрузкой")
def test_reconcile_under_load(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
"""Загрузки и отправки дольше grace, сверка с `--apply` раз в 3 с:
внесённое исправлено, ни одно подтверждённое вложение не потеряно."""
data = images.sample_png()
orphans = {str(uuid.uuid4()) for _ in range(2)}
for item in orphans:
put_object(stand, object_key(item), data)
injected = [api.upload_ready(alice, chat["id"], data) for _ in range(2)]
for item in injected:
lose_original(stand, item)
lost_before = lost_ids(stand)
deadline = time.monotonic() + CLEANER_GRACE_S + LOAD_EXTRA_S
lock = threading.Lock()
uploads: dict[str, str] = {}
messages: dict[str, list[str]] = {}
failures: list[str] = []
passes: list[Reconcile] = []
def load(worker: int) -> None:
actor = alice if worker % 2 == 0 else bob
with Api(
str(api.client.base_url),
timeout=THREAD_CLIENT_TIMEOUT_S,
) as client:
number = 0
while time.monotonic() < deadline:
number += 1
body = images.png_noise(200, 170, seed=worker * 10**6 + number)
resp = client.post_attachment(actor, chat["id"], body)
if resp.status_code != 202:
with lock:
failures.append(f"upload {resp.status_code}")
continue
attachment = assert_accepted(resp)
# загрузка подтверждена: она обязана стать ready и скачиваться
with lock:
uploads[attachment["id"]] = hashlib.sha256(
body,
).hexdigest()
message = client.send_message(
actor,
chat["id"],
"",
attachment_ids=[attachment["id"]],
)
with lock:
messages[message["id"]] = [attachment["id"]]
time.sleep(LOAD_PAUSE_S)
def reconcile() -> None:
while time.monotonic() < deadline:
started = time.monotonic()
passes.append(run_reconcile(stand, "apply"))
sleep_until(started + LOAD_RECONCILE_PERIOD_S)
def guarded(target: Callable[..., None], *args: Any) -> None:
# исключение потока — в список, а не молча в stderr
try:
target(*args)
except Exception as e:
with lock:
failures.append(repr(e))
threads = [
threading.Thread(target=guarded, args=(load, worker))
for worker in range(LOAD_THREADS)
]
threads.append(threading.Thread(target=guarded, args=(reconcile,)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert not failures, failures[:5]
assert uploads, "no uploads under load"
assert len(passes) >= 3, len(passes)
assert all(item.exit_code in (0, 3) for item in passes), [
item.exit_code for item in passes
]
# внесённые исправлены: объекты без строки удалены, потери помечены
assert not orphans & stored_ids(stand)
assert lost_ids(stand) - lost_before == {item["id"] for item in injected}
# ни одно подтверждённое вложение не потеряно
for attachment_id, digest in uploads.items():
api.wait_attachment(alice, attachment_id)
resp = api.attachment_content(alice, attachment_id)
assert resp.status_code == 200, (attachment_id, resp.status_code)
assert hashlib.sha256(resp.content).hexdigest() == digest
history = {
item["id"]: item["attachment_ids"]
for item in api.all_messages(alice, chat["id"])
}
for message_id, attachment_ids in messages.items():
assert history[message_id] == attachment_ids
@pytest.mark.spec("storage-consistency", "Плановая сверка без команды")
def test_scheduled_reconcile(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
scheduled = api.upload_ready(alice, chat["id"], images.sample_png())
lose_original(stand, scheduled)
log = stand("run-cleaner", str(SCHEDULED_RUN_S))
records = [
json.loads(line) for line in log.splitlines() if line.startswith("{")
]
passes = [item for item in records if item.get("event") == FINISHED]
assert passes, log[-2000:]
assert any(
item["apply"] and item["interrupted_by"] is None for item in passes
), passes
assert scheduled["id"] in lost_ids(stand)
# ---------------------------------------------------------------------------
# Стоимость сверки (change track2-reconcile-cost, design, решение 14):
# стратегия HEAD, глубокая сверка и класс «другой SHA-256», снятие пометки
# только при том же SHA-256, предел скорости, отказ по пределу без лишних
# перепроверок. Аргументы сверки hook передаёт команде как есть
# ---------------------------------------------------------------------------
HEAD_STRATEGY = ("--strategy", "head")
DEEP = ("--deep",)
NO_RATE_LIMIT = ("--max-objects-per-second", "0")
# сколько вложений загружает тест предела скорости: объектов в bucket
# заведомо хватает на несколько секунд прохода
RATE_UPLOADS = 8
# во сколько раз проход с пределом длиннее числа объектов на предел
# (допуск на округление `duration_ms`)
RATE_SLACK = 0.95
def same_size_bytes(data: bytes, *, seed: int) -> bytes:
"""Другие байты того же размера, от зерна теста."""
other = random.Random(seed).randbytes(len(data))
assert other != data
return other
def findings(run: Reconcile) -> set[tuple[str, str]]:
"""Все расхождения прохода: класс и ключ объекта."""
return {(item["kind"], item["object_key"]) for item in run.events(FOUND)}
@pytest.mark.spec("storage-consistency", "Стратегия HEAD находит то же")
def test_reconcile_head_strategy_same_findings(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Те же расхождения, что в `test_reconcile_finds_and_fixes`: dry-run
стратегией по умолчанию и `--strategy head` называют одно и то же, а
HEAD-проход дороже по запросам к S3 и сеансам PostgreSQL."""
data = images.sample_png()
# уборщик стенда стоит: зависшую загрузку P не уберут между проходами
with outage(stand, "stop", "cleaner"):
a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3))
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
orphan = str(uuid.uuid4())
keys = {
"o": object_key(orphan),
"b": object_key(b["id"]),
"bv": variant_key(b["id"]),
}
put_object(stand, keys["o"], data)
stand("delete-object", object_key(a["id"]))
stand("delete-row", b["id"])
stand("set-size", c["id"], str(len(data) + 1))
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
resp = upload_raw(api, alice, chat, data, filename=filename)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
time.sleep(OBJECT_AGE_WAIT_S)
ids = {a["id"], c["id"], pending}
expected = {
keys["o"]: "orphan_object",
keys["b"]: "orphan_object",
keys["bv"]: "orphan_object",
a["id"]: "missing_object",
c["id"]: "size_mismatch",
pending: "stale_pending",
}
snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
batch = run_reconcile(stand, "dry-run")
head = run_reconcile(stand, "dry-run", options=HEAD_STRATEGY)
assert batch.kinds(FOUND, ids=ids, keys=keys.values()) == expected
assert head.kinds(FOUND, ids=ids, keys=keys.values()) == expected
# не только свои: весь отчёт и итог по классам одинаковы
assert findings(head) == findings(batch)
for field in ("found", "listed_objects", "checked_rows"):
assert head.finished[field] == batch.finished[field], field
assert head.exit_code == batch.exit_code == 3
assert (batch.finished["strategy"], head.finished["strategy"]) == (
"batch",
"head",
)
assert (
head.finished["s3_requests"]["head"]
> batch.finished["s3_requests"]["head"]
), (head.finished, batch.finished)
assert head.finished["db_sessions"] > batch.finished["db_sessions"]
after = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
assert after == snapshot
# стенд — без внесённых расхождений для следующих тестов
fixed = run_reconcile(stand, "apply")
assert set(fixed.kinds(FIXED, ids=ids, keys=keys.values())) == set(
expected,
)
@pytest.mark.spec(
"storage-consistency",
"Другой SHA-256 находит глубокая сверка",
)
def test_reconcile_deep_checksum(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.png_noise(64, 48, seed=11)
a = api.upload_ready(alice, chat["id"], data)
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
other = same_size_bytes(data, seed=11)
# объект перезаписан мимо API байтами того же размера
put_object(stand, object_key(a["id"]), other)
stand("age-row", a["id"], str(AGED_S))
time.sleep(OBJECT_AGE_WAIT_S)
snapshot = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
shallow = run_reconcile(stand, "dry-run")
assert not shallow.about(FOUND, ids={a["id"]})
assert shallow.finished["deep"] is False
deep = run_reconcile(stand, "dry-run", options=DEEP)
assert deep.kinds(FOUND, ids={a["id"]}) == {a["id"]: "checksum_mismatch"}
assert deep.finished["deep"] is True
assert deep.finished["read_bytes"] >= len(data), deep.finished
assert deep.finished["s3_requests"]["get"] >= 1, deep.finished
assert deep.exit_code == 3
after = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
assert after == snapshot
# SHA-256 объектов в журнале нет
journal = json.dumps(deep.records)
for body in (data, other):
assert hashlib.sha256(body).hexdigest() not in journal
applied = run_reconcile(stand, "apply", options=DEEP)
(record,) = applied.about(FIXED, ids={a["id"]})
assert (record["kind"], record["action"]) == (
"checksum_mismatch",
"marked_lost",
)
assert a["id"] in lost_ids(stand)
assert_lost_content(api, alice, a, aged=True)
@pytest.mark.spec(
"storage-consistency",
"Вернулся другой объект того же размера",
)
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
def test_reconcile_restore_needs_same_bytes(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.png_noise(64, 48, seed=13)
restored = api.upload_ready(alice, chat["id"], data)
ids = {restored["id"]}
lose_original(stand, restored)
marked = run_reconcile(stand, "apply")
assert marked.kinds(FIXED, ids=ids) == {restored["id"]: "missing_object"}
assert restored["id"] in lost_ids(stand)
# под ключом — другие байты того же размера: пометка остаётся
put_object(
stand,
object_key(restored["id"]),
same_size_bytes(data, seed=13),
)
time.sleep(OBJECT_AGE_WAIT_S)
wrong = run_reconcile(stand, "apply")
assert not wrong.about(FIXED, ids=ids)
assert all(
record["kind"] != "restored_object"
for record in wrong.about(FOUND, ids=ids)
)
assert restored["id"] in lost_ids(stand)
assert_lost_content(api, alice, restored, aged=True)
# те же байты, что были загружены: пометка снята
put_object(stand, object_key(restored["id"]), data)
time.sleep(OBJECT_AGE_WAIT_S)
cleared = run_reconcile(stand, "apply")
(record,) = cleared.about(FIXED, ids=ids)
assert (record["kind"], record["action"]) == (
"restored_object",
"cleared_lost",
)
assert restored["id"] not in lost_ids(stand)
assert_content(
api.attachment_content(alice, restored["id"]),
data,
"image/png",
)
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids=ids)
@pytest.mark.spec("storage-consistency", "Скорость прохода ограничена")
@pytest.mark.spec("storage-consistency", "Без ограничения скорости")
def test_reconcile_rate_limit(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Предел на четверть объектов прохода: проход длится не меньше
объектов на предел, находки и код — как без предела."""
data = images.sample_png()
for _ in range(RATE_UPLOADS):
api.upload_ready(alice, chat["id"], data)
lost = api.upload_ready(alice, chat["id"], data)
ids = {lost["id"]}
lose_original(stand, lost)
# уборщик стенда стоит: между проходами стенд не меняется
with outage(stand, "stop", "cleaner"):
free = run_reconcile(stand, "dry-run", options=NO_RATE_LIMIT)
rate = max(5, math.ceil(free.finished["storage_objects"] / 4))
limited = run_reconcile(
stand,
"dry-run",
options=("--max-objects-per-second", str(rate)),
)
assert free.finished["max_objects_per_second"] == 0
assert limited.finished["max_objects_per_second"] == rate
objects = limited.finished["storage_objects"]
assert objects >= RATE_UPLOADS, limited.finished
assert (
limited.finished["duration_ms"] >= 1000 * objects / rate * RATE_SLACK
), limited.finished
assert free.finished["duration_ms"] < limited.finished["duration_ms"]
expected = {lost["id"]: "missing_object"}
assert free.kinds(FOUND, ids=ids) == expected
assert limited.kinds(FOUND, ids=ids) == expected
assert free.exit_code == limited.exit_code == 3
# стенд — без внесённой потери
applied = run_reconcile(stand, "apply")
assert applied.kinds(FIXED, ids=ids) == expected
@pytest.mark.spec(
"storage-consistency",
"Отказ по пределу без лишних перепроверок",
)
def test_reconcile_limit_stops_rechecks(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
limit = 2
batch = [
api.upload_ready(alice, chat["id"], images.sample_png())
for _ in range(6)
]
ids = {item["id"] for item in batch}
for item in batch:
lose_original(stand, item)
refused = run_reconcile(stand, "apply", limit)
assert refused.exit_code == 3
(error,) = refused.events(LIMIT_EXCEEDED)
assert error["level"] == "error"
# отказ решён на третьем подтверждённом: остальные не перепроверены
assert error["changes"] == limit + 1, error
assert error["max_changes"] == limit
assert error["unchecked"] >= len(batch) - (limit + 1), error
assert refused.finished["unchecked"] == error["unchecked"]
assert refused.finished["refused"] is True
assert not refused.about(FIXED, ids=ids)
assert not ids & lost_ids(stand)
# dry-run перепроверяет каждого и называет все шесть
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, ids=ids) == dict.fromkeys(ids, "missing_object")
assert dry.finished["unchecked"] == 0
applied = run_reconcile(stand, "apply", 100)
assert set(applied.kinds(FIXED, ids=ids)) == ids
assert ids <= lost_ids(stand)
def own_findings(
run: Reconcile,
event: str,
keys: Collection[str],
) -> set[tuple[str, str, str | None]]:
"""Записи `event` о своих ключах: класс, ключ и `id` вложения — у
строки, ссылающейся на чужой ключ, `id` и ключ из разных вложений."""
return {
(item["kind"], item["object_key"], item.get("attachment_id"))
for item in run.events(event)
if item.get("object_key") in keys
}
@pytest.mark.spec(
"storage-consistency",
"Строка ссылается на объект другого id",
)
def test_reconcile_row_with_other_id_key(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W2 verify `track2-reconcile-cost`: строка Y ссылается на
`<Z>/original` чужого `id`. Обе стратегии сравнивают Y с объектом по
её ключу: другой размер и, с `--deep`, другой SHA-256 — расхождения
Y; объекты Z и `<Y>/original` — чужие и на месте."""
data = images.png_noise(64, 48, seed=17)
y1, y2 = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
z1, z2 = (str(uuid.uuid4()) for _ in range(2))
keys = {
"y1": object_key(y1["id"]),
"y2": object_key(y2["id"]),
"z1": object_key(z1),
"z2": object_key(z2),
}
# Z1 — другого размера, Z2 — того же размера с другими байтами
put_object(stand, keys["z1"], data + b"\0")
put_object(stand, keys["z2"], same_size_bytes(data, seed=17))
try:
stand("set-key", y1["id"], keys["z1"])
stand("set-key", y2["id"], keys["z2"])
for item in (y1, y2):
stand("age-row", item["id"], str(AGED_S))
# объекты Z записаны только что: после G + 1 все старше grace
time.sleep(OBJECT_AGE_WAIT_S)
expected = {
("size_mismatch", keys["z1"], y1["id"]),
("checksum_mismatch", keys["z2"], y2["id"]),
("unknown_object", keys["z1"], z1),
("unknown_object", keys["z2"], z2),
("unknown_object", keys["y1"], y1["id"]),
("unknown_object", keys["y2"], y2["id"]),
}
batch = run_reconcile(stand, "dry-run", options=DEEP)
head = run_reconcile(
stand,
"dry-run",
options=(*DEEP, *HEAD_STRATEGY),
)
assert own_findings(batch, FOUND, keys.values()) == expected
assert own_findings(head, FOUND, keys.values()) == expected
assert batch.exit_code == head.exit_code == 3
# поверхностная сверка видит только другой размер
shallow = run_reconcile(stand, "dry-run")
assert own_findings(shallow, FOUND, keys.values()) == expected - {
("checksum_mismatch", keys["z2"], y2["id"]),
}
applied = run_reconcile(stand, "apply", options=DEEP)
assert own_findings(applied, FIXED, keys.values()) == {
("size_mismatch", keys["z1"], y1["id"]),
("checksum_mismatch", keys["z2"], y2["id"]),
}
assert applied.exit_code == 3
# ни один объект не удалён, обе строки — ready и помечены
assert {y1["id"], y2["id"], z1, z2} <= stored_ids(stand)
rows = attachment_rows(stand)
assert rows[y1["id"]] == rows[y2["id"]] == "ready"
assert {y1["id"], y2["id"]} <= lost_ids(stand)
finally:
# прежние ключи; объекты Z, на которые больше никто не ссылается,
# — прочь, чтобы следующие проходы не удаляли их как свои
stand("set-key", y1["id"], keys["y1"])
stand("set-key", y2["id"], keys["y2"])
stand("delete-object", keys["z1"])
stand("delete-object", keys["z2"])
@pytest.mark.spec(
"storage-consistency",
"Недавно перезаписанный объект не тронут",
)
def test_reconcile_fresh_overwrite_untouched(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание S1 verify `track2-reconcile-cost`: объект готового
вложения старше grace перезаписан мимо API меньше grace назад — другим
размером (A) и теми же размером с другими байтами (B). Сверка с
`--deep --apply` и HEAD-стратегией считает оба объекта свежими; после
grace — расхождения. Глубокий проход читает весь bucket стенда и к концу
набора длится дольше G: окно «свежий» — grace разового прохода
`FRESH_GRACE_S`, он меньше возраста строк."""
data = images.png_noise(64, 48, seed=19)
a, b = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
ids = {a["id"], b["id"]}
for item in (a, b):
stand("age-row", item["id"], str(AGED_S))
def overwrite() -> float:
"""Объекты A и B заново перезаписаны: моложе grace с этого момента."""
put_object(stand, object_key(a["id"]), data + b"\0")
put_object(stand, object_key(b["id"]), same_size_bytes(data, seed=19))
return time.monotonic()
grace = ("--grace", str(FRESH_GRACE_S))
try:
# перед каждым проходом объекты перезаписываются заново: глубокий
# проход читает весь bucket стенда, и два подряд дольше grace
for options, mode in (
((*DEEP, *grace), "apply"),
((*DEEP, *HEAD_STRATEGY, *grace), "dry-run"),
):
written = overwrite()
run = run_reconcile(stand, mode, options=options)
# проход — пока объекты моложе grace, иначе он ничего не
# доказывает
assert time.monotonic() - written < FRESH_GRACE_S, (
"reconcile run took longer than its grace period"
)
assert not run.about(FOUND, ids=ids), run.finished
assert not run.about(FIXED, ids=ids), run.finished
assert run.finished["fresh_objects"] >= len(ids), run.finished
assert not ids & lost_ids(stand)
assert_error(api.attachment_content(alice, a["id"]), 500)
time.sleep(max(0.0, OBJECT_AGE_WAIT_S - (time.monotonic() - written)))
aged = run_reconcile(stand, "dry-run", options=DEEP)
assert aged.kinds(FOUND, ids=ids) == {
a["id"]: "size_mismatch",
b["id"]: "checksum_mismatch",
}
finally:
# стенд — с исходными байтами: расхождений не остаётся
for item in (a, b):
put_object(stand, object_key(item["id"]), data)
# ---------------------------------------------------------------------------
# Сверка и фоновая обработка (change async-attachments, design, решение 11):
# объекты вариантов своих вложений и текущая работа — не расхождения
# ---------------------------------------------------------------------------
@pytest.mark.spec("storage-consistency", "Объекты вариантов своих вложений")
def test_reconcile_variant_objects(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
batch = [api.upload_ready(alice, chat["id"], data) for _ in range(2)]
ids = {item["id"] for item in batch}
keys = {variant_key(item["id"]) for item in batch}
for item in batch:
assert stand("variant-object-count", item["id"]) == str(
len(VARIANT_NAMES),
)
stand("age-row", item["id"], str(AGED_S))
# объекты вариантов записаны только что: после G + 1 старше grace
time.sleep(OBJECT_AGE_WAIT_S)
for options in ((), HEAD_STRATEGY):
run = run_reconcile(stand, "dry-run", options=options)
assert not run.about(FOUND, ids=ids, keys=keys), run.finished
run.assert_code_matches()
@pytest.mark.spec("storage-consistency", "Объект варианта без метаданных")
def test_reconcile_orphan_variant(
stand: Stand,
) -> None:
data = images.sample_png()
orphan = str(uuid.uuid4())
key = variant_key(orphan)
put_object(stand, key, data)
assert stand("variant-object-count", orphan) == "1"
time.sleep(OBJECT_AGE_WAIT_S)
run = run_reconcile(stand, "apply")
(record,) = run.about(FIXED, keys={key})
assert (record["kind"], record["action"]) == (
"orphan_object",
"deleted_object",
)
assert not run.about(FOUND, keys={key})
assert stand("variant-object-count", orphan) == "0"
@pytest.mark.spec(
"storage-consistency",
"Текущая работа не считается расхождением",
)
def test_reconcile_keeps_current_work(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
# уборщик стоит: срок хранения `failed` на стенде 60 с, а проход к концу набора
# длинный, и уборщик убрал бы staging строки `failed` раньше проверки
with (
outage(stand, "stop", "cleaner"),
current_work(api, stand, alice, chat) as work,
):
# конечные объекты F и P и варианты P старше grace
time.sleep(OBJECT_AGE_WAIT_S)
ids = {item["id"] for item in work.values()}
keys = {staging_key(item) for item in ids} | {
object_key(item) for item in ids
}
run = run_reconcile(stand, "apply")
assert not run.about(FOUND, ids=ids, keys=keys), run.finished
assert not run.about(FIXED, ids=ids, keys=keys), run.finished
run.assert_code_matches()
assert not ids & lost_ids(stand)
rows = attachment_rows(stand)
staged = staged_ids(stand)
for name, item in work.items():
assert rows.get(item["id"]) == name, (name, rows.get(item["id"]))
assert item["id"] in staged, name
stored = stored_ids(stand)
# конечный объект у задания, дошедшего до него, на месте
assert work["failed"]["id"] in stored
assert work["processing"]["id"] in stored
# ---------------------------------------------------------------------------
# Надёжность конвейера (change pipeline-reliability, design, решения 2–9, 14,
# 15): relay, окна сбоя без точек в коде, аренда и fencing, лимиты процесса
# обработки, backoff, реестр failed и replay, prefetch, очистка staging. Worker
# и relay запускаются в finally, триггеры и дополнительные контейнеры снимаются
# в finally
# ---------------------------------------------------------------------------
@contextmanager
def extra_worker(stand: Stand, *env: str) -> Iterator[None]:
"""Основной worker остановлен, работает второй с этим окружением."""
wait_queue_empty(stand)
with outage(stand, "stop", "worker"):
stand("run-worker", *env)
try:
yield
finally:
stand("stop-extra")
@pytest.mark.spec("job-outbox", "Ответ не ждёт публикации")
@pytest.mark.spec("job-outbox", "Crash между фиксацией и публикацией")
@pytest.mark.spec("job-outbox", "Задание без отметки от прежней версии")
def test_relay_killed_before_publish(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Relay убит: загрузка 202, сообщения нет, метаданные `queued`, задание без
отметки — то же, что строка прежней версии. После запуска — `ready` с тем же
заданием."""
wait_queue_empty(stand)
with outage(stand, "kill", "relay"):
resp, elapsed = timed(
lambda: upload_raw(api, alice, chat, images.sample_png()),
)
accepted = assert_accepted(resp)
assert elapsed < ACCEPT_MAX_S, elapsed
assert queue_state(stand)["messages"] == 0
meta = assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
assert meta["status"] == "queued", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["published_at"] is None, job
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
done = job_row(stand, accepted["id"])
assert done is not None
assert done["id"] == job["id"], done
assert done["published_at"] is not None, done
assert done["attempt"] == 1, done
def crash_relay_after_confirm(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Worker остановлен, отметка задержана триггером, брокер сообщение уже
подтвердил: relay убит, транзакция отметки откатилась. После запуска relay
публикует ещё раз; дубль — no-op."""
filename = fault_filename()
with worker_stopped(stand):
stand("fail-attachments", "relay-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
wait_queue_messages(stand, 1)
job = job_row(stand, accepted["id"])
assert job is not None
assert job["published_at"] is None, job
stand("kill", "relay")
finally:
stand("clear-fail-attachments")
stand("start", "relay")
# та же работа опубликована ещё раз: отметки не было
wait_queue_messages(stand, 2)
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, meta)
wait_queue_empty(stand)
done = job_row(stand, accepted["id"])
assert done is not None
assert done["attempt"] == 1, done
@pytest.mark.spec("job-outbox", "Crash после подтверждения до отметки")
def test_relay_killed_after_confirm(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
crash_relay_after_confirm(api, stand, alice, chat)
@pytest.mark.spec("job-outbox", "Отказ брокера во время работы relay")
def test_relay_survives_broker_outage(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = json.loads(stand("relay-state"))
seen = len(relay_records(stand))
with outage(stand, "stop", "rabbitmq"):
accepted = api.upload(alice, chat["id"], images.sample_png())
wait_relay_record(stand, "Broker is unavailable", skip=seen)
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
after = json.loads(stand("relay-state"))
assert after["running"] is True, after
assert after["started_at"] == before["started_at"], after
assert after["restart_count"] == before["restart_count"], after
@pytest.mark.spec("job-outbox", "Отказ PostgreSQL во время работы relay")
def test_relay_survives_database_outage(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = json.loads(stand("relay-state"))
seen = len(relay_records(stand))
with outage(stand, "stop", "postgres"):
wait_relay_record(stand, "Relay pass was interrupted", skip=seen)
accepted = api.upload(alice, chat["id"], images.sample_png())
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
after = json.loads(stand("relay-state"))
assert after["running"] is True, after
assert after["started_at"] == before["started_at"], after
assert after["restart_count"] == before["restart_count"], after
@pytest.mark.spec("job-outbox", "Два relay")
def test_two_relays(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
files = 6
with worker_stopped(stand):
stand("run-relay", "RELAY__POLL_INTERVAL_SECONDS=0.05")
try:
accepted = [
api.upload(alice, chat["id"], images.sample_png())
for _ in range(files)
]
wait_queue_messages(stand, files)
time.sleep(1)
assert queue_state(stand)["messages"] == files
finally:
stand("stop-extra")
for item in accepted:
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
wait_queue_empty(stand)
@pytest.mark.spec("job-outbox", "Следующая попытка не раньше срока")
def test_relay_respects_next_attempt(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
delay = 6
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
# relay стоит, пока срок отодвигается: случайная пауза короче хода теста
with relay_stopped(stand), outage(stand, "stop", "s3"):
stand("start", "worker")
wait_job(stand, accepted["id"], state="queued", attempt=1)
stand("stop", "worker")
stand("set-next-attempt", accepted["id"], str(delay))
due = time.monotonic() + delay
assert queue_state(stand)["messages"] == 0
wait_queue_messages(stand, 1, timeout=delay + PUBLISH_WAIT_S)
assert time.monotonic() >= due - SLACK_S
peeked = json.loads(stand("peek-job"))
assert json.loads(peeked["body"])["attempt"] == 2, peeked
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("job-outbox", "Очередь удалена вместе с сообщением")
def test_lost_message_republished(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
stand("delete-queue")
# срок прошёл: relay объявляет очередь и публикует снова
state = wait_queue_messages(
stand,
1,
timeout=RELAY_REPUBLISH_S + PUBLISH_WAIT_S,
)
assert state["durable"] is True, state
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("job-outbox", "Очередь с сообщениями не обрастает копиями")
def test_no_copies_while_queue_has_messages(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
files = 2
with worker_stopped(stand):
accepted = [
api.upload(alice, chat["id"], images.sample_png())
for _ in range(files)
]
wait_queue_messages(stand, files)
time.sleep(RELAY_REPUBLISH_S + 2)
assert queue_state(stand)["messages"] == files
for item in accepted:
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("job-outbox", "Очередь ограничена пределом")
def test_published_backlog_limit(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
limit, files = 2, 5
with worker_stopped(stand), relay_stopped(stand):
stand("run-relay", f"RELAY__MAX_PUBLISHED_BACKLOG={limit}")
try:
accepted = [
api.upload(alice, chat["id"], images.sample_png())
for _ in range(files)
]
wait_queue_messages(stand, limit)
time.sleep(1)
assert queue_state(stand)["messages"] == limit
jobs = [job_row(stand, item["id"]) for item in accepted]
marked = [job for job in jobs if job and job["published_at"]]
assert len(marked) == limit, jobs
finally:
stand("stop-extra")
for item in accepted:
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
wait_queue_empty(stand)
@pytest.mark.spec("job-outbox", "Брокер сдерживает публикацию")
def test_broker_memory_alarm(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
seen = len(relay_records(stand))
stand("broker-memory-limit", "1MiB")
try:
resp, elapsed = timed(
lambda: upload_raw(api, alice, chat, images.sample_png()),
)
accepted = assert_accepted(resp)
assert elapsed < ACCEPT_MAX_S, elapsed
wait_relay_record(stand, "Broker is unavailable", skip=seen)
job = job_row(stand, accepted["id"])
assert job is not None
assert job["published_at"] is None, job
finally:
stand("broker-memory-limit", BROKER_MEMORY_LIMIT)
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
wait_queue_empty(stand)
def crash_after_objects(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Все объекты записаны, фиксация ждёт в триггере: KILL. Новая попытка — по
аренде — пишет те же ключи теми же байтами."""
filename = fault_filename()
wait_queue_empty(stand)
stand("fail-attachments", "ready-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
attachment_id = accepted["id"]
api.wait_attachment(
alice,
attachment_id,
status="processing",
timeout=JOB_WAIT_S,
)
wait_variant_objects(stand, attachment_id)
# заморозка: окно триггера короче запроса к брокеру, а замёрзший клиент
# COMMIT не отправит
stand("freeze", "worker")
# ack — только после фиксации: задание восстановила бы и аренда, поэтому
# ранний ack виден только здесь
held = queue_state(stand)
assert held["messages_unacknowledged"] == 1, held
stand("kill", "worker")
before = {
name: stand("object-sha256", variant_key(attachment_id, name))
for name in VARIANT_NAMES
}
assert all(before.values()), before
finally:
stand("clear-fail-attachments")
stand("start", "worker")
meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S)
variants = assert_variants(api, stand, alice, meta)
for name, digest in before.items():
assert variants[name]["sha256"] == digest, name
done = job_row(stand, attachment_id)
assert done is not None
assert done["attempt"] == 2, done
wait_queue_empty(stand)
@pytest.mark.spec(
"attachment-processing",
"Crash после записи объектов до фиксации",
)
def test_crash_after_objects_before_commit(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
crash_after_objects(api, stand, alice, chat)
def crash_after_commit(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""COMMIT фиксации ждёт в отложенном триггере, worker заморожен (контейнер
`running`, не `paused`, поэтому KILL проходит), строка `succeeded` при
замороженном клиенте, KILL: ack не ушёл. Следующий запуск получает сообщение
снова и подтверждает его без эффекта."""
filename = fault_filename()
wait_queue_empty(stand)
stand("fail-attachments", "ready-commit-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
attachment_id = accepted["id"]
stand("wait-commit-stall")
stand("freeze", "worker")
wait_job(stand, attachment_id, state="succeeded", attempt=1)
stand("kill", "worker")
job = job_row(stand, attachment_id)
assert job is not None
ready = assert_ok(
api.attachment_meta(alice, attachment_id),
200,
"Attachment",
)
assert ready["status"] == "ready", ready
seen = len(worker_records(stand))
finally:
stand("clear-fail-attachments")
stand("start", "worker")
record = wait_worker_record(
stand,
"Skipped job message",
skip=seen,
job_id=job["id"],
reason="finished",
)
assert record["level"] == "info", record
wait_queue_empty(stand)
assert (
assert_ok(api.attachment_meta(alice, attachment_id), 200, "Attachment")
== ready
)
assert job_row(stand, attachment_id) == job
assert job["attempt"] == 1, job
assert_variants(api, stand, alice, ready)
@pytest.mark.spec(
"attachment-processing",
"Crash после фиксации до подтверждения",
)
def test_crash_after_commit_before_ack(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
crash_after_commit(api, stand, alice, chat)
@pytest.mark.spec(
"attachment-processing",
"Пауза растёт со случайной добавкой",
)
def test_retry_backoff_with_jitter(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Формула решения 4: пауза попытки N — из `[d / 2, d]`,
`d = min(max, base × 2^(N − 1))`, у двух заданий паузы разные."""
seen = len(worker_records(stand))
with worker_stopped(stand):
first = api.upload(alice, chat["id"], images.sample_png())
second = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 2)
with outage(stand, "stop", "s3"):
stand("start", "worker")
for item in (first, second):
api.wait_attachment(
alice,
item["id"],
status="failed",
timeout=JOB_WAIT_S,
)
delays: dict[str, dict[int, float]] = {first["id"]: {}, second["id"]: {}}
for record in worker_records(stand)[seen:]:
if record.get("event") == "Retrying job":
delays[record["attachment_id"]][record["attempt"]] = record[
"delay_seconds"
]
for attachment_id, pauses in delays.items():
assert sorted(pauses) == list(range(1, WORKER_MAX_ATTEMPTS)), pauses
for attempt, delay in pauses.items():
top = min(
WORKER_RETRY_MAX_S,
WORKER_RETRY_BASE_S * 2 ** (attempt - 1),
)
assert top / 2 <= delay <= top, (attachment_id, attempt, delay)
for attempt in range(1, WORKER_MAX_ATTEMPTS):
assert delays[first["id"]][attempt] != delays[second["id"]][attempt]
@pytest.mark.spec("attachment-processing", "Попытка не раньше срока")
@pytest.mark.spec(
"attachment-processing",
"Пауза не занимает место конкуренции",
)
def test_retry_waits_outside_worker(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Задание A ждёт следующую попытку: дубль его сообщения и перезапуск worker
попытку не начинают, а другое вложение становится `ready` раньше срока."""
wait = 20
with worker_stopped(stand):
waiting = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
with relay_stopped(stand), outage(stand, "stop", "s3"):
stand("start", "worker")
wait_job(stand, waiting["id"], state="queued", attempt=1)
stand("set-next-attempt", waiting["id"], str(wait))
due = time.monotonic() + wait
job = job_row(stand, waiting["id"])
assert job is not None
next_attempt_at = datetime.fromisoformat(job["next_attempt_at"])
stand("stop", "worker")
stand(
"publish-job",
stdin=job_body(job["id"], waiting["id"], attempt=2),
)
seen = len(worker_records(stand))
stand("start", "worker")
wait_worker_record(
stand,
"Skipped job message",
skip=seen,
job_id=job["id"],
reason="not_due",
)
wait_queue_empty(stand)
same = job_row(stand, waiting["id"])
assert same is not None
assert (same["attempt"], same["state"]) == (1, "queued"), same
other = api.upload_ready(alice, chat["id"], images.sample_png())
assert time.monotonic() < due, "other attachment was not ready in time"
assert other["status"] == "ready"
meta = assert_ok(
api.attachment_meta(alice, waiting["id"]),
200,
"Attachment",
)
assert meta["status"] == "queued", meta
done = api.wait_attachment(alice, waiting["id"], timeout=JOB_WAIT_S)
assert done["status"] == "ready"
final = job_row(stand, waiting["id"])
assert final is not None
assert final["attempt"] == 2, final
started = wait_worker_record(
stand,
"Started job attempt",
skip=seen,
job_id=job["id"],
attempt=2,
)
assert datetime.fromisoformat(started["timestamp"]) >= next_attempt_at
@pytest.mark.spec("attachment-processing", "Пикселей больше лимита")
def test_limit_pixels(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
huge = images.png_huge(10000, 5000)
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
put_object(stand, staging_key(accepted["id"]), huge)
stand("set-size", accepted["id"], str(len(huge)))
stand("set-sha256", accepted["id"], hashlib.sha256(huge).hexdigest())
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "invalid_image", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["last_error_code"] == "limit_pixels", job
assert job["attempt"] == 1, job
@pytest.mark.spec("attachment-processing", "Объект больше лимита не читается")
def test_limit_oversized_staging(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
put_object(
stand,
staging_key(accepted["id"]),
bytes(images.MAX_BYTES + 1),
)
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "processing_failed", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["last_error_code"] == "source_corrupted", job
assert job["attempt"] == 1, job
@pytest.mark.spec("attachment-processing", "Дорогое изображение")
def test_limit_cpu_scan_bomb(
base_url: str,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = json.loads(stand("worker-state"))
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
accepted = slow.upload(
alice,
chat["id"],
images.jpeg_scan_bomb(),
content_type="image/jpeg",
filename="bomb.jpg",
)
meta = slow.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "invalid_image", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["last_error_code"] == "limit_cpu", job
assert job["attempt"] == 1, job
state = json.loads(stand("worker-state"))
assert state["running"] is True, state
assert state["restart_count"] == before["restart_count"], state
# следующий файл обрабатывается
slow.upload_ready(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachment-processing", "Предел памяти процесса обработки")
def test_limit_memory(
base_url: str,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with (
Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow,
extra_worker(stand, "WORKER__RENDER_MEMORY_LIMIT_MIB=200"),
):
accepted = slow.upload(alice, chat["id"], images.png_rgba_large())
meta = slow.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "invalid_image", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["last_error_code"] == "limit_memory", job
assert job["attempt"] == 1, job
# процесс worker работает: следующий файл обрабатывается
slow.upload_ready(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachment-processing", "Срок процесса обработки")
def test_limit_render_timeout(
base_url: str,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with (
Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow,
extra_worker(
stand,
"WORKER__RENDER_TIMEOUT_SECONDS=1",
"WORKER__RENDER_CPU_LIMIT_SECONDS=30",
),
):
accepted = slow.upload(
alice,
chat["id"],
images.jpeg_scan_bomb(),
content_type="image/jpeg",
filename="bomb.jpg",
)
meta = slow.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "retry_exhausted", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["last_error_code"] == "timeout", job
assert job["attempt"] == WORKER_MAX_ATTEMPTS, job
assert stand("render-count", "extra") == "0"
@pytest.mark.spec("attachment-processing", "Процесс обработки убит")
@pytest.mark.spec("attachment-processing", "Секретов нет у процесса обработки")
def test_render_killed(
base_url: str,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = json.loads(stand("worker-state"))
wait_queue_empty(stand)
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
accepted = slow.upload(
alice,
chat["id"],
images.jpeg_scan_bomb(),
content_type="image/jpeg",
filename="bomb.jpg",
)
wait_render_process(stand)
names = stand("render-env").split()
assert names, "no environment of the render process"
for name in names:
assert not name.startswith(
("DATABASE__", "S3__", "BROKER__", "CACHE__"),
), names
assert stand("kill-render") != "0"
job = wait_job(stand, accepted["id"], state="queued", attempt=1)
assert job["last_error_code"] == "render_killed", job
state = json.loads(stand("worker-state"))
assert state["running"] is True, state
assert state["restart_count"] == before["restart_count"], state
# следующая попытка упирается в лимит CPU: итог постоянный
meta = slow.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "invalid_image", meta
done = job_row(stand, accepted["id"])
assert done is not None
assert done["attempt"] == 2, done
@pytest.mark.spec("attachment-processing", "Прежняя попытка при живой новой")
def test_hung_worker_lease_fencing(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Первая попытка размораживается, когда вторая жива: задание в `processing`,
блокировки строки нет. Решает только номер попытки: по `state` фиксация первой
прошла бы. Оба worker — со сроками больше времени теста, иначе при разморозке
их прервал бы таймер, а не fencing. S3 заморожен, пока вторая не взята: она
ждёт его и не успеет зафиксировать результат раньше своей заморозки. Итог даёт
только вторая. Аренду делает истекшей hook: при таком сроке её не дождаться."""
second = "worker-second"
timers = (
"WORKER__JOB_TIMEOUT_SECONDS=120",
"WORKER__RENDER_TIMEOUT_SECONDS=60",
"S3__TIMEOUT_SECONDS=60",
)
seen_relay = len(relay_records(stand))
try:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
attachment_id = accepted["id"]
wait_queue_messages(stand, 1)
with outage(stand, "pause", "s3"):
stand("run-worker", *timers)
wait_job(stand, attachment_id, state="processing", attempt=1)
wait_extra_record(
stand,
"Started job attempt",
attachment_id=attachment_id,
attempt=1,
)
stand("pause-extra", "worker")
stand("expire-lease", attachment_id)
wait_relay_record(
stand,
"Recovered expired lease",
skip=seen_relay,
attachment_id=attachment_id,
)
stand("run-worker-second", *timers)
wait_job(stand, attachment_id, state="processing", attempt=2)
wait_extra_record(
stand,
"Started job attempt",
extra=second,
attachment_id=attachment_id,
attempt=2,
)
stand("pause-extra", second)
# вторая взята и заморожена в вызове S3: транзакции нет
assert stand("attachment-free", attachment_id)
taken = job_row(stand, attachment_id)
assert taken is not None
assert taken["state"] == "processing", taken
assert taken["attempt"] == 2, taken
stand("unpause-extra", "worker")
wait_extra_record(
stand,
"Skipped job message",
attachment_id=attachment_id,
attempt=1,
reason="superseded",
)
assert job_row(stand, attachment_id) == taken
assert (
assert_ok(
api.attachment_meta(alice, attachment_id),
200,
"Attachment",
)["status"]
== "processing"
)
stand("unpause-extra", second)
meta = api.wait_attachment(
alice,
attachment_id,
timeout=JOB_WAIT_S,
)
wait_extra_record(
stand,
"Attachment became ready",
extra=second,
attachment_id=attachment_id,
attempt=2,
)
job = job_row(stand, attachment_id)
assert job is not None
assert job["state"] == "succeeded", job
assert job["attempt"] == 2, job
assert not [
record
for record in extra_records(stand)
if record.get("event") == "Attachment became ready"
and record.get("attachment_id") == attachment_id
]
finally:
stand("stop-extra")
assert_variants(api, stand, alice, meta)
@pytest.mark.spec("attachment-processing", "Зависший worker")
def test_hung_worker_thawed_after_ready(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Вторая попытка уже дала `ready`, первая разморожена: итог её не меняет.
Первый worker — со сроками больше времени теста: разморозка застаёт его
таймеры живыми, и он доходит до фиксации (с обычными сроками его прервал бы
таймер, а не fencing). Здесь решает и `state`; номер попытки —
`test_hung_worker_lease_fencing`. Второй процесс — основной worker стенда."""
seen_relay = len(relay_records(stand))
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
with outage(stand, "pause", "s3"):
stand(
"run-worker",
"WORKER__JOB_TIMEOUT_SECONDS=120",
"WORKER__RENDER_TIMEOUT_SECONDS=60",
"S3__TIMEOUT_SECONDS=60",
)
try:
wait_job(stand, accepted["id"], state="processing", attempt=1)
# попытка уже ждёт S3
wait_extra_record(
stand,
"Started job attempt",
attachment_id=accepted["id"],
)
stand("pause-extra", "worker")
except BaseException:
stand("stop-extra")
raise
try:
stand("expire-lease", accepted["id"])
wait_relay_record(
stand,
"Recovered expired lease",
skip=seen_relay,
attachment_id=accepted["id"],
)
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 2, job
stand("unpause-extra", "worker")
wait_extra_record(
stand,
"Skipped job message",
attachment_id=accepted["id"],
reason="superseded",
)
assert (
assert_ok(
api.attachment_meta(alice, accepted["id"]),
200,
"Attachment",
)
== meta
)
assert job_row(stand, accepted["id"]) == job
finally:
stand("stop-extra")
assert_variants(api, stand, alice, meta)
@pytest.mark.spec("attachment-processing", "Долгая живая попытка")
def test_long_attempt_keeps_lease(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Фиксация ждёт в триггере, попытка укладывается в срок: relay её не
возвращает."""
filename = fault_filename()
seen = len(relay_records(stand))
stand("fail-attachments", "ready-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
finally:
stand("clear-fail-attachments")
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 1, job
assert meta["status"] == "ready"
assert not [
record
for record in relay_records(stand)[seen:]
if record.get("event") == "Recovered expired lease"
and record.get("attachment_id") == accepted["id"]
]
@pytest.mark.spec("attachment-processing", "Последняя попытка потеряна")
def test_last_attempt_lost(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
with extra_worker(stand, "WORKER__MAX_ATTEMPTS=1"):
stand("fail-attachments", "ready-stall", filename)
try:
accepted = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
api.wait_attachment(
alice,
accepted["id"],
status="processing",
timeout=JOB_WAIT_S,
)
stand("kill-extra", "worker")
finally:
stand("clear-fail-attachments")
stand("run-worker", "WORKER__MAX_ATTEMPTS=1")
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "retry_exhausted", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["attempt"] == 1, job
@dataclass
class Replay:
"""Один вызов команды реестра: записи и код выхода."""
records: list[dict[str, Any]]
exit_code: int
def about(self, event: str, attachment_id: str) -> list[dict[str, Any]]:
return [
item
for item in self.records
if item.get("event") == event
and item.get("attachment_id") == attachment_id
]
@property
def finished(self) -> dict[str, Any]:
(record,) = [
item
for item in self.records
if item.get("event") == "Finished replay pass"
]
return record
def run_replay(stand: Stand, *args: str) -> Replay:
lines = stand("replay", *args).splitlines()
assert lines and lines[-1].startswith("exit_code="), lines[-5:]
records = [json.loads(line) for line in lines[:-1] if line.startswith("{")]
return Replay(records=records, exit_code=int(lines[-1].split("=")[1]))
def replay_id(stand: Stand, attachment_id: str) -> Replay:
return run_replay(
stand,
"--attachment-id",
attachment_id,
"--apply",
"--operator",
"student-tests",
)
def replays(stand: Stand, attachment_id: str) -> list[dict[str, Any]]:
return [
json.loads(line)
for line in stand("replays", attachment_id).splitlines()
]
def failed_invalid(
api: Api,
alice: Actor,
chat: dict[str, Any],
) -> dict[str, Any]:
accepted = api.upload(
alice,
chat["id"],
images.jpeg_precision_12(),
content_type="image/jpeg",
filename="image.jpg",
)
return api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
def failed_exhausted(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
*,
files: int = 1,
) -> list[dict[str, Any]]:
"""Вложения `failed` с `retry_exhausted`: S3 остановлен дольше всех попыток.
S3 при выходе возвращён."""
with worker_stopped(stand):
accepted = [
api.upload(alice, chat["id"], images.sample_png())
for _ in range(files)
]
wait_queue_messages(stand, files)
with outage(stand, "stop", "s3"):
stand("start", "worker")
done = [
api.wait_attachment(
alice,
item["id"],
status="failed",
timeout=JOB_WAIT_S,
)
for item in accepted
]
for meta in done:
assert meta["error_code"] == "retry_exhausted", meta
return done
@pytest.mark.spec("attachment-processing", "Реестр показан без изменений")
def test_replay_registry_dry_run(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
failed = failed_invalid(api, alice, chat)
job = job_row(stand, failed["id"])
assert job is not None
run = run_replay(stand, "--attachment-id", failed["id"])
assert run.exit_code == 0, run.finished
(record,) = run.about("Found failed job", failed["id"])
assert record["job_id"] == job["id"], record
assert record["error_code"] == "invalid_image", record
assert record["reason"] == "undecodable", record
assert (
assert_ok(api.attachment_meta(alice, failed["id"]), 200, "Attachment")
== failed
)
assert job_row(stand, failed["id"]) == job
assert replays(stand, failed["id"]) == []
def replay_returns_work(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Объект staging заменён — `failed` `processing_failed` — реестр — объект
возвращён — replay — `ready` с тем же заданием и аудитом."""
data = images.png_noise(64, 48, seed=37)
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], data)
put_object(
stand,
staging_key(accepted["id"]),
same_size_bytes(data, seed=37),
)
meta = api.wait_attachment(
alice,
accepted["id"],
status="failed",
timeout=JOB_WAIT_S,
)
assert meta["error_code"] == "processing_failed", meta
job = job_row(stand, accepted["id"])
assert job is not None
assert job["last_error_code"] == "source_corrupted", job
registry = run_replay(stand, "--error-code", "processing_failed")
assert registry.about("Found failed job", accepted["id"]), (
registry.finished
)
put_object(stand, staging_key(accepted["id"]), data)
run = replay_id(stand, accepted["id"])
assert run.exit_code == 0, run.finished
(record,) = run.about("Replayed job", accepted["id"])
assert record["operator"] == "student-tests", record
assert record["error_code"] == "processing_failed", record
assert record["reason"] == "source_corrupted", record
ready = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
assert_content(
api.attachment_content(alice, accepted["id"]),
data,
"image/png",
)
assert ready["sha256"] == hashlib.sha256(data).hexdigest()
assert_variants(api, stand, alice, ready)
done = job_row(stand, accepted["id"])
assert done is not None
assert done["id"] == job["id"], done
assert done["attempt"] == 2, done
(audit,) = replays(stand, accepted["id"])
assert audit["error_code"] == "processing_failed", audit
assert audit["reason"] == "source_corrupted", audit
assert audit["operator"] == "student-tests", audit
@pytest.mark.spec("attachment-processing", "Replay возвращает работу")
def test_replay_returns_work(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
replay_returns_work(api, stand, alice, chat)
def replay_ready_refused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Replay готовой работы — отказ, код 3, ничего не изменилось."""
ready = api.upload_ready(alice, chat["id"], images.sample_png())
job = job_row(stand, ready["id"])
run = replay_id(stand, ready["id"])
assert run.exit_code == 3, run.finished
(record,) = run.about("Refused job replay", ready["id"])
assert record["refusal"] == "already_ready", record
assert not run.about("Replayed job", ready["id"])
assert (
assert_ok(api.attachment_meta(alice, ready["id"]), 200, "Attachment")
== ready
)
assert job_row(stand, ready["id"]) == job
assert_variants(api, stand, alice, ready)
assert replays(stand, ready["id"]) == []
@pytest.mark.spec("attachment-processing", "Replay готового не создаёт дубль")
def test_replay_ready_refused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
replay_ready_refused(api, stand, alice, chat)
@pytest.mark.spec("attachment-processing", "Replay идущей работы")
def test_replay_in_progress_refused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
queued = api.upload(alice, chat["id"], images.sample_png())
wait_queue_messages(stand, 1)
run = replay_id(stand, queued["id"])
assert run.exit_code == 3, run.finished
(record,) = run.about("Refused job replay", queued["id"])
assert record["refusal"] == "not_failed", record
assert queue_state(stand)["messages"] == 1
job = job_row(stand, queued["id"])
assert job is not None
assert job["attempt"] == 0, job
api.wait_attachment(alice, queued["id"], timeout=JOB_WAIT_S)
filename = fault_filename()
stand("fail-attachments", "ready-stall", filename)
try:
busy = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
api.wait_attachment(
alice,
busy["id"],
status="processing",
timeout=JOB_WAIT_S,
)
run = replay_id(stand, busy["id"])
assert run.exit_code == 3, run.finished
(record,) = run.about("Refused job replay", busy["id"])
# фиксация могла успеть, пока шла команда: и то, и другое — отказ
assert record["refusal"] in ("not_failed", "already_ready"), record
meta = api.wait_attachment(alice, busy["id"], timeout=JOB_WAIT_S)
finally:
stand("clear-fail-attachments")
assert meta["status"] == "ready"
done = job_row(stand, busy["id"])
assert done is not None
assert done["attempt"] == 1, done
assert replays(stand, busy["id"]) == []
@pytest.mark.spec("attachment-processing", "Replay без исходника")
def test_replay_without_source_refused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
failed = failed_invalid(api, alice, chat)
stand("age-row", failed["id"], str(STAGING_RETENTION_S + 5))
wait_staging_gone(stand, failed["id"])
run = replay_id(stand, failed["id"])
assert run.exit_code == 3, run.finished
(record,) = run.about("Refused job replay", failed["id"])
assert record["refusal"] == "source_gone", record
meta = assert_ok(
api.attachment_meta(alice, failed["id"]),
200,
"Attachment",
)
assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image")
@pytest.mark.spec("attachment-processing", "Новый бюджет попыток")
def test_replay_new_budget(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
(failed,) = failed_exhausted(api, stand, alice, chat)
run = replay_id(stand, failed["id"])
assert run.exit_code == 0, run.finished
ready = api.wait_attachment(alice, failed["id"], timeout=JOB_WAIT_S)
assert_variants(api, stand, alice, ready)
job = job_row(stand, failed["id"])
assert job is not None
assert job["attempt"] > WORKER_MAX_ATTEMPTS, job
assert job["attempt_base"] == WORKER_MAX_ATTEMPTS, job
@pytest.mark.spec("attachment-processing", "Replay по причине")
def test_replay_by_code(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
exhausted = failed_exhausted(api, stand, alice, chat, files=2)
invalid = failed_invalid(api, alice, chat)
run = run_replay(
stand,
"--error-code",
"retry_exhausted",
"--apply",
"--operator",
"student-tests",
)
for item in exhausted:
assert run.about("Replayed job", item["id"]), run.finished
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
assert not run.about("Replayed job", invalid["id"])
meta = assert_ok(
api.attachment_meta(alice, invalid["id"]),
200,
"Attachment",
)
assert meta == invalid
@pytest.mark.spec("attachment-processing", "Prefetch больше конкуренции")
def test_prefetch_bounded(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
prefetch, files = 3, 5
filename = fault_filename()
stand("fail-attachments", "ready-stall", filename)
try:
wait_queue_empty(stand)
with outage(stand, "stop", "worker"):
ids = [
api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)["id"]
for _ in range(files)
]
wait_queue_messages(stand, files)
stand(
"run-worker",
"WORKER__CONCURRENCY=1",
f"WORKER__PREFETCH={prefetch}",
)
try:
busy_seen = 0
unacked_seen = 0
deadline = time.monotonic() + JOB_WAIT_S
while True:
rows = attachment_rows(stand)
statuses = [rows[item] for item in ids]
busy = statuses.count("processing")
assert busy <= 1, statuses
busy_seen = max(busy_seen, busy)
unacked = queue_state(stand)["messages_unacknowledged"]
assert unacked <= prefetch, unacked
unacked_seen = max(unacked_seen, unacked)
if statuses.count("ready") == files:
break
assert time.monotonic() < deadline, statuses
time.sleep(0.3)
assert busy_seen == 1
assert unacked_seen > 1, unacked_seen
finally:
stand("stop-extra")
finally:
stand("clear-fail-attachments")
wait_queue_empty(stand)
@pytest.mark.spec("attachments", "Отсутствующий вариант")
def test_missing_variant(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
ready = api.upload_ready(alice, chat["id"], images.sample_png())
stand("delete-variant", ready["id"], "crop")
assert_error(
api.attachment_content(alice, ready["id"], variant="crop"),
404,
)
assert_content(
api.attachment_content(alice, ready["id"]),
images.sample_png(),
"image/png",
)
@pytest.mark.spec(
"storage-consistency",
"Загрузка без подтверждения брокера убрана",
)
def test_cancelled_upload_of_previous_version(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Реплика прежней версии при отказе брокера отменяла задание: строка
`pending`, помеченная к уборке. Её убирает уборщик вместе с объектом."""
started = time.monotonic()
with worker_stopped(stand), relay_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
stand("cancel-to-pending", accepted["id"])
assert job_row(stand, accepted["id"]) is None
assert_invisible(api, alice, chat, accepted["id"])
wait_cleaned(stand, accepted["id"], since=started)
assert job_row(stand, accepted["id"]) is None
@pytest.mark.spec("storage-consistency", "Staging готового вложения убран")
def test_staging_removed_after_ready(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
ready = api.upload_ready(alice, chat["id"], data)
wait_staging_gone(stand, ready["id"])
assert_content(
api.attachment_content(alice, ready["id"]),
data,
"image/png",
)
assert_variants(api, stand, alice, ready)
@pytest.mark.spec(
"storage-consistency",
"Staging неудачного вложения хранится до срока",
)
@pytest.mark.spec("storage-consistency", "Срок хранения прошёл")
def test_staging_retention(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
failed = failed_invalid(api, alice, chat)
attachment_id = failed["id"]
# конечный оригинал записан до декодирования; варианты прежней попытки
for name in VARIANT_NAMES:
put_object(stand, variant_key(attachment_id, name), b"variant")
time.sleep(3 * CLEANER_INTERVAL_S + 1)
assert attachment_id in staged_ids(stand)
assert attachment_id in stored_ids(stand)
dry = run_replay(stand, "--attachment-id", attachment_id)
assert dry.about("Found failed job", attachment_id), dry.finished
stand("age-row", attachment_id, str(STAGING_RETENTION_S + 5))
wait_staging_gone(stand, attachment_id)
assert attachment_id not in stored_ids(stand)
assert stand("variant-object-count", attachment_id) == "0"
meta = assert_ok(
api.attachment_meta(alice, attachment_id),
200,
"Attachment",
)
assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image")
@pytest.mark.spec("storage-consistency", "Текущая работа не убирается")
def test_staging_keeps_current_work(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with worker_stopped(stand):
accepted = api.upload(alice, chat["id"], images.sample_png())
stand("age-row", accepted["id"], str(STAGING_RETENTION_S + 5))
time.sleep(3 * CLEANER_INTERVAL_S + 1)
assert accepted["id"] in staged_ids(stand)
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
def wait_row_held(stand: Stand, attachment_id: str) -> None:
"""Строку вложения держит транзакция уборщика. Старт уборщика при 0,25 CPU — до
нескольких секунд: без ожидания запрос прошёл бы раньше блокировки."""
deadline = time.monotonic() + 3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S + 10
while stand("attachment-free", attachment_id):
assert time.monotonic() < deadline, "cleaner does not hold the row"
time.sleep(0.1)
@pytest.mark.spec("storage-consistency", "Очистка при отказе S3")
def test_staging_cleanup_survives_outage(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Уборщик держит строку и ждёт DELETE в замороженном S3: отправка сообщения
со ссылкой на вложение берёт `FOR KEY SHARE` и не ждёт его. Очередь уборщика
сначала пуста: кандидаты идут от старых, и при замороженном S3 проход каждый раз
прерывался бы на строке прежнего теста, не дойдя до этой."""
drained = api.upload_ready(alice, chat["id"], images.sample_png())
wait_staging_gone(stand, drained["id"])
with outage(stand, "stop", "cleaner"):
ready = api.upload_ready(alice, chat["id"], images.sample_png())
assert ready["id"] in staged_ids(stand)
stand("pause", "s3")
try:
for number in range(3):
wait_row_held(stand, ready["id"])
resp, took = timed(
lambda number=number: api.post_message(
alice,
chat["id"],
json={
"text": f"пока S3 заморожен {number}",
"attachment_ids": [ready["id"]],
},
),
)
assert_ok(resp, 201, "Message")
assert took < ACCEPT_MAX_S, took
row = json.loads(stand("attachment-row", ready["id"]))
assert row["staging_key"] is not None, row
finally:
stand("unpause", "s3")
wait_staging_gone(stand, ready["id"])
@pytest.mark.spec(
"storage-consistency",
"Ключ staging другого вида не удаляется",
)
def test_staging_foreign_key_kept(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
foreign = f"{FAULT_DATABASE}/attachments/{uuid.uuid4()}/foreign"
with outage(stand, "stop", "cleaner"):
ready = api.upload_ready(alice, chat["id"], data)
put_object(stand, foreign, data)
stand("set-staging-key", ready["id"], foreign)
try:
deadline = time.monotonic() + 3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S
while True:
named = [
record
for line in stand("cleaner-log").splitlines()
if line.startswith("{")
for record in [json.loads(line)]
if record.get("event") == "Staging key is not own"
and record.get("object_key") == foreign
]
if named:
break
assert time.monotonic() < deadline, "cleaner did not name the key"
time.sleep(0.3)
assert named[0]["level"] == "warning", named[0]
digest = hashlib.sha256(data).hexdigest()
assert stand("object-sha256", foreign) == digest
row = json.loads(stand("attachment-row", ready["id"]))
assert row["staging_key"] == foreign, row
finally:
# следующий проход уберёт свой ключ, журнал стенда не засоряется
stand("set-staging-key", ready["id"], staging_key(ready["id"]))
stand("delete-object", foreign)
@pytest.mark.spec("storage-consistency", "Объект staging без метаданных")
def test_reconcile_orphan_staging(stand: Stand) -> None:
data = images.sample_png()
orphan = str(uuid.uuid4())
key = staging_key(orphan)
other = f"{FAULT_DATABASE}/staging/{orphan}/other"
put_object(stand, key, data)
put_object(stand, other, data)
young = run_reconcile(stand, "dry-run")
assert not young.about(FOUND, keys={key, other}), young.finished
assert young.finished["listed_staging"] >= 2, young.finished
# старше срока хранения: срок прохода — окружением команды
time.sleep(2)
retention = ("--staging-retention", "1")
for options in (retention, (*HEAD_STRATEGY, *retention)):
old = run_reconcile(stand, "dry-run", options=options)
assert old.kinds(FOUND, keys={key, other}) == {
key: "orphan_staging",
other: "unknown_object",
}, old.finished
assert old.exit_code == 3, old.finished
assert stand("object-sha256", key)
fixed = run_reconcile(stand, "apply", options=retention)
(record,) = fixed.about(FIXED, keys={key})
assert (record["kind"], record["action"]) == (
"orphan_staging",
"deleted_object",
)
assert stand("object-sha256", key) == ""
# чужой ключ под префиксом: назван, не тронут
assert stand("object-sha256", other)
stand("delete-object", other)
def test_defense_faults(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Спек `student-tests`, «Сценарий защиты автоматизирован», шаг 4 (вне сверки
покрытия — сценарий в докстринге), тем же порядком, что `driver.py
defense-faults`: crash после записи объектов и после фиксации до ack, relay
убит до публикации и после подтверждения до отметки, replay неудачного и отказ
replay готового."""
crash_after_objects(api, stand, alice, chat)
crash_after_commit(api, stand, alice, chat)
with outage(stand, "kill", "relay"):
accepted = api.upload(alice, chat["id"], images.sample_png())
assert queue_state(stand)["messages"] == 0
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
crash_relay_after_confirm(api, stand, alice, chat)
replay_returns_work(api, stand, alice, chat)
replay_ready_refused(api, stand, alice, chat)
# Предел незавершённых заданий (design change `track1-queue-overload`, решение
# 13). Работает пятый API стенда: у прочих предел выключен, и они копят задания,
# пока worker стоит у чужих проверок.
BACKLOG_ENV = dict(
item.split("=", 1)
for item in os.environ.get("FAULT_BACKLOG_ENV", "").split()
)
BACKLOG_LIMIT = int(BACKLOG_ENV.get("UPLOADS__MAX_UNFINISHED_JOBS", "3"))
EXPECTED_BACKLOG_ENV = {"UPLOADS__MAX_UNFINISHED_JOBS": "3"}
BACKLOG_RETRY_AFTER_S = "5"
# обе остальные проверки загрузки на пределе — без тела и без места: быстрее
BACKLOG_REJECT_MAX_S = 1
# прежние проверки могли оставить задание на следующую попытку или аренду
UNFINISHED_WAIT_S = 120
# загрузки в полёте реплики: настройки по умолчанию `[uploads]` пятого API
BACKLOG_IN_FLIGHT = 4 + 16
BACKLOG_CLIENTS = 40
def assert_backlog_stand() -> None:
"""Без малого предела пятого API тесты предела ничего не доказывают."""
assert BACKLOG_ENV == EXPECTED_BACKLOG_ENV, (
f"backlog stand started with {BACKLOG_ENV}, "
f"expected {EXPECTED_BACKLOG_ENV}"
)
def unfinished_jobs(stand: Stand) -> int:
return int(stand("unfinished-jobs"))
def wait_no_unfinished(
stand: Stand,
*,
timeout: float = UNFINISHED_WAIT_S,
) -> None:
"""Незавершённых заданий нет: worker и relay работают."""
deadline = time.monotonic() + timeout
while True:
count = unfinished_jobs(stand)
if count == 0:
return
assert time.monotonic() < deadline, f"{count} unfinished jobs left"
time.sleep(0.5)
def assert_backlog_rejected(resp: httpx.Response, elapsed: float) -> None:
assert_error(resp, 429)
assert resp.headers["Retry-After"] == BACKLOG_RETRY_AFTER_S
assert elapsed < BACKLOG_REJECT_MAX_S, f"429 after {elapsed:.2f}s"
def backlog_records(stand: Stand) -> list[dict[str, Any]]:
return [
json.loads(line)
for line in stand("backlog-api-log").splitlines()
if line.startswith("{")
]
@pytest.mark.spec("queue-overload", "Предел достигнут")
@pytest.mark.spec("queue-overload", "Порядок проверок у предела")
@pytest.mark.spec("queue-overload", "Остальной API у предела")
@pytest.mark.spec("queue-overload", "Загрузки снова принимаются")
def test_backlog_limit(
backlog_api: Api,
stand: Stand,
alice: Actor,
eve: Actor,
chat: dict[str, Any],
) -> None:
assert_backlog_stand()
data = images.sample_png()
finished = backlog_api.upload_ready(alice, chat["id"], data)
wait_no_unfinished(stand)
# 30 MiB, которые начинаются как PNG: Content-Length 31 457 280
big = images.PNG_SIGNATURE + bytes(
3 * images.MAX_BYTES - len(images.PNG_SIGNATURE),
)
with worker_stopped(stand):
accepted = [
backlog_api.upload(alice, chat["id"], data)
for _ in range(BACKLOG_LIMIT)
]
assert unfinished_jobs(stand) == BACKLOG_LIMIT
rows, objects = objects_and_rows(stand)
resp, elapsed = timed(
lambda: upload_raw(backlog_api, alice, chat, data),
)
assert_backlog_rejected(resp, elapsed)
assert resp.headers["X-Request-Id"]
assert resp.headers["X-Instance-Id"]
after_rows, after_objects = objects_and_rows(stand)
assert not new_ids(rows, after_rows)
assert not after_objects - objects
assert unfinished_jobs(stand) == BACKLOG_LIMIT
# порядок проверок: сначала дешёвые, и они раньше предела
assert_error(upload_raw(backlog_api, eve, chat, data), 404)
too_big = upload_raw(backlog_api, alice, chat, big)
assert_error(too_big, 413)
assert too_big.request.headers["Content-Length"] == "31457280"
assert_error(
upload_raw(
backlog_api,
alice,
chat,
data,
content_type="text/plain",
),
415,
)
# остальной API у предела
backlog_api.send_message(alice, chat["id"], "text at the limit")
assert_ok(
backlog_api.messages_page(alice, chat["id"]),
200,
"MessagePage",
)
assert (
backlog_api.attachment_meta(alice, finished["id"]).status_code
== 200
)
assert_content(
backlog_api.attachment_content(alice, finished["id"]),
data,
"image/png",
)
backlog_api.send_message(
alice,
chat["id"],
"",
attachment_ids=[accepted[0]["id"]],
)
ready = backlog_api.request("GET", "/health/ready")
assert ready.status_code == 200, describe(ready)
wait_no_unfinished(stand)
more = backlog_api.upload(alice, chat["id"], data)
for attachment in (*accepted, more):
backlog_api.wait_attachment(
alice,
attachment["id"],
timeout=JOB_WAIT_S,
)
records = backlog_records(stand)
started = next(r for r in records if r.get("event") == "Started API")
assert started["uploads_max_unfinished_jobs"] == BACKLOG_LIMIT
rejected = [
record
for record in records
if record.get("event") == "Rejected attachment"
and record.get("reason") == "backlog"
]
assert rejected, "no backlog rejection in the replica log"
assert rejected[-1]["limit"] == BACKLOG_LIMIT
assert rejected[-1]["unfinished"] >= BACKLOG_LIMIT
@pytest.mark.spec("queue-overload", "Завершённые задания не считаются")
def test_backlog_ignores_finished(
backlog_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_backlog_stand()
wait_no_unfinished(stand)
data = images.sample_png()
undecodable = images.jpeg_precision_12()
for _ in range(BACKLOG_LIMIT):
backlog_api.upload_ready(alice, chat["id"], data)
for _ in range(BACKLOG_LIMIT):
failed = backlog_api.upload(
alice,
chat["id"],
undecodable,
content_type="image/jpeg",
)
backlog_api.wait_attachment(
alice,
failed["id"],
status="failed",
timeout=JOB_WAIT_S,
)
wait_no_unfinished(stand)
accepted = backlog_api.upload(alice, chat["id"], data)
backlog_api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
@pytest.mark.spec("queue-overload", "Ожидание следующей попытки считается")
def test_backlog_counts_waiting_retry(
backlog_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_backlog_stand()
wait_no_unfinished(stand)
data = images.sample_png()
with worker_stopped(stand), relay_stopped(stand):
accepted = [
backlog_api.upload(alice, chat["id"], data)
for _ in range(BACKLOG_LIMIT)
]
# срок следующей попытки далеко: задание ждёт, а не работает
stand("set-next-attempt", accepted[0]["id"], "600")
job = job_row(stand, accepted[0]["id"])
assert job is not None and job["state"] == "queued", job
assert job["next_attempt_at"] is not None, job
resp, elapsed = timed(
lambda: upload_raw(backlog_api, alice, chat, data),
)
assert_backlog_rejected(resp, elapsed)
stand("set-next-attempt", accepted[0]["id"], "0")
for attachment in accepted:
backlog_api.wait_attachment(
alice,
attachment["id"],
timeout=JOB_WAIT_S,
)
@pytest.mark.spec("queue-overload", "Предел при остановленных брокере и relay")
def test_backlog_without_broker(
backlog_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_backlog_stand()
wait_no_unfinished(stand)
data = images.sample_png()
accepted = []
with (
worker_stopped(stand),
relay_stopped(stand),
outage(stand, "stop", "rabbitmq"),
):
for _ in range(BACKLOG_LIMIT):
resp, elapsed = timed(
lambda: upload_raw(backlog_api, alice, chat, data),
)
accepted.append(assert_accepted(resp))
assert elapsed < BACKLOG_REJECT_MAX_S, f"202 after {elapsed:.2f}s"
resp, elapsed = timed(
lambda: upload_raw(backlog_api, alice, chat, data),
)
assert_backlog_rejected(resp, elapsed)
for attachment in accepted:
backlog_api.wait_attachment(
alice,
attachment["id"],
timeout=JOB_WAIT_S,
)
@pytest.mark.spec("queue-overload", "Превышение ограничено")
def test_backlog_overshoot_bounded(
backlog_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_backlog_stand()
wait_no_unfinished(stand)
base_url = str(backlog_api.client.base_url)
data = images.sample_png()
barrier = threading.Barrier(BACKLOG_CLIENTS)
def upload_at_once(_: int) -> httpx.Response:
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as client:
barrier.wait(timeout=30)
return upload_raw(client, alice, chat, data)
with worker_stopped(stand), ThreadPoolExecutor(BACKLOG_CLIENTS) as pool:
answers = list(pool.map(upload_at_once, range(BACKLOG_CLIENTS)))
accepted = [resp.json() for resp in answers if resp.status_code == 202]
statuses = sorted({resp.status_code for resp in answers})
assert set(statuses) <= {202, 429}, statuses
assert (
BACKLOG_LIMIT <= len(accepted) <= BACKLOG_LIMIT + BACKLOG_IN_FLIGHT
), f"accepted {len(accepted)}, statuses {statuses}"
for resp in answers:
if resp.status_code == 429:
assert_error(resp, 429)
for attachment in accepted:
backlog_api.wait_attachment(
alice,
attachment["id"],
timeout=JOB_WAIT_S,
)
@pytest.mark.spec("queue-overload", "Предел выключен")
def test_backlog_off(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_backlog_stand()
wait_no_unfinished(stand)
data = images.sample_png()
with worker_stopped(stand):
accepted = [
api.upload(alice, chat["id"], data)
for _ in range(BACKLOG_LIMIT + 2)
]
assert unfinished_jobs(stand) == BACKLOG_LIMIT + 2
for attachment in accepted:
api.wait_attachment(alice, attachment["id"], timeout=JOB_WAIT_S)