Files
mtusi2026lab4/student-tests/test_failures.py
T
2026-10-22 21:04:01 +03:00

2199 lines
86 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
Valkey и S3, уборщик незавершённых загрузок (grace и интервал —
`FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API
(`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении),
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
соединения) и «узкий по загрузкам» (`FAULT_UPLOADS_BASE_URL`: одно место
загрузки, одна в очереди, ожидание 1 с, срок тела 3 с — change
`upload-memory`). На хранилища и API тесты действуют только через фикстуру
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
рабочее состояние. Решение 4 design change `failure-checks`, решение 10
design change `upload-consistency`, решение 11 design change
`track2-reconciler` (сверка bucket и БД — в конце модуля: её тесты
оставляют помеченные потерянными вложения, прежние тесты таких не ждут).
"""
import hashlib
import json
import os
import statistics
import threading
import time
import uuid
from collections.abc import Callable, Collection, Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any
import httpx
import pytest
import images
from api_client import API, Actor, Api, new_key
from contract import (
BEARER_CHALLENGE,
assert_content,
assert_error,
assert_ok,
assert_unauthorized,
)
pytestmark = pytest.mark.fault
type Stand = Callable[..., str]
RESTORE = {"stop": "start", "pause": "unpause"}
# запас к границам времени: планировщик Docker Desktop
SLACK_S = 0.3
# [cache] timeout_seconds и retry_after_seconds по умолчанию
CACHE_TIMEOUT_S = 0.2
CACHE_RETRY_AFTER_S = 5
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
SESSION_TTL_S = 2
IDEMPOTENCY_TTL_S = 2
POOL_TIMEOUT_S = 1
# срок проверки Postgres в /health/ready
READINESS_TIMEOUT_S = 2
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
OBJECT_STORAGE_HEALTH_S = 1
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
S3_TIMEOUT_S = 5
# сколько секунд hook держит слоты max_connections; clear-connections
# освобождает их раньше
FILL_HOLD_S = 30
ME = f"{API}/users/me"
@contextmanager
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
"""`stop` или `pause` хранилища; в `finally` — обратно."""
stand(action, service)
try:
yield
finally:
stand(RESTORE[action], service)
def timed(
call: Callable[[], httpx.Response],
) -> tuple[httpx.Response, float]:
started = time.monotonic()
resp = call()
return resp, time.monotonic() - started
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
def wait_cache_ready(
api: Api,
stand: Stand,
actor: Actor,
chat: dict[str, Any],
) -> None:
"""Дождаться, пока чтение снова пишет карточку в кэш.
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
"""
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
stand("flush-cache")
while time.monotonic() < deadline:
assert_ok(read_chat(api, actor, chat), 200, "Chat")
if stand("cache-has-chat", chat["id"]) == "1":
return
time.sleep(0.5)
raise AssertionError("cache did not come back")
def chat_titles(api: Api, actor: Actor) -> set[str]:
return {
chat["title"]
for page in api.chat_pages(actor)
for chat in page["items"]
}
# ---------------------------------------------------------------------------
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
# ---------------------------------------------------------------------------
@pytest.mark.spec("sessions", "Истёкший токен")
def test_expired_token(tight_api: Api) -> None:
actor = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
time.sleep(SESSION_TTL_S + 1)
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "Срок не продлевается")
def test_use_does_not_extend_session(tight_api: Api) -> None:
actor = tight_api.register()
logged_in = time.monotonic()
successes = 0
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
successes += 1
time.sleep(0.5)
assert successes >= 3, "too few requests before expiry"
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
actor = api.register()
with outage(stand, "stop", "postgres"):
resp = api.request("GET", ME, actor=actor)
assert_error(resp, 503)
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
# ---------------------------------------------------------------------------
# Хранение: откат, остановленная БД, исчерпанный пул
# ---------------------------------------------------------------------------
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
def test_members_failure_rolls_back_chat(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
) -> None:
title = f"fault-{uuid.uuid4().hex}"
stand("fail-members", title)
try:
resp = api.request(
"POST",
f"{API}/chats",
actor=alice,
json={"title": title, "member_ids": [alice.id, bob.id]},
)
assert_error(resp, 500)
finally:
stand("clear-fail-members")
for actor in (alice, bob):
assert title not in chat_titles(api, actor)
assert stand("count-chats", title) == "0"
@pytest.mark.spec("data-storage", "БД остановлена")
@pytest.mark.spec("data-storage", "БД вернулась")
def test_database_stopped_and_back(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
error = assert_error(resp, 503)
assert took < 2, f"503 took {took:.2f}s"
for word in ("postgres", "sql", "connect"):
assert word not in error["message"].lower()
message = api.send_message(alice, chat["id"], "после возврата")
# запись во время отказа не зафиксирована, новая — видна
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
holder = Api(str(tight_api.client.base_url), timeout=15)
first: list[int] = []
def hold_connection() -> None:
first.append(holder.request("GET", ME, actor=actor).status_code)
with outage(stand, "pause", "postgres"):
thread = threading.Thread(target=hold_connection)
thread.start()
time.sleep(SLACK_S)
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
assert_error(resp, 503)
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
thread.join()
holder.close()
assert first == [503]
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
# hook оборвал соединение пула: при выдаче pool_pre_ping
# переподключается и получает «too many clients» (SQLSTATE 53300)
stand("fill-connections", str(FILL_HOLD_S))
try:
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
error = assert_error(resp, 503)
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
for word in ("postgres", "sql", "client", "connect"):
assert word not in error["message"].lower()
finally:
stand("clear-connections")
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
# сессии 2 с за время заполнения истёк — новый вход
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
# пачка одновременных запросов больше пула обычного API (10)
BURST_THREADS = 48
BURST_AFTER_BARRIER = 4
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
STATS_FLUSH_S = 11
def burst(api: Api, actor: Actor) -> None:
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
одновременно. Все ответы — 200."""
barrier = threading.Barrier(BURST_THREADS)
clients = [
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
]
responses: list[httpx.Response] = []
errors: list[BaseException] = []
lock = threading.Lock()
def worker(client: Api) -> None:
try:
got = [client.request("GET", ME, actor=actor)]
barrier.wait()
got += [
client.request("GET", ME, actor=actor)
for _ in range(BURST_AFTER_BARRIER)
]
except BaseException as e: # noqa: BLE001 — поднимется в тесте
barrier.abort()
with lock:
errors.append(e)
return
with lock:
responses.extend(got)
threads = [
threading.Thread(target=worker, args=(client,)) for client in clients
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
for client in clients:
client.close()
assert not errors, f"burst failed: {errors[0]!r}"
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
for resp in responses:
assert_ok(resp, 200, "User")
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
def test_pool_keeps_connections(
api: Api,
tight_api: Api,
stand: Stand,
) -> None:
actor = api.register()
# соединение «тесного» API могли оборвать прежние тесты
# (fill-connections): его переподключение не должно попасть в окно
tight = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
# прогрев: пул обычного API набирает свой размер
for _ in range(2):
burst(api, actor)
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
# в общую статистику без ожидания блокировки и при занятой блокировке
# откладываются. Простаивающий backend сбрасывает их принудительно по
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
# попадали в окно по одному
time.sleep(STATS_FLUSH_S)
# калибровка: второй вызов видит сессию первого вызова hook, и только её
s1 = int(stand("db-sessions"))
s2 = int(stand("db-sessions"))
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
for _ in range(3):
burst(api, actor)
s3 = int(stand("db-sessions"))
opened = s3 - s2 - 1
assert opened == 0, (
f"PostgreSQL accepted {opened} new sessions during "
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
)
# ---------------------------------------------------------------------------
# Readiness
# ---------------------------------------------------------------------------
@pytest.mark.spec("health", "Хранилище не готово")
def test_readiness_without_database(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(lambda: api.request("GET", "/health/ready"))
assert_error(resp, 503)
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
live = api.request("GET", "/health/live")
assert live.status_code == 200
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Кэш недоступен")
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "valkey"):
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Хранилище объектов недоступно")
@pytest.mark.parametrize("action", ["stop", "pause"])
def test_readiness_without_object_storage(
api: Api,
stand: Stand,
action: str,
) -> None:
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
with outage(stand, action, "s3"):
for path in ("/health/ready", "/health/live"):
resp, took = timed(lambda path=path: api.request("GET", path))
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
# ---------------------------------------------------------------------------
# Кэш карточки чата
# ---------------------------------------------------------------------------
@pytest.mark.spec("chat-cache", "Очистка кэша")
def test_flush_cache(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
stand("flush-cache")
assert stand("cache-has-chat", chat["id"]) == "0"
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert stand("cache-has-chat", chat["id"]) == "1"
@pytest.mark.spec("chat-cache", "Valkey остановлен")
def test_valkey_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
sent = [api.send_message(alice, chat["id"], "до отказа")]
with outage(stand, "stop", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
sent.append(assert_ok(resp, 201, "Message"))
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
assert api.all_messages(bob, chat["id"]) == sent
@pytest.mark.spec("chat-cache", "Valkey завис")
def test_valkey_paused(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
baseline = statistics.median(
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
)
with outage(stand, "pause", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
f"read took {took:.2f}s, baseline {baseline:.3f}s"
)
# окно пропуска: следующий запрос таймаута не ждёт
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
@pytest.mark.spec("chat-cache", "Valkey вернулся")
def test_valkey_back(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "valkey"):
assert_ok(read_chat(api, bob, chat), 200, "Chat")
# API не перезапускался; кэш снова в работе после окна пропуска
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
assert_ok(read_chat(api, bob, chat), 200, "Chat")
assert stand("cache-has-chat", chat["id"]) == "1"
hits = int(stand("cache-hits"))
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert int(stand("cache-hits")) > hits
# ---------------------------------------------------------------------------
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
# ---------------------------------------------------------------------------
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
def test_message_failure_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
# ключ вставляется первым: триггер роняет транзакцию уже после него
text = f"fault-{uuid.uuid4().hex}"
key = new_key()
stand("fail-messages", text)
try:
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 500)
finally:
stand("clear-fail-messages")
# ключ не занят: повтор выполняется заново, а не 500 и не 409
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
def test_database_outage_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
text = "при отказе БД"
key = new_key()
with outage(stand, "stop", "postgres"):
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 503)
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Ключ после срока")
def test_key_free_after_ttl(tight_api: Api) -> None:
alice = tight_api.register()
chat = tight_api.create_chat(alice)
key = new_key()
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
# в пределах срока ключ занят: другое тело — 409
resp = tight_api.post_message(
alice,
chat["id"],
key=key,
json={"text": "после срока"},
)
assert_error(resp, 409)
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
alice = tight_api.login(alice.user, alice.password)
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
assert second["id"] != first["id"]
history = tight_api.all_messages(alice, chat["id"])
assert [item["id"] for item in history] == [first["id"], second["id"]]
# ---------------------------------------------------------------------------
# Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки
# и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает
# строки прежних тестов параллельно (design change upload-consistency,
# решение 10)
# ---------------------------------------------------------------------------
# grace и интервал уборщика стенда отказов (run_failures.py,
# failures.compose.yml)
CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20"))
CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1"))
# запас поверх grace и интервала: проход, вызов hook, планировщик Docker
# Desktop
CLEANUP_SLACK_S = 5
# задержка перевода в ready триггером stall (fault_hook.py)
STALL_S = 4
# срок клиента загрузок в потоке
THREAD_CLIENT_TIMEOUT_S = 30
def attachment_rows(stand: Stand) -> dict[str, str]:
"""Вложения стенда: `id` → статус."""
return dict(line.split() for line in stand("attachment-ids").splitlines())
def stored_ids(stand: Stand) -> set[str]:
"""`id` вложений, у которых есть объект в bucket стенда."""
return set(stand("object-ids").splitlines())
def orphan_objects(stand: Stand) -> set[str]:
"""Объекты вложений без строки. Сначала строки, потом объекты: уборщик
удаляет объект раньше строки, и убранная между вызовами строка не
даёт ложного «объекта без строки»."""
rows = attachment_rows(stand)
return stored_ids(stand) - set(rows)
def ready_without_object(stand: Stand) -> set[str]:
"""Готовые вложения без объекта: строки `ready` уборщик не трогает,
поэтому порядок вызовов здесь не важен."""
return ready_ids(attachment_rows(stand)) - stored_ids(stand)
def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]:
return set(after) - set(before)
def ready_ids(rows: dict[str, str]) -> set[str]:
return {item for item, status in rows.items() if status == "ready"}
def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None:
"""Строка и объект вложения убраны не позже grace + интервал + запас
от `since` (момент до загрузки: `created_at` строки не раньше)."""
deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
while True:
rows = attachment_rows(stand)
stored = stored_ids(stand)
if attachment_id not in rows and attachment_id not in stored:
return
if time.monotonic() > deadline:
raise AssertionError(
f"attachment {attachment_id} is not cleaned "
f"{time.monotonic() - since:.1f}s after upload: "
f"row {rows.get(attachment_id)}, "
f"object {attachment_id in stored}",
)
time.sleep(0.5)
def sleep_until(moment: float) -> None:
time.sleep(max(0.0, moment - time.monotonic()))
def fault_filename() -> str:
"""Уникальное имя загрузки, на которое срабатывает триггер hook."""
return f"fault-{uuid.uuid4().hex}.png"
def upload_raw(
api: Api,
actor: Actor,
chat: dict[str, Any],
data: Any,
**kwargs: Any,
) -> httpx.Response:
return api.post_attachment(actor, chat["id"], data, **kwargs)
def assert_invisible(
api: Api,
actor: Actor,
chat: dict[str, Any],
attachment_id: str,
) -> None:
"""Вложение не в статусе ready: метаданные, байты и ссылка в сообщении
— 404 с тем же `message`, что у неизвестного `id`."""
def calls(item: str) -> tuple[httpx.Response, ...]:
return (
api.attachment_meta(actor, item),
api.attachment_content(actor, item),
api.post_message(
actor,
chat["id"],
json={"text": "ссылка", "attachment_ids": [item]},
),
)
unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))]
hidden = [assert_error(resp, 404) for resp in calls(attachment_id)]
assert [error["message"] for error in hidden] == [
error["message"] for error in unknown
]
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
@pytest.mark.spec("attachments", "Незавершённая загрузка не видна")
def test_upload_with_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
before = attachment_rows(stand)
with outage(stand, "stop", "s3"):
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
assert_error(resp, 503)
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
after = attachment_rows(stand)
# строка-намерение осталась, готовых не прибавилось; её уберёт
# уборщик (test_cleaner_survives_outages)
(pending,) = new_ids(before, after)
assert after[pending] == "pending"
assert ready_ids(after) == ready_ids(before)
# замечание S6 verify attachments: id последней pending — от hook
assert stand("last-pending") == pending
assert_invisible(api, alice, chat, pending)
api.send_message(alice, chat["id"], "текст без S3")
api.upload(alice, chat["id"], data)
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
def test_rejected_uploads_leave_nothing(
api: Api,
stand: Stand,
alice: Actor,
eve: Actor,
chat: dict[str, Any],
) -> None:
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
def chunked() -> Iterator[bytes]:
for offset in range(0, len(big), 1024 * 1024):
yield big[offset : offset + 1024 * 1024]
# хотя бы одно готовое вложение: списки hook не пусты случайно
ready = api.upload(alice, chat["id"], images.sample_png())
before = (attachment_rows(stand), stored_ids(stand))
assert before[0].get(ready["id"]) == "ready", before[0]
assert ready["id"] in before[1], before[1]
rejected = (
(404, upload_raw(api, eve, chat, images.sample_png())),
(413, upload_raw(api, alice, chat, big)),
(413, upload_raw(api, alice, chat, chunked())),
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
(
415,
upload_raw(
api,
alice,
chat,
images.sample_png(),
content_type="text/plain",
),
),
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
)
for status, resp in rejected:
assert_error(resp, status)
assert not new_ids(before[0], attachment_rows(stand))
assert not stored_ids(stand) - before[1]
@pytest.mark.spec("attachments", "Сбой записи метаданных")
def test_metadata_insert_failure(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
before = (attachment_rows(stand), stored_ids(stand))
stand("fail-attachments", "insert", filename)
try:
resp = upload_raw(
api,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
# строки нет — объект не записывался
assert not new_ids(before[0], attachment_rows(stand))
assert not stored_ids(stand) - before[1]
class BodyAborted(Exception):
"""Клиент обрывает передачу тела загрузки."""
@pytest.mark.spec("attachments", "Обрыв передачи тела")
def test_interrupted_body(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
before = (attachment_rows(stand), stored_ids(stand))
def aborted_body() -> Iterator[bytes]:
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
raise BodyAborted
# свой клиент: его соединение обрывается вместе с телом
with (
Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own,
pytest.raises((BodyAborted, httpx.TransportError)),
):
upload_raw(own, alice, chat, aborted_body())
# API замечает обрыв не мгновенно; строки до конца тела нет в любом
# случае
time.sleep(1)
assert not new_ids(before[0], attachment_rows(stand))
assert not stored_ids(stand) - before[1]
api.upload(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachments", "Фиксация ready не удалась")
@pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута")
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
def test_ready_failure_compensated(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "ready", filename)
try:
started = time.monotonic()
resp = upload_raw(
api,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
after = attachment_rows(stand)
(pending,) = new_ids(before, after)
assert after[pending] == "pending"
assert stand("last-pending") == pending
# компенсация: объект удалён до ответа 503
assert pending not in stored_ids(stand)
assert_invisible(api, alice, chat, pending)
# моложе grace строку уборщик не трогает: к половине grace он сделал
# около G / 2 / I проходов
sleep_until(started + CLEANER_GRACE_S / 2)
assert attachment_rows(stand).get(pending) == "pending"
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
@pytest.mark.spec("storage-consistency", "Объект без фиксации убран")
def test_killed_between_put_and_ready(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
before = attachment_rows(stand)
outcome: dict[str, Any] = {}
client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
def upload() -> None:
try:
outcome["status"] = upload_raw(
client,
alice,
chat,
images.sample_png(),
filename=filename,
).status_code
except httpx.TransportError as e:
outcome["error"] = e
stand("fail-attachments", "stall", filename)
killed = False
try:
started = time.monotonic()
thread = threading.Thread(target=upload)
thread.start()
# строка pending и объект — до конца задержки перевода в ready
deadline = started + STALL_S
pending = ""
while time.monotonic() < deadline:
pending = stand("last-pending")
if (
pending
and pending not in before
and pending in stored_ids(stand)
):
break
time.sleep(0.05)
else:
raise AssertionError("no pending row with object during stall")
stand("kill-api")
killed = True
thread.join(THREAD_CLIENT_TIMEOUT_S)
assert not thread.is_alive()
# 201 до клиента не дошёл: перевод в ready ждал в триггере
assert outcome.get("status") != 201, outcome
assert attachment_rows(stand).get(pending) == "pending"
assert pending in stored_ids(stand)
finally:
client.close()
stand("clear-fail-attachments")
if killed:
stand("start-api")
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
api.upload(alice, chat["id"], images.sample_png())
@pytest.mark.spec("attachments", "Исход фиксации неизвестен")
def test_ready_commit_outcome_unknown(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Перевод в ready падает на самом COMMIT (отложенный триггер): API не
знает, зафиксировано ли вложение, и объект не трогает (verify W2)."""
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
started = time.monotonic()
resp = upload_raw(
api,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
after = attachment_rows(stand)
(pending,) = new_ids(before, after)
assert after[pending] == "pending"
# компенсации нет: вложение могло стать ready
assert pending in stored_ids(stand)
assert_invisible(api, alice, chat, pending)
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
@pytest.mark.spec("storage-consistency", "Загрузка дольше grace")
def test_upload_outlives_grace(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Строка вставлена «на час старше», перевод в ready ждёт в триггере,
пока её не уберёт уборщик стенда: загрузка, пережившая grace, без
заморозки процесса (verify W3)."""
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "outlive", filename)
# ожидание в триггере до 4,5 с: срок клиента больше
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
try:
with slow:
resp = upload_raw(
slow,
alice,
chat,
images.sample_png(),
filename=filename,
)
# не 201 (ready без объекта) и не 500 (дефект ветки)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
rows = attachment_rows(stand)
assert not new_ids(before, rows), rows
assert not stored_ids(stand) - set(before)
assert not orphan_objects(stand)
@pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта")
def test_cleaner_failure_after_object_delete(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Уборщик удалил объект, а удаление строки упало (08006), пока загрузка
ждёт перевода в ready: строка помечена до удаления объекта и ready уже
не станет (verify W1)."""
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "undeletable", filename)
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
try:
with slow:
resp = upload_raw(
slow,
alice,
chat,
images.sample_png(),
filename=filename,
)
assert_error(resp, 503)
(pending,) = new_ids(before, attachment_rows(stand))
assert attachment_rows(stand).get(pending) == "pending"
assert not ready_without_object(stand)
assert_invisible(api, alice, chat, pending)
finally:
stand("clear-fail-attachments")
# строка старше grace: следующий проход доводит уборку
returned = time.monotonic()
wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S)
assert not orphan_objects(stand)
assert not ready_without_object(stand)
@pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки")
@pytest.mark.spec("storage-consistency", "БД недоступна во время уборки")
@pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты")
def test_cleaner_survives_outages(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
ready = api.upload(alice, chat["id"], data)
with outage(stand, "stop", "s3"):
before = attachment_rows(stand)
started = time.monotonic()
assert_error(upload_raw(api, alice, chat, data), 503)
(pending,) = new_ids(before, attachment_rows(stand))
# PostgreSQL недоступен, пока загрузка моложе grace
with outage(stand, "stop", "postgres"):
time.sleep(3)
# grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте
sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S)
assert attachment_rows(stand).get(pending) == "pending"
# S3 вернулся: уборщик без перезапуска убирает строку за проход
returned = time.monotonic()
deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
while pending in attachment_rows(stand):
assert time.monotonic() < deadline, (
f"pending {pending} is still there "
f"{time.monotonic() - returned:.1f}s after S3 came back"
)
time.sleep(0.5)
assert pending not in stored_ids(stand)
assert not orphan_objects(stand)
# готовое вложение старше grace на месте
meta = assert_ok(
api.attachment_meta(alice, ready["id"]),
200,
"Attachment",
)
assert meta == ready
assert meta["status"] == "ready"
assert_content(
api.attachment_content(alice, ready["id"]),
data,
"image/png",
)
@pytest.mark.spec("attachments", "S3 остановлен")
def test_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
attachment = api.upload(alice, chat["id"], data)
with outage(stand, "stop", "s3"):
resp, elapsed = timed(
lambda: api.attachment_content(bob, attachment["id"]),
)
assert_error(resp, 503)
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
meta = assert_ok(
api.attachment_meta(bob, attachment["id"]),
200,
"Attachment",
)
assert meta == attachment
message = api.send_message(
bob,
chat["id"],
"",
attachment_ids=[attachment["id"]],
)
# история с attachment_ids — из PostgreSQL, S3 ей не нужен
history = api.all_messages(alice, chat["id"])
assert [item["id"] for item in history] == [message["id"]]
assert history[0]["attachment_ids"] == [attachment["id"]]
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
assert_content(
api.attachment_content(bob, attachment["id"]),
data,
"image/png",
)
def test_environment_does_not_change_limits(
api: Api,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Спек `service-configuration`, «Окружение не меняет лимит» (вне
сверки покрытия — сценарий в докстринге): обычный API стенда
отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1`
(`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт,
64 000 пикселей) всё равно принимается: лимиты — константы."""
limits = dict(
item.split("=", 1)
for item in os.environ.get("FAULT_LIMITS_ENV", "").split()
)
# без этих переменных на стенде тест ничего не доказывает
assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, (
f"stand started without limit variables: {limits}"
)
api.upload(alice, chat["id"], images.sample_png(), filename="sample.png")
@pytest.mark.spec("attachments", "S3 завис")
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
def test_object_storage_paused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
attachment = api.upload(alice, chat["id"], data)
before = attachment_rows(stand)
started = time.monotonic()
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
with slow, outage(stand, "pause", "s3"):
for call in (
lambda: upload_raw(slow, alice, chat, data),
lambda: slow.attachment_content(alice, attachment["id"]),
):
resp, elapsed = timed(call)
assert_error(resp, 503)
# срок вызова S3 и немного на шаги с БД, а не зависание
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
f"503 after {elapsed:.2f}s"
)
assert_content(
api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)
# загрузка во время заморозки оставила строку pending; объект после
# разморозки не дописался, строку уборщик убирает после grace
(pending,) = new_ids(before, attachment_rows(stand))
wait_cleaned(stand, pending, since=started)
assert not orphan_objects(stand)
# ---------------------------------------------------------------------------
# Места загрузки и срок тела (change upload-memory, design, решение 11):
# «узкий по загрузкам» API — одно место, одна загрузка в очереди, ожидание
# места и срок тела из FAULT_UPLOADS_ENV (run_failures.py, UPLOADS_ENV).
# Место держит загрузка, тело которой стоит на threading.Event теста
# ---------------------------------------------------------------------------
UPLOADS_ENV = dict(
item.split("=", 1)
for item in os.environ.get("FAULT_UPLOADS_ENV", "").split()
)
# окружение, с которым тесты ниже ждут «узкий» API
EXPECTED_UPLOADS_ENV = {
"UPLOADS__MAX_CONCURRENT": "1",
"UPLOADS__MAX_WAITING": "1",
"UPLOADS__WAIT_TIMEOUT_SECONDS": "1",
"UPLOADS__BODY_TIMEOUT_SECONDS": "3",
}
WAIT_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__WAIT_TIMEOUT_SECONDS", "1"))
BODY_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__BODY_TIMEOUT_SECONDS", "3"))
# отказ без ожидания места: вход, членство и ответ
IMMEDIATE_S = 0.5
# пауза, за которую запрос в соседнем потоке доходит до места или очереди
SETTLE_S = 0.2
# сколько держатель места ждёт отпускания, если тест упал раньше
HOLD_MAX_S = 10
def assert_uploads_stand() -> None:
"""Без этого окружения «узкого» API тесты мест ничего не доказывают."""
assert UPLOADS_ENV == EXPECTED_UPLOADS_ENV, (
f"uploads stand started with {UPLOADS_ENV}, "
f"expected {EXPECTED_UPLOADS_ENV}"
)
class Upload:
"""Загрузка в своём потоке и своём соединении; ответ — `result()`."""
def __init__(self, api: Api, actor: Actor, chat: dict[str, Any]) -> None:
self._base_url = str(api.client.base_url)
self._actor = actor
self._chat = chat
self._outcome: dict[str, Any] = {}
self._thread: threading.Thread | None = None
def start(self, data: Any, **kwargs: Any) -> "Upload":
def run() -> None:
with Api(self._base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
started = time.monotonic()
try:
self._outcome["resp"] = upload_raw(
own,
self._actor,
self._chat,
data,
**kwargs,
)
except (BodyAborted, httpx.TransportError) as e:
self._outcome["error"] = e
self._outcome["elapsed"] = time.monotonic() - started
self._outcome["ended"] = time.monotonic()
self._thread = threading.Thread(target=run)
self._thread.start()
return self
def result(self) -> tuple[httpx.Response, float, float]:
"""Ответ, длительность и момент ответа (`time.monotonic`)."""
assert self._thread is not None
self._thread.join(THREAD_CLIENT_TIMEOUT_S)
assert not self._thread.is_alive(), "upload thread hangs"
assert "resp" in self._outcome, self._outcome
return (
self._outcome["resp"],
self._outcome["elapsed"],
self._outcome["ended"],
)
@contextmanager
def held_slot(
api: Api,
actor: Actor,
chat: dict[str, Any],
data: bytes,
) -> Iterator[Callable[[], httpx.Response]]:
"""Загрузка A держит место: отдаёт половину тела и ждёт отпускания.
Внутри — после того, как A заняла место; вызов отдаваемой функции
отпускает A и возвращает её ответ. В `finally` A отпускается всегда.
"""
sent = threading.Event()
release = threading.Event()
def body() -> Iterator[bytes]:
yield data[: len(data) // 2]
sent.set()
if not release.wait(HOLD_MAX_S):
raise BodyAborted
yield data[len(data) // 2 :]
holder = Upload(api, actor, chat).start(body())
try:
assert sent.wait(HOLD_MAX_S), "holder did not start its body"
# вход и членство — короткие шаги БД, затем место
time.sleep(SETTLE_S)
def finish() -> httpx.Response:
release.set()
return holder.result()[0]
yield finish
finally:
release.set()
def slow_chunks(data: bytes, *, parts: int, seconds: float) -> Iterator[bytes]:
"""Тело `parts` частями, растянутое на `seconds`."""
size = -(-len(data) // parts)
for offset in range(0, len(data), size):
if offset:
time.sleep(seconds / (parts - 1))
yield data[offset : offset + size]
def objects_and_rows(stand: Stand) -> tuple[dict[str, str], set[str]]:
return attachment_rows(stand), stored_ids(stand)
@pytest.mark.spec("attachments", "Все места заняты")
@pytest.mark.spec("attachments", "Очередь ожидания полна")
@pytest.mark.spec("api-conventions", "Лимит одновременных загрузок")
def test_upload_slots_busy(
uploads_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
data = images.sample_png()
before = objects_and_rows(stand)
with held_slot(uploads_api, alice, chat, data) as finish_holder:
# B ждёт место в очереди, C застаёт полную очередь
waiting = Upload(uploads_api, alice, chat).start(data)
time.sleep(SETTLE_S)
busy, busy_s = timed(
lambda: upload_raw(uploads_api, alice, chat, data),
)
waited, waited_s, _ = waiting.result()
# A отпускается сразу: её тело ограничено сроком тела стенда
holder = finish_holder()
assert_error(busy, 429)
assert busy_s < IMMEDIATE_S, f"429 of a full queue after {busy_s:.2f}s"
assert_error(waited, 429)
assert waited_s >= WAIT_TIMEOUT_S, f"429 after {waited_s:.2f}s"
for resp in (busy, waited):
assert int(resp.headers["Retry-After"]) >= 1
attachment = assert_ok(holder, 201, "Attachment")
# из трёх загрузок осталась только принятая: у 429 ни строки, ни объекта
rows, stored = objects_and_rows(stand)
assert new_ids(before[0], rows) == {attachment["id"]}
assert stored - before[1] == {attachment["id"]}
@pytest.mark.spec("attachments", "Отказ раньше очереди")
def test_rejections_before_queue(
uploads_api: Api,
alice: Actor,
eve: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
data = images.sample_png()
# 30 MiB, которые начинаются как PNG: Content-Length 31 457 280
big = images.PNG_SIGNATURE + bytes(
3 * images.MAX_BYTES - len(images.PNG_SIGNATURE),
)
with held_slot(uploads_api, alice, chat, data) as finish_holder:
waiting = Upload(uploads_api, alice, chat).start(data)
time.sleep(SETTLE_S)
early = [
(status, *timed(call))
for status, call in (
(404, lambda: upload_raw(uploads_api, eve, chat, data)),
(413, lambda: upload_raw(uploads_api, alice, chat, big)),
(
415,
lambda: upload_raw(
uploads_api,
alice,
chat,
data,
content_type="text/plain",
),
),
)
]
early_done = time.monotonic()
waited, _, waited_at = waiting.result()
holder = finish_holder()
# очередь была полна всё время ранних отказов
assert_error(waited, 429)
assert early_done < waited_at
for status, resp, elapsed in early:
assert_error(resp, status)
# без ожидания места: ожидающий получил бы его не раньше срока
assert elapsed < WAIT_TIMEOUT_S, f"{status} after {elapsed:.2f}s"
assert early[1][1].request.headers["Content-Length"] == "31457280"
assert_ok(holder, 201, "Attachment")
@pytest.mark.spec("attachments", "Ожидание места")
@pytest.mark.spec(
"storage-consistency",
"Ожидание места не входит в возраст загрузки",
)
def test_upload_waits_for_slot(
uploads_api: Api,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
first = images.sample_png()
second = images.png(80, 60)
with held_slot(uploads_api, alice, chat, first) as finish_holder:
waiting = Upload(uploads_api, bob, chat).start(second)
time.sleep(SETTLE_S)
# A заканчивается раньше срока ожидания B
holder = finish_holder()
held = assert_ok(holder, 201, "Attachment")
waited_resp, waited_s, _ = waiting.result()
waited = assert_ok(waited_resp, 201, "Attachment")
assert waited_s < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, waited_s
# метаданные ожидавшей записаны после перевода в ready державшей место:
# ожидание прошло до строки pending (граница B уборщика прежняя)
assert waited["created_at"] > held["updated_at"], (held, waited)
for attachment, data in ((held, first), (waited, second)):
assert_content(
uploads_api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)
@pytest.mark.spec("attachments", "Место освобождается")
def test_slot_released_after_failures(
uploads_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
over = images.png_of_size(images.MAX_BYTES + 1)
def aborted_body() -> Iterator[bytes]:
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
raise BodyAborted
def next_upload_gets_slot(after: str) -> None:
resp, elapsed = timed(
lambda: upload_raw(uploads_api, alice, chat, images.sample_png()),
)
assert_ok(resp, 201, "Attachment")
assert elapsed < WAIT_TIMEOUT_S, (
f"upload after {after} waited {elapsed:.2f}s for a slot"
)
assert_error(
upload_raw(
uploads_api,
alice,
chat,
slow_chunks(over, parts=10, seconds=0),
),
413,
)
next_upload_gets_slot("413 by counter")
assert_error(upload_raw(uploads_api, alice, chat, b"not an image"), 415)
next_upload_gets_slot("415 by content")
slow = slow_chunks(
images.sample_png(),
parts=4,
seconds=BODY_TIMEOUT_S + 1,
)
assert_error(upload_raw(uploads_api, alice, chat, slow), 400)
next_upload_gets_slot("400 by body timeout")
with (
Api(
str(uploads_api.client.base_url),
timeout=THREAD_CLIENT_TIMEOUT_S,
) as own,
pytest.raises((BodyAborted, httpx.TransportError)),
):
upload_raw(own, alice, chat, aborted_body())
# API замечает обрыв не мгновенно
time.sleep(SETTLE_S)
next_upload_gets_slot("interrupted body")
with outage(stand, "stop", "s3"):
assert_error(
upload_raw(uploads_api, alice, chat, images.sample_png()),
503,
)
next_upload_gets_slot("503 with S3 stopped")
@pytest.mark.spec("attachments", "Медленный клиент")
@pytest.mark.spec("attachments", "Медленная передача в срок")
def test_slow_body(
uploads_api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
assert_uploads_stand()
data = images.sample_png()
before = objects_and_rows(stand)
late = upload_raw(
uploads_api,
alice,
chat,
slow_chunks(data, parts=6, seconds=BODY_TIMEOUT_S + 1),
)
error = assert_error(late, 400)
assert error["message"] == "Upload body was not received in time"
rows, stored = objects_and_rows(stand)
assert not new_ids(before[0], rows)
assert not stored - before[1]
# мелкие части с паузами, но в срок: тело принято как есть
in_time = slow_chunks(data, parts=20, seconds=BODY_TIMEOUT_S / 2)
resp, elapsed = timed(
lambda: upload_raw(uploads_api, alice, chat, in_time),
)
attachment = assert_ok(resp, 201, "Attachment")
assert elapsed < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, elapsed
assert attachment["size_bytes"] == len(data)
assert attachment["sha256"] == hashlib.sha256(data).hexdigest()
assert_content(
uploads_api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)
# ---------------------------------------------------------------------------
# Сверка bucket и БД (change track2-reconciler, design, решение 11). Hook
# вносит расхождения с обеих сторон и запускает разовую сверку; плановая
# сверка уборщика стенда выключена. Записи сверки фильтруются по своим id и
# ключам: остатки прежних тестов (строки pending) могут попасть в отчёт, и
# код выхода сверяется с записями того же прохода, а не с числом. Объекты
# состарить нельзя (LastModified ставит S3): тесты объектов без строки ждут
# G + 1 с, строки состаривает hook (age-row)
# ---------------------------------------------------------------------------
# имя базы стенда: ключ объекта вложения — `<база>/attachments/<id>/original`
FAULT_DATABASE = os.environ.get("FAULT_DATABASE", "messenger")
# на сколько hook состаривает строку: заведомо больше grace
AGED_S = CLEANER_GRACE_S + 60
# объекты без строки старше grace: grace и секунда точности LastModified
OBJECT_AGE_WAIT_S = CLEANER_GRACE_S + 1
FOUND = "Found storage discrepancy"
FIXED = "Fixed storage discrepancy"
FINISHED = "Finished reconcile pass"
LIMIT_EXCEEDED = "Reconcile changes exceed limit"
# сверка под нагрузкой: потоки загрузок, период сверки, пауза потока
LOAD_THREADS = 4
LOAD_EXTRA_S = 15
LOAD_RECONCILE_PERIOD_S = 3
LOAD_PAUSE_S = 0.1
# плановая сверка отдельного уборщика: столько секунд он работает
SCHEDULED_RUN_S = 6
@dataclass(frozen=True)
class Reconcile:
"""Один разовый проход сверки: записи журнала процесса и код выхода."""
records: list[dict[str, Any]]
exit_code: int
def events(self, event: str) -> list[dict[str, Any]]:
return [item for item in self.records if item.get("event") == event]
@property
def finished(self) -> dict[str, Any]:
(record,) = self.events(FINISHED)
return record
def about(
self,
event: str,
*,
ids: Collection[str] = (),
keys: Collection[str] = (),
) -> list[dict[str, Any]]:
"""Записи `event` о своих `id` вложений и ключах объектов."""
return [
item
for item in self.events(event)
if item.get("attachment_id") in ids
or item.get("object_key") in keys
]
def kinds(
self,
event: str,
*,
ids: Collection[str] = (),
keys: Collection[str] = (),
) -> dict[str, str]:
"""`id` или ключ → класс расхождения в записях `event`."""
found: dict[str, str] = {}
for item in self.about(event, ids=ids, keys=keys):
name = (
item["attachment_id"]
if item.get("attachment_id") in ids
else item["object_key"]
)
found[name] = item["kind"]
return found
def assert_code_matches(self) -> None:
"""Код выхода соответствует записям прохода: 3 — остались
расхождения (запись `Found`) или отказ по пределу, иначе 0."""
finished = self.finished
assert finished["interrupted_by"] is None, finished
expected = 3 if self.events(FOUND) or finished["refused"] else 0
assert self.exit_code == expected, (self.exit_code, finished)
def run_reconcile(
stand: Stand,
mode: str,
limit: int | None = None,
) -> Reconcile:
"""Разовая сверка hook: `dry-run` или `apply`, с пределом изменений."""
args = [mode] if limit is None else [mode, str(limit)]
lines = stand("reconcile", *args).splitlines()
assert lines and lines[-1].startswith("exit_code="), lines[-5:]
records = []
for line in lines[:-1]:
if line.startswith("{"):
records.append(json.loads(line))
return Reconcile(records=records, exit_code=int(lines[-1].split("=")[1]))
def object_key(attachment_id: str) -> str:
return f"{FAULT_DATABASE}/attachments/{attachment_id}/original"
def lost_ids(stand: Stand) -> set[str]:
return set(stand("lost-ids").splitlines())
def put_object(stand: Stand, key: str, data: bytes) -> None:
stand("put-object", key, stdin=data)
def lose_original(stand: Stand, attachment: dict[str, Any]) -> None:
"""Готовое вложение старше grace без объекта: строка состарена, объект
удалён мимо API."""
stand("age-row", attachment["id"], str(AGED_S))
stand("delete-object", object_key(attachment["id"]))
def assert_lost_content(
api: Api,
actor: Actor,
attachment: dict[str, Any],
*,
aged: bool = False,
) -> None:
"""Помеченное вложение: байты — 404, метаданные — как при загрузке.
`aged` — строку состарил hook (`age-row`): её `created_at` и
`updated_at` сдвинуты им, а не сверкой, и сравниваются остальные поля.
"""
assert_error(api.attachment_content(actor, attachment["id"]), 404)
meta = assert_ok(
api.attachment_meta(actor, attachment["id"]),
200,
"Attachment",
)
if aged:
times = ("created_at", "updated_at")
meta = {key: value for key, value in meta.items() if key not in times}
attachment = {
key: value for key, value in attachment.items() if key not in times
}
assert meta == attachment
@pytest.mark.spec("storage-consistency", "Расхождения с обеих сторон найдены")
@pytest.mark.spec("storage-consistency", "Исправление расхождений")
@pytest.mark.spec("storage-consistency", "Повторная сверка")
@pytest.mark.spec("attachments", "Потерянный оригинал")
def test_reconcile_finds_and_fixes(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
with outage(stand, "stop", "cleaner"):
a, b, c = (api.upload(alice, chat["id"], data) for _ in range(3))
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
orphan = str(uuid.uuid4())
keys = {"o": object_key(orphan), "b": object_key(b["id"])}
put_object(stand, keys["o"], data)
stand("delete-object", object_key(a["id"]))
stand("delete-row", b["id"])
stand("set-size", c["id"], str(len(data) + 1))
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
resp = upload_raw(api, alice, chat, data, filename=filename)
assert_error(resp, 503)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
assert pending in stored_ids(stand)
# последний объект записан только что: после G + 1 всё старше grace
time.sleep(OBJECT_AGE_WAIT_S)
ids = {a["id"], c["id"], pending}
snapshot = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, ids=ids, keys=keys.values()) == {
keys["o"]: "orphan_object",
keys["b"]: "orphan_object",
a["id"]: "missing_object",
c["id"]: "size_mismatch",
pending: "stale_pending",
}
for record in dry.about(FOUND, ids=ids, keys=keys.values()):
assert record["age_seconds"] >= CLEANER_GRACE_S, record
assert dry.exit_code == 3
# dry-run ничего не меняет: ни строк, ни объектов, ни пометок
after = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
assert after == snapshot
assert_error(api.attachment_content(alice, a["id"]), 500)
assert_ok(api.attachment_meta(alice, a["id"]), 200, "Attachment")
fixed = run_reconcile(stand, "apply")
actions = {
record.get("attachment_id")
if record.get("attachment_id") in ids
else record["object_key"]: record["action"]
for record in fixed.about(FIXED, ids=ids, keys=keys.values())
}
assert actions == {
keys["o"]: "deleted_object",
keys["b"]: "deleted_object",
a["id"]: "marked_lost",
c["id"]: "marked_lost",
pending: "removed_pending",
}
assert not fixed.about(FOUND, ids=ids, keys=keys.values())
fixed.assert_code_matches()
rows, stored = attachment_rows(stand), stored_ids(stand)
assert not {orphan, b["id"], pending} & stored
assert pending not in rows
assert {a["id"], c["id"]} <= lost_ids(stand)
assert rows[a["id"]] == rows[c["id"]] == "ready"
assert_lost_content(api, alice, a)
# у C размер в метаданных изменил hook (set-size), а не сверка
assert_lost_content(api, alice, c | {"size_bytes": len(data) + 1})
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids=ids, keys=keys.values())
assert not again.about(FIXED, ids=ids, keys=keys.values())
again.assert_code_matches()
@pytest.mark.spec("storage-consistency", "Свежие не тронуты")
@pytest.mark.spec("storage-consistency", "Используемое вложение не удаляется")
@pytest.mark.spec("storage-consistency", "Чужой объект не удаляется")
def test_reconcile_keeps_fresh_used_and_unknown(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
used = api.upload(alice, chat["id"], data)
message = api.send_message(
alice,
chat["id"],
"",
attachment_ids=[used["id"]],
)
lose_original(stand, used)
# ключ, которого API не пишет: под префиксом вложений, но не <id>/original
unknown = f"{FAULT_DATABASE}/attachments/x-{uuid.uuid4()}"
put_object(stand, unknown, data)
time.sleep(OBJECT_AGE_WAIT_S)
# свежие: готовое вложение без объекта и объект без строки
fresh = api.upload(alice, chat["id"], data)
stand("delete-object", object_key(fresh["id"]))
fresh_object = str(uuid.uuid4())
put_object(stand, object_key(fresh_object), data)
ready_before = ready_ids(attachment_rows(stand))
try:
run = run_reconcile(stand, "apply")
keys = {unknown, object_key(fresh_object)}
ids = {used["id"], fresh["id"], fresh_object}
assert run.kinds(FOUND, ids=ids, keys=keys) == {
unknown: "unknown_object",
}
assert run.kinds(FIXED, ids=ids, keys=keys) == {
used["id"]: "missing_object",
}
assert run.exit_code == 3
assert run.finished["fresh_objects"] >= 1, run.finished
assert run.finished["fresh_rows"] >= 1, run.finished
# свежий объект на месте, свежее вложение не помечено
assert fresh_object in stored_ids(stand)
lost = lost_ids(stand)
assert fresh["id"] not in lost
# используемое: помечено, метаданные и ссылка в истории на месте
assert used["id"] in lost
assert_lost_content(api, alice, used, aged=True)
history = api.all_messages(alice, chat["id"])
(sent,) = (item for item in history if item["id"] == message["id"])
assert sent["attachment_ids"] == [used["id"]]
assert ready_before <= ready_ids(attachment_rows(stand))
# чужой объект не тронут: следующий проход снова его видит
again = run_reconcile(stand, "dry-run")
assert again.kinds(FOUND, keys={unknown}) == {
unknown: "unknown_object",
}
finally:
# стенд — без чужого и лишнего объекта, у свежего — его байты
stand("delete-object", unknown)
stand("delete-object", object_key(fresh_object))
put_object(stand, object_key(fresh["id"]), data)
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
def test_reconcile_restored_original(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.png_noise(64, 48, seed=7)
restored = api.upload(alice, chat["id"], data)
lose_original(stand, restored)
marked = run_reconcile(stand, "apply")
assert marked.kinds(FIXED, ids={restored["id"]}) == {
restored["id"]: "missing_object",
}
assert restored["id"] in lost_ids(stand)
assert_lost_content(api, alice, restored, aged=True)
# те же байты снова под ключом вложения
put_object(stand, object_key(restored["id"]), data)
cleared = run_reconcile(stand, "apply")
(record,) = cleared.about(FIXED, ids={restored["id"]})
assert (record["kind"], record["action"]) == (
"restored_object",
"cleared_lost",
)
assert restored["id"] not in lost_ids(stand)
assert_content(
api.attachment_content(alice, restored["id"]),
data,
"image/png",
)
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids={restored["id"]})
@pytest.mark.spec("storage-consistency", "Предел изменений")
def test_reconcile_change_limit(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
limit = 2
batch = [
api.upload(alice, chat["id"], images.sample_png()) for _ in range(3)
]
ids = {item["id"] for item in batch}
for item in batch:
lose_original(stand, item)
refused = run_reconcile(stand, "apply", limit)
assert refused.exit_code == 3
(error,) = refused.events(LIMIT_EXCEEDED)
assert error["level"] == "error"
assert error["changes"] >= len(batch), error
assert error["max_changes"] == limit
assert refused.finished["refused"] is True
assert refused.finished["level"] == "warning"
assert not refused.about(FIXED, ids=ids)
assert not ids & lost_ids(stand)
assert all(
record["kind"] == "missing_object"
for record in refused.about(FOUND, ids=ids)
)
# предел не меньше числа изменений: исправлено всё
applied = run_reconcile(stand, "apply", 100)
assert set(applied.kinds(FIXED, ids=ids)) == ids
assert ids <= lost_ids(stand)
@pytest.mark.spec(
"storage-consistency",
"Объект, на который ссылается строка, не удаляется",
)
def test_reconcile_keeps_referenced_objects(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W1 verify: объект `<X>/original`, у строки которого другой
ключ, и объект `<Z>/original`, на который ссылается строка Y другого
`id`, — чужие, а не объекты без метаданных."""
data = images.sample_png()
x, y = (api.upload(alice, chat["id"], data) for _ in range(2))
z = str(uuid.uuid4())
keys = {
"x": object_key(x["id"]),
"y": object_key(y["id"]),
"z": object_key(z),
# ключ X: под префиксом вложений, объекта по нему нет
"other": f"{FAULT_DATABASE}/attachments/x-other-{uuid.uuid4()}",
}
put_object(stand, keys["z"], data)
try:
stand("set-key", x["id"], keys["other"])
stand("set-key", y["id"], keys["z"])
for item in (x, y):
stand("age-row", item["id"], str(AGED_S))
# объект Z записан только что: после G + 1 все объекты старше grace
time.sleep(OBJECT_AGE_WAIT_S)
unknown = {keys[name]: "unknown_object" for name in ("x", "y", "z")}
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, keys=keys.values()) == unknown | {
keys["other"]: "missing_object",
}
assert dry.exit_code == 3
applied = run_reconcile(stand, "apply")
assert applied.kinds(FIXED, keys=keys.values()) == {
keys["other"]: "missing_object",
}
assert applied.kinds(FOUND, keys=keys.values()) == unknown
assert applied.exit_code == 3
# ни один из трёх объектов не удалён, строки на месте
assert {x["id"], y["id"], z} <= stored_ids(stand)
rows = attachment_rows(stand)
assert rows[x["id"]] == rows[y["id"]] == "ready"
# X без объекта по своему ключу помечено, у Y объект по ключу есть
lost = lost_ids(stand)
assert x["id"] in lost
assert y["id"] not in lost
finally:
# прежние ключи; объект Z, на который больше никто не ссылается, —
# прочь, чтобы следующие проходы не удаляли его как свой
stand("set-key", x["id"], keys["x"])
stand("set-key", y["id"], keys["y"])
stand("delete-object", keys["z"])
@pytest.mark.spec(
"storage-consistency",
"Предел не останавливает уборку зависших",
)
def test_reconcile_limit_keeps_cleanup(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W1 verify: отказ по пределу не мешает уборке зависшей
загрузки — её уборщик делает и без сверки."""
data = images.sample_png()
limit = 2
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
with outage(stand, "stop", "cleaner"):
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
assert_error(
upload_raw(api, alice, chat, data, filename=filename),
503,
)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
assert pending in stored_ids(stand)
stand("age-row", pending, str(AGED_S))
batch = [api.upload(alice, chat["id"], data) for _ in range(3)]
ids = {item["id"] for item in batch}
for item in batch:
lose_original(stand, item)
refused = run_reconcile(stand, "apply", limit)
assert refused.exit_code == 3
(error,) = refused.events(LIMIT_EXCEEDED)
assert error["changes"] >= len(batch), error
assert refused.finished["refused"] is True
assert refused.kinds(FIXED, ids={pending}) == {
pending: "stale_pending",
}
(record,) = refused.about(FIXED, ids={pending})
assert record["action"] == "removed_pending"
assert pending not in attachment_rows(stand)
assert pending not in stored_ids(stand)
# изменений под пределом нет: ни одной пометки
assert not refused.about(FIXED, ids=ids)
assert not ids & lost_ids(stand)
# стенд — без внесённых потерь для следующих тестов
applied = run_reconcile(stand, "apply", 100)
assert set(applied.kinds(FIXED, ids=ids)) == ids
@pytest.mark.spec(
"storage-consistency",
"Ключ вне префикса вложений не трогается",
)
def test_reconcile_foreign_key_untouched(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
"""Замечание W2 verify: строка с ключом вне `<база>/attachments/` —
готовая и зависшая загрузка — только в отчёте как чужой объект: сверка
не делает HEAD чужого префикса, не помечает и не убирает строку."""
data = images.sample_png()
# уборщик стенда стоит: иначе зависшую загрузку убрал бы он
with outage(stand, "stop", "cleaner"):
ready = api.upload(alice, chat["id"], data)
filename = fault_filename()
before = attachment_rows(stand)
stand("fail-attachments", "commit", filename)
try:
assert_error(
upload_raw(api, alice, chat, data, filename=filename),
503,
)
finally:
stand("clear-fail-attachments")
(pending,) = new_ids(before, attachment_rows(stand))
ids = {ready["id"], pending}
try:
for item in ids:
# префикс своей базы, но не вложений: список его не видит
stand("set-key", item, f"{FAULT_DATABASE}/elsewhere/{item}")
stand("age-row", item, str(AGED_S))
unknown = dict.fromkeys(ids, "unknown_object")
dry = run_reconcile(stand, "dry-run")
assert dry.kinds(FOUND, ids=ids) == unknown
assert dry.exit_code == 3
applied = run_reconcile(stand, "apply")
assert applied.kinds(FOUND, ids=ids) == unknown
assert not applied.about(FIXED, ids=ids)
assert applied.exit_code == 3
# ни пометки, ни уборки: строки и объекты под префиксом на месте
rows = attachment_rows(stand)
assert (rows.get(ready["id"]), rows.get(pending)) == (
"ready",
"pending",
)
assert ready["id"] not in lost_ids(stand)
assert ids <= stored_ids(stand)
finally:
# прежние ключи у тех строк, что остались: зависшую загрузку
# после теста уберёт уборщик стенда вместе с её объектом
rows = attachment_rows(stand)
for item in ids & set(rows):
stand("set-key", item, object_key(item))
@pytest.mark.spec("storage-consistency", "S3 недоступен во время сверки")
def test_reconcile_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
missing = api.upload(alice, chat["id"], images.sample_png())
lose_original(stand, missing)
with outage(stand, "stop", "s3"):
interrupted = run_reconcile(stand, "apply")
assert interrupted.exit_code == 1
assert interrupted.finished["interrupted_by"] == "object_storage"
assert not interrupted.events(FIXED)
assert missing["id"] not in lost_ids(stand)
applied = run_reconcile(stand, "apply")
assert applied.kinds(FIXED, ids={missing["id"]}) == {
missing["id"]: "missing_object",
}
assert missing["id"] in lost_ids(stand)
@pytest.mark.spec(
"storage-consistency",
"Прерванная сверка доводится повтором",
)
def test_reconcile_interrupted_by_database(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
filename = fault_filename()
failing = api.upload(
alice,
chat["id"],
images.sample_png(),
filename=filename,
)
other = api.upload(alice, chat["id"], images.sample_png())
ids = {failing["id"], other["id"]}
for item in (failing, other):
lose_original(stand, item)
stand("fail-attachments", "mark", filename)
try:
interrupted = run_reconcile(stand, "apply")
assert interrupted.exit_code == 1
assert interrupted.finished["interrupted_by"] == "database"
assert failing["id"] not in lost_ids(stand)
finally:
stand("clear-fail-attachments")
applied = run_reconcile(stand, "apply")
assert ids <= lost_ids(stand)
assert not applied.about(FOUND, ids=ids)
again = run_reconcile(stand, "dry-run")
assert not again.about(FOUND, ids=ids)
@pytest.mark.spec("storage-consistency", "Сверка под нагрузкой")
def test_reconcile_under_load(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
"""Загрузки и отправки дольше grace, сверка с `--apply` раз в 3 с:
внесённое исправлено, ни одно подтверждённое вложение не потеряно."""
data = images.sample_png()
orphans = {str(uuid.uuid4()) for _ in range(2)}
for item in orphans:
put_object(stand, object_key(item), data)
injected = [api.upload(alice, chat["id"], data) for _ in range(2)]
for item in injected:
lose_original(stand, item)
lost_before = lost_ids(stand)
deadline = time.monotonic() + CLEANER_GRACE_S + LOAD_EXTRA_S
lock = threading.Lock()
uploads: dict[str, str] = {}
messages: dict[str, list[str]] = {}
failures: list[str] = []
passes: list[Reconcile] = []
def load(worker: int) -> None:
actor = alice if worker % 2 == 0 else bob
with Api(
str(api.client.base_url),
timeout=THREAD_CLIENT_TIMEOUT_S,
) as client:
number = 0
while time.monotonic() < deadline:
number += 1
body = images.png_noise(200, 170, seed=worker * 10**6 + number)
resp = client.post_attachment(actor, chat["id"], body)
if resp.status_code != 201:
with lock:
failures.append(f"upload {resp.status_code}")
continue
attachment = assert_ok(resp, 201, "Attachment")
# загрузка подтверждена: она обязана скачиваться
with lock:
uploads[attachment["id"]] = hashlib.sha256(
body,
).hexdigest()
message = client.send_message(
actor,
chat["id"],
"",
attachment_ids=[attachment["id"]],
)
with lock:
messages[message["id"]] = [attachment["id"]]
time.sleep(LOAD_PAUSE_S)
def reconcile() -> None:
while time.monotonic() < deadline:
started = time.monotonic()
passes.append(run_reconcile(stand, "apply"))
sleep_until(started + LOAD_RECONCILE_PERIOD_S)
def guarded(target: Callable[..., None], *args: Any) -> None:
# исключение потока — в список, а не молча в stderr
try:
target(*args)
except Exception as e:
with lock:
failures.append(repr(e))
threads = [
threading.Thread(target=guarded, args=(load, worker))
for worker in range(LOAD_THREADS)
]
threads.append(threading.Thread(target=guarded, args=(reconcile,)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert not failures, failures[:5]
assert uploads, "no uploads under load"
assert len(passes) >= 3, len(passes)
assert all(item.exit_code in (0, 3) for item in passes), [
item.exit_code for item in passes
]
# внесённые исправлены: объекты без строки удалены, потери помечены
assert not orphans & stored_ids(stand)
assert lost_ids(stand) - lost_before == {item["id"] for item in injected}
# ни одно подтверждённое вложение не потеряно
for attachment_id, digest in uploads.items():
resp = api.attachment_content(alice, attachment_id)
assert resp.status_code == 200, (attachment_id, resp.status_code)
assert hashlib.sha256(resp.content).hexdigest() == digest
history = {
item["id"]: item["attachment_ids"]
for item in api.all_messages(alice, chat["id"])
}
for message_id, attachment_ids in messages.items():
assert history[message_id] == attachment_ids
@pytest.mark.spec("storage-consistency", "Плановая сверка без команды")
def test_scheduled_reconcile(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
scheduled = api.upload(alice, chat["id"], images.sample_png())
lose_original(stand, scheduled)
log = stand("run-cleaner", str(SCHEDULED_RUN_S))
records = [
json.loads(line) for line in log.splitlines() if line.startswith("{")
]
passes = [item for item in records if item.get("event") == FINISHED]
assert passes, log[-2000:]
assert any(item["apply"] for item in passes), passes
assert scheduled["id"] in lost_ids(stand)