Files
mtusi2026lab4/student-tests/test_failures.py
T

755 lines
35 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Проверки отказов (уровень «4»): выполняет только `make failure-checks`.
Стенд и hook готовит `student-tests/stand.py failures`: свои PostgreSQL и
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
идемпотентности 2 с, пул из одного соединения, срок ожидания пула 1 с,
срок операции 2 с). На хранилища тесты действуют только через фикстуру
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
рабочее состояние (`outage`): упавшая проверка не ломает следующие.
Что проверяется: истечение сессии и то, что запросы её не продлевают;
недоступная БД — 503 без деталей и без вызова аутентификации, а не 401 и
не анонимный доступ; ограниченное ожидание при замёрзшей БД и
исчерпанном пуле; откат создания чата; readiness; обход кэша при отказе
Valkey и его возврат без рестарта API. С лабы 3 — ключ идемпотентности:
сбой записи сообщения откатывает и ключ, транзиентная 503 не фиксируется
результатом (повтор с тем же ключом проходит один раз), срок ключа.
С лабы 4 — вложения: отказ и заморозка S3 (503 в пределах срока, текст и
готовность работают), сбой фиксации `ready` после записанного объекта
(компенсация удаляет объект), kill реплики между PUT и фиксацией (строку
и объект убирает уборщик после grace), места загрузок (429 с
`Retry-After`) и срок тела (400), сверка bucket и БД с расхождениями на
обеих сторонах и под одновременными загрузками.
Границы времени — с запасом 0,3–1 с, но на порядок ниже «плохого»
поведения (секунды повторов redis-py, зависший до разморозки запрос).
"""
import hashlib
import os
import re
import statistics
import threading
import time
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import httpx
import pytest
import images
from helpers import (
BEARER_CHALLENGE,
CHATS,
ME,
SAMPLE_PNG,
assert_error,
assert_unauthorized,
attachment_content,
attachment_meta,
chat_ids,
create_chat,
headers,
login,
new_key,
public,
read_history,
register,
send,
upload,
uploaded,
)
pytestmark = pytest.mark.fault
type Stand = Callable[..., str]
RESTORE = {"stop": "start", "pause": "unpause"}
# запас к границам времени: планировщик Docker Desktop
SLACK = 0.3
# значения по умолчанию CACHE_TIMEOUT и CACHE_RETRY_AFTER
CACHE_TIMEOUT = 0.2
CACHE_RETRY_AFTER = 5.0
# окружение «тесного» API (TIGHT_ENV в stand.py)
SESSION_TTL = 2
IDEMPOTENCY_TTL = 2
POOL_TIMEOUT = 1.0
OPERATION_TIMEOUT = 2.0
# ограниченное закрытие соединения (DB_CLOSE_TIMEOUT) и срок readiness
CLOSE_TIMEOUT = 2.0
READINESS_TIMEOUT = 2.0
# лаба 4: срок вызова S3 (S3_TIMEOUT), уборщик стенда (CLEANER_ENV в
# stand.py) и «узкий по загрузкам» API (UPLOADS_ENV)
S3_TIMEOUT = 5.0
CLEANER_GRACE = float(os.environ.get("FAULT_CLEANER_GRACE", "25"))
CLEANER_INTERVAL = float(os.environ.get("FAULT_CLEANER_INTERVAL", "1"))
UPLOAD_WAIT_TIMEOUT = 1.0
UPLOAD_BODY_TIMEOUT = 2.0
@contextmanager
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
"""`stop` или `pause` хранилища; в `finally` — обратно."""
stand(action, service)
try:
yield
finally:
stand(RESTORE[action], service)
def timed(call: Callable[[], httpx.Response]) -> tuple[httpx.Response, float]:
started = time.monotonic()
response = call()
return response, time.monotonic() - started
def read_chat(client: httpx.Client, actor: dict[str, Any], chat: dict[str, Any]):
return client.get(f"{CHATS}/{chat['id']}", headers=headers(actor))
def wait_cache_ready(
client: httpx.Client,
stand: Stand,
actor: dict[str, Any],
chat: dict[str, Any],
) -> None:
"""Дождаться, пока чтение снова пишет карточку в кэш.
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
"""
deadline = time.monotonic() + CACHE_RETRY_AFTER + 2
stand("flush-cache")
while time.monotonic() < deadline:
assert read_chat(client, actor, chat).status_code == 200
if stand("cache-has-chat", chat["id"]) == "1":
return
time.sleep(0.5)
raise AssertionError("cache did not come back")
# --- сессии: срок 2 с на «тесном» API, БД остановлена — на обычном -----------
def test_expired_token_is_401(tight_client: httpx.Client) -> None:
actor = register(tight_client, "ttl")
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
time.sleep(SESSION_TTL + 1)
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
def test_requests_do_not_extend_session(tight_client: httpx.Client) -> None:
actor = register(tight_client, "ttl")
logged_in = time.monotonic()
successes = 0
while time.monotonic() - logged_in < SESSION_TTL - SLACK:
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
successes += 1
time.sleep(0.5)
assert successes >= 3, "too few requests before expiry"
time.sleep(max(0.0, logged_in + SESSION_TTL + 1 - time.monotonic()))
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
def test_token_while_database_stopped_is_503(
client: httpx.Client, stand: Stand
) -> None:
actor = register(client, "db")
with outage(stand, "stop", "postgres"):
response = client.get(ME, headers=headers(actor))
assert_error(response, 503, "unavailable")
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in response.headers
# тот же токен после возврата БД
assert client.get(ME, headers=headers(actor)).json() == public(actor)
# --- хранение: откат, остановленная и замёрзшая БД, исчерпанный пул ------------
def test_members_failure_rolls_back_chat(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
title = f"fault-{uuid.uuid4().hex}"
stand("fail-members", title)
try:
response = client.post(
CHATS,
json={"title": title, "member_ids": [alice["id"], bob["id"]]},
headers=headers(alice),
)
assert_error(response, 500, "internal_error")
finally:
stand("clear-fail-members")
# ни чата, ни участников: вставка строки чата откатилась вместе с ними
for actor in (alice, bob):
assert chat_ids(client, actor) == []
assert stand("count-chats", title) == "0"
def test_database_stopped_and_back(client: httpx.Client, stand: Stand) -> None:
alice = register(client, "alice")
chat = create_chat(client, alice)
with outage(stand, "stop", "postgres"):
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
error = assert_error(response, 503, "unavailable")
print(f"postgres stopped: POST message -> 503 in {took:.3f}s")
assert took < 2, f"503 took {took:.2f}s"
for word in ("postgres", "sql", "connect", "asyncpg"):
assert word not in error["message"].lower()
after = send(client, alice, chat, "после возврата")
assert after.status_code == 201, after.text
# запись во время отказа не зафиксирована, новая — видна
assert read_history(client, alice, chat, limit=50)[0] == [after.json()]
def test_frozen_database_and_exhausted_pool(
tight_client: httpx.Client,
stand: Stand,
) -> None:
"""Замёрзшая БД держит запрос не дольше срока операции и закрытия
соединения; пока пул занят, следующий запрос ждёт не дольше срока пула."""
actor = register(tight_client, "pool")
first: list[tuple[int, float]] = []
def hold_connection() -> None:
with httpx.Client(base_url=str(tight_client.base_url), timeout=20) as holder:
response, took = timed(lambda: holder.get(ME, headers=headers(actor)))
first.append((response.status_code, took))
with outage(stand, "pause", "postgres"):
thread = threading.Thread(target=hold_connection)
thread.start()
time.sleep(SLACK)
response, took = timed(lambda: tight_client.get(ME, headers=headers(actor)))
assert_error(response, 503, "unavailable")
assert took < POOL_TIMEOUT + 0.5, f"pool wait took {took:.2f}s"
thread.join()
status, held = first[0]
print(
f"postgres paused: first request -> {status} in {held:.2f}s, "
f"second (pool of 1 busy) -> {response.status_code} in {took:.2f}s",
)
assert status == 503
assert held < OPERATION_TIMEOUT + CLOSE_TIMEOUT + 1, f"frozen db held {held:.2f}s"
# срок сессии 2 с мог истечь за время паузы: пул проверяем новым входом
fresh = register(tight_client, "pool")
assert tight_client.get(ME, headers=headers(fresh)).status_code == 200
# --- ключ идемпотентности (лаба 3) ----------------------------------------------
def test_message_failure_rolls_back_key(client: httpx.Client, stand: Stand) -> None:
"""Ключ вставлен первым, сообщение упало — откат убирает оба: повтор с
тем же ключом создаёт сообщение, а не 409 и не «пустой» повтор."""
alice = register(client, "alice")
chat = create_chat(client, alice)
text = f"fault-{uuid.uuid4().hex}"
key = new_key()
stand("fail-messages", text)
try:
assert_error(send(client, alice, chat, text, key=key), 500, "internal_error")
finally:
stand("clear-fail-messages")
assert stand("count-keys", chat["id"]) == "0"
assert read_history(client, alice, chat, limit=50)[0] == []
retry = send(client, alice, chat, text, key=key)
assert retry.status_code == 201, retry.text
assert read_history(client, alice, chat, limit=50)[0] == [retry.json()]
assert stand("count-keys", chat["id"]) == "1"
def test_unavailable_is_not_saved_as_result(client: httpx.Client, stand: Stand) -> None:
"""503 — транзиентный результат: тот же ключ после возврата БД даёт 201,
повтор — то же сообщение, в истории одно."""
alice = register(client, "alice")
chat = create_chat(client, alice)
key = new_key()
with outage(stand, "stop", "postgres"):
assert_error(
send(client, alice, chat, "неопределённо", key=key), 503, "unavailable"
)
first = send(client, alice, chat, "неопределённо", key=key)
assert first.status_code == 201, first.text
again = send(client, alice, chat, "неопределённо", key=key)
assert again.json() == first.json()
assert read_history(client, alice, chat, limit=50)[0] == [first.json()]
def test_key_expires_after_ttl(tight_client: httpx.Client) -> None:
"""Срок ключа 2 с на «тесном» API: в срок — повтор и 409, после срока
тот же ключ занимается заново (другое тело — новое сообщение, не 409)."""
alice = register(tight_client, "ttl")
chat = create_chat(tight_client, alice)
key = new_key()
first = send(tight_client, alice, chat, "в срок", key=key).json()
assert send(tight_client, alice, chat, "в срок", key=key).json() == first
assert_error(send(tight_client, alice, chat, "другое", key=key), 409, "conflict")
time.sleep(IDEMPOTENCY_TTL + 1)
# срок сессии на этом API тоже 2 с: новый вход
alice["token"] = login(tight_client, alice)
late = send(tight_client, alice, chat, "после срока", key=key)
assert late.status_code == 201, late.text
assert late.json()["id"] != first["id"]
items, _ = read_history(tight_client, alice, chat, limit=50)
assert [item["text"] for item in items] == ["в срок", "после срока"]
# --- readiness -------------------------------------------------------------------
def test_readiness_without_database(client: httpx.Client, stand: Stand) -> None:
with outage(stand, "stop", "postgres"):
response, took = timed(lambda: client.get("/health/ready"))
assert_error(response, 503, "unavailable")
print(f"postgres stopped: /health/ready -> 503 in {took:.3f}s")
assert took < READINESS_TIMEOUT + 0.5, f"ready took {took:.2f}s"
assert client.get("/health/live").status_code == 200
assert client.get("/health/ready").status_code == 200
def test_readiness_without_cache(client: httpx.Client, stand: Stand) -> None:
with outage(stand, "stop", "valkey"):
assert client.get("/health/ready").status_code == 200
# --- кэш карточки чата -------------------------------------------------------------
def test_flush_cache_refills_on_read(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
wait_cache_ready(client, stand, bob, chat)
stand("flush-cache")
assert stand("cache-has-chat", chat["id"]) == "0"
assert read_chat(client, bob, chat).json() == chat
assert stand("cache-has-chat", chat["id"]) == "1"
def test_valkey_stopped_is_bypassed(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
sent = [send(client, alice, chat, "до отказа").json()]
with outage(stand, "stop", "valkey"):
response, read_took = timed(lambda: read_chat(client, bob, chat))
assert response.json() == chat
assert read_took < CACHE_TIMEOUT + SLACK, f"read took {read_took:.2f}s"
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
assert response.status_code == 201, response.text
print(
f"valkey stopped: GET chat -> 200 in {read_took:.3f}s, "
f"POST message -> 201 in {took:.3f}s",
)
assert took < CACHE_TIMEOUT + SLACK, f"send took {took:.2f}s"
sent.append(response.json())
assert read_history(client, bob, chat, limit=50)[0] == sent
def test_valkey_paused_costs_one_timeout(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
wait_cache_ready(client, stand, bob, chat)
baseline = statistics.median(
timed(lambda: read_chat(client, bob, chat))[1] for _ in range(5)
)
with outage(stand, "pause", "valkey"):
response, took = timed(lambda: read_chat(client, bob, chat))
assert response.json() == chat
assert took < baseline + CACHE_TIMEOUT + SLACK, (
f"read took {took:.2f}s, baseline {baseline:.3f}s"
)
# окно пропуска: следующий запрос срок не ждёт
response, second = timed(lambda: read_chat(client, bob, chat))
assert response.json() == chat
print(
f"valkey paused: baseline {baseline:.3f}s, first read {took:.3f}s, "
f"second read {second:.3f}s",
)
assert second < CACHE_TIMEOUT, f"second read took {second:.2f}s"
def test_valkey_back_without_api_restart(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
with outage(stand, "stop", "valkey"):
assert read_chat(client, bob, chat).status_code == 200
# API не перезапускался; кэш снова в работе после окна пропуска
time.sleep(CACHE_RETRY_AFTER + 0.5)
assert read_chat(client, bob, chat).status_code == 200
assert stand("cache-has-chat", chat["id"]) == "1"
hits = int(stand("cache-hits"))
assert read_chat(client, bob, chat).json() == chat
assert int(stand("cache-hits")) > hits
# --- лаба 4: S3 недоступен ----------------------------------------------------------
def counts(stand: Stand, chat: dict[str, Any]) -> tuple[int, int]:
"""Строк вложений чата: (pending, ready)."""
pending, ready = stand("count-attachments", chat["id"]).split()
return int(pending), int(ready)
def ready(base_url: str) -> bool:
try:
return httpx.get(f"{base_url}/health/ready", timeout=2).status_code == 200
except httpx.HTTPError:
return False
def wait_until(check: Callable[[], bool], timeout: float, step: float = 0.5) -> float:
"""Секунды до выполнения условия; не выполнилось за срок — падение."""
started = time.monotonic()
while time.monotonic() - started < timeout:
if check():
return time.monotonic() - started
time.sleep(step)
raise AssertionError(f"condition not met in {timeout:.0f}s")
def test_upload_with_s3_stopped(client: httpx.Client, stand: Stand) -> None:
"""S3 остановлен: загрузка — 503 быстро, строка `pending` остаётся
уборщику, объекта нет; текст и готовность работают."""
alice = register(client, "alice")
chat = create_chat(client, alice)
with outage(stand, "stop", "s3"):
response, took = timed(lambda: upload(client, alice, chat))
error = assert_error(response, 503, "unavailable")
print(f"s3 stopped: upload -> 503 in {took:.3f}s")
assert took < S3_TIMEOUT, f"503 took {took:.2f}s"
for word in ("s3", "boto", "bucket", "seaweed", "http://"):
assert word not in error["message"].lower()
assert counts(stand, chat) == (1, 0)
assert send(client, alice, chat, "текст без S3").status_code == 201
assert client.get("/health/ready").status_code == 200
# тот же файл после возврата — 201
uploaded(client, alice, chat)
assert counts(stand, chat)[1] == 1
def test_upload_and_download_with_s3_paused(client: httpx.Client, stand: Stand) -> None:
"""Замороженный S3 держит загрузку и скачивание не дольше срока вызова;
метаданные и ссылка в сообщении без S3 работают."""
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
ready = uploaded(client, alice, chat)
with outage(stand, "pause", "s3"):
response, upload_took = timed(lambda: upload(client, alice, chat))
assert_error(response, 503, "unavailable")
response, download_took = timed(
lambda: attachment_content(client, bob, ready["id"])
)
assert_error(response, 503, "unavailable")
print(
f"s3 paused: upload -> 503 in {upload_took:.2f}s, "
f"download -> 503 in {download_took:.2f}s",
)
assert upload_took < S3_TIMEOUT + 1.5, f"upload took {upload_took:.2f}s"
assert download_took < S3_TIMEOUT + 1.5, f"download took {download_took:.2f}s"
assert attachment_meta(client, bob, ready["id"]).json() == ready
linked = send(client, alice, chat, "", attachment_ids=[ready["id"]])
assert linked.status_code == 201, linked.text
content = attachment_content(client, bob, ready["id"])
assert content.content == SAMPLE_PNG
def test_readiness_without_object_storage(client: httpx.Client, stand: Stand) -> None:
"""Без S3 реплика остаётся готовой: иначе отказ S3 вывел бы из LB все
реплики, и текст перестал бы работать тоже."""
with outage(stand, "stop", "s3"):
assert client.get("/health/ready").status_code == 200
# --- лаба 4: согласованность БД и S3 ------------------------------------------------
def test_ready_failure_compensates_and_cleaner_removes_row(
client: httpx.Client, stand: Stand
) -> None:
"""Объект записан, а перевод в `ready` падает как отказ PostgreSQL: 503,
объект удалён до ответа (компенсация), строку `pending` убирает
уборщик после grace. Объектов без строки не остаётся."""
alice = register(client, "alice")
chat = create_chat(client, alice)
filename = f"fault-{uuid.uuid4().hex}.png"
objects_before = int(stand("count-objects"))
stand("fail-ready", filename)
try:
assert_error(upload(client, alice, chat, filename=filename), 503, "unavailable")
finally:
stand("clear-fail-ready")
assert counts(stand, chat) == (1, 0)
assert int(stand("count-objects")) == objects_before
took = wait_until(
lambda: counts(stand, chat) == (0, 0),
CLEANER_GRACE + CLEANER_INTERVAL + 10,
)
print(f"ready failed: object compensated at once, row removed after {took:.1f}s")
assert int(stand("count-objects")) == objects_before
def test_kill_between_put_and_ready(client: httpx.Client, stand: Stand) -> None:
"""Процесс API убит после PUT и до фиксации `ready`: клиент 201 не
получил, после grace уборщик убрал и строку, и объект."""
alice = register(client, "alice")
chat = create_chat(client, alice)
filename = f"stall-{uuid.uuid4().hex}.png"
objects_before = int(stand("count-objects"))
stand("stall-ready", filename, "5")
outcome: list[Any] = []
def slow_upload() -> None:
with httpx.Client(base_url=str(client.base_url), timeout=20) as own:
try:
outcome.append(upload(own, alice, chat, filename=filename).status_code)
except httpx.HTTPError as exc:
outcome.append(type(exc).__name__)
thread = threading.Thread(target=slow_upload)
try:
thread.start()
# объект уже записан, перевод в ready ждёт в триггере
wait_until(lambda: int(stand("count-objects")) == objects_before + 1, 10, 0.2)
stand("kill-api")
thread.join(timeout=20)
finally:
stand("clear-fail-ready")
stand("start-api")
print(f"killed between PUT and ready: client got {outcome}")
assert outcome
assert outcome[0] != 201
# keep-alive соединение фикстуры оборвано вместе с процессом: готовность
# — новым соединением, отказ соединения — «ещё не готов»
wait_until(lambda: ready(str(client.base_url)), 30)
assert counts(stand, chat) == (1, 0)
took = wait_until(
lambda: (
counts(stand, chat) == (0, 0)
and int(stand("count-objects")) == objects_before
),
CLEANER_GRACE + CLEANER_INTERVAL + 15,
)
print(f"cleaner removed row and object {took:.1f}s after the kill check")
# --- лаба 4: места загрузок и срок тела ---------------------------------------------
def slow_body(data: bytes, pause: float) -> Iterator[bytes]:
"""Тело в два приёма с паузой: загрузка держит место всё это время."""
half = len(data) // 2
yield data[:half]
time.sleep(pause)
yield data[half:]
def test_upload_slots_busy_is_429(uploads_client: httpx.Client) -> None:
"""Одно место и одно ожидающее: третья одновременная загрузка — сразу
429 с `Retry-After`, ожидавшая — 429 по сроку ожидания места, у отказов
нет ни строки, ни объекта. Лимиты контракта окружение не меняет."""
alice = register(uploads_client, "alice")
chat = create_chat(uploads_client, alice)
url = str(uploads_client.base_url)
results: dict[str, httpx.Response] = {}
def holder() -> None:
with httpx.Client(base_url=url, timeout=20) as own:
# медленное тело в пределах срока тела: место занято ~1,5 с
results["holder"] = upload(own, alice, chat, slow_body(SAMPLE_PNG, 1.5))
def waiter() -> None:
with httpx.Client(base_url=url, timeout=20) as own:
results["waiter"] = upload(own, alice, chat)
first = threading.Thread(target=holder)
first.start()
time.sleep(0.4)
second = threading.Thread(target=waiter)
second.start()
time.sleep(0.2)
third, took = timed(lambda: upload(uploads_client, alice, chat))
first.join()
second.join()
error = assert_error(third, 429, "rate_limited")
assert third.headers["Retry-After"] == "1"
print(f"slots busy: third upload -> 429 in {took:.3f}s ({error['message']})")
assert took < UPLOAD_WAIT_TIMEOUT, f"queue-full 429 took {took:.2f}s"
assert_error(results["waiter"], 429, "rate_limited")
assert results["holder"].status_code == 201, results["holder"].text
# MAX_UPLOAD_BYTES=1 в окружении этого API лимит не изменил
assert results["holder"].json()["size_bytes"] == len(SAMPLE_PNG)
def test_upload_body_timeout_is_400(uploads_client: httpx.Client, stand: Stand) -> None:
alice = register(uploads_client, "alice")
chat = create_chat(uploads_client, alice)
response, took = timed(
lambda: upload(
uploads_client, alice, chat, slow_body(SAMPLE_PNG, UPLOAD_BODY_TIMEOUT + 1)
)
)
error = assert_error(response, 400, "invalid_request")
print(f"slow body: -> 400 in {took:.2f}s ({error['message']})")
assert counts(stand, chat) == (0, 0)
# место освобождено: следующая загрузка проходит
assert upload(uploads_client, alice, chat).status_code == 201
# --- лаба 4, трек 2: сверка bucket и БД ---------------------------------------------
@contextmanager
def cleaner_stopped(stand: Stand) -> Iterator[None]:
"""Без уборщика: зависшие загрузки сверки он не убрал бы раньше неё."""
stand("stop-cleaner")
try:
yield
finally:
stand("start-cleaner")
def reconcile(stand: Stand, *args: str) -> tuple[int, str]:
"""Разовая сверка в контейнере уборщика: код выхода и журнал."""
output = stand("reconcile", *args)
code = int(output.rsplit("exit_code=", 1)[1])
return code, output
def findings(output: str, verb: str = "found") -> dict[str, str]:
"""object_key → kind из строк «storage discrepancy found|fixed»."""
pattern = rf"storage discrepancy {verb} kind=(\S+) object_key=(\S+)"
return {key: kind for kind, key in re.findall(pattern, output)}
def test_reconcile_finds_and_fixes_both_sides(
client: httpx.Client, stand: Stand
) -> None:
"""Расхождения с обеих сторон: dry-run находит и ничего не меняет,
`--apply` исправляет безопасное, чужой объект остаётся (код 3), повтор
находит только его; вернувшийся оригинал снимает пометку."""
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
prefix = stand("prefix")
with cleaner_stopped(stand):
healthy = uploaded(client, alice, chat)
missing = uploaded(client, alice, chat, images.png(64, 64, seed=1))
resized = uploaded(client, alice, chat, images.png(64, 64, seed=2))
linked = send(client, alice, chat, "", attachment_ids=[missing["id"]])
assert linked.status_code == 201
with outage(stand, "stop", "s3"):
assert_error(upload(client, alice, chat), 503, "unavailable")
assert counts(stand, chat) == (1, 3)
orphan_key = f"{prefix}{uuid.uuid4()}/original"
foreign_key = f"{prefix}not-an-attachment.txt"
stand("put-object", orphan_key, data=b"orphan bytes")
stand("put-object", foreign_key, data=b"foreign bytes")
missing_key = stand("attachment", missing["id"]).split()[3]
stand("delete-object", missing_key)
stand("set-size", resized["id"], str(resized["size_bytes"] + 1))
# строки «старше grace»; у объекта время записи не подделать — ждём
(pending_id,) = stand("pending-ids", chat["id"]).split()
for attachment_id in (healthy["id"], missing["id"], resized["id"], pending_id):
stand("age-row", attachment_id, str(CLEANER_GRACE + 60))
time.sleep(CLEANER_GRACE + 1)
# до пометки байты пропавшего объекта — 500, а не «чужой» 404
assert_error(
attachment_content(client, bob, missing["id"]), 500, "internal_error"
)
objects_before = int(stand("count-objects"))
code, output = reconcile(stand)
found = findings(output)
print("dry-run:", {key.rsplit("/", 2)[-2]: kind for key, kind in found.items()})
assert code == 3
assert found.get(orphan_key) == "orphan_object"
assert found.get(missing_key) == "missing_object"
assert (
found.get(stand("attachment", resized["id"]).split()[3]) == "size_mismatch"
)
assert found.get(foreign_key) == "foreign_object"
assert "stale_pending" in found.values()
assert int(stand("count-objects")) == objects_before
assert counts(stand, chat) == (1, 3)
code, output = reconcile(stand, "--apply")
fixed = findings(output, "fixed")
assert code == 3, output[-2000:]
assert fixed.get(orphan_key) == "orphan_object"
assert fixed.get(missing_key) == "missing_object"
assert stand("object", orphan_key) == "none"
assert counts(stand, chat) == (0, 3)
# потерянный оригинал: метаданные 200, байты 404, ссылка — по-прежнему
assert attachment_meta(client, bob, missing["id"]).status_code == 200
assert_error(attachment_content(client, bob, missing["id"]), 404, "not_found")
assert attachment_content(client, bob, healthy["id"]).content == SAMPLE_PNG
code, output = reconcile(stand)
assert code == 3
assert set(findings(output).values()) == {"foreign_object"}
stand("delete-object", foreign_key)
# оригинал вернулся (тот же SHA-256): пометка снимается
stand("put-object", missing_key, data=images.png(64, 64, seed=1))
code, output = reconcile(stand, "--apply")
assert findings(output, "fixed").get(missing_key) == "restored_object"
assert code == 0, output[-2000:]
restored = attachment_content(client, bob, missing["id"])
assert hashlib.sha256(restored.content).hexdigest() == missing["sha256"]
code, output = reconcile(stand)
summary = [line for line in output.splitlines() if "pass finished" in line]
print("after fixes:", summary[-1].split("pass finished", 1)[1][:400])
assert code == 0
def test_reconcile_apply_under_uploads(new_client: Any, stand: Stand) -> None:
"""`--apply` раз за разом под одновременными загрузками и отправками
четырёх клиентов: ни одно подтверждённое вложение не удалено и не
помечено — каждое 201 скачивается тем же SHA-256."""
setup = new_client()
alice, bob = register(setup, "alice"), register(setup, "bob")
chat = create_chat(setup, alice, bob)
stop = threading.Event()
done: list[tuple[str, str]] = []
errors: list[str] = []
touched: set[str] = set()
def worker(index: int) -> None:
own = new_client()
number = 0
while not stop.is_set():
body = images.png(48, 48, seed=index * 1000 + number)
number += 1
response = upload(own, alice, chat, body)
if response.status_code != 201:
errors.append(f"upload {response.status_code}")
continue
attachment = response.json()
done.append((attachment["id"], attachment["sha256"]))
sent = send(own, alice, chat, "", attachment_ids=[attachment["id"]])
if sent.status_code != 201:
errors.append(f"send {sent.status_code}")
threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)]
for thread in threads:
thread.start()
passes = []
try:
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
code, output = reconcile(stand, "--apply")
passes.append(code)
# id вложения — предпоследняя часть ключа <база>/attachments/<id>/original
touched |= {key.split("/")[-2] for key in findings(output, "fixed")}
finally:
stop.set()
for thread in threads:
thread.join()
print(f"reconcile --apply passes: {passes}; uploads: {len(done)}; errors: {errors}")
assert done
assert not errors
assert not touched & {attachment_id for attachment_id, _ in done}
for attachment_id, sha256 in done:
content = attachment_content(setup, bob, attachment_id)
assert content.status_code == 200, content.text
assert hashlib.sha256(content.content).hexdigest() == sha256