Upload answers 202 after staging, a queued job in PostgreSQL and a broker confirm; a separate worker publishes the original and a thumbnail with bounded retries. Adds the job and variant tables, the broker service, the worker S3 user, cleanup and reconcile support for staging and variants, the failure stand with broker and worker, bench and driver updates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4301 lines
165 KiB
Python
4301 lines
165 KiB
Python
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
|
||
|
||
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
|
||
Valkey и S3, уборщик незавершённых загрузок (grace и интервал —
|
||
`FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API
|
||
(`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении),
|
||
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
|
||
соединения), «узкий по загрузкам» (`FAULT_UPLOADS_BASE_URL`: одно место
|
||
загрузки, одна в очереди, ожидание 1 с, срок тела 3 с — change
|
||
`upload-memory`) и «с коротким сроком S3» (`FAULT_S3_BASE_URL`: срок вызова
|
||
S3 1 с — change `fix-upload-put-flake`), брокер и worker (4 попытки, пауза
|
||
5 с, срок попытки 10 с, срок остановки 2 с — `FAULT_WORKER_*`, change
|
||
`async-attachments`). На хранилища и API тесты действуют только через
|
||
фикстуру `stand` (hook `student-tests/fault_hook.py`) и всегда возвращают
|
||
их в рабочее состояние. Решение 4 design change `failure-checks`, решение
|
||
10 design change `upload-consistency`, решение 11 design change
|
||
`track2-reconciler` (сверка bucket и БД — в конце модуля: её тесты
|
||
оставляют помеченные потерянными вложения и объекты staging без строки,
|
||
прежние тесты таких не ждут), решение 3 design change
|
||
`fix-upload-put-flake`, решение 15 design change `async-attachments`.
|
||
"""
|
||
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import math
|
||
import os
|
||
import random
|
||
import statistics
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable, Collection, Iterator, Sequence
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from contextlib import contextmanager
|
||
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
from jsonschema import Draft202012Validator, FormatChecker
|
||
|
||
import images
|
||
from api_client import API, Actor, Api, new_key
|
||
from contract import (
|
||
BEARER_CHALLENGE,
|
||
assert_accepted,
|
||
assert_content,
|
||
assert_error,
|
||
assert_ok,
|
||
assert_unauthorized,
|
||
assert_variant_content,
|
||
describe,
|
||
)
|
||
|
||
pytestmark = pytest.mark.fault
|
||
|
||
type Stand = Callable[..., str]
|
||
|
||
RESTORE = {"stop": "start", "pause": "unpause", "kill": "start"}
|
||
# запас к границам времени: планировщик Docker Desktop
|
||
SLACK_S = 0.3
|
||
# [cache] timeout_seconds и retry_after_seconds по умолчанию
|
||
CACHE_TIMEOUT_S = 0.2
|
||
CACHE_RETRY_AFTER_S = 5
|
||
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
|
||
SESSION_TTL_S = 2
|
||
IDEMPOTENCY_TTL_S = 2
|
||
POOL_TIMEOUT_S = 1
|
||
# срок проверки Postgres в /health/ready
|
||
READINESS_TIMEOUT_S = 2
|
||
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
|
||
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
|
||
OBJECT_STORAGE_HEALTH_S = 1
|
||
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
|
||
S3_TIMEOUT_S = 5
|
||
# сколько секунд hook держит слоты max_connections; clear-connections
|
||
# освобождает их раньше
|
||
FILL_HOLD_S = 30
|
||
ME = f"{API}/users/me"
|
||
|
||
|
||
@contextmanager
|
||
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
||
"""`stop`, `pause` или `kill` сервиса; в `finally` — обратно."""
|
||
stand(action, service)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand(RESTORE[action], service)
|
||
|
||
|
||
def timed(
|
||
call: Callable[[], httpx.Response],
|
||
) -> tuple[httpx.Response, float]:
|
||
started = time.monotonic()
|
||
resp = call()
|
||
return resp, time.monotonic() - started
|
||
|
||
|
||
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
|
||
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
|
||
|
||
|
||
def wait_cache_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
||
|
||
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
||
"""
|
||
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
|
||
stand("flush-cache")
|
||
while time.monotonic() < deadline:
|
||
assert_ok(read_chat(api, actor, chat), 200, "Chat")
|
||
if stand("cache-has-chat", chat["id"]) == "1":
|
||
return
|
||
time.sleep(0.5)
|
||
raise AssertionError("cache did not come back")
|
||
|
||
|
||
def chat_titles(api: Api, actor: Actor) -> set[str]:
|
||
return {
|
||
chat["title"]
|
||
for page in api.chat_pages(actor)
|
||
for chat in page["items"]
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Истёкший токен")
|
||
def test_expired_token(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
time.sleep(SESSION_TTL_S + 1)
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Срок не продлевается")
|
||
def test_use_does_not_extend_session(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
logged_in = time.monotonic()
|
||
successes = 0
|
||
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
successes += 1
|
||
time.sleep(0.5)
|
||
assert successes >= 3, "too few requests before expiry"
|
||
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
|
||
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
|
||
actor = api.register()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.request("GET", ME, actor=actor)
|
||
assert_error(resp, 503)
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Хранение: откат, остановленная БД, исчерпанный пул
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
|
||
def test_members_failure_rolls_back_chat(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
) -> None:
|
||
title = f"fault-{uuid.uuid4().hex}"
|
||
stand("fail-members", title)
|
||
try:
|
||
resp = api.request(
|
||
"POST",
|
||
f"{API}/chats",
|
||
actor=alice,
|
||
json={"title": title, "member_ids": [alice.id, bob.id]},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-members")
|
||
for actor in (alice, bob):
|
||
assert title not in chat_titles(api, actor)
|
||
assert stand("count-chats", title) == "0"
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "БД остановлена")
|
||
@pytest.mark.spec("data-storage", "БД вернулась")
|
||
def test_database_stopped_and_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
error = assert_error(resp, 503)
|
||
assert took < 2, f"503 took {took:.2f}s"
|
||
for word in ("postgres", "sql", "connect"):
|
||
assert word not in error["message"].lower()
|
||
message = api.send_message(alice, chat["id"], "после возврата")
|
||
# запись во время отказа не зафиксирована, новая — видна
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
|
||
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
holder = Api(str(tight_api.client.base_url), timeout=15)
|
||
first: list[int] = []
|
||
|
||
def hold_connection() -> None:
|
||
first.append(holder.request("GET", ME, actor=actor).status_code)
|
||
|
||
with outage(stand, "pause", "postgres"):
|
||
thread = threading.Thread(target=hold_connection)
|
||
thread.start()
|
||
time.sleep(SLACK_S)
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
assert_error(resp, 503)
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
thread.join()
|
||
holder.close()
|
||
assert first == [503]
|
||
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
|
||
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
# hook оборвал соединение пула: при выдаче pool_pre_ping
|
||
# переподключается и получает «too many clients» (SQLSTATE 53300)
|
||
stand("fill-connections", str(FILL_HOLD_S))
|
||
try:
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
error = assert_error(resp, 503)
|
||
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
for word in ("postgres", "sql", "client", "connect"):
|
||
assert word not in error["message"].lower()
|
||
finally:
|
||
stand("clear-connections")
|
||
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
|
||
# сессии 2 с за время заполнения истёк — новый вход
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
# пачка одновременных запросов больше пула обычного API (10)
|
||
BURST_THREADS = 48
|
||
BURST_AFTER_BARRIER = 4
|
||
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
|
||
STATS_FLUSH_S = 11
|
||
|
||
|
||
def burst(api: Api, actor: Actor) -> None:
|
||
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
|
||
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
|
||
одновременно. Все ответы — 200."""
|
||
barrier = threading.Barrier(BURST_THREADS)
|
||
clients = [
|
||
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
|
||
]
|
||
responses: list[httpx.Response] = []
|
||
errors: list[BaseException] = []
|
||
lock = threading.Lock()
|
||
|
||
def worker(client: Api) -> None:
|
||
try:
|
||
got = [client.request("GET", ME, actor=actor)]
|
||
barrier.wait()
|
||
got += [
|
||
client.request("GET", ME, actor=actor)
|
||
for _ in range(BURST_AFTER_BARRIER)
|
||
]
|
||
except BaseException as e: # noqa: BLE001 — поднимется в тесте
|
||
barrier.abort()
|
||
with lock:
|
||
errors.append(e)
|
||
return
|
||
with lock:
|
||
responses.extend(got)
|
||
|
||
threads = [
|
||
threading.Thread(target=worker, args=(client,)) for client in clients
|
||
]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
for client in clients:
|
||
client.close()
|
||
assert not errors, f"burst failed: {errors[0]!r}"
|
||
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
|
||
for resp in responses:
|
||
assert_ok(resp, 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
|
||
def test_pool_keeps_connections(
|
||
api: Api,
|
||
tight_api: Api,
|
||
stand: Stand,
|
||
) -> None:
|
||
actor = api.register()
|
||
# соединение «тесного» API могли оборвать прежние тесты
|
||
# (fill-connections): его переподключение не должно попасть в окно
|
||
tight = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
|
||
# прогрев: пул обычного API набирает свой размер
|
||
for _ in range(2):
|
||
burst(api, actor)
|
||
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
|
||
# в общую статистику без ожидания блокировки и при занятой блокировке
|
||
# откладываются. Простаивающий backend сбрасывает их принудительно по
|
||
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
|
||
# попадали в окно по одному
|
||
time.sleep(STATS_FLUSH_S)
|
||
# калибровка: второй вызов видит сессию первого вызова hook, и только её
|
||
s1 = int(stand("db-sessions"))
|
||
s2 = int(stand("db-sessions"))
|
||
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
|
||
for _ in range(3):
|
||
burst(api, actor)
|
||
s3 = int(stand("db-sessions"))
|
||
opened = s3 - s2 - 1
|
||
assert opened == 0, (
|
||
f"PostgreSQL accepted {opened} new sessions during "
|
||
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Readiness
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище не готово")
|
||
def test_readiness_without_database(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(lambda: api.request("GET", "/health/ready"))
|
||
assert_error(resp, 503)
|
||
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
|
||
live = api.request("GET", "/health/live")
|
||
assert live.status_code == 200
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Кэш недоступен")
|
||
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище объектов недоступно")
|
||
@pytest.mark.parametrize("action", ["stop", "pause"])
|
||
def test_readiness_without_object_storage(
|
||
api: Api,
|
||
stand: Stand,
|
||
action: str,
|
||
) -> None:
|
||
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
|
||
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
|
||
with outage(stand, action, "s3"):
|
||
for path in ("/health/ready", "/health/live"):
|
||
resp, took = timed(lambda path=path: api.request("GET", path))
|
||
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
|
||
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Кэш карточки чата
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Очистка кэша")
|
||
def test_flush_cache(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
stand("flush-cache")
|
||
assert stand("cache-has-chat", chat["id"]) == "0"
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey остановлен")
|
||
def test_valkey_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
sent = [api.send_message(alice, chat["id"], "до отказа")]
|
||
with outage(stand, "stop", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
sent.append(assert_ok(resp, 201, "Message"))
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
|
||
assert api.all_messages(bob, chat["id"]) == sent
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey завис")
|
||
def test_valkey_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
baseline = statistics.median(
|
||
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
|
||
)
|
||
with outage(stand, "pause", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
|
||
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
||
)
|
||
# окно пропуска: следующий запрос таймаута не ждёт
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey вернулся")
|
||
def test_valkey_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
# API не перезапускался; кэш снова в работе после окна пропуска
|
||
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
hits = int(stand("cache-hits"))
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert int(stand("cache-hits")) > hits
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
|
||
def test_message_failure_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
# ключ вставляется первым: триггер роняет транзакцию уже после него
|
||
text = f"fault-{uuid.uuid4().hex}"
|
||
key = new_key()
|
||
stand("fail-messages", text)
|
||
try:
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-messages")
|
||
# ключ не занят: повтор выполняется заново, а не 500 и не 409
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
|
||
def test_database_outage_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
text = "при отказе БД"
|
||
key = new_key()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 503)
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Ключ после срока")
|
||
def test_key_free_after_ttl(tight_api: Api) -> None:
|
||
alice = tight_api.register()
|
||
chat = tight_api.create_chat(alice)
|
||
key = new_key()
|
||
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
|
||
# в пределах срока ключ занят: другое тело — 409
|
||
resp = tight_api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": "после срока"},
|
||
)
|
||
assert_error(resp, 409)
|
||
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
|
||
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
|
||
alice = tight_api.login(alice.user, alice.password)
|
||
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
|
||
assert second["id"] != first["id"]
|
||
history = tight_api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [first["id"], second["id"]]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки
|
||
# и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает
|
||
# строки прежних тестов параллельно (design change upload-consistency,
|
||
# решение 10)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# grace и интервал уборщика стенда отказов (run_failures.py,
|
||
# failures.compose.yml)
|
||
CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20"))
|
||
CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1"))
|
||
# запас поверх grace и интервала: проход, вызов hook, планировщик Docker
|
||
# Desktop
|
||
CLEANUP_SLACK_S = 5
|
||
# задержка перевода в queued триггером stall и в ready триггером ready-stall
|
||
# (fault_hook.py)
|
||
STALL_S = 4
|
||
# срок клиента загрузок в потоке
|
||
THREAD_CLIENT_TIMEOUT_S = 30
|
||
# имя базы стенда: ключи объектов — `<база>/attachments/<id>/original` и
|
||
# `<база>/staging/<id>/original`
|
||
FAULT_DATABASE = os.environ.get("FAULT_DATABASE", "messenger")
|
||
|
||
|
||
def attachment_rows(stand: Stand) -> dict[str, str]:
|
||
"""Вложения стенда: `id` → статус."""
|
||
return dict(line.split() for line in stand("attachment-ids").splitlines())
|
||
|
||
|
||
def stored_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений с объектом в конечном префиксе bucket стенда."""
|
||
return set(stand("object-ids").splitlines())
|
||
|
||
|
||
def staged_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений с объектом в staging."""
|
||
return set(stand("staging-ids").splitlines())
|
||
|
||
|
||
def object_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений с объектом в любом из двух префиксов."""
|
||
return stored_ids(stand) | staged_ids(stand)
|
||
|
||
|
||
def orphan_objects(stand: Stand) -> set[str]:
|
||
"""Объекты вложений без строки. Сначала строки, потом объекты: уборщик
|
||
удаляет объект раньше строки, и убранная между вызовами строка не
|
||
даёт ложного «объекта без строки»."""
|
||
rows = attachment_rows(stand)
|
||
return object_ids(stand) - set(rows)
|
||
|
||
|
||
def accepted_without_staging(stand: Stand) -> set[str]:
|
||
"""Принятые вложения в работе и готовые без объекта staging: у `failed`
|
||
его может не быть по самой причине отказа."""
|
||
accepted = {
|
||
item
|
||
for item, status in attachment_rows(stand).items()
|
||
if status in ("queued", "processing", "ready")
|
||
}
|
||
return accepted - staged_ids(stand)
|
||
|
||
|
||
def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]:
|
||
return set(after) - set(before)
|
||
|
||
|
||
def objects_and_rows(stand: Stand) -> tuple[dict[str, str], set[str]]:
|
||
"""Снимок: строки и объекты вложений обоих префиксов."""
|
||
return attachment_rows(stand), object_ids(stand)
|
||
|
||
|
||
def new_objects(
|
||
before: tuple[dict[str, str], set[str]],
|
||
stand: Stand,
|
||
) -> set[str]:
|
||
"""Объекты, которых не было в снимке, у `id`, которых в нём тоже не было.
|
||
Worker параллельно доводит задания прежних тестов: объект конечного ключа
|
||
у прежнего `id` — не след новой загрузки."""
|
||
return object_ids(stand) - before[1] - set(before[0])
|
||
|
||
|
||
def job_row(stand: Stand, attachment_id: str) -> dict[str, Any] | None:
|
||
"""Задание вложения; нет — `None`. Два задания дали бы две строки и
|
||
ошибку разбора: задание у вложения одно."""
|
||
output = stand("job", attachment_id)
|
||
return json.loads(output) if output else None
|
||
|
||
|
||
def ready_ids(rows: dict[str, str]) -> set[str]:
|
||
return {item for item, status in rows.items() if status == "ready"}
|
||
|
||
|
||
def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None:
|
||
"""Строка и объект вложения убраны не позже grace + интервал + запас
|
||
от `since` (момент до загрузки: `created_at` строки не раньше)."""
|
||
deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while True:
|
||
rows = attachment_rows(stand)
|
||
stored = object_ids(stand)
|
||
if attachment_id not in rows and attachment_id not in stored:
|
||
return
|
||
if time.monotonic() > deadline:
|
||
raise AssertionError(
|
||
f"attachment {attachment_id} is not cleaned "
|
||
f"{time.monotonic() - since:.1f}s after upload: "
|
||
f"row {rows.get(attachment_id)}, "
|
||
f"object {attachment_id in stored}",
|
||
)
|
||
time.sleep(0.5)
|
||
|
||
|
||
def sleep_until(moment: float) -> None:
|
||
time.sleep(max(0.0, moment - time.monotonic()))
|
||
|
||
|
||
def fault_filename() -> str:
|
||
"""Уникальное имя загрузки, на которое срабатывает триггер hook."""
|
||
return f"fault-{uuid.uuid4().hex}.png"
|
||
|
||
|
||
def upload_raw(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: Any,
|
||
**kwargs: Any,
|
||
) -> httpx.Response:
|
||
return api.post_attachment(actor, chat["id"], data, **kwargs)
|
||
|
||
|
||
def assert_invisible(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
attachment_id: str,
|
||
) -> None:
|
||
"""Незавершённая загрузка: метаданные, байты и ссылка в сообщении — 404
|
||
с тем же `message`, что у неизвестного `id`."""
|
||
|
||
def calls(item: str) -> tuple[httpx.Response, ...]:
|
||
return (
|
||
api.attachment_meta(actor, item),
|
||
api.attachment_content(actor, item),
|
||
api.post_message(
|
||
actor,
|
||
chat["id"],
|
||
json={"text": "ссылка", "attachment_ids": [item]},
|
||
),
|
||
)
|
||
|
||
unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))]
|
||
hidden = [assert_error(resp, 404) for resp in calls(attachment_id)]
|
||
assert [error["message"] for error in hidden] == [
|
||
error["message"] for error in unknown
|
||
]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
|
||
@pytest.mark.spec("attachments", "Незавершённая загрузка не видна")
|
||
def test_upload_with_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
before = attachment_rows(stand)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
after = attachment_rows(stand)
|
||
# строка-намерение осталась, готовых не прибавилось; её уберёт
|
||
# уборщик (test_cleaner_survives_outages)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
# заданий и принятых не прибавилось: новая строка одна и pending
|
||
assert job_row(stand, pending) is None
|
||
# замечание S6 verify attachments: id последней pending — от hook
|
||
assert stand("last-pending") == pending
|
||
assert_invisible(api, alice, chat, pending)
|
||
api.send_message(alice, chat["id"], "текст без S3")
|
||
api.upload(alice, chat["id"], data)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
|
||
def test_rejected_uploads_leave_nothing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
|
||
|
||
def chunked() -> Iterator[bytes]:
|
||
for offset in range(0, len(big), 1024 * 1024):
|
||
yield big[offset : offset + 1024 * 1024]
|
||
|
||
# хотя бы одно готовое вложение: списки hook не пусты случайно
|
||
ready = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
before = objects_and_rows(stand)
|
||
assert before[0].get(ready["id"]) == "ready", before[0]
|
||
assert ready["id"] in before[1], before[1]
|
||
assert ready["id"] in staged_ids(stand)
|
||
rejected = (
|
||
(404, upload_raw(api, eve, chat, images.sample_png())),
|
||
(413, upload_raw(api, alice, chat, big)),
|
||
(413, upload_raw(api, alice, chat, chunked())),
|
||
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
|
||
(
|
||
415,
|
||
upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
|
||
)
|
||
for status, resp in rejected:
|
||
assert_error(resp, status)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Сбой записи метаданных")
|
||
def test_metadata_insert_failure(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = objects_and_rows(stand)
|
||
stand("fail-attachments", "insert", filename)
|
||
try:
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строки нет — объект не записывался
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
|
||
|
||
class BodyAborted(Exception):
|
||
"""Клиент обрывает передачу тела загрузки."""
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Обрыв передачи тела")
|
||
def test_interrupted_body(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = objects_and_rows(stand)
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
# свой клиент: его соединение обрывается вместе с телом
|
||
with (
|
||
Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно; строки до конца тела нет в любом
|
||
# случае
|
||
time.sleep(1)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Фиксация задания не удалась")
|
||
@pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_queued_failure_compensated(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "queued", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert stand("last-pending") == pending
|
||
# компенсация: объект удалён до ответа 503, задания нет
|
||
assert pending not in object_ids(stand)
|
||
assert job_row(stand, pending) is None
|
||
assert_invisible(api, alice, chat, pending)
|
||
# моложе grace строку уборщик не трогает: к половине grace он сделал
|
||
# около G / 2 / I проходов
|
||
sleep_until(started + CLEANER_GRACE_S / 2)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект без фиксации убран")
|
||
def test_killed_between_put_and_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
outcome: dict[str, Any] = {}
|
||
client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
|
||
def upload() -> None:
|
||
try:
|
||
outcome["status"] = upload_raw(
|
||
client,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
).status_code
|
||
except httpx.TransportError as e:
|
||
outcome["error"] = e
|
||
|
||
stand("fail-attachments", "stall", filename)
|
||
killed = False
|
||
try:
|
||
started = time.monotonic()
|
||
thread = threading.Thread(target=upload)
|
||
thread.start()
|
||
# строка pending и объект staging — до конца задержки перевода в
|
||
# queued
|
||
deadline = started + STALL_S
|
||
pending = ""
|
||
while time.monotonic() < deadline:
|
||
pending = stand("last-pending")
|
||
if (
|
||
pending
|
||
and pending not in before
|
||
and pending in staged_ids(stand)
|
||
):
|
||
break
|
||
time.sleep(0.05)
|
||
else:
|
||
raise AssertionError("no pending row with object during stall")
|
||
stand("kill-api")
|
||
killed = True
|
||
thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not thread.is_alive()
|
||
# 202 до клиента не дошёл: перевод в queued ждал в триггере
|
||
assert outcome.get("status") != 202, outcome
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert pending in staged_ids(stand)
|
||
assert job_row(stand, pending) is None
|
||
finally:
|
||
client.close()
|
||
stand("clear-fail-attachments")
|
||
if killed:
|
||
stand("start-api")
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Исход фиксации неизвестен")
|
||
def test_queued_commit_outcome_unknown(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Перевод в queued падает на самом COMMIT (отложенный триггер): API не
|
||
знает, зафиксировано ли вложение, и объект не трогает (verify W2)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
# компенсации нет: вложение могло стать queued
|
||
assert pending in staged_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Загрузка дольше grace")
|
||
def test_upload_outlives_grace(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Строка вставлена «на час старше», перевод в queued ждёт в триггере,
|
||
пока её не уберёт уборщик стенда: загрузка, пережившая grace, без
|
||
заморозки процесса (verify W3)."""
|
||
filename = fault_filename()
|
||
before = objects_and_rows(stand)
|
||
stand("fail-attachments", "outlive", filename)
|
||
# ожидание в триггере до 4,5 с: срок клиента больше
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
# не 202 (принято без объекта) и не 500 (дефект ветки)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
rows = attachment_rows(stand)
|
||
assert not new_ids(before[0], rows), rows
|
||
assert not new_objects(before, stand)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта")
|
||
def test_cleaner_failure_after_object_delete(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Уборщик удалил объект, а удаление строки упало (08006), пока загрузка
|
||
ждёт перевода в queued: строка помечена до удаления объекта и queued уже
|
||
не станет (verify W1)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "undeletable", filename)
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert job_row(stand, pending) is None
|
||
assert pending not in staged_ids(stand)
|
||
assert not accepted_without_staging(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строка старше grace: следующий проход доводит уборку
|
||
returned = time.monotonic()
|
||
wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S)
|
||
assert not orphan_objects(stand)
|
||
assert not accepted_without_staging(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "БД недоступна во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты")
|
||
def test_cleaner_survives_outages(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
assert_error(upload_raw(api, alice, chat, data), 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
# PostgreSQL недоступен, пока загрузка моложе grace
|
||
with outage(stand, "stop", "postgres"):
|
||
time.sleep(3)
|
||
# grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте
|
||
sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
# S3 вернулся: уборщик без перезапуска убирает строку за проход
|
||
returned = time.monotonic()
|
||
deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while pending in attachment_rows(stand):
|
||
assert time.monotonic() < deadline, (
|
||
f"pending {pending} is still there "
|
||
f"{time.monotonic() - returned:.1f}s after S3 came back"
|
||
)
|
||
time.sleep(0.5)
|
||
assert pending not in object_ids(stand)
|
||
assert not orphan_objects(stand)
|
||
# готовое вложение старше grace на месте
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, ready["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == ready
|
||
assert meta["status"] == "ready"
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 остановлен")
|
||
def test_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(
|
||
lambda: api.attachment_content(bob, attachment["id"]),
|
||
)
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
meta = assert_ok(
|
||
api.attachment_meta(bob, attachment["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == attachment
|
||
message = api.send_message(
|
||
bob,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment["id"]],
|
||
)
|
||
# история с attachment_ids — из PostgreSQL, S3 ей не нужен
|
||
history = api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [message["id"]]
|
||
assert history[0]["attachment_ids"] == [attachment["id"]]
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
assert_content(
|
||
api.attachment_content(bob, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
def test_environment_does_not_change_limits(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `service-configuration`, «Окружение не меняет лимит» (вне
|
||
сверки покрытия — сценарий в докстринге): обычный API стенда
|
||
отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1`
|
||
(`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт,
|
||
64 000 пикселей) всё равно принимается: лимиты — константы."""
|
||
limits = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_LIMITS_ENV", "").split()
|
||
)
|
||
# без этих переменных на стенде тест ничего не доказывает
|
||
assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, (
|
||
f"stand started without limit variables: {limits}"
|
||
)
|
||
api.upload(alice, chat["id"], images.sample_png(), filename="sample.png")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 завис")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_object_storage_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
|
||
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
|
||
with slow, outage(stand, "pause", "s3"):
|
||
for call in (
|
||
lambda: upload_raw(slow, alice, chat, data),
|
||
lambda: slow.attachment_content(alice, attachment["id"]),
|
||
):
|
||
resp, elapsed = timed(call)
|
||
assert_error(resp, 503)
|
||
# срок вызова S3 и немного на шаги с БД, а не зависание
|
||
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
|
||
f"503 after {elapsed:.2f}s"
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
# загрузка во время заморозки оставила строку pending; объект после
|
||
# разморозки не дописался, строку уборщик убирает после grace
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Места загрузки и срок тела (change upload-memory, design, решение 11):
|
||
# «узкий по загрузкам» API — одно место, одна загрузка в очереди, ожидание
|
||
# места и срок тела из FAULT_UPLOADS_ENV (run_failures.py, UPLOADS_ENV).
|
||
# Место держит загрузка, тело которой стоит на threading.Event теста
|
||
# ---------------------------------------------------------------------------
|
||
|
||
UPLOADS_ENV = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_UPLOADS_ENV", "").split()
|
||
)
|
||
# окружение, с которым тесты ниже ждут «узкий» API
|
||
EXPECTED_UPLOADS_ENV = {
|
||
"UPLOADS__MAX_CONCURRENT": "1",
|
||
"UPLOADS__MAX_WAITING": "1",
|
||
"UPLOADS__WAIT_TIMEOUT_SECONDS": "1",
|
||
"UPLOADS__BODY_TIMEOUT_SECONDS": "3",
|
||
}
|
||
WAIT_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__WAIT_TIMEOUT_SECONDS", "1"))
|
||
BODY_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__BODY_TIMEOUT_SECONDS", "3"))
|
||
# отказ без ожидания места: вход, членство и ответ
|
||
IMMEDIATE_S = 0.5
|
||
# пауза, за которую запрос в соседнем потоке доходит до места или очереди
|
||
SETTLE_S = 0.2
|
||
# сколько держатель места ждёт отпускания, если тест упал раньше
|
||
HOLD_MAX_S = 10
|
||
|
||
|
||
def assert_uploads_stand() -> None:
|
||
"""Без этого окружения «узкого» API тесты мест ничего не доказывают."""
|
||
assert UPLOADS_ENV == EXPECTED_UPLOADS_ENV, (
|
||
f"uploads stand started with {UPLOADS_ENV}, "
|
||
f"expected {EXPECTED_UPLOADS_ENV}"
|
||
)
|
||
|
||
|
||
class Upload:
|
||
"""Загрузка в своём потоке и своём соединении; ответ — `result()`."""
|
||
|
||
def __init__(self, api: Api, actor: Actor, chat: dict[str, Any]) -> None:
|
||
self._base_url = str(api.client.base_url)
|
||
self._actor = actor
|
||
self._chat = chat
|
||
self._outcome: dict[str, Any] = {}
|
||
self._thread: threading.Thread | None = None
|
||
|
||
def start(self, data: Any, **kwargs: Any) -> "Upload":
|
||
def run() -> None:
|
||
with Api(self._base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
|
||
started = time.monotonic()
|
||
try:
|
||
self._outcome["resp"] = upload_raw(
|
||
own,
|
||
self._actor,
|
||
self._chat,
|
||
data,
|
||
**kwargs,
|
||
)
|
||
except (BodyAborted, httpx.TransportError) as e:
|
||
self._outcome["error"] = e
|
||
self._outcome["elapsed"] = time.monotonic() - started
|
||
self._outcome["ended"] = time.monotonic()
|
||
|
||
self._thread = threading.Thread(target=run)
|
||
self._thread.start()
|
||
return self
|
||
|
||
def result(self) -> tuple[httpx.Response, float, float]:
|
||
"""Ответ, длительность и момент ответа (`time.monotonic`)."""
|
||
assert self._thread is not None
|
||
self._thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not self._thread.is_alive(), "upload thread hangs"
|
||
assert "resp" in self._outcome, self._outcome
|
||
return (
|
||
self._outcome["resp"],
|
||
self._outcome["elapsed"],
|
||
self._outcome["ended"],
|
||
)
|
||
|
||
|
||
@contextmanager
|
||
def held_slot(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: bytes,
|
||
) -> Iterator[Callable[[], httpx.Response]]:
|
||
"""Загрузка A держит место: отдаёт половину тела и ждёт отпускания.
|
||
|
||
Внутри — после того, как A заняла место; вызов отдаваемой функции
|
||
отпускает A и возвращает её ответ. В `finally` A отпускается всегда.
|
||
"""
|
||
sent = threading.Event()
|
||
release = threading.Event()
|
||
|
||
def body() -> Iterator[bytes]:
|
||
yield data[: len(data) // 2]
|
||
sent.set()
|
||
if not release.wait(HOLD_MAX_S):
|
||
raise BodyAborted
|
||
yield data[len(data) // 2 :]
|
||
|
||
holder = Upload(api, actor, chat).start(body())
|
||
try:
|
||
assert sent.wait(HOLD_MAX_S), "holder did not start its body"
|
||
# вход и членство — короткие шаги БД, затем место
|
||
time.sleep(SETTLE_S)
|
||
|
||
def finish() -> httpx.Response:
|
||
release.set()
|
||
return holder.result()[0]
|
||
|
||
yield finish
|
||
finally:
|
||
release.set()
|
||
|
||
|
||
def slow_chunks(data: bytes, *, parts: int, seconds: float) -> Iterator[bytes]:
|
||
"""Тело `parts` частями, растянутое на `seconds`."""
|
||
size = -(-len(data) // parts)
|
||
for offset in range(0, len(data), size):
|
||
if offset:
|
||
time.sleep(seconds / (parts - 1))
|
||
yield data[offset : offset + size]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Все места заняты")
|
||
@pytest.mark.spec("attachments", "Очередь ожидания полна")
|
||
@pytest.mark.spec("api-conventions", "Лимит одновременных загрузок")
|
||
def test_upload_slots_busy(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
before = objects_and_rows(stand)
|
||
with held_slot(uploads_api, alice, chat, data) as finish_holder:
|
||
# B ждёт место в очереди, C застаёт полную очередь
|
||
waiting = Upload(uploads_api, alice, chat).start(data)
|
||
time.sleep(SETTLE_S)
|
||
busy, busy_s = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, data),
|
||
)
|
||
waited, waited_s, _ = waiting.result()
|
||
# A отпускается сразу: её тело ограничено сроком тела стенда
|
||
holder = finish_holder()
|
||
assert_error(busy, 429)
|
||
assert busy_s < IMMEDIATE_S, f"429 of a full queue after {busy_s:.2f}s"
|
||
assert_error(waited, 429)
|
||
assert waited_s >= WAIT_TIMEOUT_S, f"429 after {waited_s:.2f}s"
|
||
for resp in (busy, waited):
|
||
assert int(resp.headers["Retry-After"]) >= 1
|
||
attachment = assert_accepted(holder)
|
||
# из трёх загрузок осталась только принятая: у 429 ни строки, ни объекта
|
||
assert new_ids(before[0], attachment_rows(stand)) == {attachment["id"]}
|
||
assert new_objects(before, stand) == {attachment["id"]}
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отказ раньше очереди")
|
||
def test_rejections_before_queue(
|
||
uploads_api: Api,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
# 30 MiB, которые начинаются как PNG: Content-Length 31 457 280
|
||
big = images.PNG_SIGNATURE + bytes(
|
||
3 * images.MAX_BYTES - len(images.PNG_SIGNATURE),
|
||
)
|
||
with held_slot(uploads_api, alice, chat, data) as finish_holder:
|
||
waiting = Upload(uploads_api, alice, chat).start(data)
|
||
time.sleep(SETTLE_S)
|
||
early = [
|
||
(status, *timed(call))
|
||
for status, call in (
|
||
(404, lambda: upload_raw(uploads_api, eve, chat, data)),
|
||
(413, lambda: upload_raw(uploads_api, alice, chat, big)),
|
||
(
|
||
415,
|
||
lambda: upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
data,
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
)
|
||
]
|
||
early_done = time.monotonic()
|
||
waited, _, waited_at = waiting.result()
|
||
holder = finish_holder()
|
||
# очередь была полна всё время ранних отказов
|
||
assert_error(waited, 429)
|
||
assert early_done < waited_at
|
||
for status, resp, elapsed in early:
|
||
assert_error(resp, status)
|
||
# без ожидания места: ожидающий получил бы его не раньше срока
|
||
assert elapsed < WAIT_TIMEOUT_S, f"{status} after {elapsed:.2f}s"
|
||
assert early[1][1].request.headers["Content-Length"] == "31457280"
|
||
assert_accepted(holder)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Ожидание места")
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Ожидание места не входит в возраст загрузки",
|
||
)
|
||
def test_upload_waits_for_slot(
|
||
uploads_api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
first = images.sample_png()
|
||
second = images.png(80, 60)
|
||
with held_slot(uploads_api, alice, chat, first) as finish_holder:
|
||
waiting = Upload(uploads_api, bob, chat).start(second)
|
||
time.sleep(SETTLE_S)
|
||
# A заканчивается раньше срока ожидания B
|
||
holder = finish_holder()
|
||
held = assert_accepted(holder)
|
||
waited_resp, waited_s, _ = waiting.result()
|
||
waited = assert_accepted(waited_resp)
|
||
assert waited_s < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, waited_s
|
||
# метаданные ожидавшей записаны после перевода в queued державшей место:
|
||
# ожидание прошло до строки pending (граница B уборщика прежняя)
|
||
assert waited["created_at"] > held["updated_at"], (held, waited)
|
||
for attachment, data in ((held, first), (waited, second)):
|
||
uploads_api.wait_attachment(alice, attachment["id"])
|
||
assert_content(
|
||
uploads_api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Место освобождается")
|
||
def test_slot_released_after_failures(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
over = images.png_of_size(images.MAX_BYTES + 1)
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
def next_upload_gets_slot(after: str) -> None:
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, images.sample_png()),
|
||
)
|
||
assert_accepted(resp)
|
||
assert elapsed < WAIT_TIMEOUT_S, (
|
||
f"upload after {after} waited {elapsed:.2f}s for a slot"
|
||
)
|
||
|
||
assert_error(
|
||
upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
slow_chunks(over, parts=10, seconds=0),
|
||
),
|
||
413,
|
||
)
|
||
next_upload_gets_slot("413 by counter")
|
||
assert_error(upload_raw(uploads_api, alice, chat, b"not an image"), 415)
|
||
next_upload_gets_slot("415 by content")
|
||
slow = slow_chunks(
|
||
images.sample_png(),
|
||
parts=4,
|
||
seconds=BODY_TIMEOUT_S + 1,
|
||
)
|
||
assert_error(upload_raw(uploads_api, alice, chat, slow), 400)
|
||
next_upload_gets_slot("400 by body timeout")
|
||
with (
|
||
Api(
|
||
str(uploads_api.client.base_url),
|
||
timeout=THREAD_CLIENT_TIMEOUT_S,
|
||
) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно
|
||
time.sleep(SETTLE_S)
|
||
next_upload_gets_slot("interrupted body")
|
||
with outage(stand, "stop", "s3"):
|
||
assert_error(
|
||
upload_raw(uploads_api, alice, chat, images.sample_png()),
|
||
503,
|
||
)
|
||
next_upload_gets_slot("503 with S3 stopped")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Медленный клиент")
|
||
@pytest.mark.spec("attachments", "Медленная передача в срок")
|
||
def test_slow_body(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
before = objects_and_rows(stand)
|
||
late = upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
slow_chunks(data, parts=6, seconds=BODY_TIMEOUT_S + 1),
|
||
)
|
||
error = assert_error(late, 400)
|
||
assert error["message"] == "Upload body was not received in time"
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
# мелкие части с паузами, но в срок: тело принято как есть
|
||
in_time = slow_chunks(data, parts=20, seconds=BODY_TIMEOUT_S / 2)
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, in_time),
|
||
)
|
||
attachment = assert_accepted(resp)
|
||
assert elapsed < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, elapsed
|
||
assert attachment["size_bytes"] == len(data)
|
||
assert attachment["sha256"] == hashlib.sha256(data).hexdigest()
|
||
uploads_api.wait_attachment(alice, attachment["id"])
|
||
assert_content(
|
||
uploads_api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Простой S3 после скачиваний (change fix-upload-put-flake, design, решение
|
||
# 3): API «с коротким сроком S3» — срок вызова S3 из FAULT_S3_ENV
|
||
# (run_failures.py, S3_ENV), других тестов на нём нет
|
||
# ---------------------------------------------------------------------------
|
||
|
||
S3_ENV = dict(
|
||
item.split("=", 1) for item in os.environ.get("FAULT_S3_ENV", "").split()
|
||
)
|
||
# окружение, с которым тест ниже ждёт API с коротким сроком S3
|
||
EXPECTED_S3_ENV = {"S3__TIMEOUT_SECONDS": "1"}
|
||
SHORT_S3_TIMEOUT_S = float(S3_ENV.get("S3__TIMEOUT_SECONDS", "1"))
|
||
# раунды: скачивания, простой дольше срока вызова S3, загрузки
|
||
IDLE_ROUNDS = 8
|
||
IDLE_DOWNLOADS = 10
|
||
IDLE_UPLOADS = 12
|
||
IDLE_EXTRA_S = 0.5
|
||
# медленный клиент: пауза после каждого блока 64 KiB
|
||
IDLE_READ_PAUSE_S = 0.002
|
||
|
||
|
||
def assert_s3_stand() -> None:
|
||
"""Без этого окружения API с коротким сроком S3 тест ничего не
|
||
доказывает: простой 1,5 с был бы короче срока вызова."""
|
||
assert S3_ENV == EXPECTED_S3_ENV, (
|
||
f"short S3 timeout stand started with {S3_ENV}, "
|
||
f"expected {EXPECTED_S3_ENV}"
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Загрузка после простоя S3")
|
||
def test_upload_after_idle_s3(
|
||
s3_api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Скачивания большого оригинала, простой S3 дольше срока вызова,
|
||
затем загрузки — каждая 202, ни одной 503.
|
||
|
||
Раунд: 10 одновременных медленных скачиваний (своё соединение у
|
||
каждого клиента, барьер, 2 мс на блок 64 KiB) PNG ~4 MiB, простой
|
||
`S3__TIMEOUT_SECONDS + 0,5` с без запросов к этому API, 12 загрузок
|
||
`sample.png` по одной: пул клиента S3 отдаёт соединения по порядку
|
||
возврата, и 12 вызовов проходят через все ≤ 10 его соединений.
|
||
|
||
До правки клиента (срок чтения aiohttp, переживший ответ) соединение
|
||
портится в раунде с p ≈ 0,28 (design, «Как подтверждено», п. 8):
|
||
промах раунда — 0,72^10 ≈ 0,037, восьми раундов — ≈ 4e-12. После
|
||
правки 8 раундов — ~16 с.
|
||
"""
|
||
assert_s3_stand()
|
||
base_url = str(s3_api.client.base_url)
|
||
original = images.png_noise(1024, 1365, seed=4)
|
||
attachment = s3_api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
original,
|
||
filename="idle-original.png",
|
||
)
|
||
sample = images.sample_png()
|
||
|
||
def download(barrier: threading.Barrier) -> httpx.Response:
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
|
||
# соединение открыто до барьера: после него сразу скачивание
|
||
own.request("GET", "/health/live")
|
||
barrier.wait()
|
||
return own.attachment_content_slowly(
|
||
alice,
|
||
attachment["id"],
|
||
pause_s=IDLE_READ_PAUSE_S,
|
||
)
|
||
|
||
for round_number in range(1, IDLE_ROUNDS + 1):
|
||
barrier = threading.Barrier(IDLE_DOWNLOADS)
|
||
with ThreadPoolExecutor(max_workers=IDLE_DOWNLOADS) as pool:
|
||
downloads = list(
|
||
pool.map(download, [barrier] * IDLE_DOWNLOADS),
|
||
)
|
||
for resp in downloads:
|
||
assert_content(resp, original, "image/png")
|
||
time.sleep(SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S)
|
||
for upload_number in range(1, IDLE_UPLOADS + 1):
|
||
resp = upload_raw(s3_api, alice, chat, sample)
|
||
assert resp.status_code == 202, describe(
|
||
resp,
|
||
f"round {round_number}, upload {upload_number} after "
|
||
f"{SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S:.1f}s of S3 idle",
|
||
)
|
||
assert_accepted(resp)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Фоновая обработка, брокер и worker (change async-attachments, design, решение
|
||
# 15). Worker стенда: 4 попытки, пауза 5 с, срок попытки 10 с, срок остановки
|
||
# 2 с (FAULT_WORKER_*). Перед остановкой worker очередь доводится до пустой:
|
||
# задания прежних тестов не считаются в «одно сообщение». Worker и триггеры
|
||
# возвращаются в finally, упавшая проверка не ломает следующие
|
||
# ---------------------------------------------------------------------------
|
||
|
||
WORKER_MAX_ATTEMPTS = int(os.environ.get("FAULT_WORKER_MAX_ATTEMPTS", "4"))
|
||
WORKER_RETRY_DELAY_S = float(os.environ.get("FAULT_WORKER_RETRY_DELAY_S", "5"))
|
||
# [worker] concurrency по умолчанию: стенд отказов её не меняет
|
||
WORKER_CONCURRENCY = 2
|
||
# [broker] publish_timeout_seconds и запас на шаги с БД вокруг публикации
|
||
PUBLISH_TIMEOUT_S = 2
|
||
BROKER_SLACK_S = 1
|
||
# отказ брокера не держит health: быстрее секунды, как у S3
|
||
BROKER_HEALTH_S = 1
|
||
# срок ожидания итога задания: все попытки и паузы с запасом
|
||
JOB_WAIT_S = 60
|
||
JOBS_SCHEMA_PATH = (
|
||
Path(__file__).resolve().parents[1] / "contracts" / "jobs.schema.json"
|
||
)
|
||
# срок «Stopped worker» по контракту остановки контейнера
|
||
WORKER_STOP_MAX_S = 10
|
||
THUMBNAIL_VARIANTS = (None, "original", "thumbnail")
|
||
|
||
|
||
def queue_state(stand: Stand) -> dict[str, Any]:
|
||
state = json.loads(stand("queue-state"))
|
||
assert state is not None, "job queue does not exist"
|
||
return state
|
||
|
||
|
||
def wait_queue_empty(stand: Stand, *, timeout: float = 90) -> None:
|
||
"""В очереди не осталось сообщений, в том числе неподтверждённых."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
state = queue_state(stand)
|
||
if state["messages"] == 0:
|
||
return
|
||
assert time.monotonic() < deadline, f"queue is not empty: {state}"
|
||
time.sleep(0.5)
|
||
|
||
|
||
@contextmanager
|
||
def worker_stopped(stand: Stand) -> Iterator[None]:
|
||
"""Worker остановлен при пустой очереди; в `finally` запускается."""
|
||
wait_queue_empty(stand)
|
||
with outage(stand, "stop", "worker"):
|
||
yield
|
||
|
||
|
||
def worker_records(stand: Stand) -> list[dict[str, Any]]:
|
||
"""Записи журнала worker за все его запуски."""
|
||
return [
|
||
json.loads(line)
|
||
for line in stand("worker-log").splitlines()
|
||
if line.startswith("{")
|
||
]
|
||
|
||
|
||
def wait_worker_record(
|
||
stand: Stand,
|
||
event: str,
|
||
*,
|
||
timeout: float = 30,
|
||
skip: int = 0,
|
||
**fields: Any,
|
||
) -> dict[str, Any]:
|
||
"""Запись журнала worker с этими полями после первых `skip` записей."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
for record in worker_records(stand)[skip:]:
|
||
if record.get("event") == event and all(
|
||
record.get(name) == value for name, value in fields.items()
|
||
):
|
||
return record
|
||
assert time.monotonic() < deadline, (
|
||
f"no worker record {event!r} with {fields} in {timeout} s"
|
||
)
|
||
time.sleep(0.3)
|
||
|
||
|
||
def job_body(
|
||
job_id: str,
|
||
attachment_id: str,
|
||
*,
|
||
attempt: int = 1,
|
||
) -> bytes:
|
||
"""Сообщение задания по `contracts/jobs.schema.json`."""
|
||
return json.dumps(
|
||
{
|
||
"schema_version": 1,
|
||
"job_id": job_id,
|
||
"type": "attachment.process.v1",
|
||
"attachment_id": attachment_id,
|
||
"pipeline_version": 1,
|
||
"attempt": attempt,
|
||
"created_at": "2026-10-31T12:00:00.000000Z",
|
||
},
|
||
).encode()
|
||
|
||
|
||
def wait_job(
|
||
stand: Stand,
|
||
attachment_id: str,
|
||
*,
|
||
state: str,
|
||
attempt: int,
|
||
timeout: float = 20,
|
||
) -> dict[str, Any]:
|
||
"""Задание в этом состоянии с этим или большим числом попыток."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
job = job_row(stand, attachment_id)
|
||
if job and job["state"] == state and job["attempt"] >= attempt:
|
||
return job
|
||
assert time.monotonic() < deadline, (
|
||
f"job of {attachment_id} is not {state} with attempt >= "
|
||
f"{attempt}: {job}"
|
||
)
|
||
time.sleep(0.2)
|
||
|
||
|
||
def thumbnail_size(data: bytes) -> tuple[int, int]:
|
||
"""Настоящие размеры JPEG: файл открывается декодером."""
|
||
from PIL import Image # noqa: PLC0415
|
||
|
||
with Image.open(io.BytesIO(data)) as image:
|
||
image.load()
|
||
assert image.format == "JPEG", image.format
|
||
return image.size
|
||
|
||
|
||
def assert_one_thumbnail(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
meta: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
"""Один `thumbnail` в метаданных и один объект варианта с его SHA-256."""
|
||
assert meta["status"] == "ready", meta
|
||
(variant,) = meta["variants"]
|
||
assert variant["name"] == "thumbnail", variant
|
||
assert stand("variant-object-count", meta["id"]) == "1"
|
||
assert_variant_content(
|
||
api.attachment_content(actor, meta["id"], variant="thumbnail"),
|
||
variant,
|
||
)
|
||
return variant
|
||
|
||
|
||
def assert_not_ready_content(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment_id: str,
|
||
) -> None:
|
||
"""Любое скачивание вложения не в `ready` — 409."""
|
||
for variant in THUMBNAIL_VARIANTS:
|
||
assert_error(
|
||
api.attachment_content(actor, attachment_id, variant=variant),
|
||
409,
|
||
)
|
||
|
||
|
||
def staging_key(attachment_id: str) -> str:
|
||
return f"{FAULT_DATABASE}/staging/{attachment_id}/original"
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Принятое вложение в очереди")
|
||
def test_queued_while_worker_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
time.sleep(3)
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == accepted
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Обработка видна в API")
|
||
def test_processing_visible(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(alice, chat["id"], data, filename=filename)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert_error(api.attachment_content(alice, accepted["id"]), 409)
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
assert_content(
|
||
api.attachment_content(alice, accepted["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Готовое вложение не откатывается")
|
||
@pytest.mark.spec("attachment-processing", "Повторная доставка готового")
|
||
def test_redelivery_of_ready_is_noop(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
meta = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
job = job_row(stand, meta["id"])
|
||
assert job is not None
|
||
wait_queue_empty(stand)
|
||
seen = len(worker_records(stand))
|
||
stand("publish-job", stdin=job_body(job["id"], meta["id"]))
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="finished",
|
||
)
|
||
wait_queue_empty(stand)
|
||
assert (
|
||
assert_ok(api.attachment_meta(alice, meta["id"]), 200, "Attachment")
|
||
== meta
|
||
)
|
||
assert job_row(stand, meta["id"]) == job
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Что сохранено к ответу 202")
|
||
def test_durable_state_at_202(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
attachment_id = accepted["id"]
|
||
assert attachment_id in staged_ids(stand)
|
||
assert attachment_id not in stored_ids(stand)
|
||
assert attachment_rows(stand)[attachment_id] == "queued"
|
||
job = job_row(stand, attachment_id)
|
||
assert job is not None
|
||
assert (job["state"], job["attempt"]) == ("queued", 0), job
|
||
assert job["published_at"] is not None, job
|
||
assert queue_state(stand)["messages"] == 1
|
||
api.wait_attachment(alice, attachment_id)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Временная ошибка")
|
||
@pytest.mark.spec("attachment-processing", "Повторы под одним заданием")
|
||
def test_transient_error_retried(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Сначала S3 остановлен и возвращается за ~8 с: попыток больше одной.
|
||
Затем S3 заморожен на первую попытку и размораживается в паузе между
|
||
попытками: ровно две, задание то же."""
|
||
data = images.sample_png()
|
||
with worker_stopped(stand):
|
||
stopped = api.upload(alice, chat["id"], data)
|
||
first = job_row(stand, stopped["id"])
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
wait_job(stand, stopped["id"], state="queued", attempt=1)
|
||
meta = api.wait_attachment(alice, stopped["id"], timeout=JOB_WAIT_S)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
job = job_row(stand, stopped["id"])
|
||
assert job is not None and first is not None
|
||
assert job["id"] == first["id"], (job, first)
|
||
assert job["attempt"] > 1, job
|
||
assert job["state"] == "succeeded", job
|
||
|
||
with worker_stopped(stand):
|
||
paused = api.upload(alice, chat["id"], data)
|
||
first = job_row(stand, paused["id"])
|
||
with outage(stand, "pause", "s3"):
|
||
stand("start", "worker")
|
||
# срок вызова S3 5 с: первая попытка заканчивается отказом S3
|
||
wait_job(stand, paused["id"], state="queued", attempt=1)
|
||
meta = api.wait_attachment(alice, paused["id"], timeout=JOB_WAIT_S)
|
||
job = job_row(stand, paused["id"])
|
||
assert job is not None and first is not None
|
||
assert job["id"] == first["id"], (job, first)
|
||
assert job["attempt"] == 2, job
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Сообщение по схеме")
|
||
def test_job_message_matches_schema(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
schema = json.loads(JOBS_SCHEMA_PATH.read_text(encoding="utf-8"))
|
||
filename = fault_filename()
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
peeked = json.loads(stand("peek-job"))
|
||
text = peeked["body"]
|
||
body = json.loads(text)
|
||
errors = list(
|
||
Draft202012Validator(
|
||
schema,
|
||
format_checker=FormatChecker(),
|
||
).iter_errors(body),
|
||
)
|
||
assert not errors, [error.message for error in errors]
|
||
datetime.fromisoformat(body["created_at"])
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert body["attachment_id"] == accepted["id"]
|
||
assert body["job_id"] == job["id"]
|
||
assert body["attempt"] == 1
|
||
# ни имени файла, ни ключей, ни адресов, ни учётных данных
|
||
lowered = text.lower()
|
||
for word in (
|
||
filename.lower(),
|
||
os.environ.get("FAULT_S3_BUCKET", "attachments").lower(),
|
||
f"{FAULT_DATABASE}/",
|
||
"staging",
|
||
"original",
|
||
"http",
|
||
"password",
|
||
"secret",
|
||
):
|
||
assert word not in lowered, f"{word!r} in the job message"
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Очередь и сообщения устойчивы")
|
||
def test_queue_durable(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
state = queue_state(stand)
|
||
assert state["durable"] is True, state
|
||
assert state["messages"] == 1, state
|
||
peeked = json.loads(stand("peek-job"))
|
||
assert peeked["delivery_mode"] == 2, peeked
|
||
# прочитанное без подтверждения вернулось в очередь
|
||
state = queue_state(stand)
|
||
assert (state["messages"], state["messages_ready"]) == (1, 1), state
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Сообщение не по схеме")
|
||
def test_malformed_message_dropped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
marker = uuid.uuid4().hex
|
||
wait_queue_empty(stand)
|
||
seen = len(worker_records(stand))
|
||
stand(
|
||
"publish-job",
|
||
stdin=json.dumps({"schema_version": 1, "note": marker}).encode(),
|
||
)
|
||
record = wait_worker_record(
|
||
stand,
|
||
"Dropped malformed job message",
|
||
skip=seen,
|
||
)
|
||
assert record["level"] == "error", record
|
||
# содержимое сообщения в журнал не попало
|
||
assert marker not in stand("worker-log")
|
||
wait_queue_empty(stand)
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Неизвестное задание")
|
||
def test_unknown_job_acked(stand: Stand) -> None:
|
||
job_id, attachment_id = str(uuid.uuid4()), str(uuid.uuid4())
|
||
wait_queue_empty(stand)
|
||
before = objects_and_rows(stand)
|
||
seen = len(worker_records(stand))
|
||
stand("publish-job", stdin=job_body(job_id, attachment_id))
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job_id,
|
||
reason="unknown",
|
||
)
|
||
wait_queue_empty(stand)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Worker убит посреди обработки")
|
||
def test_worker_killed_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
stand("kill", "worker")
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "worker")
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 2, job
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Дубль во время обработки")
|
||
def test_duplicate_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Две ветки дубля. S3 заморожен, попытка ждёт его без блокировок БД: дубль
|
||
видит живую попытку («busy») и возвращается в очередь. Фиксация ждёт в
|
||
триггере и держит строку задания: дубль дожидается её и видит итог
|
||
(«finished»), вторая попытка не начинается."""
|
||
with worker_stopped(stand):
|
||
live = api.upload(alice, chat["id"], images.sample_png())
|
||
job = job_row(stand, live["id"])
|
||
assert job is not None
|
||
# оба сообщения в очереди: prefetch выдаёт их разом, пока попытка идёт
|
||
stand("publish-job", stdin=job_body(job["id"], live["id"]))
|
||
seen = len(worker_records(stand))
|
||
with outage(stand, "pause", "s3"):
|
||
stand("start", "worker")
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="busy",
|
||
)
|
||
meta = api.wait_attachment(alice, live["id"], timeout=JOB_WAIT_S)
|
||
wait_queue_empty(stand)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
|
||
filename = fault_filename()
|
||
seen = len(worker_records(stand))
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
stand("publish-job", stdin=job_body(job["id"], accepted["id"]))
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="finished",
|
||
)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
wait_queue_empty(stand)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["attempt"] == 1, done
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Оригинал неизменен")
|
||
def test_original_published_unchanged(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=23)
|
||
meta = api.upload_ready(alice, chat["id"], data)
|
||
assert_content(
|
||
api.attachment_content(alice, meta["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
assert meta["sha256"] == hashlib.sha256(data).hexdigest()
|
||
assert meta["id"] in stored_ids(stand)
|
||
assert meta["id"] in staged_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Испорченный staging")
|
||
def test_staging_corrupted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Те же размер и другие байты — и другой размер: обе ветки сверки."""
|
||
data = images.png_noise(64, 48, seed=29)
|
||
with worker_stopped(stand):
|
||
same_size = api.upload(alice, chat["id"], data)
|
||
other_size = api.upload(alice, chat["id"], data)
|
||
put_object(
|
||
stand,
|
||
staging_key(same_size["id"]),
|
||
same_size_bytes(data, seed=29),
|
||
)
|
||
put_object(stand, staging_key(other_size["id"]), data + b"\0")
|
||
for accepted in (same_size, other_size):
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "processing_failed", meta
|
||
assert meta["variants"] == [], meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert job["last_error_code"] == "source_corrupted", job
|
||
assert accepted["id"] not in stored_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Нет объекта staging")
|
||
def test_staging_missing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
stand("delete-object", staging_key(accepted["id"]))
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "processing_failed", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert job["last_error_code"] == "source_missing", job
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Изображение в лимите пикселей")
|
||
def test_max_pixels_fit_worker_memory(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Столько PNG RGBA 8000 × 5000, сколько заданий worker обрабатывает
|
||
одновременно: они берутся разом, и контейнер не завершён из-за памяти."""
|
||
data = images.png_rgba_large()
|
||
before = json.loads(stand("worker-state"))
|
||
# срок клиента: тело на 40 млн пикселей проверяется дольше 5 с по умолчанию
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
slow.upload(alice, chat["id"], data)
|
||
for _ in range(WORKER_CONCURRENCY)
|
||
]
|
||
for item in accepted:
|
||
meta = slow.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
(variant,) = meta["variants"]
|
||
assert (variant["width"], variant["height"]) == (256, 160), variant
|
||
content = assert_variant_content(
|
||
slow.attachment_content(
|
||
alice,
|
||
item["id"],
|
||
variant="thumbnail",
|
||
),
|
||
variant,
|
||
)
|
||
assert thumbnail_size(content) == (256, 160)
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["oom_killed"] is False, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Повтор после записи объектов")
|
||
@pytest.mark.spec("attachments", "Фиксация ready не удалась")
|
||
def test_ready_failure_retried(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Объекты записаны, фиксация падает отказом БД (08006): попытка уходит в
|
||
очередь; пока триггер стоит, вложение не готово. Триггер снимается в
|
||
паузе перед следующей доставкой."""
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
seen = len(worker_records(stand))
|
||
stand("fail-attachments", "ready-fail", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
wait_worker_record(
|
||
stand,
|
||
"Job waits for database",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
)
|
||
assert stand("variant-object-count", accepted["id"]) == "1"
|
||
assert accepted["id"] in stored_ids(stand)
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] in ("queued", "processing"), meta
|
||
assert_not_ready_content(api, alice, accepted["id"])
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["id"] == job["id"], done
|
||
assert done["attempt"] >= 2, done
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Состояние между попытками")
|
||
def test_queued_between_attempts(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
job = wait_job(stand, accepted["id"], state="queued", attempt=1)
|
||
assert job["last_error_code"] == "storage_unavailable", job
|
||
assert job["next_attempt_at"] is not None, job
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
assert meta["error_code"] is None, meta
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Попытки исчерпаны")
|
||
def test_attempts_exhausted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "retry_exhausted", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == WORKER_MAX_ATTEMPTS, job
|
||
assert job["state"] == "failed", job
|
||
wait_queue_empty(stand)
|
||
# S3 вернулся: итог окончателен
|
||
time.sleep(WORKER_RETRY_DELAY_S)
|
||
after = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert after == meta
|
||
assert job_row(stand, accepted["id"]) == job
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"PostgreSQL недоступен при обработке",
|
||
)
|
||
def test_database_outage_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("worker-state"))
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
seen = len(worker_records(stand))
|
||
with outage(stand, "stop", "postgres"):
|
||
stand("start", "worker")
|
||
# журнал называет отказ, задание ждёт в очереди
|
||
wait_worker_record(stand, "Job waits for database", skip=seen)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
wait_queue_empty(stand)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
# попытки на отказе БД не считаются
|
||
assert done["attempt"] == 1, done
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Изображение не декодируется")
|
||
def test_undecodable_becomes_failed(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
assert meta["variants"] == [], meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert job["last_error_code"] == "undecodable", job
|
||
assert_error(api.attachment_content(alice, accepted["id"]), 409)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Worker недоступен")
|
||
def test_worker_down_api_serves(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
assert accepted["status"] == "queued"
|
||
message = api.send_message(alice, chat["id"], "текст без worker")
|
||
history = api.all_messages(bob, chat["id"])
|
||
assert [item["id"] for item in history] == [message["id"]]
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Ограниченная конкуренция")
|
||
def test_concurrency_bounded(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Фиксация каждого задания ждёт 4 с в триггере: заданий больше, чем
|
||
мест, и видно, сколько worker держит разом."""
|
||
filename = fault_filename()
|
||
total = WORKER_CONCURRENCY + 3
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
with worker_stopped(stand):
|
||
ids = [
|
||
api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)["id"]
|
||
for _ in range(total)
|
||
]
|
||
api.wait_attachment(
|
||
alice,
|
||
ids[0],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
most = 0
|
||
for _ in range(4):
|
||
rows = attachment_rows(stand)
|
||
statuses = [rows[item] for item in ids]
|
||
busy = statuses.count("processing")
|
||
most = max(most, busy)
|
||
assert busy <= WORKER_CONCURRENCY, statuses
|
||
assert (
|
||
statuses.count("queued")
|
||
+ busy
|
||
+ statuses.count(
|
||
"ready",
|
||
)
|
||
== total
|
||
), statuses
|
||
state = queue_state(stand)
|
||
assert state["messages_unacknowledged"] <= WORKER_CONCURRENCY, (
|
||
state
|
||
)
|
||
time.sleep(0.3)
|
||
assert most == WORKER_CONCURRENCY, most
|
||
for item in ids:
|
||
api.wait_attachment(alice, item, timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Реплика API убита после ответа")
|
||
def test_api_killed_after_202(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
stand("kill-api")
|
||
try:
|
||
# реплики нет, а работа идёт: итог — от worker
|
||
wait_job(
|
||
stand,
|
||
accepted["id"],
|
||
state="succeeded",
|
||
attempt=1,
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
finally:
|
||
stand("start-api")
|
||
meta = api.wait_attachment(alice, accepted["id"])
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Рестарт брокера с volume")
|
||
def test_broker_restart_keeps_job(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
pass
|
||
assert queue_state(stand)["messages"] == 1
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Принудительное завершение брокера")
|
||
@pytest.mark.spec("message-broker", "Подтверждённые сообщения после KILL")
|
||
def test_broker_kill_keeps_confirmed(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
confirmed = 5
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(confirmed)
|
||
]
|
||
with outage(stand, "kill", "rabbitmq"):
|
||
pass
|
||
assert queue_state(stand)["messages"] == confirmed
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Сценарий защиты")
|
||
def test_defense_scenario(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], data)
|
||
attachment_id = accepted["id"]
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment_id],
|
||
)
|
||
assert_error(api.attachment_content(alice, attachment_id), 409)
|
||
text = api.send_message(alice, chat["id"], "текст при очереди")
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
stand("kill-api")
|
||
stand("start-api")
|
||
meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S)
|
||
assert meta["id"] == attachment_id
|
||
assert_content(
|
||
api.attachment_content(alice, attachment_id),
|
||
data,
|
||
"image/png",
|
||
)
|
||
variant = assert_one_thumbnail(api, stand, alice, meta)
|
||
content = api.attachment_content(alice, attachment_id, variant="thumbnail")
|
||
assert thumbnail_size(content.content) == (
|
||
variant["width"],
|
||
variant["height"],
|
||
)
|
||
history = {
|
||
item["id"]: item for item in api.all_messages(alice, chat["id"])
|
||
}
|
||
assert history[message["id"]]["attachment_ids"] == [attachment_id]
|
||
assert text["id"] in history
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Брокер остановлен при загрузке")
|
||
@pytest.mark.spec("message-broker", "Брокер вернулся")
|
||
@pytest.mark.spec("attachments", "Брокер не подтвердил задание")
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Загрузка без подтверждения брокера убрана",
|
||
)
|
||
def test_upload_with_broker_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
assert_error(resp, 503)
|
||
assert elapsed < PUBLISH_TIMEOUT_S + BROKER_SLACK_S, elapsed
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert job_row(stand, pending) is None
|
||
assert pending in staged_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
# обязательный путь без брокера работает
|
||
api.send_message(alice, chat["id"], "текст без брокера")
|
||
assert (
|
||
assert_ok(
|
||
api.attachment_meta(alice, ready["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
== ready
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
# реплика не перезапускалась: следующая загрузка проходит
|
||
api.upload_ready(alice, chat["id"], data)
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert job_row(stand, pending) is None
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Брокер завис")
|
||
def test_upload_with_broker_paused(
|
||
base_url: str,
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Соединение издателя живо: публикация уходит в заморозку и кончается
|
||
сроком, а исход неизвестен. Сообщение, дошедшее после разморозки, worker
|
||
подтверждает без эффекта."""
|
||
data = images.sample_png()
|
||
api.upload_ready(alice, chat["id"], data)
|
||
wait_queue_empty(stand)
|
||
before = attachment_rows(stand)
|
||
slow = Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
with slow, outage(stand, "pause", "rabbitmq"):
|
||
resp, elapsed = timed(lambda: upload_raw(slow, alice, chat, data))
|
||
assert_error(resp, 503)
|
||
assert elapsed < PUBLISH_TIMEOUT_S + BROKER_SLACK_S, elapsed
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
wait_queue_empty(stand)
|
||
assert job_row(stand, pending) is None
|
||
assert stand("variant-object-count", pending) == "0"
|
||
assert pending not in stored_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
# брокер вернулся: приём снова работает
|
||
api.upload_ready(alice, chat["id"], data)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Очередь удалена")
|
||
def test_queue_deleted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
# соединение издателя живо: его канал очередь больше не найдёт
|
||
api.upload_ready(alice, chat["id"], data)
|
||
with worker_stopped(stand):
|
||
stand("delete-queue")
|
||
assert json.loads(stand("queue-state")) is None
|
||
assert_error(upload_raw(api, alice, chat, data), 503)
|
||
accepted = api.upload(alice, chat["id"], data)
|
||
state = queue_state(stand)
|
||
assert (state["durable"], state["messages"]) == (True, 1), state
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Пересоздание контейнера")
|
||
def test_broker_recreated(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
confirmed = 2
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(confirmed)
|
||
]
|
||
stand("recreate", "rabbitmq")
|
||
state = queue_state(stand)
|
||
assert (state["durable"], state["messages"]) == (True, confirmed), (
|
||
state
|
||
)
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Старт без брокера")
|
||
def test_start_without_broker(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand), outage(stand, "stop", "rabbitmq"):
|
||
seen = len(worker_records(stand))
|
||
stand("kill-api")
|
||
stand("start-api")
|
||
stand("start", "worker")
|
||
# реплика готова и обслуживает текст
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
api.send_message(alice, chat["id"], "текст при старте без брокера")
|
||
# worker называет отказ и не завершается
|
||
wait_worker_record(stand, "Broker is unavailable", skip=seen)
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
# брокер запущен, ни реплика, ни worker не перезапускались
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Рестарт брокера при работающем worker")
|
||
def test_worker_survives_broker_restart(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("worker-state"))
|
||
assert before["running"] is True, before
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
pass
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание до готовности")
|
||
def test_content_before_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Каждое скачивание вложения в `queued`, `processing` и `failed` — 409."""
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_not_ready_content(api, alice, accepted["id"])
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert_not_ready_content(api, alice, accepted["id"])
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
failed = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
failed["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert_not_ready_content(api, alice, failed["id"])
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение в обработке")
|
||
def test_message_with_queued_attachment(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"с вложением в очереди",
|
||
attachment_ids=[accepted["id"]],
|
||
)
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
assert api.all_messages(bob, chat["id"]) == [message]
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение стало failed после отправки")
|
||
def test_attachment_failed_after_message(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[accepted["id"]],
|
||
)
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Повтор после failed вложения")
|
||
def test_replay_after_attachment_failed(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
key = new_key()
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
body = {"text": "повтор", "attachment_ids": [accepted["id"]]}
|
||
message = assert_ok(
|
||
api.post_message(alice, chat["id"], key=key, json=body),
|
||
201,
|
||
"Message",
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
replay = assert_ok(
|
||
api.post_message(alice, chat["id"], key=key, json=body),
|
||
201,
|
||
"Message",
|
||
)
|
||
assert replay == message
|
||
assert_error(api.post_message(alice, chat["id"], json=body), 409)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("health", "Брокер и worker недоступны")
|
||
@pytest.mark.parametrize("action", ["stop", "pause"])
|
||
def test_readiness_without_broker_and_worker(
|
||
api: Api,
|
||
stand: Stand,
|
||
action: str,
|
||
) -> None:
|
||
with worker_stopped(stand), outage(stand, action, "rabbitmq"):
|
||
for path in ("/health/ready", "/health/live"):
|
||
resp, took = timed(lambda path=path: api.request("GET", path))
|
||
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
|
||
assert took < BROKER_HEALTH_S, f"{path} took {took:.2f}s"
|
||
|
||
|
||
def test_worker_stop_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `graceful-shutdown`, «docker stop во время обработки» (вне сверки
|
||
покрытия — сценарий в докстринге): фиксация ждёт 4 с, а срок остановки
|
||
worker стенда 2 с. `stop` быстрее 10 с, код 0, сообщение снова в очереди,
|
||
а после запуска worker вложение становится `ready`."""
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
stopped = False
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
seen = len(worker_records(stand))
|
||
started = time.monotonic()
|
||
stand("stop", "worker")
|
||
stopped = True
|
||
assert time.monotonic() - started < WORKER_STOP_MAX_S
|
||
state = json.loads(stand("worker-state"))
|
||
assert (state["running"], state["exit_code"]) == (False, 0), state
|
||
assert queue_state(stand)["messages"] == 1
|
||
records = worker_records(stand)[seen:]
|
||
assert any(item.get("event") == "Stopped worker" for item in records)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
if not stopped:
|
||
stand("stop", "worker")
|
||
stand("start", "worker")
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_one_thumbnail(api, stand, alice, meta)
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@contextmanager
|
||
def current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> Iterator[dict[str, dict[str, Any]]]:
|
||
"""Worker убит посреди задания и лежит. Вложения F в `failed` (конечный
|
||
объект у него есть), P в `processing` (объекты записаны, фиксация не
|
||
дошла) и Q в `queued`, все старше grace по строке. Выход — worker запущен,
|
||
P и Q становятся `ready`."""
|
||
wait_queue_empty(stand)
|
||
failed = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
failed["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
work: dict[str, dict[str, Any]] = {"failed": failed}
|
||
try:
|
||
processing = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
processing["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
work["processing"] = processing
|
||
# конечный объект записан до фиксации; ждём и thumbnail
|
||
deadline = time.monotonic() + STALL_S
|
||
while stand("variant-object-count", processing["id"]) != "1":
|
||
assert time.monotonic() < deadline, "no variant object in time"
|
||
time.sleep(0.1)
|
||
stand("kill", "worker")
|
||
stand("clear-fail-attachments")
|
||
work["queued"] = api.upload(alice, chat["id"], images.sample_png())
|
||
for item in work.values():
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
yield work
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "worker")
|
||
for name in ("processing", "queued"):
|
||
api.wait_attachment(alice, work[name]["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Текущая работа не тронута")
|
||
def test_cleaner_keeps_current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with current_work(api, stand, alice, chat) as work:
|
||
# проходов уборщика за это время — несколько
|
||
time.sleep(3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S)
|
||
rows = attachment_rows(stand)
|
||
staged = staged_ids(stand)
|
||
for name, item in work.items():
|
||
assert rows.get(item["id"]) == name, (name, rows.get(item["id"]))
|
||
assert item["id"] in staged, name
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сверка bucket и БД (change track2-reconciler, design, решение 11). Hook
|
||
# вносит расхождения с обеих сторон и запускает разовую сверку; плановая
|
||
# сверка уборщика стенда выключена. Записи сверки фильтруются по своим id и
|
||
# ключам: остатки прежних тестов (строки pending) могут попасть в отчёт, и
|
||
# код выхода сверяется с записями того же прохода, а не с числом. Объекты
|
||
# состарить нельзя (LastModified ставит S3): тесты объектов без строки ждут
|
||
# G + 1 с, строки состаривает hook (age-row)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# на сколько hook состаривает строку: заведомо больше grace
|
||
AGED_S = CLEANER_GRACE_S + 60
|
||
# объекты без строки старше grace: grace и секунда точности LastModified
|
||
OBJECT_AGE_WAIT_S = CLEANER_GRACE_S + 1
|
||
FOUND = "Found storage discrepancy"
|
||
FIXED = "Fixed storage discrepancy"
|
||
FINISHED = "Finished reconcile pass"
|
||
LIMIT_EXCEEDED = "Reconcile changes exceed limit"
|
||
# сверка под нагрузкой: потоки загрузок, период сверки, пауза потока
|
||
LOAD_THREADS = 4
|
||
LOAD_EXTRA_S = 15
|
||
LOAD_RECONCILE_PERIOD_S = 3
|
||
LOAD_PAUSE_S = 0.1
|
||
# плановая сверка отдельного уборщика: столько секунд он работает
|
||
SCHEDULED_RUN_S = 6
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Reconcile:
|
||
"""Один разовый проход сверки: записи журнала процесса и код выхода."""
|
||
|
||
records: list[dict[str, Any]]
|
||
exit_code: int
|
||
|
||
def events(self, event: str) -> list[dict[str, Any]]:
|
||
return [item for item in self.records if item.get("event") == event]
|
||
|
||
@property
|
||
def finished(self) -> dict[str, Any]:
|
||
(record,) = self.events(FINISHED)
|
||
return record
|
||
|
||
def about(
|
||
self,
|
||
event: str,
|
||
*,
|
||
ids: Collection[str] = (),
|
||
keys: Collection[str] = (),
|
||
) -> list[dict[str, Any]]:
|
||
"""Записи `event` о своих `id` вложений и ключах объектов."""
|
||
return [
|
||
item
|
||
for item in self.events(event)
|
||
if item.get("attachment_id") in ids
|
||
or item.get("object_key") in keys
|
||
]
|
||
|
||
def kinds(
|
||
self,
|
||
event: str,
|
||
*,
|
||
ids: Collection[str] = (),
|
||
keys: Collection[str] = (),
|
||
) -> dict[str, str]:
|
||
"""`id` или ключ → класс расхождения в записях `event`."""
|
||
found: dict[str, str] = {}
|
||
for item in self.about(event, ids=ids, keys=keys):
|
||
name = (
|
||
item["attachment_id"]
|
||
if item.get("attachment_id") in ids
|
||
else item["object_key"]
|
||
)
|
||
found[name] = item["kind"]
|
||
return found
|
||
|
||
def assert_code_matches(self) -> None:
|
||
"""Код выхода соответствует записям прохода: 3 — остались
|
||
расхождения (запись `Found`) или отказ по пределу, иначе 0."""
|
||
finished = self.finished
|
||
assert finished["interrupted_by"] is None, finished
|
||
expected = 3 if self.events(FOUND) or finished["refused"] else 0
|
||
assert self.exit_code == expected, (self.exit_code, finished)
|
||
|
||
|
||
def run_reconcile(
|
||
stand: Stand,
|
||
mode: str,
|
||
limit: int | None = None,
|
||
*,
|
||
options: Sequence[str] = (),
|
||
) -> Reconcile:
|
||
"""Разовая сверка hook: `dry-run` или `apply`, с пределом изменений;
|
||
`options` — стратегия, глубина и предел скорости (change
|
||
`track2-reconcile-cost`) как аргументы команды."""
|
||
args = [mode] if limit is None else [mode, str(limit)]
|
||
lines = stand("reconcile", *args, *options).splitlines()
|
||
assert lines and lines[-1].startswith("exit_code="), lines[-5:]
|
||
records = []
|
||
for line in lines[:-1]:
|
||
if line.startswith("{"):
|
||
records.append(json.loads(line))
|
||
return Reconcile(records=records, exit_code=int(lines[-1].split("=")[1]))
|
||
|
||
|
||
def object_key(attachment_id: str) -> str:
|
||
return f"{FAULT_DATABASE}/attachments/{attachment_id}/original"
|
||
|
||
|
||
def variant_key(attachment_id: str) -> str:
|
||
return f"{FAULT_DATABASE}/attachments/{attachment_id}/v1-thumbnail"
|
||
|
||
|
||
def lost_ids(stand: Stand) -> set[str]:
|
||
return set(stand("lost-ids").splitlines())
|
||
|
||
|
||
def put_object(stand: Stand, key: str, data: bytes) -> None:
|
||
stand("put-object", key, stdin=data)
|
||
|
||
|
||
def lose_original(stand: Stand, attachment: dict[str, Any]) -> None:
|
||
"""Готовое вложение старше grace без объекта: строка состарена, объект
|
||
удалён мимо API."""
|
||
stand("age-row", attachment["id"], str(AGED_S))
|
||
stand("delete-object", object_key(attachment["id"]))
|
||
|
||
|
||
def assert_lost_content(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
*,
|
||
aged: bool = False,
|
||
) -> None:
|
||
"""Помеченное вложение: байты — 404, метаданные — как при загрузке.
|
||
|
||
`aged` — строку состарил hook (`age-row`): её `created_at` и
|
||
`updated_at` сдвинуты им, а не сверкой, и сравниваются остальные поля.
|
||
"""
|
||
assert_error(api.attachment_content(actor, attachment["id"]), 404)
|
||
meta = assert_ok(
|
||
api.attachment_meta(actor, attachment["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
if aged:
|
||
times = ("created_at", "updated_at")
|
||
meta = {key: value for key, value in meta.items() if key not in times}
|
||
attachment = {
|
||
key: value for key, value in attachment.items() if key not in times
|
||
}
|
||
assert meta == attachment
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Расхождения с обеих сторон найдены")
|
||
@pytest.mark.spec("storage-consistency", "Исправление расхождений")
|
||
@pytest.mark.spec("storage-consistency", "Повторная сверка")
|
||
@pytest.mark.spec("attachments", "Потерянный оригинал")
|
||
def test_reconcile_finds_and_fixes(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
|
||
with outage(stand, "stop", "cleaner"):
|
||
a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3))
|
||
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
|
||
orphan = str(uuid.uuid4())
|
||
keys = {
|
||
"o": object_key(orphan),
|
||
"b": object_key(b["id"]),
|
||
"bv": variant_key(b["id"]),
|
||
}
|
||
put_object(stand, keys["o"], data)
|
||
stand("delete-object", object_key(a["id"]))
|
||
stand("delete-row", b["id"])
|
||
stand("set-size", c["id"], str(len(data) + 1))
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
resp = upload_raw(api, alice, chat, data, filename=filename)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert pending in staged_ids(stand)
|
||
# последний объект записан только что: после G + 1 всё старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
ids = {a["id"], c["id"], pending}
|
||
snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, ids=ids, keys=keys.values()) == {
|
||
keys["o"]: "orphan_object",
|
||
keys["b"]: "orphan_object",
|
||
keys["bv"]: "orphan_object",
|
||
a["id"]: "missing_object",
|
||
c["id"]: "size_mismatch",
|
||
pending: "stale_pending",
|
||
}
|
||
for record in dry.about(FOUND, ids=ids, keys=keys.values()):
|
||
assert record["age_seconds"] >= CLEANER_GRACE_S, record
|
||
assert dry.exit_code == 3
|
||
# dry-run ничего не меняет: ни строк, ни объектов, ни пометок
|
||
after = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
assert after == snapshot
|
||
assert_error(api.attachment_content(alice, a["id"]), 500)
|
||
assert_ok(api.attachment_meta(alice, a["id"]), 200, "Attachment")
|
||
|
||
fixed = run_reconcile(stand, "apply")
|
||
actions = {
|
||
record.get("attachment_id")
|
||
if record.get("attachment_id") in ids
|
||
else record["object_key"]: record["action"]
|
||
for record in fixed.about(FIXED, ids=ids, keys=keys.values())
|
||
}
|
||
assert actions == {
|
||
keys["o"]: "deleted_object",
|
||
keys["b"]: "deleted_object",
|
||
keys["bv"]: "deleted_object",
|
||
a["id"]: "marked_lost",
|
||
c["id"]: "marked_lost",
|
||
pending: "removed_pending",
|
||
}
|
||
assert not fixed.about(FOUND, ids=ids, keys=keys.values())
|
||
fixed.assert_code_matches()
|
||
rows, stored = attachment_rows(stand), stored_ids(stand)
|
||
assert not {orphan, b["id"], pending} & stored
|
||
assert stand("variant-object-count", b["id"]) == "0"
|
||
assert pending not in staged_ids(stand)
|
||
assert pending not in rows
|
||
assert {a["id"], c["id"]} <= lost_ids(stand)
|
||
assert rows[a["id"]] == rows[c["id"]] == "ready"
|
||
assert_lost_content(api, alice, a)
|
||
# у C размер в метаданных изменил hook (set-size), а не сверка
|
||
assert_lost_content(api, alice, c | {"size_bytes": len(data) + 1})
|
||
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids=ids, keys=keys.values())
|
||
assert not again.about(FIXED, ids=ids, keys=keys.values())
|
||
again.assert_code_matches()
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Свежие не тронуты")
|
||
@pytest.mark.spec("storage-consistency", "Используемое вложение не удаляется")
|
||
@pytest.mark.spec("storage-consistency", "Чужой объект не удаляется")
|
||
def test_reconcile_keeps_fresh_used_and_unknown(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
used = api.upload_ready(alice, chat["id"], data)
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[used["id"]],
|
||
)
|
||
lose_original(stand, used)
|
||
# ключ, которого API не пишет: под префиксом вложений, но не <id>/original
|
||
unknown = f"{FAULT_DATABASE}/attachments/x-{uuid.uuid4()}"
|
||
put_object(stand, unknown, data)
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
# свежие: готовое вложение без объекта и объект без строки
|
||
fresh = api.upload_ready(alice, chat["id"], data)
|
||
stand("delete-object", object_key(fresh["id"]))
|
||
fresh_object = str(uuid.uuid4())
|
||
put_object(stand, object_key(fresh_object), data)
|
||
ready_before = ready_ids(attachment_rows(stand))
|
||
try:
|
||
run = run_reconcile(stand, "apply")
|
||
keys = {unknown, object_key(fresh_object)}
|
||
ids = {used["id"], fresh["id"], fresh_object}
|
||
assert run.kinds(FOUND, ids=ids, keys=keys) == {
|
||
unknown: "unknown_object",
|
||
}
|
||
assert run.kinds(FIXED, ids=ids, keys=keys) == {
|
||
used["id"]: "missing_object",
|
||
}
|
||
assert run.exit_code == 3
|
||
assert run.finished["fresh_objects"] >= 1, run.finished
|
||
assert run.finished["fresh_rows"] >= 1, run.finished
|
||
# свежий объект на месте, свежее вложение не помечено
|
||
assert fresh_object in stored_ids(stand)
|
||
lost = lost_ids(stand)
|
||
assert fresh["id"] not in lost
|
||
# используемое: помечено, метаданные и ссылка в истории на месте
|
||
assert used["id"] in lost
|
||
assert_lost_content(api, alice, used, aged=True)
|
||
history = api.all_messages(alice, chat["id"])
|
||
(sent,) = (item for item in history if item["id"] == message["id"])
|
||
assert sent["attachment_ids"] == [used["id"]]
|
||
assert ready_before <= ready_ids(attachment_rows(stand))
|
||
# чужой объект не тронут: следующий проход снова его видит
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert again.kinds(FOUND, keys={unknown}) == {
|
||
unknown: "unknown_object",
|
||
}
|
||
finally:
|
||
# стенд — без чужого и лишнего объекта, у свежего — его байты
|
||
stand("delete-object", unknown)
|
||
stand("delete-object", object_key(fresh_object))
|
||
put_object(stand, object_key(fresh["id"]), data)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
|
||
def test_reconcile_restored_original(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=7)
|
||
restored = api.upload_ready(alice, chat["id"], data)
|
||
lose_original(stand, restored)
|
||
marked = run_reconcile(stand, "apply")
|
||
assert marked.kinds(FIXED, ids={restored["id"]}) == {
|
||
restored["id"]: "missing_object",
|
||
}
|
||
assert restored["id"] in lost_ids(stand)
|
||
assert_lost_content(api, alice, restored, aged=True)
|
||
# те же байты снова под ключом вложения
|
||
put_object(stand, object_key(restored["id"]), data)
|
||
cleared = run_reconcile(stand, "apply")
|
||
(record,) = cleared.about(FIXED, ids={restored["id"]})
|
||
assert (record["kind"], record["action"]) == (
|
||
"restored_object",
|
||
"cleared_lost",
|
||
)
|
||
assert restored["id"] not in lost_ids(stand)
|
||
assert_content(
|
||
api.attachment_content(alice, restored["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids={restored["id"]})
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Предел изменений")
|
||
def test_reconcile_change_limit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
limit = 2
|
||
batch = [
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for _ in range(3)
|
||
]
|
||
ids = {item["id"] for item in batch}
|
||
for item in batch:
|
||
lose_original(stand, item)
|
||
refused = run_reconcile(stand, "apply", limit)
|
||
assert refused.exit_code == 3
|
||
(error,) = refused.events(LIMIT_EXCEEDED)
|
||
assert error["level"] == "error"
|
||
assert error["changes"] >= len(batch), error
|
||
assert error["max_changes"] == limit
|
||
assert refused.finished["refused"] is True
|
||
assert refused.finished["level"] == "warning"
|
||
assert not refused.about(FIXED, ids=ids)
|
||
assert not ids & lost_ids(stand)
|
||
assert all(
|
||
record["kind"] == "missing_object"
|
||
for record in refused.about(FOUND, ids=ids)
|
||
)
|
||
# предел не меньше числа изменений: исправлено всё
|
||
applied = run_reconcile(stand, "apply", 100)
|
||
assert set(applied.kinds(FIXED, ids=ids)) == ids
|
||
assert ids <= lost_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Объект, на который ссылается строка, не удаляется",
|
||
)
|
||
def test_reconcile_keeps_referenced_objects(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W1 verify: объект `<X>/original`, у строки которого другой
|
||
ключ, и объект `<Z>/original`, на который ссылается строка Y другого
|
||
`id`, — чужие, а не объекты без метаданных."""
|
||
data = images.sample_png()
|
||
x, y = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
|
||
z = str(uuid.uuid4())
|
||
keys = {
|
||
"x": object_key(x["id"]),
|
||
"y": object_key(y["id"]),
|
||
"z": object_key(z),
|
||
# ключ X: под префиксом вложений, объекта по нему нет
|
||
"other": f"{FAULT_DATABASE}/attachments/x-other-{uuid.uuid4()}",
|
||
}
|
||
put_object(stand, keys["z"], data)
|
||
try:
|
||
stand("set-key", x["id"], keys["other"])
|
||
stand("set-key", y["id"], keys["z"])
|
||
for item in (x, y):
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
# объект Z записан только что: после G + 1 все объекты старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
unknown = {keys[name]: "unknown_object" for name in ("x", "y", "z")}
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, keys=keys.values()) == unknown | {
|
||
keys["other"]: "missing_object",
|
||
}
|
||
assert dry.exit_code == 3
|
||
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FIXED, keys=keys.values()) == {
|
||
keys["other"]: "missing_object",
|
||
}
|
||
assert applied.kinds(FOUND, keys=keys.values()) == unknown
|
||
assert applied.exit_code == 3
|
||
# ни один из трёх объектов не удалён, строки на месте
|
||
assert {x["id"], y["id"], z} <= stored_ids(stand)
|
||
rows = attachment_rows(stand)
|
||
assert rows[x["id"]] == rows[y["id"]] == "ready"
|
||
# X без объекта по своему ключу помечено, у Y объект по ключу есть
|
||
lost = lost_ids(stand)
|
||
assert x["id"] in lost
|
||
assert y["id"] not in lost
|
||
finally:
|
||
# прежние ключи; объект Z, на который больше никто не ссылается, —
|
||
# прочь, чтобы следующие проходы не удаляли его как свой
|
||
stand("set-key", x["id"], keys["x"])
|
||
stand("set-key", y["id"], keys["y"])
|
||
stand("delete-object", keys["z"])
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Предел не останавливает уборку зависших",
|
||
)
|
||
def test_reconcile_limit_keeps_cleanup(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W1 verify: отказ по пределу не мешает уборке зависшей
|
||
загрузки — её уборщик делает и без сверки."""
|
||
data = images.sample_png()
|
||
limit = 2
|
||
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
|
||
with outage(stand, "stop", "cleaner"):
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
assert_error(
|
||
upload_raw(api, alice, chat, data, filename=filename),
|
||
503,
|
||
)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert pending in staged_ids(stand)
|
||
stand("age-row", pending, str(AGED_S))
|
||
batch = [api.upload_ready(alice, chat["id"], data) for _ in range(3)]
|
||
ids = {item["id"] for item in batch}
|
||
for item in batch:
|
||
lose_original(stand, item)
|
||
|
||
refused = run_reconcile(stand, "apply", limit)
|
||
assert refused.exit_code == 3
|
||
(error,) = refused.events(LIMIT_EXCEEDED)
|
||
assert error["changes"] >= len(batch), error
|
||
assert refused.finished["refused"] is True
|
||
assert refused.kinds(FIXED, ids={pending}) == {
|
||
pending: "stale_pending",
|
||
}
|
||
(record,) = refused.about(FIXED, ids={pending})
|
||
assert record["action"] == "removed_pending"
|
||
assert pending not in attachment_rows(stand)
|
||
assert pending not in staged_ids(stand)
|
||
# изменений под пределом нет: ни одной пометки
|
||
assert not refused.about(FIXED, ids=ids)
|
||
assert not ids & lost_ids(stand)
|
||
# стенд — без внесённых потерь для следующих тестов
|
||
applied = run_reconcile(stand, "apply", 100)
|
||
assert set(applied.kinds(FIXED, ids=ids)) == ids
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Ключ вне префикса вложений не трогается",
|
||
)
|
||
def test_reconcile_foreign_key_untouched(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W2 verify: строка с ключом вне `<база>/attachments/` —
|
||
готовая и зависшая загрузка — только в отчёте как чужой объект: сверка
|
||
не делает HEAD чужого префикса, не помечает и не убирает строку."""
|
||
data = images.sample_png()
|
||
# уборщик стенда стоит: иначе зависшую загрузку убрал бы он
|
||
with outage(stand, "stop", "cleaner"):
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
assert_error(
|
||
upload_raw(api, alice, chat, data, filename=filename),
|
||
503,
|
||
)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
ids = {ready["id"], pending}
|
||
try:
|
||
for item in ids:
|
||
# префикс своей базы, но не вложений: список его не видит
|
||
stand("set-key", item, f"{FAULT_DATABASE}/elsewhere/{item}")
|
||
stand("age-row", item, str(AGED_S))
|
||
unknown = dict.fromkeys(ids, "unknown_object")
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, ids=ids) == unknown
|
||
assert dry.exit_code == 3
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FOUND, ids=ids) == unknown
|
||
assert not applied.about(FIXED, ids=ids)
|
||
assert applied.exit_code == 3
|
||
# ни пометки, ни уборки: строки и объекты под префиксом на месте
|
||
rows = attachment_rows(stand)
|
||
assert (rows.get(ready["id"]), rows.get(pending)) == (
|
||
"ready",
|
||
"pending",
|
||
)
|
||
assert ready["id"] not in lost_ids(stand)
|
||
assert ready["id"] in stored_ids(stand)
|
||
assert pending in staged_ids(stand)
|
||
finally:
|
||
# прежние ключи у тех строк, что остались: зависшую загрузку
|
||
# после теста уберёт уборщик стенда вместе с её объектом
|
||
rows = attachment_rows(stand)
|
||
for item in ids & set(rows):
|
||
stand("set-key", item, object_key(item))
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "S3 недоступен во время сверки")
|
||
def test_reconcile_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
missing = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
lose_original(stand, missing)
|
||
with outage(stand, "stop", "s3"):
|
||
interrupted = run_reconcile(stand, "apply")
|
||
assert interrupted.exit_code == 1
|
||
assert interrupted.finished["interrupted_by"] == "object_storage"
|
||
assert not interrupted.events(FIXED)
|
||
assert missing["id"] not in lost_ids(stand)
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FIXED, ids={missing["id"]}) == {
|
||
missing["id"]: "missing_object",
|
||
}
|
||
assert missing["id"] in lost_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Прерванная сверка доводится повтором",
|
||
)
|
||
def test_reconcile_interrupted_by_database(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
failing = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
other = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
ids = {failing["id"], other["id"]}
|
||
for item in (failing, other):
|
||
lose_original(stand, item)
|
||
stand("fail-attachments", "mark", filename)
|
||
try:
|
||
interrupted = run_reconcile(stand, "apply")
|
||
assert interrupted.exit_code == 1
|
||
assert interrupted.finished["interrupted_by"] == "database"
|
||
assert failing["id"] not in lost_ids(stand)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
applied = run_reconcile(stand, "apply")
|
||
assert ids <= lost_ids(stand)
|
||
assert not applied.about(FOUND, ids=ids)
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids=ids)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Сверка под нагрузкой")
|
||
def test_reconcile_under_load(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Загрузки и отправки дольше grace, сверка с `--apply` раз в 3 с:
|
||
внесённое исправлено, ни одно подтверждённое вложение не потеряно."""
|
||
data = images.sample_png()
|
||
orphans = {str(uuid.uuid4()) for _ in range(2)}
|
||
for item in orphans:
|
||
put_object(stand, object_key(item), data)
|
||
injected = [api.upload_ready(alice, chat["id"], data) for _ in range(2)]
|
||
for item in injected:
|
||
lose_original(stand, item)
|
||
lost_before = lost_ids(stand)
|
||
deadline = time.monotonic() + CLEANER_GRACE_S + LOAD_EXTRA_S
|
||
lock = threading.Lock()
|
||
uploads: dict[str, str] = {}
|
||
messages: dict[str, list[str]] = {}
|
||
failures: list[str] = []
|
||
passes: list[Reconcile] = []
|
||
|
||
def load(worker: int) -> None:
|
||
actor = alice if worker % 2 == 0 else bob
|
||
with Api(
|
||
str(api.client.base_url),
|
||
timeout=THREAD_CLIENT_TIMEOUT_S,
|
||
) as client:
|
||
number = 0
|
||
while time.monotonic() < deadline:
|
||
number += 1
|
||
body = images.png_noise(200, 170, seed=worker * 10**6 + number)
|
||
resp = client.post_attachment(actor, chat["id"], body)
|
||
if resp.status_code != 202:
|
||
with lock:
|
||
failures.append(f"upload {resp.status_code}")
|
||
continue
|
||
attachment = assert_accepted(resp)
|
||
# загрузка подтверждена: она обязана стать ready и скачиваться
|
||
with lock:
|
||
uploads[attachment["id"]] = hashlib.sha256(
|
||
body,
|
||
).hexdigest()
|
||
message = client.send_message(
|
||
actor,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment["id"]],
|
||
)
|
||
with lock:
|
||
messages[message["id"]] = [attachment["id"]]
|
||
time.sleep(LOAD_PAUSE_S)
|
||
|
||
def reconcile() -> None:
|
||
while time.monotonic() < deadline:
|
||
started = time.monotonic()
|
||
passes.append(run_reconcile(stand, "apply"))
|
||
sleep_until(started + LOAD_RECONCILE_PERIOD_S)
|
||
|
||
def guarded(target: Callable[..., None], *args: Any) -> None:
|
||
# исключение потока — в список, а не молча в stderr
|
||
try:
|
||
target(*args)
|
||
except Exception as e:
|
||
with lock:
|
||
failures.append(repr(e))
|
||
|
||
threads = [
|
||
threading.Thread(target=guarded, args=(load, worker))
|
||
for worker in range(LOAD_THREADS)
|
||
]
|
||
threads.append(threading.Thread(target=guarded, args=(reconcile,)))
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
assert not failures, failures[:5]
|
||
assert uploads, "no uploads under load"
|
||
assert len(passes) >= 3, len(passes)
|
||
assert all(item.exit_code in (0, 3) for item in passes), [
|
||
item.exit_code for item in passes
|
||
]
|
||
# внесённые исправлены: объекты без строки удалены, потери помечены
|
||
assert not orphans & stored_ids(stand)
|
||
assert lost_ids(stand) - lost_before == {item["id"] for item in injected}
|
||
# ни одно подтверждённое вложение не потеряно
|
||
for attachment_id, digest in uploads.items():
|
||
api.wait_attachment(alice, attachment_id)
|
||
resp = api.attachment_content(alice, attachment_id)
|
||
assert resp.status_code == 200, (attachment_id, resp.status_code)
|
||
assert hashlib.sha256(resp.content).hexdigest() == digest
|
||
history = {
|
||
item["id"]: item["attachment_ids"]
|
||
for item in api.all_messages(alice, chat["id"])
|
||
}
|
||
for message_id, attachment_ids in messages.items():
|
||
assert history[message_id] == attachment_ids
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Плановая сверка без команды")
|
||
def test_scheduled_reconcile(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
scheduled = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
lose_original(stand, scheduled)
|
||
log = stand("run-cleaner", str(SCHEDULED_RUN_S))
|
||
records = [
|
||
json.loads(line) for line in log.splitlines() if line.startswith("{")
|
||
]
|
||
passes = [item for item in records if item.get("event") == FINISHED]
|
||
assert passes, log[-2000:]
|
||
assert any(item["apply"] for item in passes), passes
|
||
assert scheduled["id"] in lost_ids(stand)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Стоимость сверки (change track2-reconcile-cost, design, решение 14):
|
||
# стратегия HEAD, глубокая сверка и класс «другой SHA-256», снятие пометки
|
||
# только при том же SHA-256, предел скорости, отказ по пределу без лишних
|
||
# перепроверок. Аргументы сверки hook передаёт команде как есть
|
||
# ---------------------------------------------------------------------------
|
||
|
||
HEAD_STRATEGY = ("--strategy", "head")
|
||
DEEP = ("--deep",)
|
||
NO_RATE_LIMIT = ("--max-objects-per-second", "0")
|
||
# сколько вложений загружает тест предела скорости: объектов в bucket
|
||
# заведомо хватает на несколько секунд прохода
|
||
RATE_UPLOADS = 8
|
||
# во сколько раз проход с пределом длиннее числа объектов на предел
|
||
# (допуск на округление `duration_ms`)
|
||
RATE_SLACK = 0.95
|
||
|
||
|
||
def same_size_bytes(data: bytes, *, seed: int) -> bytes:
|
||
"""Другие байты того же размера, от зерна теста."""
|
||
other = random.Random(seed).randbytes(len(data))
|
||
assert other != data
|
||
return other
|
||
|
||
|
||
def findings(run: Reconcile) -> set[tuple[str, str]]:
|
||
"""Все расхождения прохода: класс и ключ объекта."""
|
||
return {(item["kind"], item["object_key"]) for item in run.events(FOUND)}
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Стратегия HEAD находит то же")
|
||
def test_reconcile_head_strategy_same_findings(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Те же расхождения, что в `test_reconcile_finds_and_fixes`: dry-run
|
||
стратегией по умолчанию и `--strategy head` называют одно и то же, а
|
||
HEAD-проход дороже по запросам к S3 и сеансам PostgreSQL."""
|
||
data = images.sample_png()
|
||
# уборщик стенда стоит: зависшую загрузку P не уберут между проходами
|
||
with outage(stand, "stop", "cleaner"):
|
||
a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3))
|
||
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
|
||
orphan = str(uuid.uuid4())
|
||
keys = {
|
||
"o": object_key(orphan),
|
||
"b": object_key(b["id"]),
|
||
"bv": variant_key(b["id"]),
|
||
}
|
||
put_object(stand, keys["o"], data)
|
||
stand("delete-object", object_key(a["id"]))
|
||
stand("delete-row", b["id"])
|
||
stand("set-size", c["id"], str(len(data) + 1))
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
resp = upload_raw(api, alice, chat, data, filename=filename)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
ids = {a["id"], c["id"], pending}
|
||
expected = {
|
||
keys["o"]: "orphan_object",
|
||
keys["b"]: "orphan_object",
|
||
keys["bv"]: "orphan_object",
|
||
a["id"]: "missing_object",
|
||
c["id"]: "size_mismatch",
|
||
pending: "stale_pending",
|
||
}
|
||
snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
|
||
batch = run_reconcile(stand, "dry-run")
|
||
head = run_reconcile(stand, "dry-run", options=HEAD_STRATEGY)
|
||
assert batch.kinds(FOUND, ids=ids, keys=keys.values()) == expected
|
||
assert head.kinds(FOUND, ids=ids, keys=keys.values()) == expected
|
||
# не только свои: весь отчёт и итог по классам одинаковы
|
||
assert findings(head) == findings(batch)
|
||
for field in ("found", "listed_objects", "checked_rows"):
|
||
assert head.finished[field] == batch.finished[field], field
|
||
assert head.exit_code == batch.exit_code == 3
|
||
assert (batch.finished["strategy"], head.finished["strategy"]) == (
|
||
"batch",
|
||
"head",
|
||
)
|
||
assert (
|
||
head.finished["s3_requests"]["head"]
|
||
> batch.finished["s3_requests"]["head"]
|
||
), (head.finished, batch.finished)
|
||
assert head.finished["db_sessions"] > batch.finished["db_sessions"]
|
||
after = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
assert after == snapshot
|
||
|
||
# стенд — без внесённых расхождений для следующих тестов
|
||
fixed = run_reconcile(stand, "apply")
|
||
assert set(fixed.kinds(FIXED, ids=ids, keys=keys.values())) == set(
|
||
expected,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Другой SHA-256 находит глубокая сверка",
|
||
)
|
||
def test_reconcile_deep_checksum(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=11)
|
||
a = api.upload_ready(alice, chat["id"], data)
|
||
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
|
||
other = same_size_bytes(data, seed=11)
|
||
# объект перезаписан мимо API байтами того же размера
|
||
put_object(stand, object_key(a["id"]), other)
|
||
stand("age-row", a["id"], str(AGED_S))
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
snapshot = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
|
||
|
||
shallow = run_reconcile(stand, "dry-run")
|
||
assert not shallow.about(FOUND, ids={a["id"]})
|
||
assert shallow.finished["deep"] is False
|
||
deep = run_reconcile(stand, "dry-run", options=DEEP)
|
||
assert deep.kinds(FOUND, ids={a["id"]}) == {a["id"]: "checksum_mismatch"}
|
||
assert deep.finished["deep"] is True
|
||
assert deep.finished["read_bytes"] >= len(data), deep.finished
|
||
assert deep.finished["s3_requests"]["get"] >= 1, deep.finished
|
||
assert deep.exit_code == 3
|
||
after = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
|
||
assert after == snapshot
|
||
# SHA-256 объектов в журнале нет
|
||
journal = json.dumps(deep.records)
|
||
for body in (data, other):
|
||
assert hashlib.sha256(body).hexdigest() not in journal
|
||
|
||
applied = run_reconcile(stand, "apply", options=DEEP)
|
||
(record,) = applied.about(FIXED, ids={a["id"]})
|
||
assert (record["kind"], record["action"]) == (
|
||
"checksum_mismatch",
|
||
"marked_lost",
|
||
)
|
||
assert a["id"] in lost_ids(stand)
|
||
assert_lost_content(api, alice, a, aged=True)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Вернулся другой объект того же размера",
|
||
)
|
||
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
|
||
def test_reconcile_restore_needs_same_bytes(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=13)
|
||
restored = api.upload_ready(alice, chat["id"], data)
|
||
ids = {restored["id"]}
|
||
lose_original(stand, restored)
|
||
marked = run_reconcile(stand, "apply")
|
||
assert marked.kinds(FIXED, ids=ids) == {restored["id"]: "missing_object"}
|
||
assert restored["id"] in lost_ids(stand)
|
||
|
||
# под ключом — другие байты того же размера: пометка остаётся
|
||
put_object(
|
||
stand,
|
||
object_key(restored["id"]),
|
||
same_size_bytes(data, seed=13),
|
||
)
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
wrong = run_reconcile(stand, "apply")
|
||
assert not wrong.about(FIXED, ids=ids)
|
||
assert all(
|
||
record["kind"] != "restored_object"
|
||
for record in wrong.about(FOUND, ids=ids)
|
||
)
|
||
assert restored["id"] in lost_ids(stand)
|
||
assert_lost_content(api, alice, restored, aged=True)
|
||
|
||
# те же байты, что были загружены: пометка снята
|
||
put_object(stand, object_key(restored["id"]), data)
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
cleared = run_reconcile(stand, "apply")
|
||
(record,) = cleared.about(FIXED, ids=ids)
|
||
assert (record["kind"], record["action"]) == (
|
||
"restored_object",
|
||
"cleared_lost",
|
||
)
|
||
assert restored["id"] not in lost_ids(stand)
|
||
assert_content(
|
||
api.attachment_content(alice, restored["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids=ids)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Скорость прохода ограничена")
|
||
@pytest.mark.spec("storage-consistency", "Без ограничения скорости")
|
||
def test_reconcile_rate_limit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Предел на четверть объектов прохода: проход длится не меньше
|
||
объектов на предел, находки и код — как без предела."""
|
||
data = images.sample_png()
|
||
for _ in range(RATE_UPLOADS):
|
||
api.upload_ready(alice, chat["id"], data)
|
||
lost = api.upload_ready(alice, chat["id"], data)
|
||
ids = {lost["id"]}
|
||
lose_original(stand, lost)
|
||
# уборщик стенда стоит: между проходами стенд не меняется
|
||
with outage(stand, "stop", "cleaner"):
|
||
free = run_reconcile(stand, "dry-run", options=NO_RATE_LIMIT)
|
||
rate = max(5, math.ceil(free.finished["storage_objects"] / 4))
|
||
limited = run_reconcile(
|
||
stand,
|
||
"dry-run",
|
||
options=("--max-objects-per-second", str(rate)),
|
||
)
|
||
assert free.finished["max_objects_per_second"] == 0
|
||
assert limited.finished["max_objects_per_second"] == rate
|
||
objects = limited.finished["storage_objects"]
|
||
assert objects >= RATE_UPLOADS, limited.finished
|
||
assert (
|
||
limited.finished["duration_ms"] >= 1000 * objects / rate * RATE_SLACK
|
||
), limited.finished
|
||
assert free.finished["duration_ms"] < limited.finished["duration_ms"]
|
||
expected = {lost["id"]: "missing_object"}
|
||
assert free.kinds(FOUND, ids=ids) == expected
|
||
assert limited.kinds(FOUND, ids=ids) == expected
|
||
assert free.exit_code == limited.exit_code == 3
|
||
# стенд — без внесённой потери
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FIXED, ids=ids) == expected
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Отказ по пределу без лишних перепроверок",
|
||
)
|
||
def test_reconcile_limit_stops_rechecks(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
limit = 2
|
||
batch = [
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for _ in range(6)
|
||
]
|
||
ids = {item["id"] for item in batch}
|
||
for item in batch:
|
||
lose_original(stand, item)
|
||
refused = run_reconcile(stand, "apply", limit)
|
||
assert refused.exit_code == 3
|
||
(error,) = refused.events(LIMIT_EXCEEDED)
|
||
assert error["level"] == "error"
|
||
# отказ решён на третьем подтверждённом: остальные не перепроверены
|
||
assert error["changes"] == limit + 1, error
|
||
assert error["max_changes"] == limit
|
||
assert error["unchecked"] >= len(batch) - (limit + 1), error
|
||
assert refused.finished["unchecked"] == error["unchecked"]
|
||
assert refused.finished["refused"] is True
|
||
assert not refused.about(FIXED, ids=ids)
|
||
assert not ids & lost_ids(stand)
|
||
# dry-run перепроверяет каждого и называет все шесть
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, ids=ids) == dict.fromkeys(ids, "missing_object")
|
||
assert dry.finished["unchecked"] == 0
|
||
applied = run_reconcile(stand, "apply", 100)
|
||
assert set(applied.kinds(FIXED, ids=ids)) == ids
|
||
assert ids <= lost_ids(stand)
|
||
|
||
|
||
def own_findings(
|
||
run: Reconcile,
|
||
event: str,
|
||
keys: Collection[str],
|
||
) -> set[tuple[str, str, str | None]]:
|
||
"""Записи `event` о своих ключах: класс, ключ и `id` вложения — у
|
||
строки, ссылающейся на чужой ключ, `id` и ключ из разных вложений."""
|
||
return {
|
||
(item["kind"], item["object_key"], item.get("attachment_id"))
|
||
for item in run.events(event)
|
||
if item.get("object_key") in keys
|
||
}
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Строка ссылается на объект другого id",
|
||
)
|
||
def test_reconcile_row_with_other_id_key(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W2 verify `track2-reconcile-cost`: строка Y ссылается на
|
||
`<Z>/original` чужого `id`. Обе стратегии сравнивают Y с объектом по
|
||
её ключу: другой размер и, с `--deep`, другой SHA-256 — расхождения
|
||
Y; объекты Z и `<Y>/original` — чужие и на месте."""
|
||
data = images.png_noise(64, 48, seed=17)
|
||
y1, y2 = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
|
||
z1, z2 = (str(uuid.uuid4()) for _ in range(2))
|
||
keys = {
|
||
"y1": object_key(y1["id"]),
|
||
"y2": object_key(y2["id"]),
|
||
"z1": object_key(z1),
|
||
"z2": object_key(z2),
|
||
}
|
||
# Z1 — другого размера, Z2 — того же размера с другими байтами
|
||
put_object(stand, keys["z1"], data + b"\0")
|
||
put_object(stand, keys["z2"], same_size_bytes(data, seed=17))
|
||
try:
|
||
stand("set-key", y1["id"], keys["z1"])
|
||
stand("set-key", y2["id"], keys["z2"])
|
||
for item in (y1, y2):
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
# объекты Z записаны только что: после G + 1 все старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
expected = {
|
||
("size_mismatch", keys["z1"], y1["id"]),
|
||
("checksum_mismatch", keys["z2"], y2["id"]),
|
||
("unknown_object", keys["z1"], z1),
|
||
("unknown_object", keys["z2"], z2),
|
||
("unknown_object", keys["y1"], y1["id"]),
|
||
("unknown_object", keys["y2"], y2["id"]),
|
||
}
|
||
batch = run_reconcile(stand, "dry-run", options=DEEP)
|
||
head = run_reconcile(
|
||
stand,
|
||
"dry-run",
|
||
options=(*DEEP, *HEAD_STRATEGY),
|
||
)
|
||
assert own_findings(batch, FOUND, keys.values()) == expected
|
||
assert own_findings(head, FOUND, keys.values()) == expected
|
||
assert batch.exit_code == head.exit_code == 3
|
||
# поверхностная сверка видит только другой размер
|
||
shallow = run_reconcile(stand, "dry-run")
|
||
assert own_findings(shallow, FOUND, keys.values()) == expected - {
|
||
("checksum_mismatch", keys["z2"], y2["id"]),
|
||
}
|
||
|
||
applied = run_reconcile(stand, "apply", options=DEEP)
|
||
assert own_findings(applied, FIXED, keys.values()) == {
|
||
("size_mismatch", keys["z1"], y1["id"]),
|
||
("checksum_mismatch", keys["z2"], y2["id"]),
|
||
}
|
||
assert applied.exit_code == 3
|
||
# ни один объект не удалён, обе строки — ready и помечены
|
||
assert {y1["id"], y2["id"], z1, z2} <= stored_ids(stand)
|
||
rows = attachment_rows(stand)
|
||
assert rows[y1["id"]] == rows[y2["id"]] == "ready"
|
||
assert {y1["id"], y2["id"]} <= lost_ids(stand)
|
||
finally:
|
||
# прежние ключи; объекты Z, на которые больше никто не ссылается,
|
||
# — прочь, чтобы следующие проходы не удаляли их как свои
|
||
stand("set-key", y1["id"], keys["y1"])
|
||
stand("set-key", y2["id"], keys["y2"])
|
||
stand("delete-object", keys["z1"])
|
||
stand("delete-object", keys["z2"])
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Недавно перезаписанный объект не тронут",
|
||
)
|
||
def test_reconcile_fresh_overwrite_untouched(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание S1 verify `track2-reconcile-cost`: объект готового
|
||
вложения старше grace перезаписан мимо API меньше grace назад — другим
|
||
размером (A) и теми же размером с другими байтами (B). Сверка с
|
||
`--deep --apply` и HEAD-стратегией считает оба объекта свежими; после
|
||
grace — расхождения."""
|
||
data = images.png_noise(64, 48, seed=19)
|
||
a, b = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
|
||
ids = {a["id"], b["id"]}
|
||
for item in (a, b):
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
|
||
def overwrite() -> float:
|
||
"""Объекты A и B заново перезаписаны: моложе grace с этого момента."""
|
||
put_object(stand, object_key(a["id"]), data + b"\0")
|
||
put_object(stand, object_key(b["id"]), same_size_bytes(data, seed=19))
|
||
return time.monotonic()
|
||
|
||
try:
|
||
# перед каждым проходом объекты перезаписываются заново: глубокий
|
||
# проход читает весь bucket стенда, и два подряд дольше grace
|
||
for options, mode in (
|
||
(DEEP, "apply"),
|
||
((*DEEP, *HEAD_STRATEGY), "dry-run"),
|
||
):
|
||
written = overwrite()
|
||
run = run_reconcile(stand, mode, options=options)
|
||
# проход — пока объекты моложе grace, иначе он ничего не
|
||
# доказывает
|
||
assert time.monotonic() - written < CLEANER_GRACE_S, (
|
||
"reconcile run took longer than the stand grace period"
|
||
)
|
||
assert not run.about(FOUND, ids=ids), run.finished
|
||
assert not run.about(FIXED, ids=ids), run.finished
|
||
assert run.finished["fresh_objects"] >= len(ids), run.finished
|
||
assert not ids & lost_ids(stand)
|
||
assert_error(api.attachment_content(alice, a["id"]), 500)
|
||
|
||
time.sleep(max(0.0, OBJECT_AGE_WAIT_S - (time.monotonic() - written)))
|
||
aged = run_reconcile(stand, "dry-run", options=DEEP)
|
||
assert aged.kinds(FOUND, ids=ids) == {
|
||
a["id"]: "size_mismatch",
|
||
b["id"]: "checksum_mismatch",
|
||
}
|
||
finally:
|
||
# стенд — с исходными байтами: расхождений не остаётся
|
||
for item in (a, b):
|
||
put_object(stand, object_key(item["id"]), data)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сверка и фоновая обработка (change async-attachments, design, решение 11):
|
||
# объекты вариантов своих вложений и текущая работа — не расхождения
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объекты вариантов своих вложений")
|
||
def test_reconcile_variant_objects(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
batch = [api.upload_ready(alice, chat["id"], data) for _ in range(2)]
|
||
ids = {item["id"] for item in batch}
|
||
keys = {variant_key(item["id"]) for item in batch}
|
||
for item in batch:
|
||
assert stand("variant-object-count", item["id"]) == "1"
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
# объекты вариантов записаны только что: после G + 1 старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
for options in ((), HEAD_STRATEGY):
|
||
run = run_reconcile(stand, "dry-run", options=options)
|
||
assert not run.about(FOUND, ids=ids, keys=keys), run.finished
|
||
run.assert_code_matches()
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект варианта без метаданных")
|
||
def test_reconcile_orphan_variant(
|
||
stand: Stand,
|
||
) -> None:
|
||
data = images.sample_png()
|
||
orphan = str(uuid.uuid4())
|
||
key = variant_key(orphan)
|
||
put_object(stand, key, data)
|
||
assert stand("variant-object-count", orphan) == "1"
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
run = run_reconcile(stand, "apply")
|
||
(record,) = run.about(FIXED, keys={key})
|
||
assert (record["kind"], record["action"]) == (
|
||
"orphan_object",
|
||
"deleted_object",
|
||
)
|
||
assert not run.about(FOUND, keys={key})
|
||
assert stand("variant-object-count", orphan) == "0"
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Текущая работа не считается расхождением",
|
||
)
|
||
def test_reconcile_keeps_current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with current_work(api, stand, alice, chat) as work:
|
||
# конечные объекты F и P и варианты P старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
ids = {item["id"] for item in work.values()}
|
||
keys = {staging_key(item) for item in ids} | {
|
||
object_key(item) for item in ids
|
||
}
|
||
run = run_reconcile(stand, "apply")
|
||
assert not run.about(FOUND, ids=ids, keys=keys), run.finished
|
||
assert not run.about(FIXED, ids=ids, keys=keys), run.finished
|
||
run.assert_code_matches()
|
||
assert not ids & lost_ids(stand)
|
||
rows = attachment_rows(stand)
|
||
staged = staged_ids(stand)
|
||
for name, item in work.items():
|
||
assert rows.get(item["id"]) == name, (name, rows.get(item["id"]))
|
||
assert item["id"] in staged, name
|
||
stored = stored_ids(stand)
|
||
# конечный объект у задания, дошедшего до него, на месте
|
||
assert work["failed"]["id"] in stored
|
||
assert work["processing"]["id"] in stored
|