Move publishing out of the upload path: a relay process publishes queued jobs with confirms. Add lease and attempt fencing, backoff with jitter, a per-job render process with CPU and memory limits, crop and optimized variants, the failed registry with audited replay, and staging cleanup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
6054 lines
230 KiB
Python
6054 lines
230 KiB
Python
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
|
||
|
||
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
|
||
Valkey и S3, уборщик незавершённых загрузок (grace и интервал —
|
||
`FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API
|
||
(`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении),
|
||
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
|
||
соединения), «узкий по загрузкам» (`FAULT_UPLOADS_BASE_URL`: одно место
|
||
загрузки, одна в очереди, ожидание 1 с, срок тела 3 с — change
|
||
`upload-memory`) и «с коротким сроком S3» (`FAULT_S3_BASE_URL`: срок вызова
|
||
S3 1 с — change `fix-upload-put-flake`), брокер и worker (4 попытки, пауза
|
||
5 с, срок попытки 10 с, срок остановки 2 с — `FAULT_WORKER_*`, change
|
||
`async-attachments`). На хранилища и API тесты действуют только через
|
||
фикстуру `stand` (hook `student-tests/fault_hook.py`) и всегда возвращают
|
||
их в рабочее состояние. Решение 4 design change `failure-checks`, решение
|
||
10 design change `upload-consistency`, решение 11 design change
|
||
`track2-reconciler` (сверка bucket и БД — в конце модуля: её тесты
|
||
оставляют помеченные потерянными вложения и объекты staging без строки,
|
||
прежние тесты таких не ждут), решение 3 design change
|
||
`fix-upload-put-flake`, решение 15 design change `async-attachments`.
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import os
|
||
import random
|
||
import statistics
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable, Collection, Iterator, Sequence
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from contextlib import contextmanager
|
||
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
from jsonschema import Draft202012Validator, FormatChecker
|
||
|
||
import images
|
||
from api_client import API, Actor, Api, new_key
|
||
from contract import (
|
||
BEARER_CHALLENGE,
|
||
assert_accepted,
|
||
assert_content,
|
||
assert_error,
|
||
assert_ok,
|
||
assert_unauthorized,
|
||
assert_variant_content,
|
||
assert_variant_image,
|
||
describe,
|
||
)
|
||
|
||
pytestmark = pytest.mark.fault
|
||
|
||
type Stand = Callable[..., str]
|
||
|
||
RESTORE = {"stop": "start", "pause": "unpause", "kill": "start"}
|
||
# запас к границам времени: планировщик Docker Desktop
|
||
SLACK_S = 0.3
|
||
# [cache] timeout_seconds и retry_after_seconds по умолчанию
|
||
CACHE_TIMEOUT_S = 0.2
|
||
CACHE_RETRY_AFTER_S = 5
|
||
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
|
||
SESSION_TTL_S = 2
|
||
IDEMPOTENCY_TTL_S = 2
|
||
POOL_TIMEOUT_S = 1
|
||
# срок проверки Postgres в /health/ready
|
||
READINESS_TIMEOUT_S = 2
|
||
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
|
||
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
|
||
OBJECT_STORAGE_HEALTH_S = 1
|
||
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
|
||
S3_TIMEOUT_S = 5
|
||
# сколько секунд hook держит слоты max_connections; clear-connections
|
||
# освобождает их раньше
|
||
FILL_HOLD_S = 30
|
||
ME = f"{API}/users/me"
|
||
|
||
|
||
@contextmanager
|
||
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
||
"""`stop`, `pause` или `kill` сервиса; в `finally` — обратно."""
|
||
stand(action, service)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand(RESTORE[action], service)
|
||
|
||
|
||
def timed(
|
||
call: Callable[[], httpx.Response],
|
||
) -> tuple[httpx.Response, float]:
|
||
started = time.monotonic()
|
||
resp = call()
|
||
return resp, time.monotonic() - started
|
||
|
||
|
||
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
|
||
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
|
||
|
||
|
||
def wait_cache_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
||
|
||
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
||
"""
|
||
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
|
||
stand("flush-cache")
|
||
while time.monotonic() < deadline:
|
||
assert_ok(read_chat(api, actor, chat), 200, "Chat")
|
||
if stand("cache-has-chat", chat["id"]) == "1":
|
||
return
|
||
time.sleep(0.5)
|
||
raise AssertionError("cache did not come back")
|
||
|
||
|
||
def chat_titles(api: Api, actor: Actor) -> set[str]:
|
||
return {
|
||
chat["title"]
|
||
for page in api.chat_pages(actor)
|
||
for chat in page["items"]
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Истёкший токен")
|
||
def test_expired_token(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
time.sleep(SESSION_TTL_S + 1)
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Срок не продлевается")
|
||
def test_use_does_not_extend_session(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
logged_in = time.monotonic()
|
||
successes = 0
|
||
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
successes += 1
|
||
time.sleep(0.5)
|
||
assert successes >= 3, "too few requests before expiry"
|
||
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
|
||
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
|
||
actor = api.register()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.request("GET", ME, actor=actor)
|
||
assert_error(resp, 503)
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Хранение: откат, остановленная БД, исчерпанный пул
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
|
||
def test_members_failure_rolls_back_chat(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
) -> None:
|
||
title = f"fault-{uuid.uuid4().hex}"
|
||
stand("fail-members", title)
|
||
try:
|
||
resp = api.request(
|
||
"POST",
|
||
f"{API}/chats",
|
||
actor=alice,
|
||
json={"title": title, "member_ids": [alice.id, bob.id]},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-members")
|
||
for actor in (alice, bob):
|
||
assert title not in chat_titles(api, actor)
|
||
assert stand("count-chats", title) == "0"
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "БД остановлена")
|
||
@pytest.mark.spec("data-storage", "БД вернулась")
|
||
def test_database_stopped_and_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
error = assert_error(resp, 503)
|
||
assert took < 2, f"503 took {took:.2f}s"
|
||
for word in ("postgres", "sql", "connect"):
|
||
assert word not in error["message"].lower()
|
||
message = api.send_message(alice, chat["id"], "после возврата")
|
||
# запись во время отказа не зафиксирована, новая — видна
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
|
||
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
holder = Api(str(tight_api.client.base_url), timeout=15)
|
||
first: list[int] = []
|
||
|
||
def hold_connection() -> None:
|
||
first.append(holder.request("GET", ME, actor=actor).status_code)
|
||
|
||
with outage(stand, "pause", "postgres"):
|
||
thread = threading.Thread(target=hold_connection)
|
||
thread.start()
|
||
time.sleep(SLACK_S)
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
assert_error(resp, 503)
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
thread.join()
|
||
holder.close()
|
||
assert first == [503]
|
||
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
|
||
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
# hook оборвал соединение пула: при выдаче pool_pre_ping
|
||
# переподключается и получает «too many clients» (SQLSTATE 53300)
|
||
stand("fill-connections", str(FILL_HOLD_S))
|
||
try:
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
error = assert_error(resp, 503)
|
||
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
for word in ("postgres", "sql", "client", "connect"):
|
||
assert word not in error["message"].lower()
|
||
finally:
|
||
stand("clear-connections")
|
||
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
|
||
# сессии 2 с за время заполнения истёк — новый вход
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
# пачка одновременных запросов больше пула обычного API (10)
|
||
BURST_THREADS = 48
|
||
BURST_AFTER_BARRIER = 4
|
||
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
|
||
STATS_FLUSH_S = 11
|
||
|
||
|
||
def burst(api: Api, actor: Actor) -> None:
|
||
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
|
||
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
|
||
одновременно. Все ответы — 200."""
|
||
barrier = threading.Barrier(BURST_THREADS)
|
||
clients = [
|
||
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
|
||
]
|
||
responses: list[httpx.Response] = []
|
||
errors: list[BaseException] = []
|
||
lock = threading.Lock()
|
||
|
||
def worker(client: Api) -> None:
|
||
try:
|
||
got = [client.request("GET", ME, actor=actor)]
|
||
barrier.wait()
|
||
got += [
|
||
client.request("GET", ME, actor=actor)
|
||
for _ in range(BURST_AFTER_BARRIER)
|
||
]
|
||
except BaseException as e: # noqa: BLE001 — поднимется в тесте
|
||
barrier.abort()
|
||
with lock:
|
||
errors.append(e)
|
||
return
|
||
with lock:
|
||
responses.extend(got)
|
||
|
||
threads = [
|
||
threading.Thread(target=worker, args=(client,)) for client in clients
|
||
]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
for client in clients:
|
||
client.close()
|
||
assert not errors, f"burst failed: {errors[0]!r}"
|
||
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
|
||
for resp in responses:
|
||
assert_ok(resp, 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
|
||
def test_pool_keeps_connections(
|
||
api: Api,
|
||
tight_api: Api,
|
||
stand: Stand,
|
||
) -> None:
|
||
actor = api.register()
|
||
# соединение «тесного» API могли оборвать прежние тесты
|
||
# (fill-connections): его переподключение не должно попасть в окно
|
||
tight = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
|
||
# прогрев: пул обычного API набирает свой размер
|
||
for _ in range(2):
|
||
burst(api, actor)
|
||
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
|
||
# в общую статистику без ожидания блокировки и при занятой блокировке
|
||
# откладываются. Простаивающий backend сбрасывает их принудительно по
|
||
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
|
||
# попадали в окно по одному
|
||
time.sleep(STATS_FLUSH_S)
|
||
# калибровка: второй вызов видит сессию первого вызова hook, и только её
|
||
s1 = int(stand("db-sessions"))
|
||
s2 = int(stand("db-sessions"))
|
||
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
|
||
for _ in range(3):
|
||
burst(api, actor)
|
||
s3 = int(stand("db-sessions"))
|
||
opened = s3 - s2 - 1
|
||
assert opened == 0, (
|
||
f"PostgreSQL accepted {opened} new sessions during "
|
||
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Readiness
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище не готово")
|
||
def test_readiness_without_database(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(lambda: api.request("GET", "/health/ready"))
|
||
assert_error(resp, 503)
|
||
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
|
||
live = api.request("GET", "/health/live")
|
||
assert live.status_code == 200
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Кэш недоступен")
|
||
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище объектов недоступно")
|
||
@pytest.mark.parametrize("action", ["stop", "pause"])
|
||
def test_readiness_without_object_storage(
|
||
api: Api,
|
||
stand: Stand,
|
||
action: str,
|
||
) -> None:
|
||
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
|
||
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
|
||
with outage(stand, action, "s3"):
|
||
for path in ("/health/ready", "/health/live"):
|
||
resp, took = timed(lambda path=path: api.request("GET", path))
|
||
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
|
||
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Кэш карточки чата
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Очистка кэша")
|
||
def test_flush_cache(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
stand("flush-cache")
|
||
assert stand("cache-has-chat", chat["id"]) == "0"
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey остановлен")
|
||
def test_valkey_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
sent = [api.send_message(alice, chat["id"], "до отказа")]
|
||
with outage(stand, "stop", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
sent.append(assert_ok(resp, 201, "Message"))
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
|
||
assert api.all_messages(bob, chat["id"]) == sent
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey завис")
|
||
def test_valkey_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
baseline = statistics.median(
|
||
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
|
||
)
|
||
with outage(stand, "pause", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
|
||
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
||
)
|
||
# окно пропуска: следующий запрос таймаута не ждёт
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey вернулся")
|
||
def test_valkey_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
# API не перезапускался; кэш снова в работе после окна пропуска
|
||
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
hits = int(stand("cache-hits"))
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert int(stand("cache-hits")) > hits
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
|
||
def test_message_failure_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
# ключ вставляется первым: триггер роняет транзакцию уже после него
|
||
text = f"fault-{uuid.uuid4().hex}"
|
||
key = new_key()
|
||
stand("fail-messages", text)
|
||
try:
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-messages")
|
||
# ключ не занят: повтор выполняется заново, а не 500 и не 409
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
|
||
def test_database_outage_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
text = "при отказе БД"
|
||
key = new_key()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 503)
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Ключ после срока")
|
||
def test_key_free_after_ttl(tight_api: Api) -> None:
|
||
alice = tight_api.register()
|
||
chat = tight_api.create_chat(alice)
|
||
key = new_key()
|
||
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
|
||
# в пределах срока ключ занят: другое тело — 409
|
||
resp = tight_api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": "после срока"},
|
||
)
|
||
assert_error(resp, 409)
|
||
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
|
||
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
|
||
alice = tight_api.login(alice.user, alice.password)
|
||
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
|
||
assert second["id"] != first["id"]
|
||
history = tight_api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [first["id"], second["id"]]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки
|
||
# и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает
|
||
# строки прежних тестов параллельно (design change upload-consistency,
|
||
# решение 10)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# grace и интервал уборщика стенда отказов (run_failures.py,
|
||
# failures.compose.yml)
|
||
CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20"))
|
||
CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1"))
|
||
# запас поверх grace и интервала: проход, вызов hook, планировщик Docker
|
||
# Desktop
|
||
CLEANUP_SLACK_S = 5
|
||
# задержка перевода в queued триггером stall и в ready триггером ready-stall
|
||
# (fault_hook.py)
|
||
STALL_S = 4
|
||
# срок клиента загрузок в потоке
|
||
THREAD_CLIENT_TIMEOUT_S = 30
|
||
# имя базы стенда: ключи объектов — `<база>/attachments/<id>/original` и
|
||
# `<база>/staging/<id>/original`
|
||
FAULT_DATABASE = os.environ.get("FAULT_DATABASE", "messenger")
|
||
|
||
|
||
def attachment_rows(stand: Stand) -> dict[str, str]:
|
||
"""Вложения стенда: `id` → статус."""
|
||
return dict(line.split() for line in stand("attachment-ids").splitlines())
|
||
|
||
|
||
def stored_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений с объектом в конечном префиксе bucket стенда."""
|
||
return set(stand("object-ids").splitlines())
|
||
|
||
|
||
def staged_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений с объектом в staging."""
|
||
return set(stand("staging-ids").splitlines())
|
||
|
||
|
||
def object_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений с объектом в любом из двух префиксов."""
|
||
return stored_ids(stand) | staged_ids(stand)
|
||
|
||
|
||
def orphan_objects(stand: Stand) -> set[str]:
|
||
"""Объекты вложений без строки. Сначала строки, потом объекты: уборщик
|
||
удаляет объект раньше строки, и убранная между вызовами строка не
|
||
даёт ложного «объекта без строки»."""
|
||
rows = attachment_rows(stand)
|
||
return object_ids(stand) - set(rows)
|
||
|
||
|
||
def accepted_without_staging(stand: Stand) -> set[str]:
|
||
"""Принятые вложения без своего объекта: в работе — staging, готовые —
|
||
конечного оригинала (staging у них убирает уборщик). У `failed` его может не
|
||
быть по самой причине отказа."""
|
||
rows = attachment_rows(stand)
|
||
working = {
|
||
item
|
||
for item, status in rows.items()
|
||
if status in ("queued", "processing")
|
||
}
|
||
return (working - staged_ids(stand)) | (
|
||
ready_ids(rows) - stored_ids(stand)
|
||
)
|
||
|
||
|
||
def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]:
|
||
return set(after) - set(before)
|
||
|
||
|
||
def objects_and_rows(stand: Stand) -> tuple[dict[str, str], set[str]]:
|
||
"""Снимок: строки и объекты вложений обоих префиксов."""
|
||
return attachment_rows(stand), object_ids(stand)
|
||
|
||
|
||
def new_objects(
|
||
before: tuple[dict[str, str], set[str]],
|
||
stand: Stand,
|
||
) -> set[str]:
|
||
"""Объекты, которых не было в снимке, у `id`, которых в нём тоже не было.
|
||
Worker параллельно доводит задания прежних тестов: объект конечного ключа
|
||
у прежнего `id` — не след новой загрузки."""
|
||
return object_ids(stand) - before[1] - set(before[0])
|
||
|
||
|
||
def job_row(stand: Stand, attachment_id: str) -> dict[str, Any] | None:
|
||
"""Задание вложения; нет — `None`. Два задания дали бы две строки и
|
||
ошибку разбора: задание у вложения одно."""
|
||
output = stand("job", attachment_id)
|
||
return json.loads(output) if output else None
|
||
|
||
|
||
def ready_ids(rows: dict[str, str]) -> set[str]:
|
||
return {item for item, status in rows.items() if status == "ready"}
|
||
|
||
|
||
def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None:
|
||
"""Строка и объект вложения убраны не позже grace + интервал + запас
|
||
от `since` (момент до загрузки: `created_at` строки не раньше)."""
|
||
deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while True:
|
||
rows = attachment_rows(stand)
|
||
stored = object_ids(stand)
|
||
if attachment_id not in rows and attachment_id not in stored:
|
||
return
|
||
if time.monotonic() > deadline:
|
||
raise AssertionError(
|
||
f"attachment {attachment_id} is not cleaned "
|
||
f"{time.monotonic() - since:.1f}s after upload: "
|
||
f"row {rows.get(attachment_id)}, "
|
||
f"object {attachment_id in stored}",
|
||
)
|
||
time.sleep(0.5)
|
||
|
||
|
||
def sleep_until(moment: float) -> None:
|
||
time.sleep(max(0.0, moment - time.monotonic()))
|
||
|
||
|
||
def fault_filename() -> str:
|
||
"""Уникальное имя загрузки, на которое срабатывает триггер hook."""
|
||
return f"fault-{uuid.uuid4().hex}.png"
|
||
|
||
|
||
def upload_raw(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: Any,
|
||
**kwargs: Any,
|
||
) -> httpx.Response:
|
||
return api.post_attachment(actor, chat["id"], data, **kwargs)
|
||
|
||
|
||
def assert_invisible(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
attachment_id: str,
|
||
) -> None:
|
||
"""Незавершённая загрузка: метаданные, байты и ссылка в сообщении — 404
|
||
с тем же `message`, что у неизвестного `id`."""
|
||
|
||
def calls(item: str) -> tuple[httpx.Response, ...]:
|
||
return (
|
||
api.attachment_meta(actor, item),
|
||
api.attachment_content(actor, item),
|
||
api.post_message(
|
||
actor,
|
||
chat["id"],
|
||
json={"text": "ссылка", "attachment_ids": [item]},
|
||
),
|
||
)
|
||
|
||
unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))]
|
||
hidden = [assert_error(resp, 404) for resp in calls(attachment_id)]
|
||
assert [error["message"] for error in hidden] == [
|
||
error["message"] for error in unknown
|
||
]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
|
||
@pytest.mark.spec("attachments", "Незавершённая загрузка не видна")
|
||
def test_upload_with_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
# хотя бы одно готовое вложение: контроль ниже не пуст случайно
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
before = attachment_rows(stand)
|
||
assert ready["id"] in ready_ids(before)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
after = attachment_rows(stand)
|
||
# строка-намерение осталась; её уберёт уборщик
|
||
# (test_cleaner_survives_outages)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
# ready не откатывается; число готовых может расти: worker
|
||
# параллельно доводит задания прежних тестов
|
||
assert ready_ids(before) <= ready_ids(after)
|
||
# заданий и принятых не прибавилось: новая строка одна и pending
|
||
assert job_row(stand, pending) is None
|
||
# замечание S6 verify attachments: id последней pending — от hook
|
||
assert stand("last-pending") == pending
|
||
assert_invisible(api, alice, chat, pending)
|
||
api.send_message(alice, chat["id"], "текст без S3")
|
||
api.upload(alice, chat["id"], data)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
|
||
def test_rejected_uploads_leave_nothing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
|
||
|
||
def chunked() -> Iterator[bytes]:
|
||
for offset in range(0, len(big), 1024 * 1024):
|
||
yield big[offset : offset + 1024 * 1024]
|
||
|
||
# хотя бы одно готовое вложение: списки hook не пусты случайно
|
||
ready = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
before = objects_and_rows(stand)
|
||
assert before[0].get(ready["id"]) == "ready", before[0]
|
||
assert ready["id"] in before[1], before[1]
|
||
rejected = (
|
||
(404, upload_raw(api, eve, chat, images.sample_png())),
|
||
(413, upload_raw(api, alice, chat, big)),
|
||
(413, upload_raw(api, alice, chat, chunked())),
|
||
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
|
||
(
|
||
415,
|
||
upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
|
||
)
|
||
for status, resp in rejected:
|
||
assert_error(resp, status)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Сбой записи метаданных")
|
||
def test_metadata_insert_failure(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = objects_and_rows(stand)
|
||
stand("fail-attachments", "insert", filename)
|
||
try:
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строки нет — объект не записывался
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
|
||
|
||
class BodyAborted(Exception):
|
||
"""Клиент обрывает передачу тела загрузки."""
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Обрыв передачи тела")
|
||
def test_interrupted_body(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = objects_and_rows(stand)
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
# свой клиент: его соединение обрывается вместе с телом
|
||
with (
|
||
Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно; строки до конца тела нет в любом
|
||
# случае
|
||
time.sleep(1)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Фиксация задания не удалась")
|
||
@pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_queued_failure_compensated(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "queued", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert stand("last-pending") == pending
|
||
# компенсация: объект удалён до ответа 503, задания нет
|
||
assert pending not in object_ids(stand)
|
||
assert job_row(stand, pending) is None
|
||
assert_invisible(api, alice, chat, pending)
|
||
# моложе grace строку уборщик не трогает: к половине grace он сделал
|
||
# около G / 2 / I проходов
|
||
sleep_until(started + CLEANER_GRACE_S / 2)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект без фиксации убран")
|
||
def test_killed_between_put_and_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
outcome: dict[str, Any] = {}
|
||
client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
|
||
def upload() -> None:
|
||
try:
|
||
outcome["status"] = upload_raw(
|
||
client,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
).status_code
|
||
except httpx.TransportError as e:
|
||
outcome["error"] = e
|
||
|
||
stand("fail-attachments", "stall", filename)
|
||
killed = False
|
||
try:
|
||
started = time.monotonic()
|
||
thread = threading.Thread(target=upload)
|
||
thread.start()
|
||
# строка pending и объект staging — до конца задержки перевода в
|
||
# queued
|
||
deadline = started + STALL_S
|
||
pending = ""
|
||
while time.monotonic() < deadline:
|
||
pending = stand("last-pending")
|
||
if (
|
||
pending
|
||
and pending not in before
|
||
and pending in staged_ids(stand)
|
||
):
|
||
break
|
||
time.sleep(0.05)
|
||
else:
|
||
raise AssertionError("no pending row with object during stall")
|
||
stand("kill-api")
|
||
killed = True
|
||
thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not thread.is_alive()
|
||
# 202 до клиента не дошёл: перевод в queued ждал в триггере
|
||
assert outcome.get("status") != 202, outcome
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert pending in staged_ids(stand)
|
||
assert job_row(stand, pending) is None
|
||
finally:
|
||
client.close()
|
||
stand("clear-fail-attachments")
|
||
if killed:
|
||
stand("start-api")
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Исход фиксации неизвестен")
|
||
def test_queued_commit_outcome_unknown(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Перевод в queued падает на самом COMMIT (отложенный триггер): API не
|
||
знает, зафиксировано ли вложение, и объект не трогает (verify W2)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
# компенсации нет: вложение могло стать queued
|
||
assert pending in staged_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Загрузка дольше grace")
|
||
def test_upload_outlives_grace(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Строка вставлена «на час старше», перевод в queued ждёт в триггере,
|
||
пока её не уберёт уборщик стенда: загрузка, пережившая grace, без
|
||
заморозки процесса (verify W3)."""
|
||
filename = fault_filename()
|
||
before = objects_and_rows(stand)
|
||
stand("fail-attachments", "outlive", filename)
|
||
# ожидание в триггере до 4,5 с: срок клиента больше
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
# не 202 (принято без объекта) и не 500 (дефект ветки)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
rows = attachment_rows(stand)
|
||
assert not new_ids(before[0], rows), rows
|
||
assert not new_objects(before, stand)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта")
|
||
def test_cleaner_failure_after_object_delete(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Уборщик удалил объект, а удаление строки упало (08006), пока загрузка
|
||
ждёт перевода в queued: строка помечена до удаления объекта и queued уже
|
||
не станет (verify W1)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "undeletable", filename)
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert job_row(stand, pending) is None
|
||
assert pending not in staged_ids(stand)
|
||
assert not accepted_without_staging(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строка старше grace: следующий проход доводит уборку
|
||
returned = time.monotonic()
|
||
wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S)
|
||
assert not orphan_objects(stand)
|
||
assert not accepted_without_staging(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "БД недоступна во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты")
|
||
def test_cleaner_survives_outages(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
assert_error(upload_raw(api, alice, chat, data), 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
# PostgreSQL недоступен, пока загрузка моложе grace
|
||
with outage(stand, "stop", "postgres"):
|
||
time.sleep(3)
|
||
# grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте
|
||
sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
# S3 вернулся: уборщик без перезапуска убирает строку за проход
|
||
returned = time.monotonic()
|
||
deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while pending in attachment_rows(stand):
|
||
assert time.monotonic() < deadline, (
|
||
f"pending {pending} is still there "
|
||
f"{time.monotonic() - returned:.1f}s after S3 came back"
|
||
)
|
||
time.sleep(0.5)
|
||
assert pending not in object_ids(stand)
|
||
assert not orphan_objects(stand)
|
||
# готовое вложение старше grace на месте
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, ready["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == ready
|
||
assert meta["status"] == "ready"
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 остановлен")
|
||
def test_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(
|
||
lambda: api.attachment_content(bob, attachment["id"]),
|
||
)
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
meta = assert_ok(
|
||
api.attachment_meta(bob, attachment["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == attachment
|
||
message = api.send_message(
|
||
bob,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment["id"]],
|
||
)
|
||
# история с attachment_ids — из PostgreSQL, S3 ей не нужен
|
||
history = api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [message["id"]]
|
||
assert history[0]["attachment_ids"] == [attachment["id"]]
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
assert_content(
|
||
api.attachment_content(bob, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
def test_environment_does_not_change_limits(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `service-configuration`, «Окружение не меняет лимит» (вне
|
||
сверки покрытия — сценарий в докстринге): обычный API стенда
|
||
отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1`
|
||
(`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт,
|
||
64 000 пикселей) всё равно принимается: лимиты — константы."""
|
||
limits = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_LIMITS_ENV", "").split()
|
||
)
|
||
# без этих переменных на стенде тест ничего не доказывает
|
||
assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, (
|
||
f"stand started without limit variables: {limits}"
|
||
)
|
||
api.upload(alice, chat["id"], images.sample_png(), filename="sample.png")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 завис")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_object_storage_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
|
||
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
|
||
with slow, outage(stand, "pause", "s3"):
|
||
for call in (
|
||
lambda: upload_raw(slow, alice, chat, data),
|
||
lambda: slow.attachment_content(alice, attachment["id"]),
|
||
):
|
||
resp, elapsed = timed(call)
|
||
assert_error(resp, 503)
|
||
# срок вызова S3 и немного на шаги с БД, а не зависание
|
||
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
|
||
f"503 after {elapsed:.2f}s"
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
# загрузка во время заморозки оставила строку pending; объект после
|
||
# разморозки не дописался, строку уборщик убирает после grace
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Места загрузки и срок тела (change upload-memory, design, решение 11):
|
||
# «узкий по загрузкам» API — одно место, одна загрузка в очереди, ожидание
|
||
# места и срок тела из FAULT_UPLOADS_ENV (run_failures.py, UPLOADS_ENV).
|
||
# Место держит загрузка, тело которой стоит на threading.Event теста
|
||
# ---------------------------------------------------------------------------
|
||
|
||
UPLOADS_ENV = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_UPLOADS_ENV", "").split()
|
||
)
|
||
# окружение, с которым тесты ниже ждут «узкий» API
|
||
EXPECTED_UPLOADS_ENV = {
|
||
"UPLOADS__MAX_CONCURRENT": "1",
|
||
"UPLOADS__MAX_WAITING": "1",
|
||
"UPLOADS__WAIT_TIMEOUT_SECONDS": "1",
|
||
"UPLOADS__BODY_TIMEOUT_SECONDS": "3",
|
||
}
|
||
WAIT_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__WAIT_TIMEOUT_SECONDS", "1"))
|
||
BODY_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__BODY_TIMEOUT_SECONDS", "3"))
|
||
# отказ без ожидания места: вход, членство и ответ
|
||
IMMEDIATE_S = 0.5
|
||
# пауза, за которую запрос в соседнем потоке доходит до места или очереди
|
||
SETTLE_S = 0.2
|
||
# сколько держатель места ждёт отпускания, если тест упал раньше
|
||
HOLD_MAX_S = 10
|
||
|
||
|
||
def assert_uploads_stand() -> None:
|
||
"""Без этого окружения «узкого» API тесты мест ничего не доказывают."""
|
||
assert UPLOADS_ENV == EXPECTED_UPLOADS_ENV, (
|
||
f"uploads stand started with {UPLOADS_ENV}, "
|
||
f"expected {EXPECTED_UPLOADS_ENV}"
|
||
)
|
||
|
||
|
||
class Upload:
|
||
"""Загрузка в своём потоке и своём соединении; ответ — `result()`."""
|
||
|
||
def __init__(self, api: Api, actor: Actor, chat: dict[str, Any]) -> None:
|
||
self._base_url = str(api.client.base_url)
|
||
self._actor = actor
|
||
self._chat = chat
|
||
self._outcome: dict[str, Any] = {}
|
||
self._thread: threading.Thread | None = None
|
||
|
||
def start(self, data: Any, **kwargs: Any) -> "Upload":
|
||
def run() -> None:
|
||
with Api(self._base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
|
||
started = time.monotonic()
|
||
try:
|
||
self._outcome["resp"] = upload_raw(
|
||
own,
|
||
self._actor,
|
||
self._chat,
|
||
data,
|
||
**kwargs,
|
||
)
|
||
except (BodyAborted, httpx.TransportError) as e:
|
||
self._outcome["error"] = e
|
||
self._outcome["elapsed"] = time.monotonic() - started
|
||
self._outcome["ended"] = time.monotonic()
|
||
|
||
self._thread = threading.Thread(target=run)
|
||
self._thread.start()
|
||
return self
|
||
|
||
def result(self) -> tuple[httpx.Response, float, float]:
|
||
"""Ответ, длительность и момент ответа (`time.monotonic`)."""
|
||
assert self._thread is not None
|
||
self._thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not self._thread.is_alive(), "upload thread hangs"
|
||
assert "resp" in self._outcome, self._outcome
|
||
return (
|
||
self._outcome["resp"],
|
||
self._outcome["elapsed"],
|
||
self._outcome["ended"],
|
||
)
|
||
|
||
|
||
@contextmanager
|
||
def held_slot(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: bytes,
|
||
) -> Iterator[Callable[[], httpx.Response]]:
|
||
"""Загрузка A держит место: отдаёт половину тела и ждёт отпускания.
|
||
|
||
Внутри — после того, как A заняла место; вызов отдаваемой функции
|
||
отпускает A и возвращает её ответ. В `finally` A отпускается всегда.
|
||
"""
|
||
sent = threading.Event()
|
||
release = threading.Event()
|
||
|
||
def body() -> Iterator[bytes]:
|
||
yield data[: len(data) // 2]
|
||
sent.set()
|
||
if not release.wait(HOLD_MAX_S):
|
||
raise BodyAborted
|
||
yield data[len(data) // 2 :]
|
||
|
||
holder = Upload(api, actor, chat).start(body())
|
||
try:
|
||
assert sent.wait(HOLD_MAX_S), "holder did not start its body"
|
||
# вход и членство — короткие шаги БД, затем место
|
||
time.sleep(SETTLE_S)
|
||
|
||
def finish() -> httpx.Response:
|
||
release.set()
|
||
return holder.result()[0]
|
||
|
||
yield finish
|
||
finally:
|
||
release.set()
|
||
|
||
|
||
def slow_chunks(data: bytes, *, parts: int, seconds: float) -> Iterator[bytes]:
|
||
"""Тело `parts` частями, растянутое на `seconds`."""
|
||
size = -(-len(data) // parts)
|
||
for offset in range(0, len(data), size):
|
||
if offset:
|
||
time.sleep(seconds / (parts - 1))
|
||
yield data[offset : offset + size]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Все места заняты")
|
||
@pytest.mark.spec("attachments", "Очередь ожидания полна")
|
||
@pytest.mark.spec("api-conventions", "Лимит одновременных загрузок")
|
||
def test_upload_slots_busy(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
before = objects_and_rows(stand)
|
||
with held_slot(uploads_api, alice, chat, data) as finish_holder:
|
||
# B ждёт место в очереди, C застаёт полную очередь
|
||
waiting = Upload(uploads_api, alice, chat).start(data)
|
||
time.sleep(SETTLE_S)
|
||
busy, busy_s = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, data),
|
||
)
|
||
waited, waited_s, _ = waiting.result()
|
||
# A отпускается сразу: её тело ограничено сроком тела стенда
|
||
holder = finish_holder()
|
||
assert_error(busy, 429)
|
||
assert busy_s < IMMEDIATE_S, f"429 of a full queue after {busy_s:.2f}s"
|
||
assert_error(waited, 429)
|
||
assert waited_s >= WAIT_TIMEOUT_S, f"429 after {waited_s:.2f}s"
|
||
for resp in (busy, waited):
|
||
assert int(resp.headers["Retry-After"]) >= 1
|
||
attachment = assert_accepted(holder)
|
||
# из трёх загрузок осталась только принятая: у 429 ни строки, ни объекта
|
||
assert new_ids(before[0], attachment_rows(stand)) == {attachment["id"]}
|
||
assert new_objects(before, stand) == {attachment["id"]}
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отказ раньше очереди")
|
||
def test_rejections_before_queue(
|
||
uploads_api: Api,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
# 30 MiB, которые начинаются как PNG: Content-Length 31 457 280
|
||
big = images.PNG_SIGNATURE + bytes(
|
||
3 * images.MAX_BYTES - len(images.PNG_SIGNATURE),
|
||
)
|
||
with held_slot(uploads_api, alice, chat, data) as finish_holder:
|
||
waiting = Upload(uploads_api, alice, chat).start(data)
|
||
time.sleep(SETTLE_S)
|
||
early = [
|
||
(status, *timed(call))
|
||
for status, call in (
|
||
(404, lambda: upload_raw(uploads_api, eve, chat, data)),
|
||
(413, lambda: upload_raw(uploads_api, alice, chat, big)),
|
||
(
|
||
415,
|
||
lambda: upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
data,
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
)
|
||
]
|
||
early_done = time.monotonic()
|
||
waited, _, waited_at = waiting.result()
|
||
holder = finish_holder()
|
||
# очередь была полна всё время ранних отказов
|
||
assert_error(waited, 429)
|
||
assert early_done < waited_at
|
||
for status, resp, elapsed in early:
|
||
assert_error(resp, status)
|
||
# без ожидания места: ожидающий получил бы его не раньше срока
|
||
assert elapsed < WAIT_TIMEOUT_S, f"{status} after {elapsed:.2f}s"
|
||
assert early[1][1].request.headers["Content-Length"] == "31457280"
|
||
assert_accepted(holder)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Ожидание места")
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Ожидание места не входит в возраст загрузки",
|
||
)
|
||
def test_upload_waits_for_slot(
|
||
uploads_api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
first = images.sample_png()
|
||
second = images.png(80, 60)
|
||
with held_slot(uploads_api, alice, chat, first) as finish_holder:
|
||
waiting = Upload(uploads_api, bob, chat).start(second)
|
||
time.sleep(SETTLE_S)
|
||
# A заканчивается раньше срока ожидания B
|
||
holder = finish_holder()
|
||
held = assert_accepted(holder)
|
||
waited_resp, waited_s, _ = waiting.result()
|
||
waited = assert_accepted(waited_resp)
|
||
assert waited_s < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, waited_s
|
||
# метаданные ожидавшей записаны после перевода в queued державшей место:
|
||
# ожидание прошло до строки pending (граница B уборщика прежняя)
|
||
assert waited["created_at"] > held["updated_at"], (held, waited)
|
||
for attachment, data in ((held, first), (waited, second)):
|
||
uploads_api.wait_attachment(alice, attachment["id"])
|
||
assert_content(
|
||
uploads_api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Место освобождается")
|
||
def test_slot_released_after_failures(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
over = images.png_of_size(images.MAX_BYTES + 1)
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
def next_upload_gets_slot(after: str) -> None:
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, images.sample_png()),
|
||
)
|
||
assert_accepted(resp)
|
||
assert elapsed < WAIT_TIMEOUT_S, (
|
||
f"upload after {after} waited {elapsed:.2f}s for a slot"
|
||
)
|
||
|
||
assert_error(
|
||
upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
slow_chunks(over, parts=10, seconds=0),
|
||
),
|
||
413,
|
||
)
|
||
next_upload_gets_slot("413 by counter")
|
||
assert_error(upload_raw(uploads_api, alice, chat, b"not an image"), 415)
|
||
next_upload_gets_slot("415 by content")
|
||
slow = slow_chunks(
|
||
images.sample_png(),
|
||
parts=4,
|
||
seconds=BODY_TIMEOUT_S + 1,
|
||
)
|
||
assert_error(upload_raw(uploads_api, alice, chat, slow), 400)
|
||
next_upload_gets_slot("400 by body timeout")
|
||
with (
|
||
Api(
|
||
str(uploads_api.client.base_url),
|
||
timeout=THREAD_CLIENT_TIMEOUT_S,
|
||
) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно
|
||
time.sleep(SETTLE_S)
|
||
next_upload_gets_slot("interrupted body")
|
||
with outage(stand, "stop", "s3"):
|
||
assert_error(
|
||
upload_raw(uploads_api, alice, chat, images.sample_png()),
|
||
503,
|
||
)
|
||
next_upload_gets_slot("503 with S3 stopped")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Медленный клиент")
|
||
@pytest.mark.spec("attachments", "Медленная передача в срок")
|
||
def test_slow_body(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
before = objects_and_rows(stand)
|
||
late = upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
slow_chunks(data, parts=6, seconds=BODY_TIMEOUT_S + 1),
|
||
)
|
||
error = assert_error(late, 400)
|
||
assert error["message"] == "Upload body was not received in time"
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
# мелкие части с паузами, но в срок: тело принято как есть
|
||
in_time = slow_chunks(data, parts=20, seconds=BODY_TIMEOUT_S / 2)
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, in_time),
|
||
)
|
||
attachment = assert_accepted(resp)
|
||
assert elapsed < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, elapsed
|
||
assert attachment["size_bytes"] == len(data)
|
||
assert attachment["sha256"] == hashlib.sha256(data).hexdigest()
|
||
uploads_api.wait_attachment(alice, attachment["id"])
|
||
assert_content(
|
||
uploads_api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Простой S3 после скачиваний (change fix-upload-put-flake, design, решение
|
||
# 3): API «с коротким сроком S3» — срок вызова S3 из FAULT_S3_ENV
|
||
# (run_failures.py, S3_ENV), других тестов на нём нет
|
||
# ---------------------------------------------------------------------------
|
||
|
||
S3_ENV = dict(
|
||
item.split("=", 1) for item in os.environ.get("FAULT_S3_ENV", "").split()
|
||
)
|
||
# окружение, с которым тест ниже ждёт API с коротким сроком S3
|
||
EXPECTED_S3_ENV = {"S3__TIMEOUT_SECONDS": "1"}
|
||
SHORT_S3_TIMEOUT_S = float(S3_ENV.get("S3__TIMEOUT_SECONDS", "1"))
|
||
# раунды: скачивания, простой дольше срока вызова S3, загрузки
|
||
IDLE_ROUNDS = 8
|
||
IDLE_DOWNLOADS = 10
|
||
IDLE_UPLOADS = 12
|
||
IDLE_EXTRA_S = 0.5
|
||
# медленный клиент: пауза после каждого блока 64 KiB
|
||
IDLE_READ_PAUSE_S = 0.002
|
||
|
||
|
||
def assert_s3_stand() -> None:
|
||
"""Без этого окружения API с коротким сроком S3 тест ничего не
|
||
доказывает: простой 1,5 с был бы короче срока вызова."""
|
||
assert S3_ENV == EXPECTED_S3_ENV, (
|
||
f"short S3 timeout stand started with {S3_ENV}, "
|
||
f"expected {EXPECTED_S3_ENV}"
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Загрузка после простоя S3")
|
||
def test_upload_after_idle_s3(
|
||
s3_api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Скачивания большого оригинала, простой S3 дольше срока вызова,
|
||
затем загрузки — каждая 202, ни одной 503.
|
||
|
||
Раунд: 10 одновременных медленных скачиваний (своё соединение у
|
||
каждого клиента, барьер, 2 мс на блок 64 KiB) PNG ~4 MiB, простой
|
||
`S3__TIMEOUT_SECONDS + 0,5` с без запросов к этому API, 12 загрузок
|
||
`sample.png` по одной: пул клиента S3 отдаёт соединения по порядку
|
||
возврата, и 12 вызовов проходят через все ≤ 10 его соединений.
|
||
|
||
До правки клиента (срок чтения aiohttp, переживший ответ) соединение
|
||
портится в раунде с p ≈ 0,28 (design, «Как подтверждено», п. 8):
|
||
промах раунда — 0,72^10 ≈ 0,037, восьми раундов — ≈ 4e-12. После
|
||
правки 8 раундов — ~16 с.
|
||
"""
|
||
assert_s3_stand()
|
||
base_url = str(s3_api.client.base_url)
|
||
original = images.png_noise(1024, 1365, seed=4)
|
||
attachment = s3_api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
original,
|
||
filename="idle-original.png",
|
||
)
|
||
sample = images.sample_png()
|
||
|
||
def download(barrier: threading.Barrier) -> httpx.Response:
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
|
||
# соединение открыто до барьера: после него сразу скачивание
|
||
own.request("GET", "/health/live")
|
||
barrier.wait()
|
||
return own.attachment_content_slowly(
|
||
alice,
|
||
attachment["id"],
|
||
pause_s=IDLE_READ_PAUSE_S,
|
||
)
|
||
|
||
for round_number in range(1, IDLE_ROUNDS + 1):
|
||
barrier = threading.Barrier(IDLE_DOWNLOADS)
|
||
with ThreadPoolExecutor(max_workers=IDLE_DOWNLOADS) as pool:
|
||
downloads = list(
|
||
pool.map(download, [barrier] * IDLE_DOWNLOADS),
|
||
)
|
||
for resp in downloads:
|
||
assert_content(resp, original, "image/png")
|
||
time.sleep(SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S)
|
||
for upload_number in range(1, IDLE_UPLOADS + 1):
|
||
resp = upload_raw(s3_api, alice, chat, sample)
|
||
assert resp.status_code == 202, describe(
|
||
resp,
|
||
f"round {round_number}, upload {upload_number} after "
|
||
f"{SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S:.1f}s of S3 idle",
|
||
)
|
||
assert_accepted(resp)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Фоновая обработка, брокер и worker (change async-attachments, design, решение
|
||
# 15). Worker стенда: 4 попытки, пауза 5 с, срок попытки 10 с, срок остановки
|
||
# 2 с (FAULT_WORKER_*). Перед остановкой worker очередь доводится до пустой:
|
||
# задания прежних тестов не считаются в «одно сообщение». Worker и триггеры
|
||
# возвращаются в finally, упавшая проверка не ломает следующие
|
||
# ---------------------------------------------------------------------------
|
||
|
||
WORKER_MAX_ATTEMPTS = int(os.environ.get("FAULT_WORKER_MAX_ATTEMPTS", "4"))
|
||
WORKER_RETRY_BASE_S = float(os.environ.get("FAULT_WORKER_RETRY_BASE_S", "4"))
|
||
WORKER_RETRY_MAX_S = float(os.environ.get("FAULT_WORKER_RETRY_MAX_S", "16"))
|
||
WORKER_JOB_TIMEOUT_S = float(
|
||
os.environ.get("FAULT_WORKER_JOB_TIMEOUT_S", "10"),
|
||
)
|
||
WORKER_LEASE_MARGIN_S = float(
|
||
os.environ.get("FAULT_WORKER_LEASE_MARGIN_S", "2"),
|
||
)
|
||
WORKER_LEASE_S = WORKER_JOB_TIMEOUT_S + WORKER_LEASE_MARGIN_S
|
||
RELAY_REPUBLISH_S = float(os.environ.get("FAULT_RELAY_REPUBLISH_S", "5"))
|
||
STAGING_RETENTION_S = float(
|
||
os.environ.get("FAULT_CLEANER_STAGING_RETENTION_S", "60"),
|
||
)
|
||
# [worker] concurrency по умолчанию: стенд отказов её не меняет
|
||
WORKER_CONCURRENCY = 2
|
||
# ответ на загрузку не ждёт брокера и relay (спек job-outbox)
|
||
ACCEPT_MAX_S = 1
|
||
# отказ брокера не держит health: быстрее секунды, как у S3
|
||
BROKER_HEALTH_S = 1
|
||
# срок ожидания итога задания: все попытки и паузы с запасом
|
||
JOB_WAIT_S = 60
|
||
# relay публикует намерение за проход; срок ожидания сообщения с отказом брокера
|
||
PUBLISH_WAIT_S = 15
|
||
JOBS_SCHEMA_PATH = (
|
||
Path(__file__).resolve().parents[1] / "contracts" / "jobs.schema.json"
|
||
)
|
||
# срок «Stopped worker» по контракту остановки контейнера
|
||
WORKER_STOP_MAX_S = 10
|
||
ALL_VARIANTS = (None, "original", "thumbnail", "crop", "optimized")
|
||
VARIANT_NAMES = {"thumbnail", "crop", "optimized"}
|
||
# предел памяти брокера в конфигурации стенда: его возвращает проба сдерживания
|
||
BROKER_MEMORY_LIMIT = "400MiB"
|
||
|
||
|
||
def queue_state(stand: Stand) -> dict[str, Any]:
|
||
state = json.loads(stand("queue-state"))
|
||
assert state is not None, "job queue does not exist"
|
||
return state
|
||
|
||
|
||
def wait_queue_messages(
|
||
stand: Stand,
|
||
count: int,
|
||
*,
|
||
timeout: float = PUBLISH_WAIT_S,
|
||
) -> dict[str, Any]:
|
||
"""В очереди столько сообщений: relay публикует намерения за проход."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
state = json.loads(stand("queue-state"))
|
||
if state is not None and state["messages"] == count:
|
||
return state
|
||
assert time.monotonic() < deadline, (
|
||
f"queue has not {count} messages: {state}"
|
||
)
|
||
time.sleep(0.2)
|
||
|
||
|
||
@contextmanager
|
||
def relay_stopped(stand: Stand) -> Iterator[None]:
|
||
"""Relay остановлен; в `finally` запускается."""
|
||
with outage(stand, "stop", "relay"):
|
||
yield
|
||
|
||
|
||
def relay_records(stand: Stand) -> list[dict[str, Any]]:
|
||
return [
|
||
json.loads(line)
|
||
for line in stand("relay-log").splitlines()
|
||
if line.startswith("{")
|
||
]
|
||
|
||
|
||
def wait_relay_record(
|
||
stand: Stand,
|
||
event: str,
|
||
*,
|
||
timeout: float = 30,
|
||
skip: int = 0,
|
||
**fields: Any,
|
||
) -> dict[str, Any]:
|
||
"""Запись журнала relay с этими полями после первых `skip` записей."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
for record in relay_records(stand)[skip:]:
|
||
if record.get("event") == event and all(
|
||
record.get(name) == value for name, value in fields.items()
|
||
):
|
||
return record
|
||
assert time.monotonic() < deadline, (
|
||
f"no relay record {event!r} with {fields} in {timeout} s"
|
||
)
|
||
time.sleep(0.3)
|
||
|
||
|
||
def extra_records(stand: Stand, name: str = "worker") -> list[dict[str, Any]]:
|
||
"""Записи журнала дополнительного процесса hook `run-worker`."""
|
||
return [
|
||
json.loads(line)
|
||
for line in stand("extra-log", name).splitlines()
|
||
if line.startswith("{")
|
||
]
|
||
|
||
|
||
def wait_extra_record(
|
||
stand: Stand,
|
||
event: str,
|
||
*,
|
||
timeout: float = 30,
|
||
**fields: Any,
|
||
) -> dict[str, Any]:
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
for record in extra_records(stand):
|
||
if record.get("event") == event and all(
|
||
record.get(name) == value for name, value in fields.items()
|
||
):
|
||
return record
|
||
assert time.monotonic() < deadline, (
|
||
f"no extra worker record {event!r} with {fields} in {timeout} s"
|
||
)
|
||
time.sleep(0.3)
|
||
|
||
|
||
def wait_queue_empty(stand: Stand, *, timeout: float = 90) -> None:
|
||
"""В очереди не осталось сообщений, в том числе неподтверждённых."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
state = queue_state(stand)
|
||
if state["messages"] == 0:
|
||
return
|
||
assert time.monotonic() < deadline, f"queue is not empty: {state}"
|
||
time.sleep(0.5)
|
||
|
||
|
||
@contextmanager
|
||
def worker_stopped(stand: Stand) -> Iterator[None]:
|
||
"""Worker остановлен при пустой очереди; в `finally` запускается."""
|
||
wait_queue_empty(stand)
|
||
with outage(stand, "stop", "worker"):
|
||
yield
|
||
|
||
|
||
def worker_records(stand: Stand) -> list[dict[str, Any]]:
|
||
"""Записи журнала worker за все его запуски."""
|
||
return [
|
||
json.loads(line)
|
||
for line in stand("worker-log").splitlines()
|
||
if line.startswith("{")
|
||
]
|
||
|
||
|
||
def wait_worker_record(
|
||
stand: Stand,
|
||
event: str,
|
||
*,
|
||
timeout: float = 30,
|
||
skip: int = 0,
|
||
**fields: Any,
|
||
) -> dict[str, Any]:
|
||
"""Запись журнала worker с этими полями после первых `skip` записей."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
for record in worker_records(stand)[skip:]:
|
||
if record.get("event") == event and all(
|
||
record.get(name) == value for name, value in fields.items()
|
||
):
|
||
return record
|
||
assert time.monotonic() < deadline, (
|
||
f"no worker record {event!r} with {fields} in {timeout} s"
|
||
)
|
||
time.sleep(0.3)
|
||
|
||
|
||
def job_body(
|
||
job_id: str,
|
||
attachment_id: str,
|
||
*,
|
||
attempt: int = 1,
|
||
) -> bytes:
|
||
"""Сообщение задания по `contracts/jobs.schema.json`."""
|
||
return json.dumps(
|
||
{
|
||
"schema_version": 1,
|
||
"job_id": job_id,
|
||
"type": "attachment.process.v1",
|
||
"attachment_id": attachment_id,
|
||
"pipeline_version": 1,
|
||
"attempt": attempt,
|
||
"created_at": "2026-10-31T12:00:00.000000Z",
|
||
},
|
||
).encode()
|
||
|
||
|
||
def wait_job(
|
||
stand: Stand,
|
||
attachment_id: str,
|
||
*,
|
||
state: str,
|
||
attempt: int,
|
||
timeout: float = 20,
|
||
) -> dict[str, Any]:
|
||
"""Задание в этом состоянии с этим или большим числом попыток."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
job = job_row(stand, attachment_id)
|
||
if job and job["state"] == state and job["attempt"] >= attempt:
|
||
return job
|
||
assert time.monotonic() < deadline, (
|
||
f"job of {attachment_id} is not {state} with attempt >= "
|
||
f"{attempt}: {job}"
|
||
)
|
||
time.sleep(0.2)
|
||
|
||
|
||
def assert_variants(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
meta: dict[str, Any],
|
||
) -> dict[str, dict[str, Any]]:
|
||
"""По одному варианту каждого имени в метаданных и по объекту на каждый, с
|
||
его SHA-256; настоящие размеры и тип. Возвращает варианты по имени."""
|
||
assert meta["status"] == "ready", meta
|
||
variants = {item["name"]: item for item in meta["variants"]}
|
||
assert len(meta["variants"]) == len(VARIANT_NAMES), meta["variants"]
|
||
assert set(variants) == VARIANT_NAMES, variants
|
||
assert stand("variant-object-count", meta["id"]) == str(len(VARIANT_NAMES))
|
||
for name, variant in variants.items():
|
||
data = assert_variant_content(
|
||
api.attachment_content(actor, meta["id"], variant=name),
|
||
variant,
|
||
)
|
||
assert_variant_image(data, variant)
|
||
return variants
|
||
|
||
|
||
def assert_not_ready_content(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment_id: str,
|
||
) -> None:
|
||
"""Любое скачивание вложения не в `ready` — 409."""
|
||
for variant in ALL_VARIANTS:
|
||
assert_error(
|
||
api.attachment_content(actor, attachment_id, variant=variant),
|
||
409,
|
||
)
|
||
|
||
|
||
def staging_key(attachment_id: str) -> str:
|
||
return f"{FAULT_DATABASE}/staging/{attachment_id}/original"
|
||
|
||
|
||
def wait_staging_gone(stand: Stand, attachment_id: str) -> None:
|
||
"""Объект staging и ключ в метаданных убраны: не позже следующего прохода
|
||
уборщика с запасом."""
|
||
deadline = time.monotonic() + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while True:
|
||
row = json.loads(stand("attachment-row", attachment_id))
|
||
if row["staging_key"] is None and attachment_id not in staged_ids(
|
||
stand,
|
||
):
|
||
return
|
||
assert time.monotonic() < deadline, (attachment_id, row)
|
||
time.sleep(0.3)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Принятое вложение в очереди")
|
||
def test_queued_while_worker_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
time.sleep(3)
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == accepted
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Обработка видна в API")
|
||
def test_processing_visible(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(alice, chat["id"], data, filename=filename)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert_error(api.attachment_content(alice, accepted["id"]), 409)
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
assert_content(
|
||
api.attachment_content(alice, accepted["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Готовое вложение не откатывается")
|
||
@pytest.mark.spec("attachment-processing", "Повторная доставка готового")
|
||
def test_redelivery_of_ready_is_noop(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
meta = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
job = job_row(stand, meta["id"])
|
||
assert job is not None
|
||
wait_queue_empty(stand)
|
||
seen = len(worker_records(stand))
|
||
stand("publish-job", stdin=job_body(job["id"], meta["id"]))
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="finished",
|
||
)
|
||
wait_queue_empty(stand)
|
||
assert (
|
||
assert_ok(api.attachment_meta(alice, meta["id"]), 200, "Attachment")
|
||
== meta
|
||
)
|
||
assert job_row(stand, meta["id"]) == job
|
||
assert_variants(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Что сохранено к ответу 202")
|
||
def test_durable_state_at_202(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
with relay_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
attachment_id = accepted["id"]
|
||
assert attachment_id in staged_ids(stand)
|
||
assert attachment_id not in stored_ids(stand)
|
||
assert attachment_rows(stand)[attachment_id] == "queued"
|
||
job = job_row(stand, attachment_id)
|
||
assert job is not None
|
||
assert (job["state"], job["attempt"]) == ("queued", 0), job
|
||
assert job["published_at"] is None, job
|
||
assert queue_state(stand)["messages"] == 0
|
||
wait_queue_messages(stand, 1)
|
||
published = job_row(stand, attachment_id)
|
||
assert published is not None
|
||
assert published["published_at"] is not None, published
|
||
api.wait_attachment(alice, attachment_id)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Временная ошибка")
|
||
@pytest.mark.spec("attachment-processing", "Повторы под одним заданием")
|
||
def test_transient_error_retried(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Сначала S3 остановлен и возвращается за ~8 с: попыток больше одной.
|
||
Затем S3 заморожен на первую попытку и размораживается в паузе между
|
||
попытками: ровно две, задание то же."""
|
||
data = images.sample_png()
|
||
with worker_stopped(stand):
|
||
stopped = api.upload(alice, chat["id"], data)
|
||
first = job_row(stand, stopped["id"])
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
wait_job(stand, stopped["id"], state="queued", attempt=1)
|
||
meta = api.wait_attachment(alice, stopped["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
job = job_row(stand, stopped["id"])
|
||
assert job is not None and first is not None
|
||
assert job["id"] == first["id"], (job, first)
|
||
assert job["attempt"] > 1, job
|
||
assert job["state"] == "succeeded", job
|
||
|
||
with worker_stopped(stand):
|
||
paused = api.upload(alice, chat["id"], data)
|
||
wait_queue_messages(stand, 1)
|
||
first = job_row(stand, paused["id"])
|
||
with relay_stopped(stand), outage(stand, "pause", "s3"):
|
||
stand("start", "worker")
|
||
# срок вызова S3 5 с: первая попытка заканчивается отказом S3.
|
||
# Срок следующей отодвинут, пока S3 размораживается: случайная
|
||
# пауза могла бы её начать раньше, relay стоит до этого
|
||
wait_job(stand, paused["id"], state="queued", attempt=1)
|
||
stand("set-next-attempt", paused["id"], "6")
|
||
meta = api.wait_attachment(alice, paused["id"], timeout=JOB_WAIT_S)
|
||
job = job_row(stand, paused["id"])
|
||
assert job is not None and first is not None
|
||
assert job["id"] == first["id"], (job, first)
|
||
assert job["attempt"] == 2, job
|
||
assert_variants(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Сообщение по схеме")
|
||
def test_job_message_matches_schema(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
schema = json.loads(JOBS_SCHEMA_PATH.read_text(encoding="utf-8"))
|
||
filename = fault_filename()
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
wait_queue_messages(stand, 1)
|
||
peeked = json.loads(stand("peek-job"))
|
||
text = peeked["body"]
|
||
body = json.loads(text)
|
||
errors = list(
|
||
Draft202012Validator(
|
||
schema,
|
||
format_checker=FormatChecker(),
|
||
).iter_errors(body),
|
||
)
|
||
assert not errors, [error.message for error in errors]
|
||
datetime.fromisoformat(body["created_at"])
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert body["attachment_id"] == accepted["id"]
|
||
assert body["job_id"] == job["id"]
|
||
assert body["attempt"] == 1
|
||
# ни имени файла, ни ключей, ни адресов, ни учётных данных
|
||
lowered = text.lower()
|
||
for word in (
|
||
filename.lower(),
|
||
os.environ.get("FAULT_S3_BUCKET", "attachments").lower(),
|
||
f"{FAULT_DATABASE}/",
|
||
"staging",
|
||
"original",
|
||
"http",
|
||
"password",
|
||
"secret",
|
||
):
|
||
assert word not in lowered, f"{word!r} in the job message"
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Очередь и сообщения устойчивы")
|
||
def test_queue_durable(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
state = wait_queue_messages(stand, 1)
|
||
assert state["durable"] is True, state
|
||
peeked = json.loads(stand("peek-job"))
|
||
assert peeked["delivery_mode"] == 2, peeked
|
||
# прочитанное без подтверждения вернулось в очередь
|
||
state = queue_state(stand)
|
||
assert (state["messages"], state["messages_ready"]) == (1, 1), state
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Сообщение не по схеме")
|
||
def test_malformed_message_dropped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
marker = uuid.uuid4().hex
|
||
wait_queue_empty(stand)
|
||
seen = len(worker_records(stand))
|
||
stand(
|
||
"publish-job",
|
||
stdin=json.dumps({"schema_version": 1, "note": marker}).encode(),
|
||
)
|
||
record = wait_worker_record(
|
||
stand,
|
||
"Dropped malformed job message",
|
||
skip=seen,
|
||
)
|
||
assert record["level"] == "error", record
|
||
# содержимое сообщения в журнал не попало
|
||
assert marker not in stand("worker-log")
|
||
wait_queue_empty(stand)
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Неизвестное задание")
|
||
def test_unknown_job_acked(stand: Stand) -> None:
|
||
job_id, attachment_id = str(uuid.uuid4()), str(uuid.uuid4())
|
||
wait_queue_empty(stand)
|
||
before = objects_and_rows(stand)
|
||
seen = len(worker_records(stand))
|
||
stand("publish-job", stdin=job_body(job_id, attachment_id))
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job_id,
|
||
reason="unknown",
|
||
)
|
||
wait_queue_empty(stand)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not new_objects(before, stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Worker убит посреди обработки")
|
||
def test_worker_killed_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
stand("kill", "worker")
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "worker")
|
||
# сообщение вернулось, но аренда жива: новую попытку начнёт её истечение
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 2, job
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Дубль во время обработки")
|
||
def test_duplicate_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Две ветки дубля. S3 заморожен короче срока вызова, попытка ждёт его без
|
||
блокировок БД: дубли видят живую аренду («busy») и подтверждаются, а после
|
||
разморозки та же попытка доходит до `ready`. Фиксация ждёт в триггере и
|
||
держит строку задания: дубль дожидается её и видит итог («finished»)."""
|
||
with worker_stopped(stand):
|
||
live = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
job = job_row(stand, live["id"])
|
||
assert job is not None
|
||
stand("publish-job", stdin=job_body(job["id"], live["id"]))
|
||
stand("publish-job", stdin=job_body(job["id"], live["id"]))
|
||
wait_queue_messages(stand, 3)
|
||
seen = len(worker_records(stand))
|
||
with outage(stand, "pause", "s3"):
|
||
stand("start", "worker")
|
||
deadline = time.monotonic() + S3_TIMEOUT_S
|
||
while True:
|
||
busy = [
|
||
item
|
||
for item in worker_records(stand)[seen:]
|
||
if item.get("event") == "Skipped job message"
|
||
and item.get("job_id") == job["id"]
|
||
and item.get("reason") == "busy"
|
||
]
|
||
if len(busy) == 2:
|
||
break
|
||
assert time.monotonic() < deadline, busy
|
||
time.sleep(0.1)
|
||
meta = api.wait_attachment(alice, live["id"], timeout=JOB_WAIT_S)
|
||
wait_queue_empty(stand)
|
||
assert_variants(api, stand, alice, meta)
|
||
done = job_row(stand, live["id"])
|
||
assert done is not None
|
||
assert done["attempt"] == 1, done
|
||
|
||
filename = fault_filename()
|
||
seen = len(worker_records(stand))
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
stand("publish-job", stdin=job_body(job["id"], accepted["id"]))
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="finished",
|
||
)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
wait_queue_empty(stand)
|
||
assert_variants(api, stand, alice, meta)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["attempt"] == 1, done
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Оригинал неизменен")
|
||
def test_original_published_unchanged(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=23)
|
||
meta = api.upload_ready(alice, chat["id"], data)
|
||
assert_content(
|
||
api.attachment_content(alice, meta["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
assert meta["sha256"] == hashlib.sha256(data).hexdigest()
|
||
assert meta["id"] in stored_ids(stand)
|
||
wait_staging_gone(stand, meta["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Испорченный staging")
|
||
def test_staging_corrupted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Те же размер и другие байты — и другой размер: обе ветки сверки."""
|
||
data = images.png_noise(64, 48, seed=29)
|
||
with worker_stopped(stand):
|
||
same_size = api.upload(alice, chat["id"], data)
|
||
other_size = api.upload(alice, chat["id"], data)
|
||
put_object(
|
||
stand,
|
||
staging_key(same_size["id"]),
|
||
same_size_bytes(data, seed=29),
|
||
)
|
||
put_object(stand, staging_key(other_size["id"]), data + b"\0")
|
||
for accepted in (same_size, other_size):
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "processing_failed", meta
|
||
assert meta["variants"] == [], meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert job["last_error_code"] == "source_corrupted", job
|
||
assert accepted["id"] not in stored_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Нет объекта staging")
|
||
def test_staging_missing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
stand("delete-object", staging_key(accepted["id"]))
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "processing_failed", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert job["last_error_code"] == "source_missing", job
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Изображение в лимите пикселей")
|
||
def test_max_pixels_fit_worker_memory(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Столько PNG RGBA 8000 × 5000, сколько заданий worker обрабатывает
|
||
одновременно: они берутся разом, и контейнер не завершён из-за памяти."""
|
||
data = images.png_rgba_large()
|
||
before = json.loads(stand("worker-state"))
|
||
# срок клиента: тело на 40 млн пикселей проверяется дольше 5 с по умолчанию
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
slow.upload(alice, chat["id"], data)
|
||
for _ in range(WORKER_CONCURRENCY)
|
||
]
|
||
sizes = {
|
||
"thumbnail": (256, 160),
|
||
"crop": (128, 128),
|
||
"optimized": (1280, 800),
|
||
}
|
||
for item in accepted:
|
||
meta = slow.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
variants = assert_variants(slow, stand, alice, meta)
|
||
for name, size in sizes.items():
|
||
variant = variants[name]
|
||
assert (variant["width"], variant["height"]) == size, variant
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["oom_killed"] is False, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Повтор после записи объектов")
|
||
@pytest.mark.spec("attachments", "Фиксация ready не удалась")
|
||
def test_ready_failure_retried(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Объекты записаны, фиксация падает отказом БД (08006): сообщение уходит в
|
||
очередь; пока триггер стоит, вложение не готово. Аренда жива: следующую
|
||
попытку начинает её истечение, а триггер снимается раньше."""
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
seen = len(worker_records(stand))
|
||
stand("fail-attachments", "ready-fail", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
wait_worker_record(
|
||
stand,
|
||
"Job waits for database",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
)
|
||
assert stand("variant-object-count", accepted["id"]) == str(
|
||
len(VARIANT_NAMES),
|
||
)
|
||
assert accepted["id"] in stored_ids(stand)
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] in ("queued", "processing"), meta
|
||
assert_not_ready_content(api, alice, accepted["id"])
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["id"] == job["id"], done
|
||
assert done["attempt"] >= 2, done
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Состояние между попытками")
|
||
def test_queued_between_attempts(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
job = wait_job(stand, accepted["id"], state="queued", attempt=1)
|
||
assert job["last_error_code"] == "storage_unavailable", job
|
||
assert job["next_attempt_at"] is not None, job
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
assert meta["error_code"] is None, meta
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Попытки исчерпаны")
|
||
def test_attempts_exhausted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "retry_exhausted", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == WORKER_MAX_ATTEMPTS, job
|
||
assert job["state"] == "failed", job
|
||
wait_queue_empty(stand)
|
||
# S3 вернулся: итог окончателен
|
||
time.sleep(WORKER_RETRY_BASE_S)
|
||
after = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert after == meta
|
||
assert job_row(stand, accepted["id"]) == job
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"PostgreSQL недоступен при обработке",
|
||
)
|
||
def test_database_outage_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("worker-state"))
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
seen = len(worker_records(stand))
|
||
with outage(stand, "stop", "postgres"):
|
||
stand("start", "worker")
|
||
# журнал называет отказ, задание ждёт в очереди
|
||
wait_worker_record(stand, "Job waits for database", skip=seen)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
wait_queue_empty(stand)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
# попытки на отказе БД не считаются
|
||
assert done["attempt"] == 1, done
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Изображение не декодируется")
|
||
def test_undecodable_becomes_failed(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
assert meta["variants"] == [], meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert job["last_error_code"] == "undecodable", job
|
||
assert_error(api.attachment_content(alice, accepted["id"]), 409)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Worker недоступен")
|
||
def test_worker_down_api_serves(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
assert accepted["status"] == "queued"
|
||
message = api.send_message(alice, chat["id"], "текст без worker")
|
||
history = api.all_messages(bob, chat["id"])
|
||
assert [item["id"] for item in history] == [message["id"]]
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Ограниченная конкуренция")
|
||
def test_concurrency_bounded(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Фиксация каждого задания ждёт 4 с в триггере: заданий больше, чем
|
||
мест, и видно, сколько worker держит разом."""
|
||
filename = fault_filename()
|
||
total = WORKER_CONCURRENCY + 3
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
with worker_stopped(stand):
|
||
ids = [
|
||
api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)["id"]
|
||
for _ in range(total)
|
||
]
|
||
api.wait_attachment(
|
||
alice,
|
||
ids[0],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
most = 0
|
||
for _ in range(4):
|
||
rows = attachment_rows(stand)
|
||
statuses = [rows[item] for item in ids]
|
||
busy = statuses.count("processing")
|
||
most = max(most, busy)
|
||
assert busy <= WORKER_CONCURRENCY, statuses
|
||
assert (
|
||
statuses.count("queued")
|
||
+ busy
|
||
+ statuses.count(
|
||
"ready",
|
||
)
|
||
== total
|
||
), statuses
|
||
state = queue_state(stand)
|
||
assert state["messages_unacknowledged"] <= WORKER_CONCURRENCY, (
|
||
state
|
||
)
|
||
time.sleep(0.3)
|
||
assert most == WORKER_CONCURRENCY, most
|
||
for item in ids:
|
||
api.wait_attachment(alice, item, timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Реплика API убита после ответа")
|
||
def test_api_killed_after_202(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
stand("kill-api")
|
||
try:
|
||
# реплики нет, а работа идёт: итог — от worker
|
||
wait_job(
|
||
stand,
|
||
accepted["id"],
|
||
state="succeeded",
|
||
attempt=1,
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
finally:
|
||
stand("start-api")
|
||
meta = api.wait_attachment(alice, accepted["id"])
|
||
assert_variants(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Рестарт брокера с volume")
|
||
def test_broker_restart_keeps_job(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
pass
|
||
assert queue_state(stand)["messages"] == 1
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Принудительное завершение брокера")
|
||
@pytest.mark.spec("message-broker", "Подтверждённые сообщения после KILL")
|
||
def test_broker_kill_keeps_confirmed(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
confirmed = 5
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(confirmed)
|
||
]
|
||
wait_queue_messages(stand, confirmed)
|
||
with outage(stand, "kill", "rabbitmq"):
|
||
pass
|
||
assert queue_state(stand)["messages"] == confirmed
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Сценарий защиты")
|
||
def test_defense_scenario(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], data)
|
||
attachment_id = accepted["id"]
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment_id],
|
||
)
|
||
assert_error(api.attachment_content(alice, attachment_id), 409)
|
||
text = api.send_message(alice, chat["id"], "текст при очереди")
|
||
wait_queue_messages(stand, 1)
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
stand("kill-api")
|
||
stand("start-api")
|
||
meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S)
|
||
assert meta["id"] == attachment_id
|
||
assert_content(
|
||
api.attachment_content(alice, attachment_id),
|
||
data,
|
||
"image/png",
|
||
)
|
||
assert_variants(api, stand, alice, meta)
|
||
history = {
|
||
item["id"]: item for item in api.all_messages(alice, chat["id"])
|
||
}
|
||
assert history[message["id"]]["attachment_ids"] == [attachment_id]
|
||
assert text["id"] in history
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Брокер остановлен при загрузке")
|
||
@pytest.mark.spec("message-broker", "Брокер вернулся")
|
||
@pytest.mark.spec("attachments", "Брокер не подтвердил задание")
|
||
@pytest.mark.spec("job-outbox", "Загрузка при остановленном брокере")
|
||
def test_upload_with_broker_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
accepted = assert_accepted(resp)
|
||
assert elapsed < ACCEPT_MAX_S, elapsed
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["published_at"] is None, job
|
||
assert accepted["id"] in staged_ids(stand)
|
||
# обязательный путь без брокера работает
|
||
api.send_message(alice, chat["id"], "текст без брокера")
|
||
assert (
|
||
assert_ok(
|
||
api.attachment_meta(alice, ready["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
== ready
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
# relay и worker не перезапускались: принятое без брокера доходит до ready,
|
||
# и следующая загрузка тоже
|
||
done = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, done)
|
||
api.upload_ready(alice, chat["id"], data)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Брокер завис")
|
||
@pytest.mark.spec("job-outbox", "Загрузка при замороженном брокере")
|
||
def test_upload_with_broker_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Соединение издателя живо: публикация relay уходит в заморозку и кончается
|
||
сроком, исход неизвестен. Приём этого не видит, а дубль после разморозки
|
||
worker подтверждает без эффекта."""
|
||
data = images.sample_png()
|
||
api.upload_ready(alice, chat["id"], data)
|
||
wait_queue_empty(stand)
|
||
with outage(stand, "pause", "rabbitmq"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
accepted = assert_accepted(resp)
|
||
assert elapsed < ACCEPT_MAX_S, elapsed
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
wait_queue_empty(stand)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Очередь удалена")
|
||
def test_queue_deleted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
# соединение издателя живо: его канал очередь больше не найдёт, а новое
|
||
# соединение объявит её заново
|
||
api.upload_ready(alice, chat["id"], data)
|
||
with worker_stopped(stand):
|
||
stand("delete-queue")
|
||
assert json.loads(stand("queue-state")) is None
|
||
accepted = [api.upload(alice, chat["id"], data) for _ in range(2)]
|
||
state = wait_queue_messages(stand, 2)
|
||
assert state["durable"] is True, state
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Пересоздание контейнера")
|
||
def test_broker_recreated(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
confirmed = 2
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(confirmed)
|
||
]
|
||
wait_queue_messages(stand, confirmed)
|
||
stand("recreate", "rabbitmq")
|
||
state = queue_state(stand)
|
||
assert (state["durable"], state["messages"]) == (True, confirmed), (
|
||
state
|
||
)
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Старт без брокера")
|
||
def test_start_without_broker(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with (
|
||
worker_stopped(stand),
|
||
relay_stopped(stand),
|
||
outage(stand, "stop", "rabbitmq"),
|
||
):
|
||
seen = len(worker_records(stand))
|
||
seen_relay = len(relay_records(stand))
|
||
stand("kill-api")
|
||
stand("start-api")
|
||
stand("start", "worker")
|
||
stand("start", "relay")
|
||
# реплика готова и принимает файл
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
api.send_message(alice, chat["id"], "текст при старте без брокера")
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
# worker и relay называют отказ и не завершаются
|
||
wait_worker_record(stand, "Broker is unavailable", skip=seen)
|
||
wait_relay_record(stand, "Broker is unavailable", skip=seen_relay)
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
state = json.loads(stand("relay-state"))
|
||
assert state["running"] is True, state
|
||
# брокер запущен, ни реплика, ни worker не перезапускались
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("message-broker", "Рестарт брокера при работающем worker")
|
||
def test_worker_survives_broker_restart(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("worker-state"))
|
||
assert before["running"] is True, before
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
pass
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание до готовности")
|
||
def test_content_before_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Каждое скачивание вложения в `queued`, `processing` и `failed` — 409."""
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_not_ready_content(api, alice, accepted["id"])
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert_not_ready_content(api, alice, accepted["id"])
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
failed = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
failed["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert_not_ready_content(api, alice, failed["id"])
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение в обработке")
|
||
def test_message_with_queued_attachment(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"с вложением в очереди",
|
||
attachment_ids=[accepted["id"]],
|
||
)
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
assert api.all_messages(bob, chat["id"]) == [message]
|
||
api.wait_attachment(alice, accepted["id"])
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение стало failed после отправки")
|
||
def test_attachment_failed_after_message(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[accepted["id"]],
|
||
)
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Повтор после failed вложения")
|
||
def test_replay_after_attachment_failed(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
key = new_key()
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
body = {"text": "повтор", "attachment_ids": [accepted["id"]]}
|
||
message = assert_ok(
|
||
api.post_message(alice, chat["id"], key=key, json=body),
|
||
201,
|
||
"Message",
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
replay = assert_ok(
|
||
api.post_message(alice, chat["id"], key=key, json=body),
|
||
201,
|
||
"Message",
|
||
)
|
||
assert replay == message
|
||
assert_error(api.post_message(alice, chat["id"], json=body), 409)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("health", "Брокер и worker недоступны")
|
||
@pytest.mark.parametrize("action", ["stop", "pause"])
|
||
def test_readiness_without_broker_and_worker(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
action: str,
|
||
) -> None:
|
||
with (
|
||
worker_stopped(stand),
|
||
relay_stopped(stand),
|
||
outage(stand, action, "rabbitmq"),
|
||
):
|
||
for path in ("/health/ready", "/health/live"):
|
||
resp, took = timed(lambda path=path: api.request("GET", path))
|
||
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
|
||
assert took < BROKER_HEALTH_S, f"{path} took {took:.2f}s"
|
||
# загрузка от них не зависит
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
def test_worker_stop_during_processing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `graceful-shutdown`, «docker stop во время обработки» (вне сверки
|
||
покрытия — сценарий в докстринге): фиксация ждёт 4 с, а срок остановки
|
||
worker стенда 2 с. `stop` быстрее 10 с, код 0, сообщение снова в очереди,
|
||
а после запуска worker вложение становится `ready`."""
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
stopped = False
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
seen = len(worker_records(stand))
|
||
started = time.monotonic()
|
||
stand("stop", "worker")
|
||
stopped = True
|
||
assert time.monotonic() - started < WORKER_STOP_MAX_S
|
||
state = json.loads(stand("worker-state"))
|
||
assert (state["running"], state["exit_code"]) == (False, 0), state
|
||
assert queue_state(stand)["messages"] == 1
|
||
records = worker_records(stand)[seen:]
|
||
assert any(item.get("event") == "Stopped worker" for item in records)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
if not stopped:
|
||
stand("stop", "worker")
|
||
stand("start", "worker")
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
def test_worker_stop_during_render(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `graceful-shutdown`, «docker stop во время декодирования» (вне сверки
|
||
покрытия — сценарий в докстринге): дорогое изображение декодируется дольше
|
||
срока остановки worker стенда (2 с). `stop` быстрее 10 с, код 0, процессов
|
||
обработки нет, а задание получает следующую попытку по аренде."""
|
||
wait_queue_empty(stand)
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
|
||
accepted = slow.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_scan_bomb(),
|
||
content_type="image/jpeg",
|
||
filename="bomb.jpg",
|
||
)
|
||
wait_render_process(stand)
|
||
started = time.monotonic()
|
||
stand("stop", "worker")
|
||
try:
|
||
assert time.monotonic() - started < WORKER_STOP_MAX_S
|
||
state = json.loads(stand("worker-state"))
|
||
assert (state["running"], state["exit_code"]) == (False, 0), state
|
||
finally:
|
||
stand("start", "worker")
|
||
# контейнер снова запущен: процессов обработки прежней попытки нет,
|
||
# попытка 2 дойдёт до лимита CPU
|
||
meta = slow.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 2, job
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
assert job["last_error_code"] == "limit_cpu", job
|
||
|
||
|
||
def wait_render_process(stand: Stand, *args: str, timeout: float = 15) -> None:
|
||
"""В контейнере worker идёт процесс обработки."""
|
||
deadline = time.monotonic() + timeout
|
||
while stand("render-count", *args) == "0":
|
||
assert time.monotonic() < deadline, "no render process in time"
|
||
time.sleep(0.05)
|
||
|
||
|
||
def wait_variant_objects(stand: Stand, attachment_id: str) -> None:
|
||
"""Объекты всех вариантов записаны: пока фиксация ждёт в триггере."""
|
||
deadline = time.monotonic() + STALL_S
|
||
while stand("variant-object-count", attachment_id) != str(
|
||
len(VARIANT_NAMES),
|
||
):
|
||
assert time.monotonic() < deadline, "no variant objects in time"
|
||
time.sleep(0.1)
|
||
|
||
|
||
@contextmanager
|
||
def current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> Iterator[dict[str, dict[str, Any]]]:
|
||
"""Worker убит посреди задания и лежит, relay остановлен: иначе он вернул бы
|
||
P по истечении аренды. Вложения F в `failed` (конечный объект у него есть) —
|
||
старше grace, но моложе срока хранения staging, P в `processing` (объекты
|
||
записаны, фиксация не дошла) и Q в `queued` — старше grace и срока хранения
|
||
по строке. Выход — worker и relay запущены, P и Q становятся `ready`."""
|
||
wait_queue_empty(stand)
|
||
failed = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
failed["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
work: dict[str, dict[str, Any]] = {"failed": failed}
|
||
try:
|
||
processing = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
processing["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
work["processing"] = processing
|
||
# конечный объект записан до фиксации; ждём и варианты
|
||
wait_variant_objects(stand, processing["id"])
|
||
stand("kill", "worker")
|
||
stand("stop", "relay")
|
||
stand("clear-fail-attachments")
|
||
work["queued"] = api.upload(alice, chat["id"], images.sample_png())
|
||
for name, item in work.items():
|
||
age = FAILED_AGED_S if name == "failed" else AGED_S
|
||
stand("age-row", item["id"], str(age))
|
||
yield work
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "relay")
|
||
stand("start", "worker")
|
||
for name in ("processing", "queued"):
|
||
api.wait_attachment(alice, work[name]["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Текущая работа не тронута")
|
||
def test_cleaner_keeps_current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with current_work(api, stand, alice, chat) as work:
|
||
# проходов уборщика за это время — несколько
|
||
time.sleep(3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S)
|
||
rows = attachment_rows(stand)
|
||
staged = staged_ids(stand)
|
||
for name, item in work.items():
|
||
assert rows.get(item["id"]) == name, (name, rows.get(item["id"]))
|
||
assert item["id"] in staged, name
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сверка bucket и БД (change track2-reconciler, design, решение 11). Hook
|
||
# вносит расхождения с обеих сторон и запускает разовую сверку; плановая
|
||
# сверка уборщика стенда выключена. Записи сверки фильтруются по своим id и
|
||
# ключам: остатки прежних тестов (строки pending) могут попасть в отчёт, и
|
||
# код выхода сверяется с записями того же прохода, а не с числом. Объекты
|
||
# состарить нельзя (LastModified ставит S3): тесты объектов без строки ждут
|
||
# G + 1 с, строки состаривает hook (age-row)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# на сколько hook состаривает строку: заведомо больше grace
|
||
AGED_S = CLEANER_GRACE_S + 60
|
||
# строка failed: старше grace, а срок хранения staging (60 с) тест не перейдёт
|
||
FAILED_AGED_S = CLEANER_GRACE_S + 5
|
||
# объекты без строки старше grace: grace и секунда точности LastModified
|
||
OBJECT_AGE_WAIT_S = CLEANER_GRACE_S + 1
|
||
# grace разового прохода, когда он сам дольше grace стенда; меньше AGED_S
|
||
FRESH_GRACE_S = int(AGED_S) - 20
|
||
FOUND = "Found storage discrepancy"
|
||
FIXED = "Fixed storage discrepancy"
|
||
FINISHED = "Finished reconcile pass"
|
||
LIMIT_EXCEEDED = "Reconcile changes exceed limit"
|
||
# сверка под нагрузкой: потоки загрузок, период сверки, пауза потока
|
||
LOAD_THREADS = 4
|
||
LOAD_EXTRA_S = 15
|
||
LOAD_RECONCILE_PERIOD_S = 3
|
||
LOAD_PAUSE_S = 0.1
|
||
# плановая сверка отдельного уборщика: столько секунд он работает
|
||
SCHEDULED_RUN_S = 60
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Reconcile:
|
||
"""Один разовый проход сверки: записи журнала процесса и код выхода."""
|
||
|
||
records: list[dict[str, Any]]
|
||
exit_code: int
|
||
|
||
def events(self, event: str) -> list[dict[str, Any]]:
|
||
return [item for item in self.records if item.get("event") == event]
|
||
|
||
@property
|
||
def finished(self) -> dict[str, Any]:
|
||
(record,) = self.events(FINISHED)
|
||
return record
|
||
|
||
def about(
|
||
self,
|
||
event: str,
|
||
*,
|
||
ids: Collection[str] = (),
|
||
keys: Collection[str] = (),
|
||
) -> list[dict[str, Any]]:
|
||
"""Записи `event` о своих `id` вложений и ключах объектов."""
|
||
return [
|
||
item
|
||
for item in self.events(event)
|
||
if item.get("attachment_id") in ids
|
||
or item.get("object_key") in keys
|
||
]
|
||
|
||
def kinds(
|
||
self,
|
||
event: str,
|
||
*,
|
||
ids: Collection[str] = (),
|
||
keys: Collection[str] = (),
|
||
) -> dict[str, str]:
|
||
"""`id` или ключ → класс расхождения в записях `event`."""
|
||
found: dict[str, str] = {}
|
||
for item in self.about(event, ids=ids, keys=keys):
|
||
name = (
|
||
item["attachment_id"]
|
||
if item.get("attachment_id") in ids
|
||
else item["object_key"]
|
||
)
|
||
found[name] = item["kind"]
|
||
return found
|
||
|
||
def assert_code_matches(self) -> None:
|
||
"""Код выхода соответствует записям прохода: 3 — остались
|
||
расхождения (запись `Found`) или отказ по пределу, иначе 0."""
|
||
finished = self.finished
|
||
assert finished["interrupted_by"] is None, finished
|
||
expected = 3 if self.events(FOUND) or finished["refused"] else 0
|
||
assert self.exit_code == expected, (self.exit_code, finished)
|
||
|
||
|
||
def run_reconcile(
|
||
stand: Stand,
|
||
mode: str,
|
||
limit: int | None = None,
|
||
*,
|
||
options: Sequence[str] = (),
|
||
) -> Reconcile:
|
||
"""Разовая сверка hook: `dry-run` или `apply`, с пределом изменений;
|
||
`options` — стратегия, глубина и предел скорости (change
|
||
`track2-reconcile-cost`) как аргументы команды."""
|
||
args = [mode] if limit is None else [mode, str(limit)]
|
||
lines = stand("reconcile", *args, *options).splitlines()
|
||
assert lines and lines[-1].startswith("exit_code="), lines[-5:]
|
||
records = []
|
||
for line in lines[:-1]:
|
||
if line.startswith("{"):
|
||
records.append(json.loads(line))
|
||
return Reconcile(records=records, exit_code=int(lines[-1].split("=")[1]))
|
||
|
||
|
||
def object_key(attachment_id: str) -> str:
|
||
return f"{FAULT_DATABASE}/attachments/{attachment_id}/original"
|
||
|
||
|
||
def variant_key(attachment_id: str, name: str = "thumbnail") -> str:
|
||
return f"{FAULT_DATABASE}/attachments/{attachment_id}/v1-{name}"
|
||
|
||
|
||
def lost_ids(stand: Stand) -> set[str]:
|
||
return set(stand("lost-ids").splitlines())
|
||
|
||
|
||
def put_object(stand: Stand, key: str, data: bytes) -> None:
|
||
stand("put-object", key, stdin=data)
|
||
|
||
|
||
def lose_original(stand: Stand, attachment: dict[str, Any]) -> None:
|
||
"""Готовое вложение старше grace без объекта: строка состарена, объект
|
||
удалён мимо API."""
|
||
stand("age-row", attachment["id"], str(AGED_S))
|
||
stand("delete-object", object_key(attachment["id"]))
|
||
|
||
|
||
def assert_lost_content(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
*,
|
||
aged: bool = False,
|
||
) -> None:
|
||
"""Помеченное вложение: байты — 404, метаданные — как при загрузке.
|
||
|
||
`aged` — строку состарил hook (`age-row`): её `created_at` и
|
||
`updated_at` сдвинуты им, а не сверкой, и сравниваются остальные поля.
|
||
"""
|
||
assert_error(api.attachment_content(actor, attachment["id"]), 404)
|
||
meta = assert_ok(
|
||
api.attachment_meta(actor, attachment["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
if aged:
|
||
times = ("created_at", "updated_at")
|
||
meta = {key: value for key, value in meta.items() if key not in times}
|
||
attachment = {
|
||
key: value for key, value in attachment.items() if key not in times
|
||
}
|
||
assert meta == attachment
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Расхождения с обеих сторон найдены")
|
||
@pytest.mark.spec("storage-consistency", "Исправление расхождений")
|
||
@pytest.mark.spec("storage-consistency", "Повторная сверка")
|
||
@pytest.mark.spec("attachments", "Потерянный оригинал")
|
||
def test_reconcile_finds_and_fixes(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
|
||
with outage(stand, "stop", "cleaner"):
|
||
a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3))
|
||
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
|
||
orphan = str(uuid.uuid4())
|
||
keys = {
|
||
"o": object_key(orphan),
|
||
"b": object_key(b["id"]),
|
||
"bv": variant_key(b["id"]),
|
||
}
|
||
put_object(stand, keys["o"], data)
|
||
stand("delete-object", object_key(a["id"]))
|
||
stand("delete-row", b["id"])
|
||
stand("set-size", c["id"], str(len(data) + 1))
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
resp = upload_raw(api, alice, chat, data, filename=filename)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert pending in staged_ids(stand)
|
||
# последний объект записан только что: после G + 1 всё старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
ids = {a["id"], c["id"], pending}
|
||
snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, ids=ids, keys=keys.values()) == {
|
||
keys["o"]: "orphan_object",
|
||
keys["b"]: "orphan_object",
|
||
keys["bv"]: "orphan_object",
|
||
a["id"]: "missing_object",
|
||
c["id"]: "size_mismatch",
|
||
pending: "stale_pending",
|
||
}
|
||
for record in dry.about(FOUND, ids=ids, keys=keys.values()):
|
||
assert record["age_seconds"] >= CLEANER_GRACE_S, record
|
||
assert dry.exit_code == 3
|
||
# dry-run ничего не меняет: ни строк, ни объектов, ни пометок
|
||
after = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
assert after == snapshot
|
||
assert_error(api.attachment_content(alice, a["id"]), 500)
|
||
assert_ok(api.attachment_meta(alice, a["id"]), 200, "Attachment")
|
||
|
||
fixed = run_reconcile(stand, "apply")
|
||
actions = {
|
||
record.get("attachment_id")
|
||
if record.get("attachment_id") in ids
|
||
else record["object_key"]: record["action"]
|
||
for record in fixed.about(FIXED, ids=ids, keys=keys.values())
|
||
}
|
||
assert actions == {
|
||
keys["o"]: "deleted_object",
|
||
keys["b"]: "deleted_object",
|
||
keys["bv"]: "deleted_object",
|
||
a["id"]: "marked_lost",
|
||
c["id"]: "marked_lost",
|
||
pending: "removed_pending",
|
||
}
|
||
assert not fixed.about(FOUND, ids=ids, keys=keys.values())
|
||
fixed.assert_code_matches()
|
||
rows, stored = attachment_rows(stand), stored_ids(stand)
|
||
assert not {orphan, b["id"], pending} & stored
|
||
assert stand("variant-object-count", b["id"]) == "0"
|
||
assert pending not in staged_ids(stand)
|
||
assert pending not in rows
|
||
assert {a["id"], c["id"]} <= lost_ids(stand)
|
||
assert rows[a["id"]] == rows[c["id"]] == "ready"
|
||
assert_lost_content(api, alice, a)
|
||
# у C размер в метаданных изменил hook (set-size), а не сверка
|
||
assert_lost_content(api, alice, c | {"size_bytes": len(data) + 1})
|
||
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids=ids, keys=keys.values())
|
||
assert not again.about(FIXED, ids=ids, keys=keys.values())
|
||
again.assert_code_matches()
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Свежие не тронуты")
|
||
@pytest.mark.spec("storage-consistency", "Используемое вложение не удаляется")
|
||
@pytest.mark.spec("storage-consistency", "Чужой объект не удаляется")
|
||
def test_reconcile_keeps_fresh_used_and_unknown(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
used = api.upload_ready(alice, chat["id"], data)
|
||
message = api.send_message(
|
||
alice,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[used["id"]],
|
||
)
|
||
lose_original(stand, used)
|
||
# ключ, которого API не пишет: под префиксом вложений, но не <id>/original
|
||
unknown = f"{FAULT_DATABASE}/attachments/x-{uuid.uuid4()}"
|
||
put_object(stand, unknown, data)
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
# свежие: готовое вложение без объекта и объект без строки
|
||
fresh = api.upload_ready(alice, chat["id"], data)
|
||
stand("delete-object", object_key(fresh["id"]))
|
||
fresh_object = str(uuid.uuid4())
|
||
put_object(stand, object_key(fresh_object), data)
|
||
ready_before = ready_ids(attachment_rows(stand))
|
||
try:
|
||
run = run_reconcile(stand, "apply")
|
||
keys = {unknown, object_key(fresh_object)}
|
||
ids = {used["id"], fresh["id"], fresh_object}
|
||
assert run.kinds(FOUND, ids=ids, keys=keys) == {
|
||
unknown: "unknown_object",
|
||
}
|
||
assert run.kinds(FIXED, ids=ids, keys=keys) == {
|
||
used["id"]: "missing_object",
|
||
}
|
||
assert run.exit_code == 3
|
||
assert run.finished["fresh_objects"] >= 1, run.finished
|
||
assert run.finished["fresh_rows"] >= 1, run.finished
|
||
# свежий объект на месте, свежее вложение не помечено
|
||
assert fresh_object in stored_ids(stand)
|
||
lost = lost_ids(stand)
|
||
assert fresh["id"] not in lost
|
||
# используемое: помечено, метаданные и ссылка в истории на месте
|
||
assert used["id"] in lost
|
||
assert_lost_content(api, alice, used, aged=True)
|
||
history = api.all_messages(alice, chat["id"])
|
||
(sent,) = (item for item in history if item["id"] == message["id"])
|
||
assert sent["attachment_ids"] == [used["id"]]
|
||
assert ready_before <= ready_ids(attachment_rows(stand))
|
||
# чужой объект не тронут: следующий проход снова его видит
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert again.kinds(FOUND, keys={unknown}) == {
|
||
unknown: "unknown_object",
|
||
}
|
||
finally:
|
||
# стенд — без чужого и лишнего объекта, у свежего — его байты
|
||
stand("delete-object", unknown)
|
||
stand("delete-object", object_key(fresh_object))
|
||
put_object(stand, object_key(fresh["id"]), data)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
|
||
def test_reconcile_restored_original(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=7)
|
||
restored = api.upload_ready(alice, chat["id"], data)
|
||
lose_original(stand, restored)
|
||
marked = run_reconcile(stand, "apply")
|
||
assert marked.kinds(FIXED, ids={restored["id"]}) == {
|
||
restored["id"]: "missing_object",
|
||
}
|
||
assert restored["id"] in lost_ids(stand)
|
||
assert_lost_content(api, alice, restored, aged=True)
|
||
# те же байты снова под ключом вложения
|
||
put_object(stand, object_key(restored["id"]), data)
|
||
cleared = run_reconcile(stand, "apply")
|
||
(record,) = cleared.about(FIXED, ids={restored["id"]})
|
||
assert (record["kind"], record["action"]) == (
|
||
"restored_object",
|
||
"cleared_lost",
|
||
)
|
||
assert restored["id"] not in lost_ids(stand)
|
||
assert_content(
|
||
api.attachment_content(alice, restored["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids={restored["id"]})
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Предел изменений")
|
||
def test_reconcile_change_limit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
limit = 2
|
||
batch = [
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for _ in range(3)
|
||
]
|
||
ids = {item["id"] for item in batch}
|
||
for item in batch:
|
||
lose_original(stand, item)
|
||
refused = run_reconcile(stand, "apply", limit)
|
||
assert refused.exit_code == 3
|
||
(error,) = refused.events(LIMIT_EXCEEDED)
|
||
assert error["level"] == "error"
|
||
assert error["changes"] >= len(batch), error
|
||
assert error["max_changes"] == limit
|
||
assert refused.finished["refused"] is True
|
||
assert refused.finished["level"] == "warning"
|
||
assert not refused.about(FIXED, ids=ids)
|
||
assert not ids & lost_ids(stand)
|
||
assert all(
|
||
record["kind"] == "missing_object"
|
||
for record in refused.about(FOUND, ids=ids)
|
||
)
|
||
# предел не меньше числа изменений: исправлено всё
|
||
applied = run_reconcile(stand, "apply", 100)
|
||
assert set(applied.kinds(FIXED, ids=ids)) == ids
|
||
assert ids <= lost_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Объект, на который ссылается строка, не удаляется",
|
||
)
|
||
def test_reconcile_keeps_referenced_objects(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W1 verify: объект `<X>/original`, у строки которого другой
|
||
ключ, и объект `<Z>/original`, на который ссылается строка Y другого
|
||
`id`, — чужие, а не объекты без метаданных."""
|
||
data = images.sample_png()
|
||
x, y = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
|
||
z = str(uuid.uuid4())
|
||
keys = {
|
||
"x": object_key(x["id"]),
|
||
"y": object_key(y["id"]),
|
||
"z": object_key(z),
|
||
# ключ X: под префиксом вложений, объекта по нему нет
|
||
"other": f"{FAULT_DATABASE}/attachments/x-other-{uuid.uuid4()}",
|
||
}
|
||
put_object(stand, keys["z"], data)
|
||
try:
|
||
stand("set-key", x["id"], keys["other"])
|
||
stand("set-key", y["id"], keys["z"])
|
||
for item in (x, y):
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
# объект Z записан только что: после G + 1 все объекты старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
unknown = {keys[name]: "unknown_object" for name in ("x", "y", "z")}
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, keys=keys.values()) == unknown | {
|
||
keys["other"]: "missing_object",
|
||
}
|
||
assert dry.exit_code == 3
|
||
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FIXED, keys=keys.values()) == {
|
||
keys["other"]: "missing_object",
|
||
}
|
||
assert applied.kinds(FOUND, keys=keys.values()) == unknown
|
||
assert applied.exit_code == 3
|
||
# ни один из трёх объектов не удалён, строки на месте
|
||
assert {x["id"], y["id"], z} <= stored_ids(stand)
|
||
rows = attachment_rows(stand)
|
||
assert rows[x["id"]] == rows[y["id"]] == "ready"
|
||
# X без объекта по своему ключу помечено, у Y объект по ключу есть
|
||
lost = lost_ids(stand)
|
||
assert x["id"] in lost
|
||
assert y["id"] not in lost
|
||
finally:
|
||
# прежние ключи; объект Z, на который больше никто не ссылается, —
|
||
# прочь, чтобы следующие проходы не удаляли его как свой
|
||
stand("set-key", x["id"], keys["x"])
|
||
stand("set-key", y["id"], keys["y"])
|
||
stand("delete-object", keys["z"])
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Предел не останавливает уборку зависших",
|
||
)
|
||
def test_reconcile_limit_keeps_cleanup(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W1 verify: отказ по пределу не мешает уборке зависшей
|
||
загрузки — её уборщик делает и без сверки."""
|
||
data = images.sample_png()
|
||
limit = 2
|
||
# уборщик стенда стоит: зависшую загрузку P должна убрать сверка
|
||
with outage(stand, "stop", "cleaner"):
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
assert_error(
|
||
upload_raw(api, alice, chat, data, filename=filename),
|
||
503,
|
||
)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert pending in staged_ids(stand)
|
||
stand("age-row", pending, str(AGED_S))
|
||
batch = [api.upload_ready(alice, chat["id"], data) for _ in range(3)]
|
||
ids = {item["id"] for item in batch}
|
||
for item in batch:
|
||
lose_original(stand, item)
|
||
|
||
refused = run_reconcile(stand, "apply", limit)
|
||
assert refused.exit_code == 3
|
||
(error,) = refused.events(LIMIT_EXCEEDED)
|
||
assert error["changes"] >= len(batch), error
|
||
assert refused.finished["refused"] is True
|
||
assert refused.kinds(FIXED, ids={pending}) == {
|
||
pending: "stale_pending",
|
||
}
|
||
(record,) = refused.about(FIXED, ids={pending})
|
||
assert record["action"] == "removed_pending"
|
||
assert pending not in attachment_rows(stand)
|
||
assert pending not in staged_ids(stand)
|
||
# изменений под пределом нет: ни одной пометки
|
||
assert not refused.about(FIXED, ids=ids)
|
||
assert not ids & lost_ids(stand)
|
||
# стенд — без внесённых потерь для следующих тестов
|
||
applied = run_reconcile(stand, "apply", 100)
|
||
assert set(applied.kinds(FIXED, ids=ids)) == ids
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Ключ вне префикса вложений не трогается",
|
||
)
|
||
def test_reconcile_foreign_key_untouched(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W2 verify: строка с ключом вне `<база>/attachments/` —
|
||
готовая и зависшая загрузка — только в отчёте как чужой объект: сверка
|
||
не делает HEAD чужого префикса, не помечает и не убирает строку."""
|
||
data = images.sample_png()
|
||
# уборщик стенда стоит: иначе зависшую загрузку убрал бы он
|
||
with outage(stand, "stop", "cleaner"):
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
assert_error(
|
||
upload_raw(api, alice, chat, data, filename=filename),
|
||
503,
|
||
)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
ids = {ready["id"], pending}
|
||
try:
|
||
for item in ids:
|
||
# префикс своей базы, но не вложений: список его не видит
|
||
stand("set-key", item, f"{FAULT_DATABASE}/elsewhere/{item}")
|
||
stand("age-row", item, str(AGED_S))
|
||
unknown = dict.fromkeys(ids, "unknown_object")
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, ids=ids) == unknown
|
||
assert dry.exit_code == 3
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FOUND, ids=ids) == unknown
|
||
assert not applied.about(FIXED, ids=ids)
|
||
assert applied.exit_code == 3
|
||
# ни пометки, ни уборки: строки и объекты под префиксом на месте
|
||
rows = attachment_rows(stand)
|
||
assert (rows.get(ready["id"]), rows.get(pending)) == (
|
||
"ready",
|
||
"pending",
|
||
)
|
||
assert ready["id"] not in lost_ids(stand)
|
||
assert ready["id"] in stored_ids(stand)
|
||
assert pending in staged_ids(stand)
|
||
finally:
|
||
# прежние ключи у тех строк, что остались: зависшую загрузку
|
||
# после теста уберёт уборщик стенда вместе с её объектом
|
||
rows = attachment_rows(stand)
|
||
for item in ids & set(rows):
|
||
stand("set-key", item, object_key(item))
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "S3 недоступен во время сверки")
|
||
def test_reconcile_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
missing = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
lose_original(stand, missing)
|
||
with outage(stand, "stop", "s3"):
|
||
interrupted = run_reconcile(stand, "apply")
|
||
assert interrupted.exit_code == 1
|
||
assert interrupted.finished["interrupted_by"] == "object_storage"
|
||
assert not interrupted.events(FIXED)
|
||
assert missing["id"] not in lost_ids(stand)
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FIXED, ids={missing["id"]}) == {
|
||
missing["id"]: "missing_object",
|
||
}
|
||
assert missing["id"] in lost_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Прерванная сверка доводится повтором",
|
||
)
|
||
def test_reconcile_interrupted_by_database(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
failing = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
other = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
ids = {failing["id"], other["id"]}
|
||
for item in (failing, other):
|
||
lose_original(stand, item)
|
||
stand("fail-attachments", "mark", filename)
|
||
try:
|
||
interrupted = run_reconcile(stand, "apply")
|
||
assert interrupted.exit_code == 1
|
||
assert interrupted.finished["interrupted_by"] == "database"
|
||
assert failing["id"] not in lost_ids(stand)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
applied = run_reconcile(stand, "apply")
|
||
assert ids <= lost_ids(stand)
|
||
assert not applied.about(FOUND, ids=ids)
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids=ids)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Сверка под нагрузкой")
|
||
def test_reconcile_under_load(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Загрузки и отправки дольше grace, сверка с `--apply` раз в 3 с:
|
||
внесённое исправлено, ни одно подтверждённое вложение не потеряно."""
|
||
data = images.sample_png()
|
||
orphans = {str(uuid.uuid4()) for _ in range(2)}
|
||
for item in orphans:
|
||
put_object(stand, object_key(item), data)
|
||
injected = [api.upload_ready(alice, chat["id"], data) for _ in range(2)]
|
||
for item in injected:
|
||
lose_original(stand, item)
|
||
lost_before = lost_ids(stand)
|
||
deadline = time.monotonic() + CLEANER_GRACE_S + LOAD_EXTRA_S
|
||
lock = threading.Lock()
|
||
uploads: dict[str, str] = {}
|
||
messages: dict[str, list[str]] = {}
|
||
failures: list[str] = []
|
||
passes: list[Reconcile] = []
|
||
|
||
def load(worker: int) -> None:
|
||
actor = alice if worker % 2 == 0 else bob
|
||
with Api(
|
||
str(api.client.base_url),
|
||
timeout=THREAD_CLIENT_TIMEOUT_S,
|
||
) as client:
|
||
number = 0
|
||
while time.monotonic() < deadline:
|
||
number += 1
|
||
body = images.png_noise(200, 170, seed=worker * 10**6 + number)
|
||
resp = client.post_attachment(actor, chat["id"], body)
|
||
if resp.status_code != 202:
|
||
with lock:
|
||
failures.append(f"upload {resp.status_code}")
|
||
continue
|
||
attachment = assert_accepted(resp)
|
||
# загрузка подтверждена: она обязана стать ready и скачиваться
|
||
with lock:
|
||
uploads[attachment["id"]] = hashlib.sha256(
|
||
body,
|
||
).hexdigest()
|
||
message = client.send_message(
|
||
actor,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment["id"]],
|
||
)
|
||
with lock:
|
||
messages[message["id"]] = [attachment["id"]]
|
||
time.sleep(LOAD_PAUSE_S)
|
||
|
||
def reconcile() -> None:
|
||
while time.monotonic() < deadline:
|
||
started = time.monotonic()
|
||
passes.append(run_reconcile(stand, "apply"))
|
||
sleep_until(started + LOAD_RECONCILE_PERIOD_S)
|
||
|
||
def guarded(target: Callable[..., None], *args: Any) -> None:
|
||
# исключение потока — в список, а не молча в stderr
|
||
try:
|
||
target(*args)
|
||
except Exception as e:
|
||
with lock:
|
||
failures.append(repr(e))
|
||
|
||
threads = [
|
||
threading.Thread(target=guarded, args=(load, worker))
|
||
for worker in range(LOAD_THREADS)
|
||
]
|
||
threads.append(threading.Thread(target=guarded, args=(reconcile,)))
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
assert not failures, failures[:5]
|
||
assert uploads, "no uploads under load"
|
||
assert len(passes) >= 3, len(passes)
|
||
assert all(item.exit_code in (0, 3) for item in passes), [
|
||
item.exit_code for item in passes
|
||
]
|
||
# внесённые исправлены: объекты без строки удалены, потери помечены
|
||
assert not orphans & stored_ids(stand)
|
||
assert lost_ids(stand) - lost_before == {item["id"] for item in injected}
|
||
# ни одно подтверждённое вложение не потеряно
|
||
for attachment_id, digest in uploads.items():
|
||
api.wait_attachment(alice, attachment_id)
|
||
resp = api.attachment_content(alice, attachment_id)
|
||
assert resp.status_code == 200, (attachment_id, resp.status_code)
|
||
assert hashlib.sha256(resp.content).hexdigest() == digest
|
||
history = {
|
||
item["id"]: item["attachment_ids"]
|
||
for item in api.all_messages(alice, chat["id"])
|
||
}
|
||
for message_id, attachment_ids in messages.items():
|
||
assert history[message_id] == attachment_ids
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Плановая сверка без команды")
|
||
def test_scheduled_reconcile(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
scheduled = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
lose_original(stand, scheduled)
|
||
log = stand("run-cleaner", str(SCHEDULED_RUN_S))
|
||
records = [
|
||
json.loads(line) for line in log.splitlines() if line.startswith("{")
|
||
]
|
||
passes = [item for item in records if item.get("event") == FINISHED]
|
||
assert passes, log[-2000:]
|
||
assert any(
|
||
item["apply"] and item["interrupted_by"] is None for item in passes
|
||
), passes
|
||
assert scheduled["id"] in lost_ids(stand)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Стоимость сверки (change track2-reconcile-cost, design, решение 14):
|
||
# стратегия HEAD, глубокая сверка и класс «другой SHA-256», снятие пометки
|
||
# только при том же SHA-256, предел скорости, отказ по пределу без лишних
|
||
# перепроверок. Аргументы сверки hook передаёт команде как есть
|
||
# ---------------------------------------------------------------------------
|
||
|
||
HEAD_STRATEGY = ("--strategy", "head")
|
||
DEEP = ("--deep",)
|
||
NO_RATE_LIMIT = ("--max-objects-per-second", "0")
|
||
# сколько вложений загружает тест предела скорости: объектов в bucket
|
||
# заведомо хватает на несколько секунд прохода
|
||
RATE_UPLOADS = 8
|
||
# во сколько раз проход с пределом длиннее числа объектов на предел
|
||
# (допуск на округление `duration_ms`)
|
||
RATE_SLACK = 0.95
|
||
|
||
|
||
def same_size_bytes(data: bytes, *, seed: int) -> bytes:
|
||
"""Другие байты того же размера, от зерна теста."""
|
||
other = random.Random(seed).randbytes(len(data))
|
||
assert other != data
|
||
return other
|
||
|
||
|
||
def findings(run: Reconcile) -> set[tuple[str, str]]:
|
||
"""Все расхождения прохода: класс и ключ объекта."""
|
||
return {(item["kind"], item["object_key"]) for item in run.events(FOUND)}
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Стратегия HEAD находит то же")
|
||
def test_reconcile_head_strategy_same_findings(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Те же расхождения, что в `test_reconcile_finds_and_fixes`: dry-run
|
||
стратегией по умолчанию и `--strategy head` называют одно и то же, а
|
||
HEAD-проход дороже по запросам к S3 и сеансам PostgreSQL."""
|
||
data = images.sample_png()
|
||
# уборщик стенда стоит: зависшую загрузку P не уберут между проходами
|
||
with outage(stand, "stop", "cleaner"):
|
||
a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3))
|
||
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
|
||
orphan = str(uuid.uuid4())
|
||
keys = {
|
||
"o": object_key(orphan),
|
||
"b": object_key(b["id"]),
|
||
"bv": variant_key(b["id"]),
|
||
}
|
||
put_object(stand, keys["o"], data)
|
||
stand("delete-object", object_key(a["id"]))
|
||
stand("delete-row", b["id"])
|
||
stand("set-size", c["id"], str(len(data) + 1))
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
resp = upload_raw(api, alice, chat, data, filename=filename)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
ids = {a["id"], c["id"], pending}
|
||
expected = {
|
||
keys["o"]: "orphan_object",
|
||
keys["b"]: "orphan_object",
|
||
keys["bv"]: "orphan_object",
|
||
a["id"]: "missing_object",
|
||
c["id"]: "size_mismatch",
|
||
pending: "stale_pending",
|
||
}
|
||
snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
|
||
batch = run_reconcile(stand, "dry-run")
|
||
head = run_reconcile(stand, "dry-run", options=HEAD_STRATEGY)
|
||
assert batch.kinds(FOUND, ids=ids, keys=keys.values()) == expected
|
||
assert head.kinds(FOUND, ids=ids, keys=keys.values()) == expected
|
||
# не только свои: весь отчёт и итог по классам одинаковы
|
||
assert findings(head) == findings(batch)
|
||
for field in ("found", "listed_objects", "checked_rows"):
|
||
assert head.finished[field] == batch.finished[field], field
|
||
assert head.exit_code == batch.exit_code == 3
|
||
assert (batch.finished["strategy"], head.finished["strategy"]) == (
|
||
"batch",
|
||
"head",
|
||
)
|
||
assert (
|
||
head.finished["s3_requests"]["head"]
|
||
> batch.finished["s3_requests"]["head"]
|
||
), (head.finished, batch.finished)
|
||
assert head.finished["db_sessions"] > batch.finished["db_sessions"]
|
||
after = (attachment_rows(stand), object_ids(stand), lost_ids(stand))
|
||
assert after == snapshot
|
||
|
||
# стенд — без внесённых расхождений для следующих тестов
|
||
fixed = run_reconcile(stand, "apply")
|
||
assert set(fixed.kinds(FIXED, ids=ids, keys=keys.values())) == set(
|
||
expected,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Другой SHA-256 находит глубокая сверка",
|
||
)
|
||
def test_reconcile_deep_checksum(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=11)
|
||
a = api.upload_ready(alice, chat["id"], data)
|
||
api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]])
|
||
other = same_size_bytes(data, seed=11)
|
||
# объект перезаписан мимо API байтами того же размера
|
||
put_object(stand, object_key(a["id"]), other)
|
||
stand("age-row", a["id"], str(AGED_S))
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
snapshot = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
|
||
|
||
shallow = run_reconcile(stand, "dry-run")
|
||
assert not shallow.about(FOUND, ids={a["id"]})
|
||
assert shallow.finished["deep"] is False
|
||
deep = run_reconcile(stand, "dry-run", options=DEEP)
|
||
assert deep.kinds(FOUND, ids={a["id"]}) == {a["id"]: "checksum_mismatch"}
|
||
assert deep.finished["deep"] is True
|
||
assert deep.finished["read_bytes"] >= len(data), deep.finished
|
||
assert deep.finished["s3_requests"]["get"] >= 1, deep.finished
|
||
assert deep.exit_code == 3
|
||
after = (attachment_rows(stand), stored_ids(stand), lost_ids(stand))
|
||
assert after == snapshot
|
||
# SHA-256 объектов в журнале нет
|
||
journal = json.dumps(deep.records)
|
||
for body in (data, other):
|
||
assert hashlib.sha256(body).hexdigest() not in journal
|
||
|
||
applied = run_reconcile(stand, "apply", options=DEEP)
|
||
(record,) = applied.about(FIXED, ids={a["id"]})
|
||
assert (record["kind"], record["action"]) == (
|
||
"checksum_mismatch",
|
||
"marked_lost",
|
||
)
|
||
assert a["id"] in lost_ids(stand)
|
||
assert_lost_content(api, alice, a, aged=True)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Вернулся другой объект того же размера",
|
||
)
|
||
@pytest.mark.spec("storage-consistency", "Оригинал вернулся")
|
||
def test_reconcile_restore_needs_same_bytes(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.png_noise(64, 48, seed=13)
|
||
restored = api.upload_ready(alice, chat["id"], data)
|
||
ids = {restored["id"]}
|
||
lose_original(stand, restored)
|
||
marked = run_reconcile(stand, "apply")
|
||
assert marked.kinds(FIXED, ids=ids) == {restored["id"]: "missing_object"}
|
||
assert restored["id"] in lost_ids(stand)
|
||
|
||
# под ключом — другие байты того же размера: пометка остаётся
|
||
put_object(
|
||
stand,
|
||
object_key(restored["id"]),
|
||
same_size_bytes(data, seed=13),
|
||
)
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
wrong = run_reconcile(stand, "apply")
|
||
assert not wrong.about(FIXED, ids=ids)
|
||
assert all(
|
||
record["kind"] != "restored_object"
|
||
for record in wrong.about(FOUND, ids=ids)
|
||
)
|
||
assert restored["id"] in lost_ids(stand)
|
||
assert_lost_content(api, alice, restored, aged=True)
|
||
|
||
# те же байты, что были загружены: пометка снята
|
||
put_object(stand, object_key(restored["id"]), data)
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
cleared = run_reconcile(stand, "apply")
|
||
(record,) = cleared.about(FIXED, ids=ids)
|
||
assert (record["kind"], record["action"]) == (
|
||
"restored_object",
|
||
"cleared_lost",
|
||
)
|
||
assert restored["id"] not in lost_ids(stand)
|
||
assert_content(
|
||
api.attachment_content(alice, restored["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
again = run_reconcile(stand, "dry-run")
|
||
assert not again.about(FOUND, ids=ids)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Скорость прохода ограничена")
|
||
@pytest.mark.spec("storage-consistency", "Без ограничения скорости")
|
||
def test_reconcile_rate_limit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Предел на четверть объектов прохода: проход длится не меньше
|
||
объектов на предел, находки и код — как без предела."""
|
||
data = images.sample_png()
|
||
for _ in range(RATE_UPLOADS):
|
||
api.upload_ready(alice, chat["id"], data)
|
||
lost = api.upload_ready(alice, chat["id"], data)
|
||
ids = {lost["id"]}
|
||
lose_original(stand, lost)
|
||
# уборщик стенда стоит: между проходами стенд не меняется
|
||
with outage(stand, "stop", "cleaner"):
|
||
free = run_reconcile(stand, "dry-run", options=NO_RATE_LIMIT)
|
||
rate = max(5, math.ceil(free.finished["storage_objects"] / 4))
|
||
limited = run_reconcile(
|
||
stand,
|
||
"dry-run",
|
||
options=("--max-objects-per-second", str(rate)),
|
||
)
|
||
assert free.finished["max_objects_per_second"] == 0
|
||
assert limited.finished["max_objects_per_second"] == rate
|
||
objects = limited.finished["storage_objects"]
|
||
assert objects >= RATE_UPLOADS, limited.finished
|
||
assert (
|
||
limited.finished["duration_ms"] >= 1000 * objects / rate * RATE_SLACK
|
||
), limited.finished
|
||
assert free.finished["duration_ms"] < limited.finished["duration_ms"]
|
||
expected = {lost["id"]: "missing_object"}
|
||
assert free.kinds(FOUND, ids=ids) == expected
|
||
assert limited.kinds(FOUND, ids=ids) == expected
|
||
assert free.exit_code == limited.exit_code == 3
|
||
# стенд — без внесённой потери
|
||
applied = run_reconcile(stand, "apply")
|
||
assert applied.kinds(FIXED, ids=ids) == expected
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Отказ по пределу без лишних перепроверок",
|
||
)
|
||
def test_reconcile_limit_stops_rechecks(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
limit = 2
|
||
batch = [
|
||
api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for _ in range(6)
|
||
]
|
||
ids = {item["id"] for item in batch}
|
||
for item in batch:
|
||
lose_original(stand, item)
|
||
refused = run_reconcile(stand, "apply", limit)
|
||
assert refused.exit_code == 3
|
||
(error,) = refused.events(LIMIT_EXCEEDED)
|
||
assert error["level"] == "error"
|
||
# отказ решён на третьем подтверждённом: остальные не перепроверены
|
||
assert error["changes"] == limit + 1, error
|
||
assert error["max_changes"] == limit
|
||
assert error["unchecked"] >= len(batch) - (limit + 1), error
|
||
assert refused.finished["unchecked"] == error["unchecked"]
|
||
assert refused.finished["refused"] is True
|
||
assert not refused.about(FIXED, ids=ids)
|
||
assert not ids & lost_ids(stand)
|
||
# dry-run перепроверяет каждого и называет все шесть
|
||
dry = run_reconcile(stand, "dry-run")
|
||
assert dry.kinds(FOUND, ids=ids) == dict.fromkeys(ids, "missing_object")
|
||
assert dry.finished["unchecked"] == 0
|
||
applied = run_reconcile(stand, "apply", 100)
|
||
assert set(applied.kinds(FIXED, ids=ids)) == ids
|
||
assert ids <= lost_ids(stand)
|
||
|
||
|
||
def own_findings(
|
||
run: Reconcile,
|
||
event: str,
|
||
keys: Collection[str],
|
||
) -> set[tuple[str, str, str | None]]:
|
||
"""Записи `event` о своих ключах: класс, ключ и `id` вложения — у
|
||
строки, ссылающейся на чужой ключ, `id` и ключ из разных вложений."""
|
||
return {
|
||
(item["kind"], item["object_key"], item.get("attachment_id"))
|
||
for item in run.events(event)
|
||
if item.get("object_key") in keys
|
||
}
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Строка ссылается на объект другого id",
|
||
)
|
||
def test_reconcile_row_with_other_id_key(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание W2 verify `track2-reconcile-cost`: строка Y ссылается на
|
||
`<Z>/original` чужого `id`. Обе стратегии сравнивают Y с объектом по
|
||
её ключу: другой размер и, с `--deep`, другой SHA-256 — расхождения
|
||
Y; объекты Z и `<Y>/original` — чужие и на месте."""
|
||
data = images.png_noise(64, 48, seed=17)
|
||
y1, y2 = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
|
||
z1, z2 = (str(uuid.uuid4()) for _ in range(2))
|
||
keys = {
|
||
"y1": object_key(y1["id"]),
|
||
"y2": object_key(y2["id"]),
|
||
"z1": object_key(z1),
|
||
"z2": object_key(z2),
|
||
}
|
||
# Z1 — другого размера, Z2 — того же размера с другими байтами
|
||
put_object(stand, keys["z1"], data + b"\0")
|
||
put_object(stand, keys["z2"], same_size_bytes(data, seed=17))
|
||
try:
|
||
stand("set-key", y1["id"], keys["z1"])
|
||
stand("set-key", y2["id"], keys["z2"])
|
||
for item in (y1, y2):
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
# объекты Z записаны только что: после G + 1 все старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
expected = {
|
||
("size_mismatch", keys["z1"], y1["id"]),
|
||
("checksum_mismatch", keys["z2"], y2["id"]),
|
||
("unknown_object", keys["z1"], z1),
|
||
("unknown_object", keys["z2"], z2),
|
||
("unknown_object", keys["y1"], y1["id"]),
|
||
("unknown_object", keys["y2"], y2["id"]),
|
||
}
|
||
batch = run_reconcile(stand, "dry-run", options=DEEP)
|
||
head = run_reconcile(
|
||
stand,
|
||
"dry-run",
|
||
options=(*DEEP, *HEAD_STRATEGY),
|
||
)
|
||
assert own_findings(batch, FOUND, keys.values()) == expected
|
||
assert own_findings(head, FOUND, keys.values()) == expected
|
||
assert batch.exit_code == head.exit_code == 3
|
||
# поверхностная сверка видит только другой размер
|
||
shallow = run_reconcile(stand, "dry-run")
|
||
assert own_findings(shallow, FOUND, keys.values()) == expected - {
|
||
("checksum_mismatch", keys["z2"], y2["id"]),
|
||
}
|
||
|
||
applied = run_reconcile(stand, "apply", options=DEEP)
|
||
assert own_findings(applied, FIXED, keys.values()) == {
|
||
("size_mismatch", keys["z1"], y1["id"]),
|
||
("checksum_mismatch", keys["z2"], y2["id"]),
|
||
}
|
||
assert applied.exit_code == 3
|
||
# ни один объект не удалён, обе строки — ready и помечены
|
||
assert {y1["id"], y2["id"], z1, z2} <= stored_ids(stand)
|
||
rows = attachment_rows(stand)
|
||
assert rows[y1["id"]] == rows[y2["id"]] == "ready"
|
||
assert {y1["id"], y2["id"]} <= lost_ids(stand)
|
||
finally:
|
||
# прежние ключи; объекты Z, на которые больше никто не ссылается,
|
||
# — прочь, чтобы следующие проходы не удаляли их как свои
|
||
stand("set-key", y1["id"], keys["y1"])
|
||
stand("set-key", y2["id"], keys["y2"])
|
||
stand("delete-object", keys["z1"])
|
||
stand("delete-object", keys["z2"])
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Недавно перезаписанный объект не тронут",
|
||
)
|
||
def test_reconcile_fresh_overwrite_untouched(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Замечание S1 verify `track2-reconcile-cost`: объект готового
|
||
вложения старше grace перезаписан мимо API меньше grace назад — другим
|
||
размером (A) и теми же размером с другими байтами (B). Сверка с
|
||
`--deep --apply` и HEAD-стратегией считает оба объекта свежими; после
|
||
grace — расхождения. Глубокий проход читает весь bucket стенда и к концу
|
||
набора длится дольше G: окно «свежий» — grace разового прохода
|
||
`FRESH_GRACE_S`, он меньше возраста строк."""
|
||
data = images.png_noise(64, 48, seed=19)
|
||
a, b = (api.upload_ready(alice, chat["id"], data) for _ in range(2))
|
||
ids = {a["id"], b["id"]}
|
||
for item in (a, b):
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
|
||
def overwrite() -> float:
|
||
"""Объекты A и B заново перезаписаны: моложе grace с этого момента."""
|
||
put_object(stand, object_key(a["id"]), data + b"\0")
|
||
put_object(stand, object_key(b["id"]), same_size_bytes(data, seed=19))
|
||
return time.monotonic()
|
||
|
||
grace = ("--grace", str(FRESH_GRACE_S))
|
||
try:
|
||
# перед каждым проходом объекты перезаписываются заново: глубокий
|
||
# проход читает весь bucket стенда, и два подряд дольше grace
|
||
for options, mode in (
|
||
((*DEEP, *grace), "apply"),
|
||
((*DEEP, *HEAD_STRATEGY, *grace), "dry-run"),
|
||
):
|
||
written = overwrite()
|
||
run = run_reconcile(stand, mode, options=options)
|
||
# проход — пока объекты моложе grace, иначе он ничего не
|
||
# доказывает
|
||
assert time.monotonic() - written < FRESH_GRACE_S, (
|
||
"reconcile run took longer than its grace period"
|
||
)
|
||
assert not run.about(FOUND, ids=ids), run.finished
|
||
assert not run.about(FIXED, ids=ids), run.finished
|
||
assert run.finished["fresh_objects"] >= len(ids), run.finished
|
||
assert not ids & lost_ids(stand)
|
||
assert_error(api.attachment_content(alice, a["id"]), 500)
|
||
|
||
time.sleep(max(0.0, OBJECT_AGE_WAIT_S - (time.monotonic() - written)))
|
||
aged = run_reconcile(stand, "dry-run", options=DEEP)
|
||
assert aged.kinds(FOUND, ids=ids) == {
|
||
a["id"]: "size_mismatch",
|
||
b["id"]: "checksum_mismatch",
|
||
}
|
||
finally:
|
||
# стенд — с исходными байтами: расхождений не остаётся
|
||
for item in (a, b):
|
||
put_object(stand, object_key(item["id"]), data)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сверка и фоновая обработка (change async-attachments, design, решение 11):
|
||
# объекты вариантов своих вложений и текущая работа — не расхождения
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объекты вариантов своих вложений")
|
||
def test_reconcile_variant_objects(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
batch = [api.upload_ready(alice, chat["id"], data) for _ in range(2)]
|
||
ids = {item["id"] for item in batch}
|
||
keys = {variant_key(item["id"]) for item in batch}
|
||
for item in batch:
|
||
assert stand("variant-object-count", item["id"]) == str(
|
||
len(VARIANT_NAMES),
|
||
)
|
||
stand("age-row", item["id"], str(AGED_S))
|
||
# объекты вариантов записаны только что: после G + 1 старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
for options in ((), HEAD_STRATEGY):
|
||
run = run_reconcile(stand, "dry-run", options=options)
|
||
assert not run.about(FOUND, ids=ids, keys=keys), run.finished
|
||
run.assert_code_matches()
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект варианта без метаданных")
|
||
def test_reconcile_orphan_variant(
|
||
stand: Stand,
|
||
) -> None:
|
||
data = images.sample_png()
|
||
orphan = str(uuid.uuid4())
|
||
key = variant_key(orphan)
|
||
put_object(stand, key, data)
|
||
assert stand("variant-object-count", orphan) == "1"
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
run = run_reconcile(stand, "apply")
|
||
(record,) = run.about(FIXED, keys={key})
|
||
assert (record["kind"], record["action"]) == (
|
||
"orphan_object",
|
||
"deleted_object",
|
||
)
|
||
assert not run.about(FOUND, keys={key})
|
||
assert stand("variant-object-count", orphan) == "0"
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Текущая работа не считается расхождением",
|
||
)
|
||
def test_reconcile_keeps_current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
# уборщик стоит: срок хранения `failed` на стенде 60 с, а проход к концу набора
|
||
# длинный, и уборщик убрал бы staging строки `failed` раньше проверки
|
||
with (
|
||
outage(stand, "stop", "cleaner"),
|
||
current_work(api, stand, alice, chat) as work,
|
||
):
|
||
# конечные объекты F и P и варианты P старше grace
|
||
time.sleep(OBJECT_AGE_WAIT_S)
|
||
ids = {item["id"] for item in work.values()}
|
||
keys = {staging_key(item) for item in ids} | {
|
||
object_key(item) for item in ids
|
||
}
|
||
run = run_reconcile(stand, "apply")
|
||
assert not run.about(FOUND, ids=ids, keys=keys), run.finished
|
||
assert not run.about(FIXED, ids=ids, keys=keys), run.finished
|
||
run.assert_code_matches()
|
||
assert not ids & lost_ids(stand)
|
||
rows = attachment_rows(stand)
|
||
staged = staged_ids(stand)
|
||
for name, item in work.items():
|
||
assert rows.get(item["id"]) == name, (name, rows.get(item["id"]))
|
||
assert item["id"] in staged, name
|
||
stored = stored_ids(stand)
|
||
# конечный объект у задания, дошедшего до него, на месте
|
||
assert work["failed"]["id"] in stored
|
||
assert work["processing"]["id"] in stored
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Надёжность конвейера (change pipeline-reliability, design, решения 2–9, 14,
|
||
# 15): relay, окна сбоя без точек в коде, аренда и fencing, лимиты процесса
|
||
# обработки, backoff, реестр failed и replay, prefetch, очистка staging. Worker
|
||
# и relay запускаются в finally, триггеры и дополнительные контейнеры снимаются
|
||
# в finally
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@contextmanager
|
||
def extra_worker(stand: Stand, *env: str) -> Iterator[None]:
|
||
"""Основной worker остановлен, работает второй с этим окружением."""
|
||
wait_queue_empty(stand)
|
||
with outage(stand, "stop", "worker"):
|
||
stand("run-worker", *env)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand("stop-extra")
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Ответ не ждёт публикации")
|
||
@pytest.mark.spec("job-outbox", "Crash между фиксацией и публикацией")
|
||
@pytest.mark.spec("job-outbox", "Задание без отметки от прежней версии")
|
||
def test_relay_killed_before_publish(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Relay убит: загрузка 202, сообщения нет, метаданные `queued`, задание без
|
||
отметки — то же, что строка прежней версии. После запуска — `ready` с тем же
|
||
заданием."""
|
||
wait_queue_empty(stand)
|
||
with outage(stand, "kill", "relay"):
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(api, alice, chat, images.sample_png()),
|
||
)
|
||
accepted = assert_accepted(resp)
|
||
assert elapsed < ACCEPT_MAX_S, elapsed
|
||
assert queue_state(stand)["messages"] == 0
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["published_at"] is None, job
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["id"] == job["id"], done
|
||
assert done["published_at"] is not None, done
|
||
assert done["attempt"] == 1, done
|
||
|
||
|
||
def crash_relay_after_confirm(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Worker остановлен, отметка задержана триггером, брокер сообщение уже
|
||
подтвердил: relay убит, транзакция отметки откатилась. После запуска relay
|
||
публикует ещё раз; дубль — no-op."""
|
||
filename = fault_filename()
|
||
with worker_stopped(stand):
|
||
stand("fail-attachments", "relay-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
wait_queue_messages(stand, 1)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["published_at"] is None, job
|
||
stand("kill", "relay")
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "relay")
|
||
# та же работа опубликована ещё раз: отметки не было
|
||
wait_queue_messages(stand, 2)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, meta)
|
||
wait_queue_empty(stand)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["attempt"] == 1, done
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Crash после подтверждения до отметки")
|
||
def test_relay_killed_after_confirm(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
crash_relay_after_confirm(api, stand, alice, chat)
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Отказ брокера во время работы relay")
|
||
def test_relay_survives_broker_outage(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("relay-state"))
|
||
seen = len(relay_records(stand))
|
||
with outage(stand, "stop", "rabbitmq"):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_relay_record(stand, "Broker is unavailable", skip=seen)
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
after = json.loads(stand("relay-state"))
|
||
assert after["running"] is True, after
|
||
assert after["started_at"] == before["started_at"], after
|
||
assert after["restart_count"] == before["restart_count"], after
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Отказ PostgreSQL во время работы relay")
|
||
def test_relay_survives_database_outage(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("relay-state"))
|
||
seen = len(relay_records(stand))
|
||
with outage(stand, "stop", "postgres"):
|
||
wait_relay_record(stand, "Relay pass was interrupted", skip=seen)
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
after = json.loads(stand("relay-state"))
|
||
assert after["running"] is True, after
|
||
assert after["started_at"] == before["started_at"], after
|
||
assert after["restart_count"] == before["restart_count"], after
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Два relay")
|
||
def test_two_relays(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
files = 6
|
||
with worker_stopped(stand):
|
||
stand("run-relay", "RELAY__POLL_INTERVAL_SECONDS=0.05")
|
||
try:
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(files)
|
||
]
|
||
wait_queue_messages(stand, files)
|
||
time.sleep(1)
|
||
assert queue_state(stand)["messages"] == files
|
||
finally:
|
||
stand("stop-extra")
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Следующая попытка не раньше срока")
|
||
def test_relay_respects_next_attempt(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
delay = 6
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
# relay стоит, пока срок отодвигается: случайная пауза короче хода теста
|
||
with relay_stopped(stand), outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
wait_job(stand, accepted["id"], state="queued", attempt=1)
|
||
stand("stop", "worker")
|
||
stand("set-next-attempt", accepted["id"], str(delay))
|
||
due = time.monotonic() + delay
|
||
assert queue_state(stand)["messages"] == 0
|
||
wait_queue_messages(stand, 1, timeout=delay + PUBLISH_WAIT_S)
|
||
assert time.monotonic() >= due - SLACK_S
|
||
peeked = json.loads(stand("peek-job"))
|
||
assert json.loads(peeked["body"])["attempt"] == 2, peeked
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Очередь удалена вместе с сообщением")
|
||
def test_lost_message_republished(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
stand("delete-queue")
|
||
# срок прошёл: relay объявляет очередь и публикует снова
|
||
state = wait_queue_messages(
|
||
stand,
|
||
1,
|
||
timeout=RELAY_REPUBLISH_S + PUBLISH_WAIT_S,
|
||
)
|
||
assert state["durable"] is True, state
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Очередь с сообщениями не обрастает копиями")
|
||
def test_no_copies_while_queue_has_messages(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
files = 2
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(files)
|
||
]
|
||
wait_queue_messages(stand, files)
|
||
time.sleep(RELAY_REPUBLISH_S + 2)
|
||
assert queue_state(stand)["messages"] == files
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Очередь ограничена пределом")
|
||
def test_published_backlog_limit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
limit, files = 2, 5
|
||
with worker_stopped(stand), relay_stopped(stand):
|
||
stand("run-relay", f"RELAY__MAX_PUBLISHED_BACKLOG={limit}")
|
||
try:
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(files)
|
||
]
|
||
wait_queue_messages(stand, limit)
|
||
time.sleep(1)
|
||
assert queue_state(stand)["messages"] == limit
|
||
jobs = [job_row(stand, item["id"]) for item in accepted]
|
||
marked = [job for job in jobs if job and job["published_at"]]
|
||
assert len(marked) == limit, jobs
|
||
finally:
|
||
stand("stop-extra")
|
||
for item in accepted:
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("job-outbox", "Брокер сдерживает публикацию")
|
||
def test_broker_memory_alarm(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
seen = len(relay_records(stand))
|
||
stand("broker-memory-limit", "1MiB")
|
||
try:
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(api, alice, chat, images.sample_png()),
|
||
)
|
||
accepted = assert_accepted(resp)
|
||
assert elapsed < ACCEPT_MAX_S, elapsed
|
||
wait_relay_record(stand, "Broker is unavailable", skip=seen)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["published_at"] is None, job
|
||
finally:
|
||
stand("broker-memory-limit", BROKER_MEMORY_LIMIT)
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
def crash_after_objects(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Все объекты записаны, фиксация ждёт в триггере: KILL. Новая попытка — по
|
||
аренде — пишет те же ключи теми же байтами."""
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
attachment_id = accepted["id"]
|
||
api.wait_attachment(
|
||
alice,
|
||
attachment_id,
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
wait_variant_objects(stand, attachment_id)
|
||
# заморозка: окно триггера короче запроса к брокеру, а замёрзший клиент
|
||
# COMMIT не отправит
|
||
stand("freeze", "worker")
|
||
# ack — только после фиксации: задание восстановила бы и аренда, поэтому
|
||
# ранний ack виден только здесь
|
||
held = queue_state(stand)
|
||
assert held["messages_unacknowledged"] == 1, held
|
||
stand("kill", "worker")
|
||
before = {
|
||
name: stand("object-sha256", variant_key(attachment_id, name))
|
||
for name in VARIANT_NAMES
|
||
}
|
||
assert all(before.values()), before
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "worker")
|
||
meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S)
|
||
variants = assert_variants(api, stand, alice, meta)
|
||
for name, digest in before.items():
|
||
assert variants[name]["sha256"] == digest, name
|
||
done = job_row(stand, attachment_id)
|
||
assert done is not None
|
||
assert done["attempt"] == 2, done
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Crash после записи объектов до фиксации",
|
||
)
|
||
def test_crash_after_objects_before_commit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
crash_after_objects(api, stand, alice, chat)
|
||
|
||
|
||
def crash_after_commit(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""COMMIT фиксации ждёт в отложенном триггере, worker заморожен (контейнер
|
||
`running`, не `paused`, поэтому KILL проходит), строка `succeeded` при
|
||
замороженном клиенте, KILL: ack не ушёл. Следующий запуск получает сообщение
|
||
снова и подтверждает его без эффекта."""
|
||
filename = fault_filename()
|
||
wait_queue_empty(stand)
|
||
stand("fail-attachments", "ready-commit-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
attachment_id = accepted["id"]
|
||
stand("wait-commit-stall")
|
||
stand("freeze", "worker")
|
||
wait_job(stand, attachment_id, state="succeeded", attempt=1)
|
||
stand("kill", "worker")
|
||
job = job_row(stand, attachment_id)
|
||
assert job is not None
|
||
ready = assert_ok(
|
||
api.attachment_meta(alice, attachment_id),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert ready["status"] == "ready", ready
|
||
seen = len(worker_records(stand))
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("start", "worker")
|
||
record = wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="finished",
|
||
)
|
||
assert record["level"] == "info", record
|
||
wait_queue_empty(stand)
|
||
assert (
|
||
assert_ok(api.attachment_meta(alice, attachment_id), 200, "Attachment")
|
||
== ready
|
||
)
|
||
assert job_row(stand, attachment_id) == job
|
||
assert job["attempt"] == 1, job
|
||
assert_variants(api, stand, alice, ready)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Crash после фиксации до подтверждения",
|
||
)
|
||
def test_crash_after_commit_before_ack(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
crash_after_commit(api, stand, alice, chat)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Пауза растёт со случайной добавкой",
|
||
)
|
||
def test_retry_backoff_with_jitter(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Формула решения 4: пауза попытки N — из `[d / 2, d]`,
|
||
`d = min(max, base × 2^(N − 1))`, у двух заданий паузы разные."""
|
||
seen = len(worker_records(stand))
|
||
with worker_stopped(stand):
|
||
first = api.upload(alice, chat["id"], images.sample_png())
|
||
second = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 2)
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
for item in (first, second):
|
||
api.wait_attachment(
|
||
alice,
|
||
item["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
delays: dict[str, dict[int, float]] = {first["id"]: {}, second["id"]: {}}
|
||
for record in worker_records(stand)[seen:]:
|
||
if record.get("event") == "Retrying job":
|
||
delays[record["attachment_id"]][record["attempt"]] = record[
|
||
"delay_seconds"
|
||
]
|
||
for attachment_id, pauses in delays.items():
|
||
assert sorted(pauses) == list(range(1, WORKER_MAX_ATTEMPTS)), pauses
|
||
for attempt, delay in pauses.items():
|
||
top = min(
|
||
WORKER_RETRY_MAX_S,
|
||
WORKER_RETRY_BASE_S * 2 ** (attempt - 1),
|
||
)
|
||
assert top / 2 <= delay <= top, (attachment_id, attempt, delay)
|
||
for attempt in range(1, WORKER_MAX_ATTEMPTS):
|
||
assert delays[first["id"]][attempt] != delays[second["id"]][attempt]
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Попытка не раньше срока")
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Пауза не занимает место конкуренции",
|
||
)
|
||
def test_retry_waits_outside_worker(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Задание A ждёт следующую попытку: дубль его сообщения и перезапуск worker
|
||
попытку не начинают, а другое вложение становится `ready` раньше срока."""
|
||
wait = 20
|
||
with worker_stopped(stand):
|
||
waiting = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
with relay_stopped(stand), outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
wait_job(stand, waiting["id"], state="queued", attempt=1)
|
||
stand("set-next-attempt", waiting["id"], str(wait))
|
||
due = time.monotonic() + wait
|
||
job = job_row(stand, waiting["id"])
|
||
assert job is not None
|
||
next_attempt_at = datetime.fromisoformat(job["next_attempt_at"])
|
||
stand("stop", "worker")
|
||
stand(
|
||
"publish-job",
|
||
stdin=job_body(job["id"], waiting["id"], attempt=2),
|
||
)
|
||
seen = len(worker_records(stand))
|
||
stand("start", "worker")
|
||
wait_worker_record(
|
||
stand,
|
||
"Skipped job message",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
reason="not_due",
|
||
)
|
||
wait_queue_empty(stand)
|
||
same = job_row(stand, waiting["id"])
|
||
assert same is not None
|
||
assert (same["attempt"], same["state"]) == (1, "queued"), same
|
||
other = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
assert time.monotonic() < due, "other attachment was not ready in time"
|
||
assert other["status"] == "ready"
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, waiting["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta["status"] == "queued", meta
|
||
done = api.wait_attachment(alice, waiting["id"], timeout=JOB_WAIT_S)
|
||
assert done["status"] == "ready"
|
||
final = job_row(stand, waiting["id"])
|
||
assert final is not None
|
||
assert final["attempt"] == 2, final
|
||
started = wait_worker_record(
|
||
stand,
|
||
"Started job attempt",
|
||
skip=seen,
|
||
job_id=job["id"],
|
||
attempt=2,
|
||
)
|
||
assert datetime.fromisoformat(started["timestamp"]) >= next_attempt_at
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Пикселей больше лимита")
|
||
def test_limit_pixels(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
huge = images.png_huge(10000, 5000)
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
put_object(stand, staging_key(accepted["id"]), huge)
|
||
stand("set-size", accepted["id"], str(len(huge)))
|
||
stand("set-sha256", accepted["id"], hashlib.sha256(huge).hexdigest())
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["last_error_code"] == "limit_pixels", job
|
||
assert job["attempt"] == 1, job
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Объект больше лимита не читается")
|
||
def test_limit_oversized_staging(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
put_object(
|
||
stand,
|
||
staging_key(accepted["id"]),
|
||
bytes(images.MAX_BYTES + 1),
|
||
)
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "processing_failed", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["last_error_code"] == "source_corrupted", job
|
||
assert job["attempt"] == 1, job
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Дорогое изображение")
|
||
def test_limit_cpu_scan_bomb(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("worker-state"))
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
|
||
accepted = slow.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_scan_bomb(),
|
||
content_type="image/jpeg",
|
||
filename="bomb.jpg",
|
||
)
|
||
meta = slow.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["last_error_code"] == "limit_cpu", job
|
||
assert job["attempt"] == 1, job
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
# следующий файл обрабатывается
|
||
slow.upload_ready(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Предел памяти процесса обработки")
|
||
def test_limit_memory(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with (
|
||
Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow,
|
||
extra_worker(stand, "WORKER__RENDER_MEMORY_LIMIT_MIB=200"),
|
||
):
|
||
accepted = slow.upload(alice, chat["id"], images.png_rgba_large())
|
||
meta = slow.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["last_error_code"] == "limit_memory", job
|
||
assert job["attempt"] == 1, job
|
||
# процесс worker работает: следующий файл обрабатывается
|
||
slow.upload_ready(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Срок процесса обработки")
|
||
def test_limit_render_timeout(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with (
|
||
Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow,
|
||
extra_worker(
|
||
stand,
|
||
"WORKER__RENDER_TIMEOUT_SECONDS=1",
|
||
"WORKER__RENDER_CPU_LIMIT_SECONDS=30",
|
||
),
|
||
):
|
||
accepted = slow.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_scan_bomb(),
|
||
content_type="image/jpeg",
|
||
filename="bomb.jpg",
|
||
)
|
||
meta = slow.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "retry_exhausted", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["last_error_code"] == "timeout", job
|
||
assert job["attempt"] == WORKER_MAX_ATTEMPTS, job
|
||
assert stand("render-count", "extra") == "0"
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Процесс обработки убит")
|
||
@pytest.mark.spec("attachment-processing", "Секретов нет у процесса обработки")
|
||
def test_render_killed(
|
||
base_url: str,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = json.loads(stand("worker-state"))
|
||
wait_queue_empty(stand)
|
||
with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow:
|
||
accepted = slow.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_scan_bomb(),
|
||
content_type="image/jpeg",
|
||
filename="bomb.jpg",
|
||
)
|
||
wait_render_process(stand)
|
||
names = stand("render-env").split()
|
||
assert names, "no environment of the render process"
|
||
for name in names:
|
||
assert not name.startswith(
|
||
("DATABASE__", "S3__", "BROKER__", "CACHE__"),
|
||
), names
|
||
assert stand("kill-render") != "0"
|
||
job = wait_job(stand, accepted["id"], state="queued", attempt=1)
|
||
assert job["last_error_code"] == "render_killed", job
|
||
state = json.loads(stand("worker-state"))
|
||
assert state["running"] is True, state
|
||
assert state["restart_count"] == before["restart_count"], state
|
||
# следующая попытка упирается в лимит CPU: итог постоянный
|
||
meta = slow.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "invalid_image", meta
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["attempt"] == 2, done
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Зависший worker")
|
||
def test_hung_worker_lease_fencing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Первый worker — со сроками больше времени теста: разморозка застаёт его
|
||
таймеры живыми, и он доходит до фиксации, где решает только номер попытки (с
|
||
обычными сроками его прервал бы таймер, а не fencing). Аренду делает истекшей
|
||
hook, а не ожидание: при таком сроке попытки её не дождаться. Второй процесс —
|
||
основной worker стенда."""
|
||
seen_relay = len(relay_records(stand))
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
with outage(stand, "pause", "s3"):
|
||
stand(
|
||
"run-worker",
|
||
"WORKER__JOB_TIMEOUT_SECONDS=120",
|
||
"WORKER__RENDER_TIMEOUT_SECONDS=60",
|
||
"S3__TIMEOUT_SECONDS=60",
|
||
)
|
||
try:
|
||
wait_job(stand, accepted["id"], state="processing", attempt=1)
|
||
# попытка уже ждёт S3
|
||
wait_extra_record(
|
||
stand,
|
||
"Started job attempt",
|
||
attachment_id=accepted["id"],
|
||
)
|
||
stand("pause-extra", "worker")
|
||
except BaseException:
|
||
stand("stop-extra")
|
||
raise
|
||
try:
|
||
stand("expire-lease", accepted["id"])
|
||
wait_relay_record(
|
||
stand,
|
||
"Recovered expired lease",
|
||
skip=seen_relay,
|
||
attachment_id=accepted["id"],
|
||
)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 2, job
|
||
stand("unpause-extra", "worker")
|
||
wait_extra_record(
|
||
stand,
|
||
"Skipped job message",
|
||
attachment_id=accepted["id"],
|
||
reason="superseded",
|
||
)
|
||
assert (
|
||
assert_ok(
|
||
api.attachment_meta(alice, accepted["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
== meta
|
||
)
|
||
assert job_row(stand, accepted["id"]) == job
|
||
finally:
|
||
stand("stop-extra")
|
||
assert_variants(api, stand, alice, meta)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Долгая живая попытка")
|
||
def test_long_attempt_keeps_lease(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Фиксация ждёт в триггере, попытка укладывается в срок: relay её не
|
||
возвращает."""
|
||
filename = fault_filename()
|
||
seen = len(relay_records(stand))
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
assert meta["status"] == "ready"
|
||
assert not [
|
||
record
|
||
for record in relay_records(stand)[seen:]
|
||
if record.get("event") == "Recovered expired lease"
|
||
and record.get("attachment_id") == accepted["id"]
|
||
]
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Последняя попытка потеряна")
|
||
def test_last_attempt_lost(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
with extra_worker(stand, "WORKER__MAX_ATTEMPTS=1"):
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
stand("kill-extra", "worker")
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
stand("run-worker", "WORKER__MAX_ATTEMPTS=1")
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "retry_exhausted", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 1, job
|
||
|
||
|
||
@dataclass
|
||
class Replay:
|
||
"""Один вызов команды реестра: записи и код выхода."""
|
||
|
||
records: list[dict[str, Any]]
|
||
exit_code: int
|
||
|
||
def about(self, event: str, attachment_id: str) -> list[dict[str, Any]]:
|
||
return [
|
||
item
|
||
for item in self.records
|
||
if item.get("event") == event
|
||
and item.get("attachment_id") == attachment_id
|
||
]
|
||
|
||
@property
|
||
def finished(self) -> dict[str, Any]:
|
||
(record,) = [
|
||
item
|
||
for item in self.records
|
||
if item.get("event") == "Finished replay pass"
|
||
]
|
||
return record
|
||
|
||
|
||
def run_replay(stand: Stand, *args: str) -> Replay:
|
||
lines = stand("replay", *args).splitlines()
|
||
assert lines and lines[-1].startswith("exit_code="), lines[-5:]
|
||
records = [json.loads(line) for line in lines[:-1] if line.startswith("{")]
|
||
return Replay(records=records, exit_code=int(lines[-1].split("=")[1]))
|
||
|
||
|
||
def replay_id(stand: Stand, attachment_id: str) -> Replay:
|
||
return run_replay(
|
||
stand,
|
||
"--attachment-id",
|
||
attachment_id,
|
||
"--apply",
|
||
"--operator",
|
||
"student-tests",
|
||
)
|
||
|
||
|
||
def replays(stand: Stand, attachment_id: str) -> list[dict[str, Any]]:
|
||
return [
|
||
json.loads(line)
|
||
for line in stand("replays", attachment_id).splitlines()
|
||
]
|
||
|
||
|
||
def failed_invalid(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
filename="image.jpg",
|
||
)
|
||
return api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
|
||
|
||
def failed_exhausted(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
*,
|
||
files: int = 1,
|
||
) -> list[dict[str, Any]]:
|
||
"""Вложения `failed` с `retry_exhausted`: S3 остановлен дольше всех попыток.
|
||
S3 при выходе возвращён."""
|
||
with worker_stopped(stand):
|
||
accepted = [
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
for _ in range(files)
|
||
]
|
||
wait_queue_messages(stand, files)
|
||
with outage(stand, "stop", "s3"):
|
||
stand("start", "worker")
|
||
done = [
|
||
api.wait_attachment(
|
||
alice,
|
||
item["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
for item in accepted
|
||
]
|
||
for meta in done:
|
||
assert meta["error_code"] == "retry_exhausted", meta
|
||
return done
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Реестр показан без изменений")
|
||
def test_replay_registry_dry_run(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
failed = failed_invalid(api, alice, chat)
|
||
job = job_row(stand, failed["id"])
|
||
assert job is not None
|
||
run = run_replay(stand, "--attachment-id", failed["id"])
|
||
assert run.exit_code == 0, run.finished
|
||
(record,) = run.about("Found failed job", failed["id"])
|
||
assert record["job_id"] == job["id"], record
|
||
assert record["error_code"] == "invalid_image", record
|
||
assert record["reason"] == "undecodable", record
|
||
assert (
|
||
assert_ok(api.attachment_meta(alice, failed["id"]), 200, "Attachment")
|
||
== failed
|
||
)
|
||
assert job_row(stand, failed["id"]) == job
|
||
assert replays(stand, failed["id"]) == []
|
||
|
||
|
||
def replay_returns_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Объект staging заменён — `failed` `processing_failed` — реестр — объект
|
||
возвращён — replay — `ready` с тем же заданием и аудитом."""
|
||
data = images.png_noise(64, 48, seed=37)
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], data)
|
||
put_object(
|
||
stand,
|
||
staging_key(accepted["id"]),
|
||
same_size_bytes(data, seed=37),
|
||
)
|
||
meta = api.wait_attachment(
|
||
alice,
|
||
accepted["id"],
|
||
status="failed",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
assert meta["error_code"] == "processing_failed", meta
|
||
job = job_row(stand, accepted["id"])
|
||
assert job is not None
|
||
assert job["last_error_code"] == "source_corrupted", job
|
||
registry = run_replay(stand, "--error-code", "processing_failed")
|
||
assert registry.about("Found failed job", accepted["id"]), (
|
||
registry.finished
|
||
)
|
||
put_object(stand, staging_key(accepted["id"]), data)
|
||
run = replay_id(stand, accepted["id"])
|
||
assert run.exit_code == 0, run.finished
|
||
(record,) = run.about("Replayed job", accepted["id"])
|
||
assert record["operator"] == "student-tests", record
|
||
assert record["error_code"] == "processing_failed", record
|
||
assert record["reason"] == "source_corrupted", record
|
||
ready = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
assert_content(
|
||
api.attachment_content(alice, accepted["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
assert ready["sha256"] == hashlib.sha256(data).hexdigest()
|
||
assert_variants(api, stand, alice, ready)
|
||
done = job_row(stand, accepted["id"])
|
||
assert done is not None
|
||
assert done["id"] == job["id"], done
|
||
assert done["attempt"] == 2, done
|
||
(audit,) = replays(stand, accepted["id"])
|
||
assert audit["error_code"] == "processing_failed", audit
|
||
assert audit["reason"] == "source_corrupted", audit
|
||
assert audit["operator"] == "student-tests", audit
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Replay возвращает работу")
|
||
def test_replay_returns_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
replay_returns_work(api, stand, alice, chat)
|
||
|
||
|
||
def replay_ready_refused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Replay готовой работы — отказ, код 3, ничего не изменилось."""
|
||
ready = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
job = job_row(stand, ready["id"])
|
||
run = replay_id(stand, ready["id"])
|
||
assert run.exit_code == 3, run.finished
|
||
(record,) = run.about("Refused job replay", ready["id"])
|
||
assert record["refusal"] == "already_ready", record
|
||
assert not run.about("Replayed job", ready["id"])
|
||
assert (
|
||
assert_ok(api.attachment_meta(alice, ready["id"]), 200, "Attachment")
|
||
== ready
|
||
)
|
||
assert job_row(stand, ready["id"]) == job
|
||
assert_variants(api, stand, alice, ready)
|
||
assert replays(stand, ready["id"]) == []
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Replay готового не создаёт дубль")
|
||
def test_replay_ready_refused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
replay_ready_refused(api, stand, alice, chat)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Replay идущей работы")
|
||
def test_replay_in_progress_refused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
queued = api.upload(alice, chat["id"], images.sample_png())
|
||
wait_queue_messages(stand, 1)
|
||
run = replay_id(stand, queued["id"])
|
||
assert run.exit_code == 3, run.finished
|
||
(record,) = run.about("Refused job replay", queued["id"])
|
||
assert record["refusal"] == "not_failed", record
|
||
assert queue_state(stand)["messages"] == 1
|
||
job = job_row(stand, queued["id"])
|
||
assert job is not None
|
||
assert job["attempt"] == 0, job
|
||
api.wait_attachment(alice, queued["id"], timeout=JOB_WAIT_S)
|
||
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
busy = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
api.wait_attachment(
|
||
alice,
|
||
busy["id"],
|
||
status="processing",
|
||
timeout=JOB_WAIT_S,
|
||
)
|
||
run = replay_id(stand, busy["id"])
|
||
assert run.exit_code == 3, run.finished
|
||
(record,) = run.about("Refused job replay", busy["id"])
|
||
# фиксация могла успеть, пока шла команда: и то, и другое — отказ
|
||
assert record["refusal"] in ("not_failed", "already_ready"), record
|
||
meta = api.wait_attachment(alice, busy["id"], timeout=JOB_WAIT_S)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
assert meta["status"] == "ready"
|
||
done = job_row(stand, busy["id"])
|
||
assert done is not None
|
||
assert done["attempt"] == 1, done
|
||
assert replays(stand, busy["id"]) == []
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Replay без исходника")
|
||
def test_replay_without_source_refused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
failed = failed_invalid(api, alice, chat)
|
||
stand("age-row", failed["id"], str(STAGING_RETENTION_S + 5))
|
||
wait_staging_gone(stand, failed["id"])
|
||
run = replay_id(stand, failed["id"])
|
||
assert run.exit_code == 3, run.finished
|
||
(record,) = run.about("Refused job replay", failed["id"])
|
||
assert record["refusal"] == "source_gone", record
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, failed["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image")
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Новый бюджет попыток")
|
||
def test_replay_new_budget(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
(failed,) = failed_exhausted(api, stand, alice, chat)
|
||
run = replay_id(stand, failed["id"])
|
||
assert run.exit_code == 0, run.finished
|
||
ready = api.wait_attachment(alice, failed["id"], timeout=JOB_WAIT_S)
|
||
assert_variants(api, stand, alice, ready)
|
||
job = job_row(stand, failed["id"])
|
||
assert job is not None
|
||
assert job["attempt"] > WORKER_MAX_ATTEMPTS, job
|
||
assert job["attempt_base"] == WORKER_MAX_ATTEMPTS, job
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Replay по причине")
|
||
def test_replay_by_code(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
exhausted = failed_exhausted(api, stand, alice, chat, files=2)
|
||
invalid = failed_invalid(api, alice, chat)
|
||
run = run_replay(
|
||
stand,
|
||
"--error-code",
|
||
"retry_exhausted",
|
||
"--apply",
|
||
"--operator",
|
||
"student-tests",
|
||
)
|
||
for item in exhausted:
|
||
assert run.about("Replayed job", item["id"]), run.finished
|
||
api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S)
|
||
assert not run.about("Replayed job", invalid["id"])
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, invalid["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == invalid
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Prefetch больше конкуренции")
|
||
def test_prefetch_bounded(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
prefetch, files = 3, 5
|
||
filename = fault_filename()
|
||
stand("fail-attachments", "ready-stall", filename)
|
||
try:
|
||
wait_queue_empty(stand)
|
||
with outage(stand, "stop", "worker"):
|
||
ids = [
|
||
api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)["id"]
|
||
for _ in range(files)
|
||
]
|
||
wait_queue_messages(stand, files)
|
||
stand(
|
||
"run-worker",
|
||
"WORKER__CONCURRENCY=1",
|
||
f"WORKER__PREFETCH={prefetch}",
|
||
)
|
||
try:
|
||
busy_seen = 0
|
||
unacked_seen = 0
|
||
deadline = time.monotonic() + JOB_WAIT_S
|
||
while True:
|
||
rows = attachment_rows(stand)
|
||
statuses = [rows[item] for item in ids]
|
||
busy = statuses.count("processing")
|
||
assert busy <= 1, statuses
|
||
busy_seen = max(busy_seen, busy)
|
||
unacked = queue_state(stand)["messages_unacknowledged"]
|
||
assert unacked <= prefetch, unacked
|
||
unacked_seen = max(unacked_seen, unacked)
|
||
if statuses.count("ready") == files:
|
||
break
|
||
assert time.monotonic() < deadline, statuses
|
||
time.sleep(0.3)
|
||
assert busy_seen == 1
|
||
assert unacked_seen > 1, unacked_seen
|
||
finally:
|
||
stand("stop-extra")
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
wait_queue_empty(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отсутствующий вариант")
|
||
def test_missing_variant(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
ready = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
stand("delete-variant", ready["id"], "crop")
|
||
assert_error(
|
||
api.attachment_content(alice, ready["id"], variant="crop"),
|
||
404,
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
images.sample_png(),
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Загрузка без подтверждения брокера убрана",
|
||
)
|
||
def test_cancelled_upload_of_previous_version(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Реплика прежней версии при отказе брокера отменяла задание: строка
|
||
`pending`, помеченная к уборке. Её убирает уборщик вместе с объектом."""
|
||
started = time.monotonic()
|
||
with worker_stopped(stand), relay_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
stand("cancel-to-pending", accepted["id"])
|
||
assert job_row(stand, accepted["id"]) is None
|
||
assert_invisible(api, alice, chat, accepted["id"])
|
||
wait_cleaned(stand, accepted["id"], since=started)
|
||
assert job_row(stand, accepted["id"]) is None
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Staging готового вложения убран")
|
||
def test_staging_removed_after_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
wait_staging_gone(stand, ready["id"])
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
assert_variants(api, stand, alice, ready)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Staging неудачного вложения хранится до срока",
|
||
)
|
||
@pytest.mark.spec("storage-consistency", "Срок хранения прошёл")
|
||
def test_staging_retention(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
failed = failed_invalid(api, alice, chat)
|
||
attachment_id = failed["id"]
|
||
# конечный оригинал записан до декодирования; варианты прежней попытки
|
||
for name in VARIANT_NAMES:
|
||
put_object(stand, variant_key(attachment_id, name), b"variant")
|
||
time.sleep(3 * CLEANER_INTERVAL_S + 1)
|
||
assert attachment_id in staged_ids(stand)
|
||
assert attachment_id in stored_ids(stand)
|
||
dry = run_replay(stand, "--attachment-id", attachment_id)
|
||
assert dry.about("Found failed job", attachment_id), dry.finished
|
||
|
||
stand("age-row", attachment_id, str(STAGING_RETENTION_S + 5))
|
||
wait_staging_gone(stand, attachment_id)
|
||
assert attachment_id not in stored_ids(stand)
|
||
assert stand("variant-object-count", attachment_id) == "0"
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, attachment_id),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image")
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Текущая работа не убирается")
|
||
def test_staging_keeps_current_work(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with worker_stopped(stand):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
stand("age-row", accepted["id"], str(STAGING_RETENTION_S + 5))
|
||
time.sleep(3 * CLEANER_INTERVAL_S + 1)
|
||
assert accepted["id"] in staged_ids(stand)
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
|
||
|
||
def wait_row_held(stand: Stand, attachment_id: str) -> None:
|
||
"""Строку вложения держит транзакция уборщика. Старт уборщика при 0,25 CPU — до
|
||
нескольких секунд: без ожидания запрос прошёл бы раньше блокировки."""
|
||
deadline = time.monotonic() + 3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S + 10
|
||
while stand("attachment-free", attachment_id):
|
||
assert time.monotonic() < deadline, "cleaner does not hold the row"
|
||
time.sleep(0.1)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Очистка при отказе S3")
|
||
def test_staging_cleanup_survives_outage(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Уборщик держит строку и ждёт DELETE в замороженном S3: отправка сообщения
|
||
со ссылкой на вложение берёт `FOR KEY SHARE` и не ждёт его. Очередь уборщика
|
||
сначала пуста: кандидаты идут от старых, и при замороженном S3 проход каждый раз
|
||
прерывался бы на строке прежнего теста, не дойдя до этой."""
|
||
drained = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
wait_staging_gone(stand, drained["id"])
|
||
with outage(stand, "stop", "cleaner"):
|
||
ready = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
assert ready["id"] in staged_ids(stand)
|
||
stand("pause", "s3")
|
||
try:
|
||
for number in range(3):
|
||
wait_row_held(stand, ready["id"])
|
||
resp, took = timed(
|
||
lambda number=number: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={
|
||
"text": f"пока S3 заморожен {number}",
|
||
"attachment_ids": [ready["id"]],
|
||
},
|
||
),
|
||
)
|
||
assert_ok(resp, 201, "Message")
|
||
assert took < ACCEPT_MAX_S, took
|
||
row = json.loads(stand("attachment-row", ready["id"]))
|
||
assert row["staging_key"] is not None, row
|
||
finally:
|
||
stand("unpause", "s3")
|
||
wait_staging_gone(stand, ready["id"])
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Ключ staging другого вида не удаляется",
|
||
)
|
||
def test_staging_foreign_key_kept(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
foreign = f"{FAULT_DATABASE}/attachments/{uuid.uuid4()}/foreign"
|
||
with outage(stand, "stop", "cleaner"):
|
||
ready = api.upload_ready(alice, chat["id"], data)
|
||
put_object(stand, foreign, data)
|
||
stand("set-staging-key", ready["id"], foreign)
|
||
try:
|
||
deadline = time.monotonic() + 3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while True:
|
||
named = [
|
||
record
|
||
for line in stand("cleaner-log").splitlines()
|
||
if line.startswith("{")
|
||
for record in [json.loads(line)]
|
||
if record.get("event") == "Staging key is not own"
|
||
and record.get("object_key") == foreign
|
||
]
|
||
if named:
|
||
break
|
||
assert time.monotonic() < deadline, "cleaner did not name the key"
|
||
time.sleep(0.3)
|
||
assert named[0]["level"] == "warning", named[0]
|
||
digest = hashlib.sha256(data).hexdigest()
|
||
assert stand("object-sha256", foreign) == digest
|
||
row = json.loads(stand("attachment-row", ready["id"]))
|
||
assert row["staging_key"] == foreign, row
|
||
finally:
|
||
# следующий проход уберёт свой ключ, журнал стенда не засоряется
|
||
stand("set-staging-key", ready["id"], staging_key(ready["id"]))
|
||
stand("delete-object", foreign)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект staging без метаданных")
|
||
def test_reconcile_orphan_staging(stand: Stand) -> None:
|
||
data = images.sample_png()
|
||
orphan = str(uuid.uuid4())
|
||
key = staging_key(orphan)
|
||
other = f"{FAULT_DATABASE}/staging/{orphan}/other"
|
||
put_object(stand, key, data)
|
||
put_object(stand, other, data)
|
||
young = run_reconcile(stand, "dry-run")
|
||
assert not young.about(FOUND, keys={key, other}), young.finished
|
||
assert young.finished["listed_staging"] >= 2, young.finished
|
||
# старше срока хранения: срок прохода — окружением команды
|
||
time.sleep(2)
|
||
retention = ("--staging-retention", "1")
|
||
for options in (retention, (*HEAD_STRATEGY, *retention)):
|
||
old = run_reconcile(stand, "dry-run", options=options)
|
||
assert old.kinds(FOUND, keys={key, other}) == {
|
||
key: "orphan_staging",
|
||
other: "unknown_object",
|
||
}, old.finished
|
||
assert old.exit_code == 3, old.finished
|
||
assert stand("object-sha256", key)
|
||
fixed = run_reconcile(stand, "apply", options=retention)
|
||
(record,) = fixed.about(FIXED, keys={key})
|
||
assert (record["kind"], record["action"]) == (
|
||
"orphan_staging",
|
||
"deleted_object",
|
||
)
|
||
assert stand("object-sha256", key) == ""
|
||
# чужой ключ под префиксом: назван, не тронут
|
||
assert stand("object-sha256", other)
|
||
stand("delete-object", other)
|
||
|
||
|
||
def test_defense_faults(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `student-tests`, «Сценарий защиты автоматизирован», шаг 4 (вне сверки
|
||
покрытия — сценарий в докстринге), тем же порядком, что `driver.py
|
||
defense-faults`: crash после записи объектов и после фиксации до ack, relay
|
||
убит до публикации и после подтверждения до отметки, replay неудачного и отказ
|
||
replay готового."""
|
||
crash_after_objects(api, stand, alice, chat)
|
||
crash_after_commit(api, stand, alice, chat)
|
||
with outage(stand, "kill", "relay"):
|
||
accepted = api.upload(alice, chat["id"], images.sample_png())
|
||
assert queue_state(stand)["messages"] == 0
|
||
api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S)
|
||
crash_relay_after_confirm(api, stand, alice, chat)
|
||
replay_returns_work(api, stand, alice, chat)
|
||
replay_ready_refused(api, stand, alice, chat)
|