Files
mtusi2026lab4/student-tests/test_storage_consistency.py
T

189 lines
7.1 KiB
Python

"""
Согласованность S3 и PostgreSQL на живом стенде.
Сбои внедряются напрямую в хранилища: строка pending с объектом — процесс
упал между PUT в S3 и подтверждением в БД; объект без строки — сирота.
Затем janitor (тот же код и ключи, что в сервисе janitor) выполняет проход
с коротким grace period. Обрыв загрузки идёт напрямую в API-реплику, минуя
буферизацию nginx.
Запуск явно: make test-storage (RUN_STORAGE_TESTS=1).
"""
import json
import os
import time
import uuid
import httpx
import pytest
from helpers import auth, container_json, png_bytes, psql, run_janitor
pytestmark = pytest.mark.skipif(
os.environ.get("RUN_STORAGE_TESTS") != "1",
reason="меняет данные стенда напрямую: запускайте через make test-storage",
)
# Запас на время docker exec janitor (импорт кода под лимитом CPU 0.25)
GRACE_SECONDS = 20
STALE_ROW_AGE_SECONDS = 300
PUT_OBJECTS = """
import asyncio, json
from infrastructure.dependencies import storage_manager
async def main():
s3 = await storage_manager.create_client()
for key in {keys!r}:
await s3.put_object(Bucket=storage_manager.bucket, Key=key, Body=b"x")
await storage_manager.close()
print(json.dumps({{"put": len({keys!r})}}))
asyncio.run(main())
"""
EXISTING_OBJECTS = """
import asyncio, json
from infrastructure.dependencies import storage_manager
async def main():
s3 = await storage_manager.create_client()
found = {{}}
for key in {keys!r}:
try:
await s3.head_object(Bucket=storage_manager.bucket, Key=key)
found[key] = True
except Exception:
found[key] = False
await storage_manager.close()
print(json.dumps(found))
asyncio.run(main())
"""
def put_objects(*keys: str) -> None:
container_json("janitor", PUT_OBJECTS.format(keys=list(keys)))
def existing_objects(*keys: str) -> dict[str, bool]:
return container_json("janitor", EXISTING_OBJECTS.format(keys=list(keys)))
def insert_pending(chat: dict, owner: dict, key: str, age_seconds: int) -> None:
"""Строка, какую оставляет падение API между PUT и подтверждением"""
psql(
f"""
INSERT INTO attachments
(chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, created_at, updated_at)
VALUES
('{chat["id"]}', '{owner["id"]}', 'crash.png', 'image/png', 1,
sha256('x'), '{key}', 'pending',
clock_timestamp() - make_interval(secs => {age_seconds}),
clock_timestamp());
"""
)
def row_status(key: str) -> str:
return psql(f"SELECT coalesce(max(status), 'absent') FROM attachments WHERE object_key = '{key}';")
def test_janitor_cleans_only_stale_unaccounted_data(
api: httpx.Client, make_user, make_chat, upload, send_attachments
) -> None:
alice = make_user("alice")
chat = make_chat(alice, [alice])
ready = upload(alice, chat, png_bytes(12, 12))
linked = upload(alice, chat, png_bytes(13, 13))
assert send_attachments(alice, chat, [linked["id"]]).status_code == 201
run_id = uuid.uuid4().hex
crashed = f"originals/test-crashed-{run_id}"
crashed_no_object = f"originals/test-crashed-before-put-{run_id}"
orphan_old = f"originals/test-orphan-old-{run_id}"
fresh_pending = f"originals/test-fresh-pending-{run_id}"
orphan_fresh = f"originals/test-orphan-fresh-{run_id}"
# «Старые» сбои: строка старше grace по часам БД, объект — по времени ожидания.
# Возраст меньше grace фонового сервиса janitor (900 с): строки забирает
# только проход теста, без гонки с сервисом
insert_pending(chat, alice, crashed, age_seconds=STALE_ROW_AGE_SECONDS)
insert_pending(chat, alice, crashed_no_object, age_seconds=STALE_ROW_AGE_SECONDS)
put_objects(crashed, orphan_old)
time.sleep(GRACE_SECONDS + 2)
dry = run_janitor("--dry-run", "--grace-seconds", str(GRACE_SECONDS))
assert dry["unfinished_claimed"] >= 2 and dry["orphan_objects_found"] >= 1
assert existing_objects(crashed, orphan_old) == {crashed: True, orphan_old: True}
# «Свежие»: загрузка, которая могла бы ещё идти
insert_pending(chat, alice, fresh_pending, age_seconds=0)
put_objects(fresh_pending, orphan_fresh)
report = run_janitor("--grace-seconds", str(GRACE_SECONDS))
print("\nJANITOR REPORT:", json.dumps(report))
assert existing_objects(crashed, orphan_old, fresh_pending, orphan_fresh) == {
crashed: False,
orphan_old: False,
fresh_pending: True,
orphan_fresh: True,
}
assert row_status(crashed) == row_status(crashed_no_object) == "absent"
assert row_status(fresh_pending) == "pending"
# Готовые вложения (в том числе привязанные к сообщению) не тронуты
for attachment in (ready, linked):
response = api.get(
f"/api/v1/attachments/{attachment['id']}/content", headers=auth(alice)
)
assert response.status_code == 200
# Свежие данные становятся «старыми» и тоже убираются
time.sleep(GRACE_SECONDS + 1)
run_janitor("--grace-seconds", str(GRACE_SECONDS))
assert existing_objects(fresh_pending, orphan_fresh) == {
fresh_pending: False,
orphan_fresh: False,
}
assert row_status(fresh_pending) == "absent"
again = run_janitor("--dry-run", "--grace-seconds", str(GRACE_SECONDS))
assert again["orphan_objects_found"] == 0
INTERRUPTED_UPLOAD = """
import json, socket
body = {body!r}
request = (
"POST /api/v1/chats/{chat_id}/attachments?filename=cut.png HTTP/1.1\\r\\n"
"Host: api\\r\\n"
"Authorization: Bearer {token}\\r\\n"
"Content-Type: image/png\\r\\n"
"Content-Length: " + str(len(body)) + "\\r\\n\\r\\n"
).encode()
sent = 0
with socket.create_connection(("api", 8080), timeout=5) as connection:
connection.sendall(request)
connection.sendall(body[: len(body) // 3])
sent = len(body) // 3
print(json.dumps({{"sent": sent, "declared": len(body)}}))
"""
def test_interrupted_upload_leaves_nothing(make_user, make_chat) -> None:
"""Клиент оборвал соединение посреди тела, прямо к реплике API"""
alice = make_user("alice")
chat = make_chat(alice, [alice])
body = list(png_bytes(400, 400, noise=True))
result = container_json(
"janitor",
INTERRUPTED_UPLOAD.format(
body=bytes(body), chat_id=chat["id"], token=alice["token"]
),
)
assert 0 < result["sent"] < result["declared"]
time.sleep(1)
rows = psql(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat['id']}';")
assert rows == "0"