forked from mtusi2026/mtusi2026lab5
189 lines
7.1 KiB
Python
189 lines
7.1 KiB
Python
"""
|
|
Согласованность S3 и PostgreSQL на живом стенде.
|
|
|
|
Сбои внедряются напрямую в хранилища: строка pending с объектом — процесс
|
|
упал между PUT в S3 и подтверждением в БД; объект без строки — сирота.
|
|
Затем janitor (тот же код и ключи, что в сервисе janitor) выполняет проход
|
|
с коротким grace period. Обрыв загрузки идёт напрямую в API-реплику, минуя
|
|
буферизацию nginx.
|
|
|
|
Запуск явно: make test-storage (RUN_STORAGE_TESTS=1).
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
import uuid
|
|
|
|
import httpx
|
|
import pytest
|
|
from helpers import auth, container_json, png_bytes, psql, run_janitor
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
os.environ.get("RUN_STORAGE_TESTS") != "1",
|
|
reason="меняет данные стенда напрямую: запускайте через make test-storage",
|
|
)
|
|
|
|
# Запас на время docker exec janitor (импорт кода под лимитом CPU 0.25)
|
|
GRACE_SECONDS = 20
|
|
STALE_ROW_AGE_SECONDS = 300
|
|
|
|
PUT_OBJECTS = """
|
|
import asyncio, json
|
|
from infrastructure.dependencies import storage_manager
|
|
|
|
async def main():
|
|
s3 = await storage_manager.create_client()
|
|
for key in {keys!r}:
|
|
await s3.put_object(Bucket=storage_manager.bucket, Key=key, Body=b"x")
|
|
await storage_manager.close()
|
|
print(json.dumps({{"put": len({keys!r})}}))
|
|
|
|
asyncio.run(main())
|
|
"""
|
|
|
|
EXISTING_OBJECTS = """
|
|
import asyncio, json
|
|
from infrastructure.dependencies import storage_manager
|
|
|
|
async def main():
|
|
s3 = await storage_manager.create_client()
|
|
found = {{}}
|
|
for key in {keys!r}:
|
|
try:
|
|
await s3.head_object(Bucket=storage_manager.bucket, Key=key)
|
|
found[key] = True
|
|
except Exception:
|
|
found[key] = False
|
|
await storage_manager.close()
|
|
print(json.dumps(found))
|
|
|
|
asyncio.run(main())
|
|
"""
|
|
|
|
|
|
def put_objects(*keys: str) -> None:
|
|
container_json("janitor", PUT_OBJECTS.format(keys=list(keys)))
|
|
|
|
|
|
def existing_objects(*keys: str) -> dict[str, bool]:
|
|
return container_json("janitor", EXISTING_OBJECTS.format(keys=list(keys)))
|
|
|
|
|
|
def insert_pending(chat: dict, owner: dict, key: str, age_seconds: int) -> None:
|
|
"""Строка, какую оставляет падение API между PUT и подтверждением"""
|
|
psql(
|
|
f"""
|
|
INSERT INTO attachments
|
|
(chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
|
object_key, status, created_at, updated_at)
|
|
VALUES
|
|
('{chat["id"]}', '{owner["id"]}', 'crash.png', 'image/png', 1,
|
|
sha256('x'), '{key}', 'pending',
|
|
clock_timestamp() - make_interval(secs => {age_seconds}),
|
|
clock_timestamp());
|
|
"""
|
|
)
|
|
|
|
|
|
def row_status(key: str) -> str:
|
|
return psql(f"SELECT coalesce(max(status), 'absent') FROM attachments WHERE object_key = '{key}';")
|
|
|
|
|
|
def test_janitor_cleans_only_stale_unaccounted_data(
|
|
api: httpx.Client, make_user, make_chat, upload, send_attachments
|
|
) -> None:
|
|
alice = make_user("alice")
|
|
chat = make_chat(alice, [alice])
|
|
ready = upload(alice, chat, png_bytes(12, 12))
|
|
linked = upload(alice, chat, png_bytes(13, 13))
|
|
assert send_attachments(alice, chat, [linked["id"]]).status_code == 201
|
|
|
|
run_id = uuid.uuid4().hex
|
|
crashed = f"originals/test-crashed-{run_id}"
|
|
crashed_no_object = f"originals/test-crashed-before-put-{run_id}"
|
|
orphan_old = f"originals/test-orphan-old-{run_id}"
|
|
fresh_pending = f"originals/test-fresh-pending-{run_id}"
|
|
orphan_fresh = f"originals/test-orphan-fresh-{run_id}"
|
|
|
|
# «Старые» сбои: строка старше grace по часам БД, объект — по времени ожидания.
|
|
# Возраст меньше grace фонового сервиса janitor (900 с): строки забирает
|
|
# только проход теста, без гонки с сервисом
|
|
insert_pending(chat, alice, crashed, age_seconds=STALE_ROW_AGE_SECONDS)
|
|
insert_pending(chat, alice, crashed_no_object, age_seconds=STALE_ROW_AGE_SECONDS)
|
|
put_objects(crashed, orphan_old)
|
|
time.sleep(GRACE_SECONDS + 2)
|
|
|
|
dry = run_janitor("--dry-run", "--grace-seconds", str(GRACE_SECONDS))
|
|
assert dry["unfinished_claimed"] >= 2 and dry["orphan_objects_found"] >= 1
|
|
assert existing_objects(crashed, orphan_old) == {crashed: True, orphan_old: True}
|
|
|
|
# «Свежие»: загрузка, которая могла бы ещё идти
|
|
insert_pending(chat, alice, fresh_pending, age_seconds=0)
|
|
put_objects(fresh_pending, orphan_fresh)
|
|
report = run_janitor("--grace-seconds", str(GRACE_SECONDS))
|
|
print("\nJANITOR REPORT:", json.dumps(report))
|
|
|
|
assert existing_objects(crashed, orphan_old, fresh_pending, orphan_fresh) == {
|
|
crashed: False,
|
|
orphan_old: False,
|
|
fresh_pending: True,
|
|
orphan_fresh: True,
|
|
}
|
|
assert row_status(crashed) == row_status(crashed_no_object) == "absent"
|
|
assert row_status(fresh_pending) == "pending"
|
|
|
|
# Готовые вложения (в том числе привязанные к сообщению) не тронуты
|
|
for attachment in (ready, linked):
|
|
response = api.get(
|
|
f"/api/v1/attachments/{attachment['id']}/content", headers=auth(alice)
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
# Свежие данные становятся «старыми» и тоже убираются
|
|
time.sleep(GRACE_SECONDS + 1)
|
|
run_janitor("--grace-seconds", str(GRACE_SECONDS))
|
|
assert existing_objects(fresh_pending, orphan_fresh) == {
|
|
fresh_pending: False,
|
|
orphan_fresh: False,
|
|
}
|
|
assert row_status(fresh_pending) == "absent"
|
|
again = run_janitor("--dry-run", "--grace-seconds", str(GRACE_SECONDS))
|
|
assert again["orphan_objects_found"] == 0
|
|
|
|
|
|
INTERRUPTED_UPLOAD = """
|
|
import json, socket
|
|
body = {body!r}
|
|
request = (
|
|
"POST /api/v1/chats/{chat_id}/attachments?filename=cut.png HTTP/1.1\\r\\n"
|
|
"Host: api\\r\\n"
|
|
"Authorization: Bearer {token}\\r\\n"
|
|
"Content-Type: image/png\\r\\n"
|
|
"Content-Length: " + str(len(body)) + "\\r\\n\\r\\n"
|
|
).encode()
|
|
sent = 0
|
|
with socket.create_connection(("api", 8080), timeout=5) as connection:
|
|
connection.sendall(request)
|
|
connection.sendall(body[: len(body) // 3])
|
|
sent = len(body) // 3
|
|
print(json.dumps({{"sent": sent, "declared": len(body)}}))
|
|
"""
|
|
|
|
|
|
def test_interrupted_upload_leaves_nothing(make_user, make_chat) -> None:
|
|
"""Клиент оборвал соединение посреди тела, прямо к реплике API"""
|
|
alice = make_user("alice")
|
|
chat = make_chat(alice, [alice])
|
|
body = list(png_bytes(400, 400, noise=True))
|
|
result = container_json(
|
|
"janitor",
|
|
INTERRUPTED_UPLOAD.format(
|
|
body=bytes(body), chat_id=chat["id"], token=alice["token"]
|
|
),
|
|
)
|
|
assert 0 < result["sent"] < result["declared"]
|
|
time.sleep(1)
|
|
rows = psql(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat['id']}';")
|
|
assert rows == "0"
|