Files
mtusi2026lab4/student-tests/test_storage_failures.py
T
Alexey BodanyukandClaude Opus 5.5 a2d867bec6 lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key;
  dry-run read-only конструктивно (default_transaction_read_only + S3 без записи),
  --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо),
  перепроверка под FOR UPDATE, предохранитель от массового удаления;
- attachment_ids хранится в строке сообщения (история остаётся одним index range scan),
  message_attachments — для FK и обратного поиска;
- семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY);
- тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки,
  приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler;
- бенчмарки памяти загрузок и стоимости обхода reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 14:05:03 +03:00

108 lines
5.1 KiB
Python

"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4).
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
stand is waited for until HAProxy sees all replicas UP again."""
from __future__ import annotations
import time
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
@pytest.fixture()
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
"""S3 down (stopped: connection refused; paused: no answer at all):
* readiness is 503 on every replica (and through the LB), quickly;
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
* after recovery the earlier file is intact."""
alice, bob, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
with service_down("s3", how):
deadline = time.monotonic() + 15
while api.client.get("/health/ready").status_code != 503:
assert time.monotonic() < deadline, "readiness did not turn 503"
time.sleep(0.5)
for replica in running_replicas():
assert probe_inside(replica, "/health/ready") == 503
assert probe_inside(replica, "/health/ready/core") == 200
started = time.monotonic()
error = api.upload(alice, chat, SAMPLE, 503).json()
assert error["error"]["code"] == "unavailable"
assert time.monotonic() - started < 8
started = time.monotonic()
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
assert time.monotonic() - started < 8
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
api.send(alice, chat, "текст работает без S3") # degraded mode
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.content == SAMPLE
api.upload(alice, chat, SAMPLE)
def test_upload_survives_failover_of_the_serving_replica(api, pair):
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
(nothing of the file lives in the replica)."""
alice, bob, chat = pair
data = imagegen.png_of_size(500_000)
response = api.upload(alice, chat, data)
att, served_by = response.json(), response.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica, "kill"):
for _ in range(5):
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.headers["x-instance-id"] != served_by and got.content == data
def test_bucket_is_private_outside_the_api(api, pair):
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
unsigned LIST are refused; the same GET signed with the API key pair works (control).
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
alice, _, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
key = attachment_row(att["id"])["object_key"]
probe = (
"import urllib.request as u, urllib.error as e, socket\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"for port in (9333, 8888, 8080):\n"
" s = socket.socket(); s.settimeout(2)\n"
" print(port, s.connect_ex(('s3', port)))\n")
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
get_status, list_status = out[0].split()
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
for line in out[1:4]:
port, code = line.split()
assert code != "0", f"admin port {port} reachable"
signed = compose("exec", "-T", "api", "python", "-c",
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
assert signed == str(len(SAMPLE))
def test_s3_restart_keeps_objects(api, pair):
alice, bob, chat = pair
data = imagegen.png_of_size(200_000)
att = api.upload(alice, chat, data).json()
with service_down("s3", "stop"):
pass
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data