forked from mtusi2026/mtusi2026lab4
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key; dry-run read-only конструктивно (default_transaction_read_only + S3 без записи), --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо), перепроверка под FOR UPDATE, предохранитель от массового удаления; - attachment_ids хранится в строке сообщения (история остаётся одним index range scan), message_attachments — для FK и обратного поиска; - семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY); - тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки, приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler; - бенчмарки памяти загрузок и стоимости обхода reconciler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
5.1 KiB
Python
108 lines
5.1 KiB
Python
"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4).
|
|
|
|
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
|
|
stand is waited for until HAProxy sees all replicas UP again."""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import imagegen
|
|
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
|
|
|
|
@pytest.fixture()
|
|
def pair(api):
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
return alice, bob, api.chat(alice, bob)
|
|
|
|
|
|
@pytest.mark.parametrize("how", ["stop", "pause"])
|
|
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
|
|
"""S3 down (stopped: connection refused; paused: no answer at all):
|
|
* readiness is 503 on every replica (and through the LB), quickly;
|
|
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
|
|
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
|
|
* after recovery the earlier file is intact."""
|
|
alice, bob, chat = pair
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
with service_down("s3", how):
|
|
deadline = time.monotonic() + 15
|
|
while api.client.get("/health/ready").status_code != 503:
|
|
assert time.monotonic() < deadline, "readiness did not turn 503"
|
|
time.sleep(0.5)
|
|
for replica in running_replicas():
|
|
assert probe_inside(replica, "/health/ready") == 503
|
|
assert probe_inside(replica, "/health/ready/core") == 200
|
|
started = time.monotonic()
|
|
error = api.upload(alice, chat, SAMPLE, 503).json()
|
|
assert error["error"]["code"] == "unavailable"
|
|
assert time.monotonic() - started < 8
|
|
started = time.monotonic()
|
|
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
|
|
assert time.monotonic() - started < 8
|
|
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
|
|
api.send(alice, chat, "текст работает без S3") # degraded mode
|
|
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
|
assert got.content == SAMPLE
|
|
api.upload(alice, chat, SAMPLE)
|
|
|
|
|
|
def test_upload_survives_failover_of_the_serving_replica(api, pair):
|
|
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
|
|
(nothing of the file lives in the replica)."""
|
|
alice, bob, chat = pair
|
|
data = imagegen.png_of_size(500_000)
|
|
response = api.upload(alice, chat, data)
|
|
att, served_by = response.json(), response.headers["x-instance-id"]
|
|
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
|
|
with replica_stopped(replica, "kill"):
|
|
for _ in range(5):
|
|
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
|
assert got.headers["x-instance-id"] != served_by and got.content == data
|
|
|
|
|
|
def test_bucket_is_private_outside_the_api(api, pair):
|
|
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
|
|
unsigned LIST are refused; the same GET signed with the API key pair works (control).
|
|
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
|
|
alice, _, chat = pair
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
key = attachment_row(att["id"])["object_key"]
|
|
probe = (
|
|
"import urllib.request as u, urllib.error as e, socket\n"
|
|
"def st(url):\n"
|
|
" try: return u.urlopen(url, timeout=3).status\n"
|
|
" except e.HTTPError as x: return x.code\n"
|
|
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
|
"for port in (9333, 8888, 8080):\n"
|
|
" s = socket.socket(); s.settimeout(2)\n"
|
|
" print(port, s.connect_ex(('s3', port)))\n")
|
|
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
|
|
get_status, list_status = out[0].split()
|
|
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
|
|
for line in out[1:4]:
|
|
port, code = line.split()
|
|
assert code != "0", f"admin port {port} reachable"
|
|
signed = compose("exec", "-T", "api", "python", "-c",
|
|
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
|
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
|
|
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
|
|
assert signed == str(len(SAMPLE))
|
|
|
|
|
|
def test_s3_restart_keeps_objects(api, pair):
|
|
alice, bob, chat = pair
|
|
data = imagegen.png_of_size(200_000)
|
|
att = api.upload(alice, chat, data).json()
|
|
with service_down("s3", "stop"):
|
|
pass
|
|
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data
|