"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4). Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the stand is waited for until HAProxy sees all replicas UP again.""" from __future__ import annotations import time from pathlib import Path import pytest import imagegen from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down pytestmark = pytest.mark.stack ROOT = Path(__file__).resolve().parents[1] SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() @pytest.fixture() def pair(api): alice, bob = api.user("alice"), api.user("bob") return alice, bob, api.chat(alice, bob) @pytest.mark.parametrize("how", ["stop", "pause"]) def test_s3_outage_is_503_and_text_keeps_working(api, pair, how): """S3 down (stopped: connection refused; paused: no answer at all): * readiness is 503 on every replica (and through the LB), quickly; * uploads/downloads are 503 `unavailable` within the S3 deadline, never 500; * the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core); * after recovery the earlier file is intact.""" alice, bob, chat = pair att = api.upload(alice, chat, SAMPLE).json() with service_down("s3", how): deadline = time.monotonic() + 15 while api.client.get("/health/ready").status_code != 503: assert time.monotonic() < deadline, "readiness did not turn 503" time.sleep(0.5) for replica in running_replicas(): assert probe_inside(replica, "/health/ready") == 503 assert probe_inside(replica, "/health/ready/core") == 200 started = time.monotonic() error = api.upload(alice, chat, SAMPLE, 503).json() assert error["error"]["code"] == "unavailable" assert time.monotonic() - started < 8 started = time.monotonic() api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) assert time.monotonic() - started < 8 api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only api.send(alice, chat, "текст работает без S3") # degraded mode got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) assert got.content == SAMPLE api.upload(alice, chat, SAMPLE) def test_upload_survives_failover_of_the_serving_replica(api, pair): """Upload -> stop the replica that served it -> another replica downloads the same bytes (nothing of the file lives in the replica).""" alice, bob, chat = pair data = imagegen.png_of_size(500_000) response = api.upload(alice, chat, data) att, served_by = response.json(), response.headers["x-instance-id"] replica = next(r for r in running_replicas() if r["hostname"] == served_by) with replica_stopped(replica, "kill"): for _ in range(5): got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) assert got.headers["x-instance-id"] != served_by and got.content == data def test_bucket_is_private_outside_the_api(api, pair): """Inside the storage network (where the API is): an UNSIGNED GET of a real object and an unsigned LIST are refused; the same GET signed with the API key pair works (control). SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable.""" alice, _, chat = pair att = api.upload(alice, chat, SAMPLE).json() key = attachment_row(att["id"])["object_key"] probe = ( "import urllib.request as u, urllib.error as e, socket\n" "def st(url):\n" " try: return u.urlopen(url, timeout=3).status\n" " except e.HTTPError as x: return x.code\n" f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n" "for port in (9333, 8888, 8080):\n" " s = socket.socket(); s.settimeout(2)\n" " print(port, s.connect_ex(('s3', port)))\n") out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n") get_status, list_status = out[0].split() assert get_status in ("401", "403") and list_status in ("401", "403"), out[0] for line in out[1:4]: port, code = line.split() assert code != "0", f"admin port {port} reachable" signed = compose("exec", "-T", "api", "python", "-c", "import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n" "async def m():\n s=Storage(Settings.from_env()); await s.open()\n" f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip() assert signed == str(len(SAMPLE)) def test_s3_restart_keeps_objects(api, pair): alice, bob, chat = pair data = imagegen.png_of_size(200_000) att = api.upload(alice, chat, data).json() with service_down("s3", "stop"): pass assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data