Files
mtusi2026lab4/student-tests/test_s3_failures.py
T

132 lines
6.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Отказы S3 и согласованность S3 <-> PostgreSQL (уровень 4). Запуск: python student-tests/test_s3_failures.py
Тесты останавливают/запускают контейнеры и пересоздают api с внедрением сбоев (volumes сохраняются).
"""
import json
import os
import subprocess
import sys
import time
import unittest
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, call, psql, register # noqa: E402
from test_attachments import PNG, download, upload # noqa: E402
def compose(*args, env=None, check=True):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True,
env=dict(os.environ, **(env or {})), check=check)
def wait(predicate, seconds=90):
deadline = time.time() + seconds
while time.time() < deadline:
try:
if predicate():
return
except Exception:
pass
time.sleep(1)
raise AssertionError("условие не выполнено за %d с" % seconds)
def recreate_api(env=None):
compose("up", "-d", "--no-deps", "--force-recreate", "api", env=env)
wait(lambda: call("GET", "/health/ready")[0] == 200)
time.sleep(3)
def reconcile(*args):
out = compose("run", "--rm", "-T", "api", "python", "-m", "app.reconcile", *args).stdout
return json.loads(out[out.index("{"):])
def s3_python(code):
return compose("exec", "-T", "api", "python", "-c", code).stdout.strip()
class S3FailureTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
compose("start", "seaweedfs", check=False)
recreate_api()
def setUp(self):
compose("start", "seaweedfs", check=False)
wait(lambda: call("GET", "/health/ready")[0] == 200)
self.a = register("s3f")
self.chat = call("POST", "/api/v1/chats", {"title": "s3", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
def test_s3_down_gives_503_text_path_keeps_working(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
count = psql("SELECT count(*) FROM attachments")
compose("stop", "seaweedfs")
r = upload(self.a["token"], self.chat["id"], PNG)
self.assertEqual(r[0], 503, r[2])
self.assertEqual(r[2]["error"]["code"], "unavailable")
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
self.assertEqual(download(self.a["token"], att["id"])[0], 503)
self.assertEqual(download(self.a["token"], att["id"], "")[0], 200) # метаданные — из PostgreSQL
self.assertEqual(call("POST", "/api/v1/chats/%s/messages" % self.chat["id"], {"text": "текст без S3"}, self.a["token"])[0], 201)
self.assertEqual(call("GET", "/health/ready")[0], 200) # readiness не зависит от S3
self.assertEqual(psql("SELECT count(*) FROM attachments"), count, "неудачная загрузка не должна оставлять метаданные")
compose("start", "seaweedfs")
wait(lambda: download(self.a["token"], att["id"])[0] == 200, 90)
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG)[0], 201)
def test_db_failure_after_s3_write_is_compensated(self):
recreate_api({"FAULT_INJECTION": "db_after_put"})
before = reconcile("--grace", "0")["s3_objects"]
r = upload(self.a["token"], self.chat["id"], PNG)
self.assertEqual(r[0], 500)
self.assertEqual(r[2]["error"]["code"], "internal_error")
after = reconcile("--grace", "0")
self.assertEqual(after["s3_objects"], before, "объект должен быть удалён компенсацией")
self.assertEqual(after["orphan_objects_count"], 0)
recreate_api()
def test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely(self):
keeper = upload(self.a["token"], self.chat["id"], PNG)[2] # используемое вложение
path = "/api/v1/chats/%s/messages" % self.chat["id"]
sent = call("POST", path, {"text": "keeper", "attachment_ids": [keeper["id"]]}, self.a["token"])
self.assertEqual(sent[0], 201, sent[2])
recreate_api({"FAULT_INJECTION": "db_after_put,s3_delete_fails"})
r = upload(self.a["token"], self.chat["id"], PNG)
self.assertEqual(r[0], 500)
recreate_api()
fresh = reconcile("--grace", "900") # grace period: свежий объект не трогаем
self.assertGreaterEqual(fresh["orphans_skipped_fresh"], 1)
self.assertEqual(fresh["orphan_objects_count"], 0)
protected = reconcile("--grace", "900", "--apply")
self.assertGreaterEqual(protected["orphans_skipped_fresh"], 1)
self.assertEqual(protected["fixed"], {"deleted_orphans": 0, "marked_failed": 0})
dry = reconcile("--grace", "0") # dry-run находит, но не удаляет
self.assertGreaterEqual(dry["orphan_objects_count"], 1)
self.assertEqual(dry["fixed"], {"deleted_orphans": 0, "marked_failed": 0})
self.assertEqual(reconcile("--grace", "0")["orphan_objects_count"], dry["orphan_objects_count"]) # dry-run ничего не меняет
applied = reconcile("--grace", "0", "--apply")
self.assertEqual(applied["fixed"]["deleted_orphans"], dry["orphan_objects_count"])
self.assertEqual(reconcile("--grace", "0")["orphan_objects_count"], 0) # повторный запуск идемпотентен
self.assertEqual(download(self.a["token"], keeper["id"])[0], 200, "используемое вложение удалять нельзя")
messages = call("GET", path, token=self.a["token"])[2]["items"]
self.assertEqual(messages[0]["id"], sent[2]["id"])
self.assertEqual(messages[0]["attachment_ids"], [keeper["id"]])
def test_missing_object_is_reported_and_attachment_marked_failed(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
code = ("import asyncio\nfrom app.s3 import S3\nasync def m():\n s=S3(); await s.start(); await s.delete('%s'); await s.stop()\n"
"asyncio.run(m())" % key)
s3_python(code)
dry = reconcile("--grace", "0")
self.assertIn(att["id"], dry["missing_objects"])
self.assertEqual(psql("SELECT status FROM attachments WHERE id = '%s'" % att["id"]), "ready") # dry-run не меняет
reconcile("--grace", "0", "--apply")
self.assertEqual(psql("SELECT status || ':' || error_code FROM attachments WHERE id = '%s'" % att["id"]), "failed:object_missing")
self.assertEqual(download(self.a["token"], att["id"])[0], 409)
if __name__ == "__main__":
unittest.main()