132 lines
6.9 KiB
Python
132 lines
6.9 KiB
Python
"""Отказы S3 и согласованность S3 <-> PostgreSQL (уровень 4). Запуск: python student-tests/test_s3_failures.py
|
||
Тесты останавливают/запускают контейнеры и пересоздают api с внедрением сбоев (volumes сохраняются).
|
||
"""
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import unittest
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
from test_api import ROOT, call, psql, register # noqa: E402
|
||
from test_attachments import PNG, download, upload # noqa: E402
|
||
|
||
|
||
def compose(*args, env=None, check=True):
|
||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True,
|
||
env=dict(os.environ, **(env or {})), check=check)
|
||
|
||
|
||
def wait(predicate, seconds=90):
|
||
deadline = time.time() + seconds
|
||
while time.time() < deadline:
|
||
try:
|
||
if predicate():
|
||
return
|
||
except Exception:
|
||
pass
|
||
time.sleep(1)
|
||
raise AssertionError("условие не выполнено за %d с" % seconds)
|
||
|
||
|
||
def recreate_api(env=None):
|
||
compose("up", "-d", "--no-deps", "--force-recreate", "api", env=env)
|
||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||
time.sleep(3)
|
||
|
||
|
||
def reconcile(*args):
|
||
out = compose("run", "--rm", "-T", "api", "python", "-m", "app.reconcile", *args).stdout
|
||
return json.loads(out[out.index("{"):])
|
||
|
||
|
||
def s3_python(code):
|
||
return compose("exec", "-T", "api", "python", "-c", code).stdout.strip()
|
||
|
||
|
||
class S3FailureTest(unittest.TestCase):
|
||
@classmethod
|
||
def tearDownClass(cls):
|
||
compose("start", "seaweedfs", check=False)
|
||
recreate_api()
|
||
|
||
def setUp(self):
|
||
compose("start", "seaweedfs", check=False)
|
||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||
self.a = register("s3f")
|
||
self.chat = call("POST", "/api/v1/chats", {"title": "s3", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
|
||
|
||
def test_s3_down_gives_503_text_path_keeps_working(self):
|
||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||
count = psql("SELECT count(*) FROM attachments")
|
||
compose("stop", "seaweedfs")
|
||
r = upload(self.a["token"], self.chat["id"], PNG)
|
||
self.assertEqual(r[0], 503, r[2])
|
||
self.assertEqual(r[2]["error"]["code"], "unavailable")
|
||
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
|
||
self.assertEqual(download(self.a["token"], att["id"])[0], 503)
|
||
self.assertEqual(download(self.a["token"], att["id"], "")[0], 200) # метаданные — из PostgreSQL
|
||
self.assertEqual(call("POST", "/api/v1/chats/%s/messages" % self.chat["id"], {"text": "текст без S3"}, self.a["token"])[0], 201)
|
||
self.assertEqual(call("GET", "/health/ready")[0], 200) # readiness не зависит от S3
|
||
self.assertEqual(psql("SELECT count(*) FROM attachments"), count, "неудачная загрузка не должна оставлять метаданные")
|
||
compose("start", "seaweedfs")
|
||
wait(lambda: download(self.a["token"], att["id"])[0] == 200, 90)
|
||
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG)[0], 201)
|
||
|
||
def test_db_failure_after_s3_write_is_compensated(self):
|
||
recreate_api({"FAULT_INJECTION": "db_after_put"})
|
||
before = reconcile("--grace", "0")["s3_objects"]
|
||
r = upload(self.a["token"], self.chat["id"], PNG)
|
||
self.assertEqual(r[0], 500)
|
||
self.assertEqual(r[2]["error"]["code"], "internal_error")
|
||
after = reconcile("--grace", "0")
|
||
self.assertEqual(after["s3_objects"], before, "объект должен быть удалён компенсацией")
|
||
self.assertEqual(after["orphan_objects_count"], 0)
|
||
recreate_api()
|
||
|
||
def test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely(self):
|
||
keeper = upload(self.a["token"], self.chat["id"], PNG)[2] # используемое вложение
|
||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||
sent = call("POST", path, {"text": "keeper", "attachment_ids": [keeper["id"]]}, self.a["token"])
|
||
self.assertEqual(sent[0], 201, sent[2])
|
||
recreate_api({"FAULT_INJECTION": "db_after_put,s3_delete_fails"})
|
||
r = upload(self.a["token"], self.chat["id"], PNG)
|
||
self.assertEqual(r[0], 500)
|
||
recreate_api()
|
||
fresh = reconcile("--grace", "900") # grace period: свежий объект не трогаем
|
||
self.assertGreaterEqual(fresh["orphans_skipped_fresh"], 1)
|
||
self.assertEqual(fresh["orphan_objects_count"], 0)
|
||
protected = reconcile("--grace", "900", "--apply")
|
||
self.assertGreaterEqual(protected["orphans_skipped_fresh"], 1)
|
||
self.assertEqual(protected["fixed"], {"deleted_orphans": 0, "marked_failed": 0})
|
||
dry = reconcile("--grace", "0") # dry-run находит, но не удаляет
|
||
self.assertGreaterEqual(dry["orphan_objects_count"], 1)
|
||
self.assertEqual(dry["fixed"], {"deleted_orphans": 0, "marked_failed": 0})
|
||
self.assertEqual(reconcile("--grace", "0")["orphan_objects_count"], dry["orphan_objects_count"]) # dry-run ничего не меняет
|
||
applied = reconcile("--grace", "0", "--apply")
|
||
self.assertEqual(applied["fixed"]["deleted_orphans"], dry["orphan_objects_count"])
|
||
self.assertEqual(reconcile("--grace", "0")["orphan_objects_count"], 0) # повторный запуск идемпотентен
|
||
self.assertEqual(download(self.a["token"], keeper["id"])[0], 200, "используемое вложение удалять нельзя")
|
||
|
||
messages = call("GET", path, token=self.a["token"])[2]["items"]
|
||
self.assertEqual(messages[0]["id"], sent[2]["id"])
|
||
self.assertEqual(messages[0]["attachment_ids"], [keeper["id"]])
|
||
|
||
def test_missing_object_is_reported_and_attachment_marked_failed(self):
|
||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
|
||
code = ("import asyncio\nfrom app.s3 import S3\nasync def m():\n s=S3(); await s.start(); await s.delete('%s'); await s.stop()\n"
|
||
"asyncio.run(m())" % key)
|
||
s3_python(code)
|
||
dry = reconcile("--grace", "0")
|
||
self.assertIn(att["id"], dry["missing_objects"])
|
||
self.assertEqual(psql("SELECT status FROM attachments WHERE id = '%s'" % att["id"]), "ready") # dry-run не меняет
|
||
reconcile("--grace", "0", "--apply")
|
||
self.assertEqual(psql("SELECT status || ':' || error_code FROM attachments WHERE id = '%s'" % att["id"]), "failed:object_missing")
|
||
self.assertEqual(download(self.a["token"], att["id"])[0], 409)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|