forked from mtusi2026/mtusi2026lab4
Добавить вложения: S3, метаданные в PostgreSQL и очистка незавершённых загрузок
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
"""Управление стендом для тестов отказов: останавливает и запускает сервисы через docker compose."""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
|
||||
from api import BASE_URL, call # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose.test.yaml", *args]
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise AssertionError("compose %s: %s" % (" ".join(args[:4]), result.stderr[-600:]))
|
||||
return result.stdout
|
||||
|
||||
|
||||
def sql(query):
|
||||
"""Выполняет запрос через psql внутри контейнера postgres и возвращает вывод без оформления."""
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query).strip()
|
||||
|
||||
|
||||
def wait_ready(base_url=BASE_URL, seconds=60):
|
||||
"""Ждёт устойчивой готовности: балансировщик мог ещё не отметить отказ, поэтому одного ответа 200 мало."""
|
||||
deadline = time.monotonic() + seconds
|
||||
stable_since = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
ready = call("GET", "/health/ready", base_url=base_url)[0] == 200
|
||||
except OSError:
|
||||
ready = False
|
||||
if not ready:
|
||||
stable_since = None
|
||||
elif stable_since is None:
|
||||
stable_since = time.monotonic()
|
||||
elif time.monotonic() - stable_since >= 3:
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise AssertionError("стенд не стал готов за %d секунд" % seconds)
|
||||
|
||||
|
||||
def timed(*args, **kwargs):
|
||||
"""Как call, но первым элементом возвращает время ответа в секундах."""
|
||||
started = time.monotonic()
|
||||
result = call(*args, **kwargs)
|
||||
return (time.monotonic() - started, *result)
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Недоступность Redis и PostgreSQL через балансировщик.
|
||||
|
||||
Пока реплики не признаны неготовыми, 503 отвечает API, потом сам балансировщик. Формат ошибки у них один.
|
||||
"""
|
||||
import unittest
|
||||
|
||||
from stand import call, compose, timed, wait_ready
|
||||
from api import basic, history, new_chat, new_key, new_user, send, user_body
|
||||
|
||||
# таймауты в .env.example: 2 с на Redis, 3 с на ожидание соединения с БД; берём запас
|
||||
MAX_SECONDS = 6
|
||||
|
||||
|
||||
class DependencyDownTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
wait_ready()
|
||||
self.user = new_user("dep")
|
||||
self.chat = new_chat(self.user)
|
||||
send(self.user, self.chat, "сообщение до отказа")
|
||||
self.messages = "/api/v1/chats/" + self.chat["id"] + "/messages"
|
||||
|
||||
def assert_unavailable(self, result):
|
||||
elapsed, status, body, headers = result
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(body["error"]["code"], "unavailable")
|
||||
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
||||
self.assertLess(elapsed, MAX_SECONDS)
|
||||
|
||||
def test_redis_down(self):
|
||||
compose("stop", "redis")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "redis")
|
||||
|
||||
# сессию проверить нечем: запрос с настоящим токеном не выполняется, но и чужим не становится
|
||||
self.assert_unavailable(timed("GET", "/api/v1/users/me", user=self.user))
|
||||
self.assert_unavailable(timed("GET", self.messages, user=self.user))
|
||||
key = {"Idempotency-Key": new_key()}
|
||||
self.assert_unavailable(timed("POST", self.messages, {"text": "во время отказа"}, user=self.user, headers=key))
|
||||
# поддельный токен тоже не принимается за действующий
|
||||
self.assert_unavailable(timed("GET", "/api/v1/users/me", user={"token": "x" * 43}))
|
||||
# без токена Redis не нужен: API ответит 401, балансировщик без готовых реплик 503, но не данные
|
||||
self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503))
|
||||
self.assertIn(call("GET", self.messages)[0], (401, 503))
|
||||
# новую сессию создать негде
|
||||
self.assert_unavailable(timed("POST", "/api/v1/auth/sessions",
|
||||
headers=basic(self.user["username"], self.user["password"])))
|
||||
self.assert_unavailable(timed("GET", "/health/ready"))
|
||||
|
||||
compose("start", "redis")
|
||||
wait_ready()
|
||||
# прежний токен снова работает: сессия пережила остановку Redis, а сообщение во время отказа не записалось
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=self.user)[1], self.user["user"])
|
||||
self.assertEqual([item["text"] for item in history(self.user, self.chat, 50)[0]], ["сообщение до отказа"])
|
||||
|
||||
def test_uncertain_write_can_be_repeated_with_same_key(self):
|
||||
compose("stop", "postgres")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "postgres")
|
||||
key = new_key()
|
||||
self.assertEqual(send(self.user, self.chat, "важное", key=key)[0], 503)
|
||||
compose("start", "postgres")
|
||||
wait_ready()
|
||||
first = send(self.user, self.chat, "важное", key=key)
|
||||
second = send(self.user, self.chat, "важное", key=key)
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[1], second[1])
|
||||
|
||||
def test_postgres_down(self):
|
||||
compose("stop", "postgres")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "postgres")
|
||||
|
||||
self.assert_unavailable(timed("GET", "/api/v1/users/me", user=self.user))
|
||||
self.assert_unavailable(timed("GET", self.messages, user=self.user))
|
||||
key = {"Idempotency-Key": new_key()}
|
||||
self.assert_unavailable(timed("POST", self.messages, {"text": "во время отказа"}, user=self.user, headers=key))
|
||||
self.assert_unavailable(timed("POST", "/api/v1/users", user_body()))
|
||||
self.assert_unavailable(timed("POST", "/api/v1/auth/sessions",
|
||||
headers=basic(self.user["username"], self.user["password"])))
|
||||
self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503))
|
||||
self.assert_unavailable(timed("GET", "/health/ready"))
|
||||
|
||||
compose("start", "postgres")
|
||||
wait_ready()
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=self.user)[1], self.user["user"])
|
||||
self.assertEqual([item["text"] for item in history(self.user, self.chat, 50)[0]], ["сообщение до отказа"])
|
||||
self.assertEqual(send(self.user, self.chat, "после восстановления")[0], 201)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Истечение сессии и ключа. Проверяется на отдельном API с TTL 2 секунды (порт 8081, только compose.test.yaml)."""
|
||||
import datetime
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from stand import call, compose, wait_ready
|
||||
from api import login, new_key, new_user, send
|
||||
|
||||
SHORT_TTL_URL = "http://localhost:8081"
|
||||
|
||||
|
||||
class SessionExpiryTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--no-build", "api-short-ttl")
|
||||
cls.addClassCleanup(compose, "rm", "-sf", "api-short-ttl")
|
||||
wait_ready(SHORT_TTL_URL)
|
||||
|
||||
def test_session_expires(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)[0], 200)
|
||||
time.sleep(3)
|
||||
status, body, headers = call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(body["error"]["code"], "unauthorized")
|
||||
self.assertTrue(headers["WWW-Authenticate"].startswith("Bearer "))
|
||||
# logout с истёкшим токеном тоже 401
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", user=user, base_url=SHORT_TTL_URL)[0], 401)
|
||||
|
||||
def test_idempotency_key_is_free_after_ttl(self):
|
||||
user = new_user("idem", base_url=SHORT_TTL_URL)
|
||||
chat = new_chat_short(user)
|
||||
key = new_key()
|
||||
first = send(user, chat, "первое", key=key, base_url=SHORT_TTL_URL)
|
||||
repeat = send(user, chat, "первое", key=key, base_url=SHORT_TTL_URL)
|
||||
self.assertEqual((first[0], repeat[1]), (201, first[1]))
|
||||
time.sleep(2.2)
|
||||
# сессия живёт столько же, поэтому нужен новый вход
|
||||
_, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
|
||||
again = send({**user, "token": session["token"]}, chat, "уже другое", key=key, base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(again[0], 201)
|
||||
self.assertNotEqual(again[1]["id"], first[1]["id"])
|
||||
|
||||
def test_expires_at_matches_ttl(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
_, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
|
||||
expires_at = datetime.datetime.fromisoformat(session["expires_at"][:-1] + "+00:00")
|
||||
left = (expires_at - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
|
||||
self.assertTrue(0 < left <= 2.5, left)
|
||||
|
||||
def test_requests_do_not_extend_session(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
deadline = time.monotonic() + 3
|
||||
while time.monotonic() < deadline:
|
||||
call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)[0], 401)
|
||||
|
||||
def test_login_after_expiry(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
time.sleep(3)
|
||||
status, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=session, base_url=SHORT_TTL_URL)[0], 200)
|
||||
|
||||
|
||||
def new_chat_short(owner):
|
||||
body = {"title": "Чат", "member_ids": [owner["id"]]}
|
||||
status, chat, _ = call("POST", "/api/v1/chats", body, user=owner, base_url=SHORT_TTL_URL)
|
||||
assert status == 201, status
|
||||
return chat
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Согласованность S3 и PostgreSQL, отказ S3, приватность bucket. Состояние после сбоя создаётся настоящими сбоями."""
|
||||
import hashlib
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from stand import BASE_URL, call, compose, sql, timed, wait_ready
|
||||
from api import new_chat, new_user, request_bytes, send
|
||||
from samples import noisy_png
|
||||
|
||||
FIXTURE = (Path(__file__).resolve().parents[2] / "tests/fixtures/sample.png").read_bytes()
|
||||
LIST_OBJECTS = """
|
||||
import config
|
||||
from objects import ObjectStore
|
||||
store = ObjectStore(config.read_secret("S3_JANITOR_ACCESS_KEY_ID"), config.read_secret("S3_JANITOR_SECRET_ACCESS_KEY"))
|
||||
for page in store.pages():
|
||||
for item in page:
|
||||
print(item["Key"])
|
||||
"""
|
||||
PROBE_API_KEY = """
|
||||
import urllib.error, urllib.request
|
||||
import config
|
||||
from botocore.exceptions import ClientError
|
||||
from objects import ObjectStore
|
||||
store = ObjectStore(config.read_secret("S3_ACCESS_KEY_ID"), config.read_secret("S3_SECRET_ACCESS_KEY"))
|
||||
def attempt(name, action):
|
||||
try:
|
||||
action()
|
||||
print(name, "allowed")
|
||||
except ClientError as error:
|
||||
print(name, error.response["Error"]["Code"])
|
||||
attempt("list", lambda: store.client.list_objects_v2(Bucket=store.bucket))
|
||||
attempt("create_bucket", lambda: store.client.create_bucket(Bucket="other"))
|
||||
attempt("delete_bucket", lambda: store.client.delete_bucket(Bucket=store.bucket))
|
||||
try:
|
||||
urllib.request.urlopen(config.S3_ENDPOINT + "/" + store.bucket + "/KEY", timeout=3)
|
||||
print("anonymous allowed")
|
||||
except urllib.error.HTTPError as error:
|
||||
print("anonymous", error.code)
|
||||
"""
|
||||
|
||||
|
||||
def upload(user, chat, data=FIXTURE):
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"]
|
||||
return call("POST", path, raw=data, user=user, headers={"Content-Type": "image/png"}, timeout=30)
|
||||
|
||||
|
||||
def bucket_keys():
|
||||
return set(compose("exec", "-T", "janitor", "python", "-c", LIST_OBJECTS).split())
|
||||
|
||||
|
||||
def janitor(*args):
|
||||
# логи идут в stdout контейнера; сервисный проход на проверки не влияет: его период 300 секунд
|
||||
return compose("run", "--rm", "--no-deps", "janitor", "python", "janitor.py", "--once", *args)
|
||||
|
||||
|
||||
def in_api_replica(*command):
|
||||
# по id контейнера: после поочерёдной замены номера реплик уже не 1 и 2
|
||||
container = compose("ps", "-q", "api").split()[0]
|
||||
return subprocess.run(["docker", "exec", container, *command], check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def wait_s3(seconds=90):
|
||||
"""S3 готов, когда контейнер healthy: том данных зарегистрирован, запросы к объектам не зависают."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while compose("ps", "--format", "{{.Health}}", "s3").strip() != "healthy":
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError("S3 не стал готов за %d секунд" % seconds)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def rows(condition="true"):
|
||||
output = sql("SELECT object_key FROM attachments WHERE " + condition)
|
||||
return set(output.split())
|
||||
|
||||
|
||||
class StorageBase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
wait_ready()
|
||||
self.user = new_user("files")
|
||||
self.chat = new_chat(self.user)
|
||||
|
||||
def tearDown(self):
|
||||
# следы теста не должны влиять на сверку в следующем; S3 к этому моменту может ещё запускаться
|
||||
compose("start", "s3")
|
||||
wait_s3()
|
||||
janitor("--grace-seconds", "0")
|
||||
|
||||
|
||||
class ConsistencyTest(StorageBase):
|
||||
def test_commit_lost_after_s3_put(self):
|
||||
"""Состояние как после падения между PUT и подтверждением: объект есть, строка pending, клиент не получил 201."""
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
key = "originals/" + attachment["id"]
|
||||
self.assertIn(key, bucket_keys())
|
||||
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
|
||||
|
||||
janitor()
|
||||
|
||||
self.assertNotIn(key, bucket_keys())
|
||||
self.assertEqual(rows("id = '%s'" % attachment["id"]), set())
|
||||
|
||||
def test_grace_period_protects_fresh_upload(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
key = "originals/" + attachment["id"]
|
||||
sql("UPDATE attachments SET state = 'pending' WHERE id = '%s'" % attachment["id"])
|
||||
|
||||
janitor() # строка свежая, загрузка могла ещё идти
|
||||
|
||||
self.assertIn(key, bucket_keys())
|
||||
self.assertEqual(rows("id = '%s'" % attachment["id"]), {key})
|
||||
# строку не видно снаружи, пока она pending
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 404)
|
||||
|
||||
def test_ready_attachments_are_never_removed(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
janitor("--grace-seconds", "0")
|
||||
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
||||
self.assertEqual((status, body), (200, FIXTURE))
|
||||
|
||||
def test_orphan_object_is_removed_only_after_grace(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
key = "originals/" + attachment["id"]
|
||||
sql("DELETE FROM attachments WHERE id = '%s'" % attachment["id"])
|
||||
|
||||
janitor()
|
||||
self.assertIn(key, bucket_keys())
|
||||
|
||||
janitor("--grace-seconds", "0", "--dry-run")
|
||||
self.assertIn(key, bucket_keys())
|
||||
|
||||
janitor("--grace-seconds", "0")
|
||||
self.assertNotIn(key, bucket_keys())
|
||||
|
||||
def test_second_pass_changes_nothing(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
|
||||
janitor()
|
||||
before = bucket_keys(), rows()
|
||||
janitor()
|
||||
self.assertEqual((bucket_keys(), rows()), before)
|
||||
|
||||
def test_interrupted_upload_leaves_nothing(self):
|
||||
before = bucket_keys(), rows()
|
||||
address = urllib.parse.urlsplit(BASE_URL)
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
||||
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n"
|
||||
"Authorization: Bearer %s\r\n\r\n" % (path, self.user["token"])).encode()
|
||||
for _ in range(4):
|
||||
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
|
||||
connection.sendall(head + b"\x89PNG" + b"x" * 200000)
|
||||
time.sleep(0.2)
|
||||
time.sleep(1)
|
||||
self.assertEqual((bucket_keys(), rows()), before)
|
||||
|
||||
def test_postgres_lost_during_parallel_uploads(self):
|
||||
"""Настоящий сбой: база пропадает, пока идут загрузки. Часть объектов попадает в S3 без подтверждения."""
|
||||
data = noisy_png(3 * 1024 * 1024)
|
||||
# подтверждённые до сбоя файлы: они обязаны уцелеть при любом исходе гонки ниже
|
||||
confirmed_before = [upload(self.user, self.chat, data)[1] for _ in range(3)]
|
||||
before_keys = bucket_keys()
|
||||
barrier = threading.Barrier(13)
|
||||
|
||||
def post(_):
|
||||
barrier.wait()
|
||||
confirmed = []
|
||||
for _ in range(6):
|
||||
try:
|
||||
status, attachment, _ = upload(self.user, self.chat, data)
|
||||
except OSError:
|
||||
continue
|
||||
if status == 201:
|
||||
confirmed.append(attachment)
|
||||
return confirmed
|
||||
|
||||
with ThreadPoolExecutor(max_workers=12) as pool:
|
||||
jobs = [pool.submit(post, number) for number in range(12)]
|
||||
barrier.wait()
|
||||
time.sleep(0.25)
|
||||
compose("kill", "postgres")
|
||||
confirmed = confirmed_before + [item for job in jobs for item in job.result()]
|
||||
compose("start", "postgres")
|
||||
self.addCleanup(wait_ready)
|
||||
wait_ready()
|
||||
|
||||
leftovers = bucket_keys() - before_keys - {"originals/" + item["id"] for item in confirmed}
|
||||
janitor("--grace-seconds", "0")
|
||||
|
||||
self.assertEqual(rows("state = 'pending'"), set())
|
||||
self.assertEqual(bucket_keys(), rows("state = 'ready'"))
|
||||
self.assertGreaterEqual(len(confirmed), 3)
|
||||
for attachment in confirmed:
|
||||
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
||||
self.assertEqual((status, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
|
||||
print("\nподтверждено %d, в S3 без подтверждения %d, после очистки расхождений нет" % (len(confirmed), len(leftovers)))
|
||||
|
||||
|
||||
class S3DownTest(StorageBase):
|
||||
def test_s3_down(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
compose("stop", "s3")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "s3")
|
||||
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
||||
elapsed, status, body, headers = timed("POST", path, raw=FIXTURE, user=self.user, headers={"Content-Type": "image/png"})
|
||||
self.assertEqual((status, body["error"]["code"]), (503, "unavailable"))
|
||||
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
||||
self.assertLess(elapsed, 10)
|
||||
self.assertEqual(request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)[0], 503)
|
||||
|
||||
# без S3 мессенджер продолжает работать: текст, метаданные и готовность реплик от него не зависят
|
||||
self.assertEqual(send(self.user, self.chat, "текст без S3")[0], 201)
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 200)
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 200)
|
||||
|
||||
compose("start", "s3")
|
||||
wait_s3()
|
||||
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
||||
self.assertEqual((status, body), (200, FIXTURE))
|
||||
|
||||
def test_failed_upload_is_cleaned(self):
|
||||
compose("stop", "s3")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "s3")
|
||||
self.assertEqual(upload(self.user, self.chat)[0], 503)
|
||||
self.assertTrue(rows("state = 'pending'")) # строка осталась, клиент получил ошибку
|
||||
compose("start", "s3")
|
||||
wait_s3()
|
||||
janitor("--grace-seconds", "0")
|
||||
self.assertEqual(rows("state = 'pending'"), set())
|
||||
|
||||
|
||||
class PrivateBucketTest(StorageBase):
|
||||
def test_bucket_is_private_and_api_key_is_limited(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
script = PROBE_API_KEY.replace("/KEY", "/originals/" + attachment["id"])
|
||||
output = in_api_replica("python", "-c", script)
|
||||
self.assertIn("list AccessDenied", output)
|
||||
self.assertIn("create_bucket AccessDenied", output)
|
||||
self.assertIn("delete_bucket AccessDenied", output)
|
||||
self.assertIn("anonymous 403", output)
|
||||
|
||||
def test_bucket_bootstrap_can_be_repeated(self):
|
||||
before = bucket_keys()
|
||||
output = compose("run", "--rm", "--no-deps", "s3-init")
|
||||
self.assertIn("bucket уже есть", output)
|
||||
self.assertIn("пробная запись прошла", output)
|
||||
self.assertEqual(bucket_keys(), before)
|
||||
|
||||
def test_no_file_on_api_replica(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
output = in_api_replica("sh", "-c", "find / -xdev -name '*%s*' 2>/dev/null | wc -l" % attachment["id"])
|
||||
self.assertEqual(output.split()[-1], "0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Проверки на уровне БД: откат составной операции, гонка регистрации, повторный запуск миграций."""
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from stand import call, compose, sql, wait_ready
|
||||
from api import new_chat, new_key, new_user, send, user_body
|
||||
|
||||
|
||||
class TransactionTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
wait_ready()
|
||||
cls.alice, cls.bob = new_user("alice"), new_user("bob")
|
||||
|
||||
def create_chat(self, title, member_ids):
|
||||
return call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=self.alice)
|
||||
|
||||
def test_chat_with_unknown_member_is_rolled_back(self):
|
||||
title = "rollback-" + uuid.uuid4().hex
|
||||
# Алиса и Боб существуют, третьего участника нет: вставка чата и двух участников уже выполнена,
|
||||
# на третьем срабатывает внешний ключ, и транзакция должна откатить всё
|
||||
status, body, _ = self.create_chat(title, [self.alice["id"], self.bob["id"], str(uuid.uuid4())])
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(body["error"]["code"], "invalid_request")
|
||||
self.assertEqual(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title), "0")
|
||||
self.assertEqual(sql("SELECT count(*) FROM chat_members m LEFT JOIN chats c ON c.id = m.chat_id WHERE c.id IS NULL"), "0")
|
||||
|
||||
def test_valid_chat_is_committed_whole(self):
|
||||
title = "commit-" + uuid.uuid4().hex
|
||||
status, chat, _ = self.create_chat(title, [self.alice["id"], self.bob["id"]])
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title), "1")
|
||||
self.assertEqual(sql("SELECT count(*) FROM chat_members WHERE chat_id = '%s'" % chat["id"]), "2")
|
||||
|
||||
def test_username_race_creates_one_row(self):
|
||||
body = user_body(username="race_" + uuid.uuid4().hex[:8])
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def register(_):
|
||||
barrier.wait()
|
||||
return call("POST", "/api/v1/users", body)[0]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = sorted(pool.map(register, range(10)))
|
||||
self.assertEqual(statuses, [201] + [409] * 9)
|
||||
self.assertEqual(sql("SELECT count(*) FROM users WHERE username = '%s'" % body["username"]), "1")
|
||||
|
||||
def test_same_key_in_parallel_creates_one_row(self):
|
||||
chat = new_chat(self.alice)
|
||||
key = new_key()
|
||||
barrier = threading.Barrier(20)
|
||||
|
||||
def post(_):
|
||||
barrier.wait()
|
||||
return send(self.alice, chat, "один раз", key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as pool:
|
||||
results = list(pool.map(post, range(20)))
|
||||
self.assertEqual({status for status, _, _ in results}, {201})
|
||||
self.assertEqual(len({message["id"] for _, message, _ in results}), 1)
|
||||
self.assertEqual(sql("SELECT count(*) FROM messages WHERE chat_id = '%s'" % chat["id"]), "1")
|
||||
self.assertEqual(sql("SELECT count(*) FROM idempotency_keys WHERE chat_id = '%s'" % chat["id"]), "1")
|
||||
|
||||
def test_failed_message_leaves_no_key(self):
|
||||
# чужой чат: сообщение не создано, и ключ тоже не занят
|
||||
eve = new_user("eve")
|
||||
chat = new_chat(self.alice)
|
||||
key = new_key()
|
||||
self.assertEqual(send(eve, chat, "чужое", key=key)[0], 404)
|
||||
self.assertEqual(sql("SELECT count(*) FROM idempotency_keys WHERE chat_id = '%s'" % chat["id"]), "0")
|
||||
|
||||
def test_password_is_stored_as_argon2id_hash(self):
|
||||
stored = sql("SELECT password_hash FROM users WHERE id = '%s'" % self.alice["id"])
|
||||
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
|
||||
self.assertNotIn(self.alice["password"], stored)
|
||||
|
||||
def test_migrations_can_be_repeated(self):
|
||||
before = sql("SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), (SELECT count(*) FROM messages)")
|
||||
output = compose("run", "--rm", "migrate")
|
||||
self.assertIn("уже применена: 0001_init.sql", output)
|
||||
self.assertIn("уже применена: 0003_attachments.sql", output)
|
||||
self.assertEqual(sql("SELECT count(*) FROM schema_migrations"), "3")
|
||||
after = sql("SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), (SELECT count(*) FROM messages)")
|
||||
self.assertEqual(before, after)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user