Files
mtusi2026lab4/student-tests/failures/test_storage.py
T

264 lines
12 KiB
Python

"""Согласованность S3 и PostgreSQL, отказ S3, приватность bucket. Состояние после сбоя создаётся настоящими сбоями."""
import hashlib
import socket
import subprocess
import threading
import time
import unittest
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from stand import BASE_URL, call, compose, sql, timed, wait_ready
from api import new_chat, new_user, request_bytes, send
from samples import noisy_png
FIXTURE = (Path(__file__).resolve().parents[2] / "tests/fixtures/sample.png").read_bytes()
LIST_OBJECTS = """
import config
from objects import ObjectStore
store = ObjectStore(config.read_secret("S3_JANITOR_ACCESS_KEY_ID"), config.read_secret("S3_JANITOR_SECRET_ACCESS_KEY"))
for page in store.pages():
for item in page:
print(item["Key"])
"""
PROBE_API_KEY = """
import urllib.error, urllib.request
import config
from botocore.exceptions import ClientError
from objects import ObjectStore
store = ObjectStore(config.read_secret("S3_ACCESS_KEY_ID"), config.read_secret("S3_SECRET_ACCESS_KEY"))
def attempt(name, action):
try:
action()
print(name, "allowed")
except ClientError as error:
print(name, error.response["Error"]["Code"])
attempt("list", lambda: store.client.list_objects_v2(Bucket=store.bucket))
attempt("create_bucket", lambda: store.client.create_bucket(Bucket="other"))
attempt("delete_bucket", lambda: store.client.delete_bucket(Bucket=store.bucket))
try:
urllib.request.urlopen(config.S3_ENDPOINT + "/" + store.bucket + "/KEY", timeout=3)
print("anonymous allowed")
except urllib.error.HTTPError as error:
print("anonymous", error.code)
"""
def upload(user, chat, data=FIXTURE):
path = "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"]
return call("POST", path, raw=data, user=user, headers={"Content-Type": "image/png"}, timeout=30)
def bucket_keys():
return set(compose("exec", "-T", "janitor", "python", "-c", LIST_OBJECTS).split())
def janitor(*args):
# логи идут в stdout контейнера; сервисный проход на проверки не влияет: его период 300 секунд
return compose("run", "--rm", "--no-deps", "janitor", "python", "janitor.py", "--once", *args)
def in_api_replica(*command):
# по id контейнера: после поочерёдной замены номера реплик уже не 1 и 2
container = compose("ps", "-q", "api").split()[0]
return subprocess.run(["docker", "exec", container, *command], check=True, capture_output=True, text=True).stdout
def wait_s3(seconds=90):
"""S3 готов, когда контейнер healthy: том данных зарегистрирован, запросы к объектам не зависают."""
deadline = time.monotonic() + seconds
while compose("ps", "--format", "{{.Health}}", "s3").strip() != "healthy":
if time.monotonic() > deadline:
raise AssertionError("S3 не стал готов за %d секунд" % seconds)
time.sleep(1)
def rows(condition="true"):
output = sql("SELECT object_key FROM attachments WHERE " + condition)
return set(output.split())
class StorageBase(unittest.TestCase):
def setUp(self):
wait_ready()
self.user = new_user("files")
self.chat = new_chat(self.user)
def tearDown(self):
# следы теста не должны влиять на сверку в следующем; S3 к этому моменту может ещё запускаться
compose("start", "s3")
wait_s3()
janitor("--grace-seconds", "0")
class ConsistencyTest(StorageBase):
def test_commit_lost_after_s3_put(self):
"""Состояние как после падения между PUT и подтверждением: объект есть, строка pending, клиент не получил 201."""
attachment = upload(self.user, self.chat)[1]
key = "originals/" + attachment["id"]
self.assertIn(key, bucket_keys())
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
janitor()
self.assertNotIn(key, bucket_keys())
self.assertEqual(rows("id = '%s'" % attachment["id"]), set())
def test_grace_period_protects_fresh_upload(self):
attachment = upload(self.user, self.chat)[1]
key = "originals/" + attachment["id"]
sql("UPDATE attachments SET state = 'pending' WHERE id = '%s'" % attachment["id"])
janitor() # строка свежая, загрузка могла ещё идти
self.assertIn(key, bucket_keys())
self.assertEqual(rows("id = '%s'" % attachment["id"]), {key})
# строку не видно снаружи, пока она pending
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 404)
def test_ready_attachments_are_never_removed(self):
attachment = upload(self.user, self.chat)[1]
janitor("--grace-seconds", "0")
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
self.assertEqual((status, body), (200, FIXTURE))
def test_orphan_object_is_removed_only_after_grace(self):
attachment = upload(self.user, self.chat)[1]
key = "originals/" + attachment["id"]
sql("DELETE FROM attachments WHERE id = '%s'" % attachment["id"])
janitor()
self.assertIn(key, bucket_keys())
janitor("--grace-seconds", "0", "--dry-run")
self.assertIn(key, bucket_keys())
janitor("--grace-seconds", "0")
self.assertNotIn(key, bucket_keys())
def test_second_pass_changes_nothing(self):
attachment = upload(self.user, self.chat)[1]
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
janitor()
before = bucket_keys(), rows()
janitor()
self.assertEqual((bucket_keys(), rows()), before)
def test_interrupted_upload_leaves_nothing(self):
before = bucket_keys(), rows()
address = urllib.parse.urlsplit(BASE_URL)
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n"
"Authorization: Bearer %s\r\n\r\n" % (path, self.user["token"])).encode()
for _ in range(4):
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
connection.sendall(head + b"\x89PNG" + b"x" * 200000)
time.sleep(0.2)
time.sleep(1)
self.assertEqual((bucket_keys(), rows()), before)
def test_postgres_lost_during_parallel_uploads(self):
"""Настоящий сбой: база пропадает, пока идут загрузки. Часть объектов попадает в S3 без подтверждения."""
data = noisy_png(3 * 1024 * 1024)
# подтверждённые до сбоя файлы: они обязаны уцелеть при любом исходе гонки ниже
confirmed_before = [upload(self.user, self.chat, data)[1] for _ in range(3)]
before_keys = bucket_keys()
barrier = threading.Barrier(13)
def post(_):
barrier.wait()
confirmed = []
for _ in range(6):
try:
status, attachment, _ = upload(self.user, self.chat, data)
except OSError:
continue
if status == 201:
confirmed.append(attachment)
return confirmed
with ThreadPoolExecutor(max_workers=12) as pool:
jobs = [pool.submit(post, number) for number in range(12)]
barrier.wait()
time.sleep(0.25)
compose("kill", "postgres")
confirmed = confirmed_before + [item for job in jobs for item in job.result()]
compose("start", "postgres")
self.addCleanup(wait_ready)
wait_ready()
leftovers = bucket_keys() - before_keys - {"originals/" + item["id"] for item in confirmed}
janitor("--grace-seconds", "0")
self.assertEqual(rows("state = 'pending'"), set())
self.assertEqual(bucket_keys(), rows("state = 'ready'"))
self.assertGreaterEqual(len(confirmed), 3)
for attachment in confirmed:
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
self.assertEqual((status, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
print("\nподтверждено %d, в S3 без подтверждения %d, после очистки расхождений нет" % (len(confirmed), len(leftovers)))
class S3DownTest(StorageBase):
def test_s3_down(self):
attachment = upload(self.user, self.chat)[1]
compose("stop", "s3")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "s3")
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
elapsed, status, body, headers = timed("POST", path, raw=FIXTURE, user=self.user, headers={"Content-Type": "image/png"})
self.assertEqual((status, body["error"]["code"]), (503, "unavailable"))
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
self.assertLess(elapsed, 10)
self.assertEqual(request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)[0], 503)
# без S3 мессенджер продолжает работать: текст, метаданные и готовность реплик от него не зависят
self.assertEqual(send(self.user, self.chat, "текст без S3")[0], 201)
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 200)
self.assertEqual(call("GET", "/health/ready")[0], 200)
compose("start", "s3")
wait_s3()
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
self.assertEqual((status, body), (200, FIXTURE))
def test_failed_upload_is_cleaned(self):
compose("stop", "s3")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "s3")
self.assertEqual(upload(self.user, self.chat)[0], 503)
self.assertTrue(rows("state = 'pending'")) # строка осталась, клиент получил ошибку
compose("start", "s3")
wait_s3()
janitor("--grace-seconds", "0")
self.assertEqual(rows("state = 'pending'"), set())
class PrivateBucketTest(StorageBase):
def test_bucket_is_private_and_api_key_is_limited(self):
attachment = upload(self.user, self.chat)[1]
script = PROBE_API_KEY.replace("/KEY", "/originals/" + attachment["id"])
output = in_api_replica("python", "-c", script)
self.assertIn("list AccessDenied", output)
self.assertIn("create_bucket AccessDenied", output)
self.assertIn("delete_bucket AccessDenied", output)
self.assertIn("anonymous 403", output)
def test_bucket_bootstrap_can_be_repeated(self):
before = bucket_keys()
output = compose("run", "--rm", "--no-deps", "s3-init")
self.assertIn("bucket уже есть", output)
self.assertIn("пробная запись прошла", output)
self.assertEqual(bucket_keys(), before)
def test_no_file_on_api_replica(self):
attachment = upload(self.user, self.chat)[1]
output = in_api_replica("sh", "-c", "find / -xdev -name '*%s*' 2>/dev/null | wc -l" % attachment["id"])
self.assertEqual(output.split()[-1], "0")
if __name__ == "__main__":
unittest.main()