forked from mtusi2026/mtusi2026lab4
264 lines
12 KiB
Python
264 lines
12 KiB
Python
"""Согласованность S3 и PostgreSQL, отказ S3, приватность bucket. Состояние после сбоя создаётся настоящими сбоями."""
|
|
import hashlib
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import unittest
|
|
import urllib.parse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
from stand import BASE_URL, call, compose, sql, timed, wait_ready
|
|
from api import new_chat, new_user, request_bytes, send
|
|
from samples import noisy_png
|
|
|
|
FIXTURE = (Path(__file__).resolve().parents[2] / "tests/fixtures/sample.png").read_bytes()
|
|
LIST_OBJECTS = """
|
|
import config
|
|
from objects import ObjectStore
|
|
store = ObjectStore(config.read_secret("S3_JANITOR_ACCESS_KEY_ID"), config.read_secret("S3_JANITOR_SECRET_ACCESS_KEY"))
|
|
for page in store.pages():
|
|
for item in page:
|
|
print(item["Key"])
|
|
"""
|
|
PROBE_API_KEY = """
|
|
import urllib.error, urllib.request
|
|
import config
|
|
from botocore.exceptions import ClientError
|
|
from objects import ObjectStore
|
|
store = ObjectStore(config.read_secret("S3_ACCESS_KEY_ID"), config.read_secret("S3_SECRET_ACCESS_KEY"))
|
|
def attempt(name, action):
|
|
try:
|
|
action()
|
|
print(name, "allowed")
|
|
except ClientError as error:
|
|
print(name, error.response["Error"]["Code"])
|
|
attempt("list", lambda: store.client.list_objects_v2(Bucket=store.bucket))
|
|
attempt("create_bucket", lambda: store.client.create_bucket(Bucket="other"))
|
|
attempt("delete_bucket", lambda: store.client.delete_bucket(Bucket=store.bucket))
|
|
try:
|
|
urllib.request.urlopen(config.S3_ENDPOINT + "/" + store.bucket + "/KEY", timeout=3)
|
|
print("anonymous allowed")
|
|
except urllib.error.HTTPError as error:
|
|
print("anonymous", error.code)
|
|
"""
|
|
|
|
|
|
def upload(user, chat, data=FIXTURE):
|
|
path = "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"]
|
|
return call("POST", path, raw=data, user=user, headers={"Content-Type": "image/png"}, timeout=30)
|
|
|
|
|
|
def bucket_keys():
|
|
return set(compose("exec", "-T", "janitor", "python", "-c", LIST_OBJECTS).split())
|
|
|
|
|
|
def janitor(*args):
|
|
# логи идут в stdout контейнера; сервисный проход на проверки не влияет: его период 300 секунд
|
|
return compose("run", "--rm", "--no-deps", "janitor", "python", "janitor.py", "--once", *args)
|
|
|
|
|
|
def in_api_replica(*command):
|
|
# по id контейнера: после поочерёдной замены номера реплик уже не 1 и 2
|
|
container = compose("ps", "-q", "api").split()[0]
|
|
return subprocess.run(["docker", "exec", container, *command], check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
def wait_s3(seconds=90):
|
|
"""S3 готов, когда контейнер healthy: том данных зарегистрирован, запросы к объектам не зависают."""
|
|
deadline = time.monotonic() + seconds
|
|
while compose("ps", "--format", "{{.Health}}", "s3").strip() != "healthy":
|
|
if time.monotonic() > deadline:
|
|
raise AssertionError("S3 не стал готов за %d секунд" % seconds)
|
|
time.sleep(1)
|
|
|
|
|
|
def rows(condition="true"):
|
|
output = sql("SELECT object_key FROM attachments WHERE " + condition)
|
|
return set(output.split())
|
|
|
|
|
|
class StorageBase(unittest.TestCase):
|
|
def setUp(self):
|
|
wait_ready()
|
|
self.user = new_user("files")
|
|
self.chat = new_chat(self.user)
|
|
|
|
def tearDown(self):
|
|
# следы теста не должны влиять на сверку в следующем; S3 к этому моменту может ещё запускаться
|
|
compose("start", "s3")
|
|
wait_s3()
|
|
janitor("--grace-seconds", "0")
|
|
|
|
|
|
class ConsistencyTest(StorageBase):
|
|
def test_commit_lost_after_s3_put(self):
|
|
"""Состояние как после падения между PUT и подтверждением: объект есть, строка pending, клиент не получил 201."""
|
|
attachment = upload(self.user, self.chat)[1]
|
|
key = "originals/" + attachment["id"]
|
|
self.assertIn(key, bucket_keys())
|
|
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
|
|
|
|
janitor()
|
|
|
|
self.assertNotIn(key, bucket_keys())
|
|
self.assertEqual(rows("id = '%s'" % attachment["id"]), set())
|
|
|
|
def test_grace_period_protects_fresh_upload(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
key = "originals/" + attachment["id"]
|
|
sql("UPDATE attachments SET state = 'pending' WHERE id = '%s'" % attachment["id"])
|
|
|
|
janitor() # строка свежая, загрузка могла ещё идти
|
|
|
|
self.assertIn(key, bucket_keys())
|
|
self.assertEqual(rows("id = '%s'" % attachment["id"]), {key})
|
|
# строку не видно снаружи, пока она pending
|
|
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 404)
|
|
|
|
def test_ready_attachments_are_never_removed(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
janitor("--grace-seconds", "0")
|
|
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
|
self.assertEqual((status, body), (200, FIXTURE))
|
|
|
|
def test_orphan_object_is_removed_only_after_grace(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
key = "originals/" + attachment["id"]
|
|
sql("DELETE FROM attachments WHERE id = '%s'" % attachment["id"])
|
|
|
|
janitor()
|
|
self.assertIn(key, bucket_keys())
|
|
|
|
janitor("--grace-seconds", "0", "--dry-run")
|
|
self.assertIn(key, bucket_keys())
|
|
|
|
janitor("--grace-seconds", "0")
|
|
self.assertNotIn(key, bucket_keys())
|
|
|
|
def test_second_pass_changes_nothing(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
|
|
janitor()
|
|
before = bucket_keys(), rows()
|
|
janitor()
|
|
self.assertEqual((bucket_keys(), rows()), before)
|
|
|
|
def test_interrupted_upload_leaves_nothing(self):
|
|
before = bucket_keys(), rows()
|
|
address = urllib.parse.urlsplit(BASE_URL)
|
|
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
|
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n"
|
|
"Authorization: Bearer %s\r\n\r\n" % (path, self.user["token"])).encode()
|
|
for _ in range(4):
|
|
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
|
|
connection.sendall(head + b"\x89PNG" + b"x" * 200000)
|
|
time.sleep(0.2)
|
|
time.sleep(1)
|
|
self.assertEqual((bucket_keys(), rows()), before)
|
|
|
|
def test_postgres_lost_during_parallel_uploads(self):
|
|
"""Настоящий сбой: база пропадает, пока идут загрузки. Часть объектов попадает в S3 без подтверждения."""
|
|
data = noisy_png(3 * 1024 * 1024)
|
|
# подтверждённые до сбоя файлы: они обязаны уцелеть при любом исходе гонки ниже
|
|
confirmed_before = [upload(self.user, self.chat, data)[1] for _ in range(3)]
|
|
before_keys = bucket_keys()
|
|
barrier = threading.Barrier(13)
|
|
|
|
def post(_):
|
|
barrier.wait()
|
|
confirmed = []
|
|
for _ in range(6):
|
|
try:
|
|
status, attachment, _ = upload(self.user, self.chat, data)
|
|
except OSError:
|
|
continue
|
|
if status == 201:
|
|
confirmed.append(attachment)
|
|
return confirmed
|
|
|
|
with ThreadPoolExecutor(max_workers=12) as pool:
|
|
jobs = [pool.submit(post, number) for number in range(12)]
|
|
barrier.wait()
|
|
time.sleep(0.25)
|
|
compose("kill", "postgres")
|
|
confirmed = confirmed_before + [item for job in jobs for item in job.result()]
|
|
compose("start", "postgres")
|
|
self.addCleanup(wait_ready)
|
|
wait_ready()
|
|
|
|
leftovers = bucket_keys() - before_keys - {"originals/" + item["id"] for item in confirmed}
|
|
janitor("--grace-seconds", "0")
|
|
|
|
self.assertEqual(rows("state = 'pending'"), set())
|
|
self.assertEqual(bucket_keys(), rows("state = 'ready'"))
|
|
self.assertGreaterEqual(len(confirmed), 3)
|
|
for attachment in confirmed:
|
|
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
|
self.assertEqual((status, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
|
|
print("\nподтверждено %d, в S3 без подтверждения %d, после очистки расхождений нет" % (len(confirmed), len(leftovers)))
|
|
|
|
|
|
class S3DownTest(StorageBase):
|
|
def test_s3_down(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
compose("stop", "s3")
|
|
self.addCleanup(wait_ready)
|
|
self.addCleanup(compose, "start", "s3")
|
|
|
|
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
|
elapsed, status, body, headers = timed("POST", path, raw=FIXTURE, user=self.user, headers={"Content-Type": "image/png"})
|
|
self.assertEqual((status, body["error"]["code"]), (503, "unavailable"))
|
|
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
|
self.assertLess(elapsed, 10)
|
|
self.assertEqual(request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)[0], 503)
|
|
|
|
# без S3 мессенджер продолжает работать: текст, метаданные и готовность реплик от него не зависят
|
|
self.assertEqual(send(self.user, self.chat, "текст без S3")[0], 201)
|
|
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 200)
|
|
self.assertEqual(call("GET", "/health/ready")[0], 200)
|
|
|
|
compose("start", "s3")
|
|
wait_s3()
|
|
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
|
self.assertEqual((status, body), (200, FIXTURE))
|
|
|
|
def test_failed_upload_is_cleaned(self):
|
|
compose("stop", "s3")
|
|
self.addCleanup(wait_ready)
|
|
self.addCleanup(compose, "start", "s3")
|
|
self.assertEqual(upload(self.user, self.chat)[0], 503)
|
|
self.assertTrue(rows("state = 'pending'")) # строка осталась, клиент получил ошибку
|
|
compose("start", "s3")
|
|
wait_s3()
|
|
janitor("--grace-seconds", "0")
|
|
self.assertEqual(rows("state = 'pending'"), set())
|
|
|
|
|
|
class PrivateBucketTest(StorageBase):
|
|
def test_bucket_is_private_and_api_key_is_limited(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
script = PROBE_API_KEY.replace("/KEY", "/originals/" + attachment["id"])
|
|
output = in_api_replica("python", "-c", script)
|
|
self.assertIn("list AccessDenied", output)
|
|
self.assertIn("create_bucket AccessDenied", output)
|
|
self.assertIn("delete_bucket AccessDenied", output)
|
|
self.assertIn("anonymous 403", output)
|
|
|
|
def test_bucket_bootstrap_can_be_repeated(self):
|
|
before = bucket_keys()
|
|
output = compose("run", "--rm", "--no-deps", "s3-init")
|
|
self.assertIn("bucket уже есть", output)
|
|
self.assertIn("пробная запись прошла", output)
|
|
self.assertEqual(bucket_keys(), before)
|
|
|
|
def test_no_file_on_api_replica(self):
|
|
attachment = upload(self.user, self.chat)[1]
|
|
output = in_api_replica("sh", "-c", "find / -xdev -name '*%s*' 2>/dev/null | wc -l" % attachment["id"])
|
|
self.assertEqual(output.split()[-1], "0")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|