forked from mtusi2026/mtusi2026lab6
311 lines
15 KiB
Python
311 lines
15 KiB
Python
"""Согласованность PostgreSQL <-> S3 для вложений (нужен прямой доступ к БД и S3).
|
||
|
||
lab5: API пишет байты в staging/ (а не originals/), строка pending ссылается на
|
||
staging_key; успешная загрузка отвечает 202. Логика сборщика та же.
|
||
|
||
Запуск внутри compose-сети (БД и S3 наружу не публикуются):
|
||
python scripts/ops.py in-network consistency
|
||
Без DATABASE_URL(_FILE) и S3-настроек тесты пропускаются.
|
||
|
||
Что доказывается:
|
||
* сбой между S3 PUT и COMMIT (pending + объект) обнаруживается dry-run'ом,
|
||
исправляется --apply, повторный запуск ничего не меняет (идемпотентность);
|
||
* свежие pending (моложе grace) и ready/используемые вложения не удаляются;
|
||
* объект-сирота (нет строки) удаляется только после grace;
|
||
* ready без объекта — только отчёт (missing_objects), без авто-удаления;
|
||
* обрыв загрузки клиентом не оставляет ни строки, ни объекта;
|
||
* внедрённые отказы в API (если заданы FAULT_*_URL — экземпляры api-fault-*):
|
||
ошибка S3 PUT -> 503 и компенсация; ошибка БД после PUT -> 503, объект удалён;
|
||
падение процесса после PUT -> остаётся pending+объект, их убирает сборщик.
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import socket
|
||
import sys
|
||
import time
|
||
import unittest
|
||
import uuid
|
||
from pathlib import Path
|
||
from urllib.parse import urlparse
|
||
from urllib.request import Request, urlopen
|
||
from urllib.error import HTTPError, URLError
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(HERE.parent))
|
||
sys.path.insert(0, str(HERE))
|
||
|
||
HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE"))
|
||
HAS_S3 = bool(os.environ.get("S3_ENDPOINT"))
|
||
GRACE = 900
|
||
|
||
|
||
@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured")
|
||
class AttachmentConsistencyTests(unittest.TestCase):
|
||
|
||
@classmethod
|
||
def setUpClass(cls):
|
||
from src.config import load_settings
|
||
from src.db import create_pool
|
||
from src.objectstore import ObjectStore
|
||
from src.security import hash_password
|
||
from src.storage import Storage
|
||
from src import reconcile
|
||
|
||
cls.settings = load_settings()
|
||
cls.pool = create_pool(cls.settings)
|
||
cls.pool.wait(timeout=10)
|
||
cls.storage = Storage(cls.pool)
|
||
cls.store = ObjectStore(cls.settings)
|
||
cls.rec = reconcile
|
||
user = cls.storage.create_user("cons_" + uuid.uuid4().hex[:16], "Consistency", hash_password("Consistency-pw-1"))
|
||
cls.user = user
|
||
cls.chat = cls.storage.create_chat("consistency", [user.id])
|
||
|
||
@classmethod
|
||
def tearDownClass(cls):
|
||
cls.pool.close()
|
||
|
||
# helpers
|
||
|
||
def pending_with_object(self, age_seconds=0, put=True):
|
||
att_id = uuid.uuid4()
|
||
key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex)
|
||
data = b"\x89PNG fake bytes " + uuid.uuid4().bytes
|
||
ok = self.storage.create_pending_attachment(att_id, self.chat.id, self.user.id, "p.png", "image/png",
|
||
len(data), hashlib.sha256(data).hexdigest(), 1, 1, key)
|
||
self.assertTrue(ok)
|
||
if put:
|
||
self.store.put(key, data, "image/png", hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest())
|
||
if age_seconds:
|
||
self.backdate(att_id, age_seconds)
|
||
return att_id, key
|
||
|
||
def backdate(self, att_id, seconds):
|
||
with self.pool.connection() as conn:
|
||
conn.execute("UPDATE attachments SET created_at = created_at - make_interval(secs => %s) WHERE id = %s",
|
||
(seconds, att_id))
|
||
|
||
def row(self, att_id):
|
||
with self.pool.connection() as conn:
|
||
return conn.execute("SELECT state FROM attachments WHERE id = %s", (att_id,)).fetchone()
|
||
|
||
def run_gc(self, apply, grace=GRACE, sweep=True):
|
||
report = None
|
||
for _ in range(50): # фоновый GC реплики мог держать lock
|
||
report = self.rec.run_once(self.pool, self.store, grace, apply, orphan_sweep=sweep)
|
||
if report is not None:
|
||
return report
|
||
time.sleep(0.2)
|
||
self.fail("reconciler lock is busy")
|
||
|
||
# tests
|
||
|
||
def test_crash_between_put_and_commit_dry_run_then_apply_then_noop(self):
|
||
att_id, key = self.pending_with_object(age_seconds=GRACE + 60)
|
||
dry = self.run_gc(apply=False, sweep=False)
|
||
self.assertIn(str(att_id), dry["stale_pending_ids"])
|
||
self.assertIsNotNone(self.row(att_id), "dry-run must not change anything")
|
||
self.assertIsNotNone(self.store.head(key))
|
||
applied = self.run_gc(apply=True, sweep=False)
|
||
self.assertGreaterEqual(applied["stale_pending_rows_deleted"], 1)
|
||
self.assertIsNone(self.row(att_id))
|
||
self.assertIsNone(self.store.head(key))
|
||
again = self.run_gc(apply=True, sweep=False)
|
||
self.assertNotIn(str(att_id), again["stale_pending_ids"])
|
||
|
||
def test_fresh_pending_is_kept(self):
|
||
att_id, key = self.pending_with_object(age_seconds=0)
|
||
self.run_gc(apply=True)
|
||
self.assertEqual(self.row(att_id), ("pending",))
|
||
self.assertIsNotNone(self.store.head(key))
|
||
self.storage.delete_pending_attachment(att_id)
|
||
self.store.delete(key)
|
||
|
||
def test_ready_and_referenced_are_never_collected(self):
|
||
att_id, key = self.pending_with_object()
|
||
self.assertIsNotNone(self.storage.mark_attachment_ready(att_id))
|
||
msg, _ = self.storage.create_message(self.chat.id, self.user.id, "", uuid.uuid4().hex, os.urandom(32),
|
||
86400, [att_id])
|
||
self.assertEqual(msg.attachment_ids, [att_id])
|
||
self.backdate(att_id, 10 * GRACE)
|
||
self.run_gc(apply=True, grace=GRACE)
|
||
self.assertEqual(self.row(att_id), ("ready",))
|
||
self.assertIsNotNone(self.store.head(key))
|
||
# Даже прямое удаление строки запрещено БД, пока на неё ссылается сообщение.
|
||
from psycopg import errors
|
||
with self.assertRaises(errors.ForeignKeyViolation):
|
||
with self.pool.connection() as conn:
|
||
conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,))
|
||
|
||
def test_orphan_object_deleted_only_after_grace(self):
|
||
key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex)
|
||
self.store.put(key, b"orphan", "image/png", hashlib.md5(b"orphan").digest(), "0" * 64)
|
||
report = self.run_gc(apply=True, grace=GRACE)
|
||
self.assertGreaterEqual(report["orphan_objects_within_grace"], 1)
|
||
self.assertIsNotNone(self.store.head(key), "young orphan must survive (grace period)")
|
||
report = {}
|
||
self.rec.sweep_orphans(self.pool, self.store, 0, False, report, verify_ready=False)
|
||
self.assertIn(key, report["orphan_objects_sample"])
|
||
self.assertIsNotNone(self.store.head(key), "dry-run must not delete")
|
||
report = {}
|
||
self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False)
|
||
self.assertIsNone(self.store.head(key))
|
||
report = {}
|
||
self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False)
|
||
self.assertNotIn(key, report["orphan_objects_sample"])
|
||
|
||
def test_missing_object_is_reported_not_deleted(self):
|
||
att_id, key = self.pending_with_object()
|
||
self.storage.mark_attachment_ready(att_id)
|
||
self.store.delete(key) # внедрённая потеря байтов
|
||
report = self.run_gc(apply=True)
|
||
self.assertIn(str(att_id), report["missing_object_ids"])
|
||
self.assertEqual(self.row(att_id), ("ready",))
|
||
with self.pool.connection() as conn: # убрать искусственную запись, на неё нет ссылок
|
||
conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,))
|
||
|
||
def test_pending_cannot_be_attached(self):
|
||
att_id, key = self.pending_with_object()
|
||
from src.storage import AttachmentNotFound
|
||
with self.assertRaises(AttachmentNotFound):
|
||
self.storage.create_message(self.chat.id, self.user.id, "x", uuid.uuid4().hex, os.urandom(32),
|
||
86400, [att_id])
|
||
self.storage.delete_pending_attachment(att_id)
|
||
self.store.delete(key)
|
||
|
||
|
||
# ------------------------------------------------------------------ HTTP + DB
|
||
|
||
def _http(base, method, path, token=None, body=None, headers=None, timeout=20):
|
||
h = dict(headers or {})
|
||
if token:
|
||
h["Authorization"] = "Bearer " + token
|
||
data = body
|
||
if isinstance(body, dict):
|
||
data = json.dumps(body).encode()
|
||
h["Content-Type"] = "application/json"
|
||
req = Request(base.rstrip("/") + path, data=data, headers=h, method=method)
|
||
try:
|
||
with urlopen(req, timeout=timeout) as r:
|
||
return r.status, json.loads(r.read() or b"null")
|
||
except HTTPError as e:
|
||
return e.code, json.loads(e.read() or b"null")
|
||
|
||
|
||
def _actor(base):
|
||
import base64
|
||
name = "flt_" + uuid.uuid4().hex[:16]
|
||
pw = "Fault-" + uuid.uuid4().hex
|
||
s, user = _http(base, "POST", "/api/v1/users", body={"username": name, "display_name": "F", "password": pw})
|
||
assert s == 201, (s, user)
|
||
cred = base64.b64encode((name + ":" + pw).encode()).decode()
|
||
s, sess = _http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||
assert s == 201, (s, sess)
|
||
s, chat = _http(base, "POST", "/api/v1/chats", sess["token"], {"title": "faults", "member_ids": [user["id"]]})
|
||
assert s == 201, (s, chat)
|
||
return user, sess["token"], chat
|
||
|
||
|
||
@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured")
|
||
class UploadFailureTests(unittest.TestCase):
|
||
"""Отказы на пути загрузки, наблюдаемые со стороны БД и бакета."""
|
||
|
||
@classmethod
|
||
def setUpClass(cls):
|
||
AttachmentConsistencyTests.setUpClass.__func__(cls)
|
||
import imagegen
|
||
cls.png = imagegen.sample_png()
|
||
|
||
@classmethod
|
||
def tearDownClass(cls):
|
||
cls.pool.close()
|
||
|
||
def rows_of(self, owner_id):
|
||
with self.pool.connection() as conn:
|
||
return conn.execute("SELECT id, state, COALESCE(staging_key, object_key) FROM attachments "
|
||
"WHERE owner_id = %s",
|
||
(owner_id,)).fetchall()
|
||
|
||
def object_count(self):
|
||
total, token = 0, None
|
||
while True:
|
||
items, token = self.store.list_page("staging/", token)
|
||
total += len(items)
|
||
if not token:
|
||
return total
|
||
|
||
def upload(self, base, token, chat):
|
||
return _http(base, "POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], token,
|
||
self.png, {"Content-Type": "image/png"})
|
||
|
||
@unittest.skipUnless(os.environ.get("BASE_URL"), "BASE_URL is not set")
|
||
def test_aborted_upload_leaves_no_row_and_no_object(self):
|
||
base = os.environ["BASE_URL"]
|
||
user, token, chat = _actor(base)
|
||
before = self.object_count()
|
||
u = urlparse(base)
|
||
sock = socket.create_connection((u.hostname, u.port or 80), timeout=5)
|
||
head = ("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: %s\r\n"
|
||
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
|
||
% (chat["id"], u.hostname, token, 5 * 1024 * 1024)).encode()
|
||
sock.sendall(head + self.png + b"\x00" * 200000) # ~0.2 MiB из заявленных 5 MiB
|
||
time.sleep(0.5)
|
||
sock.close() # обрыв
|
||
time.sleep(2)
|
||
self.assertEqual(self.rows_of(user["id"]), [])
|
||
self.assertEqual(self.object_count(), before)
|
||
s, att = self.upload(base, token, chat) # сервис продолжает работать
|
||
self.assertEqual(s, 202, att)
|
||
|
||
@unittest.skipUnless(os.environ.get("FAULT_S3_URL"), "FAULT_S3_URL (api-fault-s3) is not set")
|
||
def test_s3_put_failure_returns_503_and_compensates(self):
|
||
base = os.environ["FAULT_S3_URL"]
|
||
user, token, chat = _actor(base)
|
||
s, body = self.upload(base, token, chat)
|
||
self.assertEqual((s, body["error"]["code"]), (503, "unavailable"))
|
||
self.assertEqual(self.rows_of(user["id"]), [], "pending row must be removed by compensation")
|
||
|
||
@unittest.skipUnless(os.environ.get("FAULT_DB_URL"), "FAULT_DB_URL (api-fault-db) is not set")
|
||
def test_db_failure_after_put_returns_503_and_deletes_object(self):
|
||
base = os.environ["FAULT_DB_URL"]
|
||
user, token, chat = _actor(base)
|
||
before = self.object_count()
|
||
s, body = self.upload(base, token, chat)
|
||
self.assertEqual((s, body["error"]["code"]), (503, "unavailable"))
|
||
self.assertEqual(self.rows_of(user["id"]), [])
|
||
self.assertEqual(self.object_count(), before, "object must be deleted by compensation")
|
||
|
||
@unittest.skipUnless(os.environ.get("FAULT_CRASH_URL"), "FAULT_CRASH_URL (api-fault-crash) is not set")
|
||
def test_process_crash_after_put_is_collected_by_gc(self):
|
||
base = os.environ["FAULT_CRASH_URL"]
|
||
user, token, chat = _actor(base)
|
||
try:
|
||
self.upload(base, token, chat)
|
||
self.fail("instance was expected to crash")
|
||
except (URLError, OSError, ConnectionError):
|
||
pass # процесс умер между S3 PUT и COMMIT
|
||
rows = self.rows_of(user["id"])
|
||
self.assertEqual(len(rows), 1)
|
||
att_id, state, key = rows[0]
|
||
self.assertEqual(state, "pending")
|
||
self.assertIsNotNone(self.store.head(key), "S3 PUT happened before the crash")
|
||
# Через grace сборщик убирает и строку, и объект (ускоряем, сдвигая created_at).
|
||
with self.pool.connection() as conn:
|
||
conn.execute("UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = %s",
|
||
(att_id,))
|
||
from src import reconcile
|
||
report = None
|
||
for _ in range(50):
|
||
report = reconcile.run_once(self.pool, self.store, GRACE, True, orphan_sweep=False)
|
||
if report:
|
||
break
|
||
time.sleep(0.2)
|
||
self.assertIn(str(att_id), report["stale_pending_ids"])
|
||
self.assertEqual(self.rows_of(user["id"]), [])
|
||
self.assertIsNone(self.store.head(key))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|