Files
mtusi2026lab6-fork/student-tests/test_attachment_consistency.py
T

311 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Согласованность PostgreSQL <-> S3 для вложений (нужен прямой доступ к БД и S3).
lab5: API пишет байты в staging/ (а не originals/), строка pending ссылается на
staging_key; успешная загрузка отвечает 202. Логика сборщика та же.
Запуск внутри compose-сети (БД и S3 наружу не публикуются):
python scripts/ops.py in-network consistency
Без DATABASE_URL(_FILE) и S3-настроек тесты пропускаются.
Что доказывается:
* сбой между S3 PUT и COMMIT (pending + объект) обнаруживается dry-run'ом,
исправляется --apply, повторный запуск ничего не меняет (идемпотентность);
* свежие pending (моложе grace) и ready/используемые вложения не удаляются;
* объект-сирота (нет строки) удаляется только после grace;
* ready без объекта — только отчёт (missing_objects), без авто-удаления;
* обрыв загрузки клиентом не оставляет ни строки, ни объекта;
* внедрённые отказы в API (если заданы FAULT_*_URL — экземпляры api-fault-*):
ошибка S3 PUT -> 503 и компенсация; ошибка БД после PUT -> 503, объект удалён;
падение процесса после PUT -> остаётся pending+объект, их убирает сборщик.
"""
import hashlib
import json
import os
import socket
import sys
import time
import unittest
import uuid
from pathlib import Path
from urllib.parse import urlparse
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent))
sys.path.insert(0, str(HERE))
HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE"))
HAS_S3 = bool(os.environ.get("S3_ENDPOINT"))
GRACE = 900
@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured")
class AttachmentConsistencyTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
from src.config import load_settings
from src.db import create_pool
from src.objectstore import ObjectStore
from src.security import hash_password
from src.storage import Storage
from src import reconcile
cls.settings = load_settings()
cls.pool = create_pool(cls.settings)
cls.pool.wait(timeout=10)
cls.storage = Storage(cls.pool)
cls.store = ObjectStore(cls.settings)
cls.rec = reconcile
user = cls.storage.create_user("cons_" + uuid.uuid4().hex[:16], "Consistency", hash_password("Consistency-pw-1"))
cls.user = user
cls.chat = cls.storage.create_chat("consistency", [user.id])
@classmethod
def tearDownClass(cls):
cls.pool.close()
# helpers
def pending_with_object(self, age_seconds=0, put=True):
att_id = uuid.uuid4()
key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex)
data = b"\x89PNG fake bytes " + uuid.uuid4().bytes
ok = self.storage.create_pending_attachment(att_id, self.chat.id, self.user.id, "p.png", "image/png",
len(data), hashlib.sha256(data).hexdigest(), 1, 1, key)
self.assertTrue(ok)
if put:
self.store.put(key, data, "image/png", hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest())
if age_seconds:
self.backdate(att_id, age_seconds)
return att_id, key
def backdate(self, att_id, seconds):
with self.pool.connection() as conn:
conn.execute("UPDATE attachments SET created_at = created_at - make_interval(secs => %s) WHERE id = %s",
(seconds, att_id))
def row(self, att_id):
with self.pool.connection() as conn:
return conn.execute("SELECT state FROM attachments WHERE id = %s", (att_id,)).fetchone()
def run_gc(self, apply, grace=GRACE, sweep=True):
report = None
for _ in range(50): # фоновый GC реплики мог держать lock
report = self.rec.run_once(self.pool, self.store, grace, apply, orphan_sweep=sweep)
if report is not None:
return report
time.sleep(0.2)
self.fail("reconciler lock is busy")
# tests
def test_crash_between_put_and_commit_dry_run_then_apply_then_noop(self):
att_id, key = self.pending_with_object(age_seconds=GRACE + 60)
dry = self.run_gc(apply=False, sweep=False)
self.assertIn(str(att_id), dry["stale_pending_ids"])
self.assertIsNotNone(self.row(att_id), "dry-run must not change anything")
self.assertIsNotNone(self.store.head(key))
applied = self.run_gc(apply=True, sweep=False)
self.assertGreaterEqual(applied["stale_pending_rows_deleted"], 1)
self.assertIsNone(self.row(att_id))
self.assertIsNone(self.store.head(key))
again = self.run_gc(apply=True, sweep=False)
self.assertNotIn(str(att_id), again["stale_pending_ids"])
def test_fresh_pending_is_kept(self):
att_id, key = self.pending_with_object(age_seconds=0)
self.run_gc(apply=True)
self.assertEqual(self.row(att_id), ("pending",))
self.assertIsNotNone(self.store.head(key))
self.storage.delete_pending_attachment(att_id)
self.store.delete(key)
def test_ready_and_referenced_are_never_collected(self):
att_id, key = self.pending_with_object()
self.assertIsNotNone(self.storage.mark_attachment_ready(att_id))
msg, _ = self.storage.create_message(self.chat.id, self.user.id, "", uuid.uuid4().hex, os.urandom(32),
86400, [att_id])
self.assertEqual(msg.attachment_ids, [att_id])
self.backdate(att_id, 10 * GRACE)
self.run_gc(apply=True, grace=GRACE)
self.assertEqual(self.row(att_id), ("ready",))
self.assertIsNotNone(self.store.head(key))
# Даже прямое удаление строки запрещено БД, пока на неё ссылается сообщение.
from psycopg import errors
with self.assertRaises(errors.ForeignKeyViolation):
with self.pool.connection() as conn:
conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,))
def test_orphan_object_deleted_only_after_grace(self):
key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex)
self.store.put(key, b"orphan", "image/png", hashlib.md5(b"orphan").digest(), "0" * 64)
report = self.run_gc(apply=True, grace=GRACE)
self.assertGreaterEqual(report["orphan_objects_within_grace"], 1)
self.assertIsNotNone(self.store.head(key), "young orphan must survive (grace period)")
report = {}
self.rec.sweep_orphans(self.pool, self.store, 0, False, report, verify_ready=False)
self.assertIn(key, report["orphan_objects_sample"])
self.assertIsNotNone(self.store.head(key), "dry-run must not delete")
report = {}
self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False)
self.assertIsNone(self.store.head(key))
report = {}
self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False)
self.assertNotIn(key, report["orphan_objects_sample"])
def test_missing_object_is_reported_not_deleted(self):
att_id, key = self.pending_with_object()
self.storage.mark_attachment_ready(att_id)
self.store.delete(key) # внедрённая потеря байтов
report = self.run_gc(apply=True)
self.assertIn(str(att_id), report["missing_object_ids"])
self.assertEqual(self.row(att_id), ("ready",))
with self.pool.connection() as conn: # убрать искусственную запись, на неё нет ссылок
conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,))
def test_pending_cannot_be_attached(self):
att_id, key = self.pending_with_object()
from src.storage import AttachmentNotFound
with self.assertRaises(AttachmentNotFound):
self.storage.create_message(self.chat.id, self.user.id, "x", uuid.uuid4().hex, os.urandom(32),
86400, [att_id])
self.storage.delete_pending_attachment(att_id)
self.store.delete(key)
# ------------------------------------------------------------------ HTTP + DB
def _http(base, method, path, token=None, body=None, headers=None, timeout=20):
h = dict(headers or {})
if token:
h["Authorization"] = "Bearer " + token
data = body
if isinstance(body, dict):
data = json.dumps(body).encode()
h["Content-Type"] = "application/json"
req = Request(base.rstrip("/") + path, data=data, headers=h, method=method)
try:
with urlopen(req, timeout=timeout) as r:
return r.status, json.loads(r.read() or b"null")
except HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def _actor(base):
import base64
name = "flt_" + uuid.uuid4().hex[:16]
pw = "Fault-" + uuid.uuid4().hex
s, user = _http(base, "POST", "/api/v1/users", body={"username": name, "display_name": "F", "password": pw})
assert s == 201, (s, user)
cred = base64.b64encode((name + ":" + pw).encode()).decode()
s, sess = _http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert s == 201, (s, sess)
s, chat = _http(base, "POST", "/api/v1/chats", sess["token"], {"title": "faults", "member_ids": [user["id"]]})
assert s == 201, (s, chat)
return user, sess["token"], chat
@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured")
class UploadFailureTests(unittest.TestCase):
"""Отказы на пути загрузки, наблюдаемые со стороны БД и бакета."""
@classmethod
def setUpClass(cls):
AttachmentConsistencyTests.setUpClass.__func__(cls)
import imagegen
cls.png = imagegen.sample_png()
@classmethod
def tearDownClass(cls):
cls.pool.close()
def rows_of(self, owner_id):
with self.pool.connection() as conn:
return conn.execute("SELECT id, state, COALESCE(staging_key, object_key) FROM attachments "
"WHERE owner_id = %s",
(owner_id,)).fetchall()
def object_count(self):
total, token = 0, None
while True:
items, token = self.store.list_page("staging/", token)
total += len(items)
if not token:
return total
def upload(self, base, token, chat):
return _http(base, "POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], token,
self.png, {"Content-Type": "image/png"})
@unittest.skipUnless(os.environ.get("BASE_URL"), "BASE_URL is not set")
def test_aborted_upload_leaves_no_row_and_no_object(self):
base = os.environ["BASE_URL"]
user, token, chat = _actor(base)
before = self.object_count()
u = urlparse(base)
sock = socket.create_connection((u.hostname, u.port or 80), timeout=5)
head = ("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: %s\r\n"
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
% (chat["id"], u.hostname, token, 5 * 1024 * 1024)).encode()
sock.sendall(head + self.png + b"\x00" * 200000) # ~0.2 MiB из заявленных 5 MiB
time.sleep(0.5)
sock.close() # обрыв
time.sleep(2)
self.assertEqual(self.rows_of(user["id"]), [])
self.assertEqual(self.object_count(), before)
s, att = self.upload(base, token, chat) # сервис продолжает работать
self.assertEqual(s, 202, att)
@unittest.skipUnless(os.environ.get("FAULT_S3_URL"), "FAULT_S3_URL (api-fault-s3) is not set")
def test_s3_put_failure_returns_503_and_compensates(self):
base = os.environ["FAULT_S3_URL"]
user, token, chat = _actor(base)
s, body = self.upload(base, token, chat)
self.assertEqual((s, body["error"]["code"]), (503, "unavailable"))
self.assertEqual(self.rows_of(user["id"]), [], "pending row must be removed by compensation")
@unittest.skipUnless(os.environ.get("FAULT_DB_URL"), "FAULT_DB_URL (api-fault-db) is not set")
def test_db_failure_after_put_returns_503_and_deletes_object(self):
base = os.environ["FAULT_DB_URL"]
user, token, chat = _actor(base)
before = self.object_count()
s, body = self.upload(base, token, chat)
self.assertEqual((s, body["error"]["code"]), (503, "unavailable"))
self.assertEqual(self.rows_of(user["id"]), [])
self.assertEqual(self.object_count(), before, "object must be deleted by compensation")
@unittest.skipUnless(os.environ.get("FAULT_CRASH_URL"), "FAULT_CRASH_URL (api-fault-crash) is not set")
def test_process_crash_after_put_is_collected_by_gc(self):
base = os.environ["FAULT_CRASH_URL"]
user, token, chat = _actor(base)
try:
self.upload(base, token, chat)
self.fail("instance was expected to crash")
except (URLError, OSError, ConnectionError):
pass # процесс умер между S3 PUT и COMMIT
rows = self.rows_of(user["id"])
self.assertEqual(len(rows), 1)
att_id, state, key = rows[0]
self.assertEqual(state, "pending")
self.assertIsNotNone(self.store.head(key), "S3 PUT happened before the crash")
# Через grace сборщик убирает и строку, и объект (ускоряем, сдвигая created_at).
with self.pool.connection() as conn:
conn.execute("UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = %s",
(att_id,))
from src import reconcile
report = None
for _ in range(50):
report = reconcile.run_once(self.pool, self.store, GRACE, True, orphan_sweep=False)
if report:
break
time.sleep(0.2)
self.assertIn(str(att_id), report["stale_pending_ids"])
self.assertEqual(self.rows_of(user["id"]), [])
self.assertIsNone(self.store.head(key))
if __name__ == "__main__":
unittest.main()