Files
mtusi2026lab5/student-tests/db/test_cleanup.py
T
2026-10-26 19:40:34 +03:00

250 lines
15 KiB
Python

"""Замечание к MR: уборщик S3 не удаляет текущую работу — решение под блокировками в настоящей PostgreSQL.
Выполняется ВНУТРИ контейнера приложения вместе с остальными тестами БД:
make db-test == docker compose --profile tools run --rm db-test
Уборщик проверяется без S3: вместо удаления объекта ему передаётся функция-«барьер», которая
запоминает ключ и, если нужно, ждёт сигнала теста. Так порядок событий задаётся точно, как
в воспроизведении из замечания (барьер в тестовом драйвере), а не надеждой на тайминги:
1. снимок ссылок -> replay коммитит queued -> уборщик продолжает: staging НЕ удаляется
(решение перепроверяется по текущему состоянию под блокировкой);
2. уборщик под блокировкой уже пометил staging истёкшего failed-вложения удалённым, COMMIT ещё
нет -> replay ждёт и безопасно отказывает source_missing, задание остаётся failed, в журнале
обе записи; сам объект удаляется после COMMIT;
3. replay держит блокировку (проверяет исходник) -> уборщик ждёт и видит queued: не удаляет;
4. загрузка записала staging, COMMIT строки ещё не было -> уборщик вложение пропускает;
5. задание в processing (worker пишет оригинал и варианты) -> ничего не удаляется;
6. обычная уборка не сломана: «сирота» без строки, staging готового и объекты failed-вложения
после retention удаляются; опубликованное и всё failed-вложения до конца retention — нет.
"""
import threading
import time
import unittest
import uuid
from messenger import config, db, faults, jobs
from messenger.store import PostgresStore
RETENTION = 3600
class Barrier:
"""«Удаление объекта» для уборщика: запоминает ключи; может остановиться и ждать сигнала."""
def __init__(self, hold: bool = False):
self.deleted, self.entered, self.release = [], threading.Event(), threading.Event()
if not hold:
self.release.set()
def __call__(self, key):
self.deleted.append(key)
self.entered.set()
assert self.release.wait(20), "the test never released the barrier"
class CleanupRaceTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
config.validate()
db.migrate()
cls.pool = db.create_pool(max_size=6)
cls.pool.open(wait=True)
cls.store = PostgresStore(cls.pool)
cls.jobs = jobs.JobStore(cls.pool, "operator:db-test")
cls.worker = jobs.JobStore(cls.pool, "worker-db-test")
cls.user = cls.store.create_user("cl_" + uuid.uuid4().hex[:12], "Cleanup test", "not-a-real-hash")
cls.chat = cls.store.create_chat(cls.user.id, "cleanup", [cls.user.id])
@classmethod
def tearDownClass(cls):
cls.pool.close()
def scalar(self, sql, *params):
with self.pool.connection() as conn:
return conn.execute(sql, params).fetchone()[0]
def execute(self, sql, *params):
with self.pool.connection() as conn:
conn.execute(sql, params)
def queued(self):
attachment_id = str(uuid.uuid4())
self.store.create_queued_attachment(
attachment_id, self.user.id, self.chat.id, "c.png", "image/png", 10, "0" * 64, 2, 2,
staging_key="staging/" + attachment_id, storage_key="originals/" + attachment_id, actor="db-test")
return attachment_id
def failed(self, expired: bool):
"""failed-вложение; expired — retention истёк (как в замечании: возраст через updated_at)."""
attachment_id = self.queued()
job_id = str(self.scalar("SELECT job_id FROM attachment_jobs WHERE attachment_id = %s", uuid.UUID(attachment_id)))
message = {"job_id": job_id, "attachment_id": attachment_id, "pipeline_version": 1, "attempt": 1}
lease = self.worker.claim(message, "host/1").lease
self.worker.fail(lease, "invalid_image", "test")
if expired:
self.execute("UPDATE attachments SET updated_at = now() - interval '2 days' WHERE id = %s",
uuid.UUID(attachment_id))
return attachment_id, job_id
def state(self, attachment_id):
with self.pool.connection() as conn:
return conn.execute("""SELECT a.status, a.staging_key IS NOT NULL, j.status FROM attachments a
JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id = %s""",
(uuid.UUID(attachment_id),)).fetchone()
def events(self, job_id):
with self.pool.connection() as conn:
return [r[0] for r in conn.execute("SELECT event FROM job_events WHERE job_id = %s ORDER BY id",
(uuid.UUID(job_id),))]
def test_1_replay_after_snapshot_keeps_staging(self):
attachment_id, job_id = self.failed(expired=True)
staging = "staging/" + attachment_id
refs = self.store.storage_references(RETENTION) # уборщик прочитал ссылки...
self.assertNotIn(staging, refs["needed"]) # ...по снимку staging лишний
self.assertEqual(self.jobs.replay(attachment_id, lambda key: True)["result"], "queued") # барьер: replay
barrier = Barrier()
outcome = self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier) # уборщик продолжает
self.assertEqual((outcome["delete"], outcome["keep"]), ([], [(staging, "current_work")]))
self.assertEqual(barrier.deleted, [])
self.assertEqual(self.state(attachment_id), ("queued", True, "queued"))
print(f"\n snapshot -> replay (queued) -> cleanup: kept {outcome['keep']}, nothing deleted")
def test_2_cleanup_first_then_replay_waits_and_refuses(self):
attachment_id, job_id = self.failed(expired=True)
staging = "staging/" + attachment_id
locked, release = threading.Event(), threading.Event()
real_pause = faults.pause
def pause(name): # барьер внутри транзакции уборщика (точка cleanup_pause_locked)
if name == "cleanup_pause_locked":
locked.set()
release.wait(20)
faults.pause = pause
try:
barrier, cleanup = Barrier(), {}
janitor = threading.Thread(target=lambda: cleanup.update(
self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier)))
janitor.start()
self.assertTrue(locked.wait(10), "cleanup did not reach its decision") # держит блокировки
replay = {}
operator = threading.Thread(target=lambda: replay.update(self.jobs.replay(attachment_id, lambda key: True)))
operator.start()
time.sleep(1.0)
self.assertTrue(operator.is_alive(), "replay must wait for the cleanup's lock")
self.assertEqual(barrier.deleted, []) # до COMMIT в S3 ничего не удаляется
release.set()
janitor.join(10)
operator.join(10)
finally:
faults.pause = real_pause
self.assertEqual(cleanup["delete"], [(staging, "staging_failed_expired")])
self.assertEqual(barrier.deleted, [staging]) # удалён после COMMIT
self.assertEqual((replay["result"], replay["reason"]), ("refused", "source_missing"))
self.assertEqual(self.state(attachment_id), ("failed", False, "failed"))
# Порядок в журнале: сначала уборщик, потом отказ replay (записи worker стенда, если он
# успел увидеть тестовое задание, сюда не относятся).
ours = [e for e in self.events(job_id) if e in ("staging_expired", "replay_refused", "replay_requested")]
self.assertEqual(ours, ["staging_expired", "replay_refused"])
print(f"\n cleanup marked the staging deleted, holds the lock -> replay waited 1 s+ -> {replay['result']} "
f"({replay['reason']}); job stays failed; journal: staging_expired, replay_refused")
def test_3_replay_first_then_cleanup_waits_and_keeps(self):
attachment_id, job_id = self.failed(expired=True)
staging = "staging/" + attachment_id
checking, release = threading.Event(), threading.Event()
def slow_head(key): # replay проверяет исходник в S3, держа блокировки
checking.set()
return release.wait(20)
replay = {}
operator = threading.Thread(target=lambda: replay.update(self.jobs.replay(attachment_id, slow_head)))
operator.start()
self.assertTrue(checking.wait(10))
cleanup, barrier = {}, Barrier()
janitor = threading.Thread(target=lambda: cleanup.update(
self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier) or {"busy": True}))
janitor.start()
time.sleep(1.0)
self.assertTrue(janitor.is_alive(), "cleanup must wait for the replay's lock")
release.set()
operator.join(10)
janitor.join(10)
self.assertEqual(replay["result"], "queued")
self.assertEqual(cleanup.get("keep"), [(staging, "current_work")])
self.assertEqual(barrier.deleted, [])
print(f"\n replay holds the lock -> cleanup waited -> kept {cleanup['keep']}")
def test_4_upload_in_progress_is_skipped(self):
attachment_id = str(uuid.uuid4())
staging = "staging/" + attachment_id
inside, finish = threading.Event(), threading.Event()
def upload(): # как attachments.accept: PUT в staging, потом строка, COMMIT
with self.store.receiving(attachment_id) as conn:
inside.set() # «PUT в staging прошёл»
finish.wait(20)
self.store.create_queued_attachment(
attachment_id, self.user.id, self.chat.id, "u.png", "image/png", 10, "0" * 64, 2, 2,
staging_key=staging, storage_key="originals/" + attachment_id, actor="db-test", conn=conn)
api = threading.Thread(target=upload)
api.start()
self.assertTrue(inside.wait(10))
barrier = Barrier()
self.assertIsNone(self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier)) # занято
finish.set()
api.join(10)
after = self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier)
self.assertEqual((after["keep"], barrier.deleted), ([(staging, "current_work")], []))
self.assertEqual(self.state(attachment_id), ("queued", True, "queued"))
print("\n upload between PUT and COMMIT -> cleanup skipped it (busy); after COMMIT -> current work, kept")
def test_5_objects_of_processing_job_are_kept(self):
attachment_id = self.queued()
job_id = str(self.scalar("SELECT job_id FROM attachment_jobs WHERE attachment_id = %s", uuid.UUID(attachment_id)))
self.worker.claim({"job_id": job_id, "attachment_id": attachment_id, "pipeline_version": 1, "attempt": 1},
"host/1")
keys = [f"originals/{attachment_id}", f"variants/{attachment_id}/v1/thumbnail.jpg", f"staging/{attachment_id}"]
barrier = Barrier()
outcome = self.store.cleanup_attachment(attachment_id, keys, RETENTION, barrier)
self.assertEqual((outcome["delete"], barrier.deleted), ([], []))
self.assertEqual({why for _key, why in outcome["keep"]}, {"current_work"})
print("\n processing job (worker writes the original and variants before COMMIT) -> all 3 objects kept")
def test_6_regular_cleanup_still_works(self):
barrier = Barrier()
orphan = str(uuid.uuid4()) # строки нет, загрузка не идёт
out = self.store.cleanup_attachment(orphan, [f"staging/{orphan}"], RETENTION, barrier)
self.assertEqual(out["delete"], [(f"staging/{orphan}", "orphan")])
kept_id, _job = self.failed(expired=False) # retention не истёк: нужен для replay
out = self.store.cleanup_attachment(kept_id, [f"staging/{kept_id}", f"originals/{kept_id}"], RETENTION, barrier)
self.assertEqual(out["keep"], [(f"staging/{kept_id}", "staging_kept_for_replay"),
(f"originals/{kept_id}", "failed_kept_for_replay")]) # replay перезапишет
dead_id, dead_job = self.failed(expired=True) # retention истёк: удаляется всё
out = self.store.cleanup_attachment(dead_id, [f"originals/{dead_id}"], RETENTION, barrier)
self.assertEqual(out["delete"], [(f"originals/{dead_id}", "orphan")])
self.assertEqual(self.state(dead_id)[1], False) # staging помечен удалённым до удаления
self.assertEqual(self.jobs.replay(dead_id, lambda key: True)["reason"], "source_missing")
ready = self.queued()
job_id = str(self.scalar("SELECT job_id FROM attachment_jobs WHERE attachment_id = %s", uuid.UUID(ready)))
lease = self.worker.claim({"job_id": job_id, "attachment_id": ready, "pipeline_version": 1, "attempt": 1},
"host/1").lease
variants = [{"name": n, "storage_key": f"variants/{ready}/v1/{n}.jpg", "content_type": "image/jpeg",
"size_bytes": 5, "width": 1, "height": 1, "sha256": "a" * 64} for n in ("thumbnail", "crop")]
self.assertTrue(self.worker.complete(lease, variants))
keys = [f"staging/{ready}", f"originals/{ready}", f"variants/{ready}/v1/thumbnail.jpg",
f"variants/{ready}/v1/stale.jpg"]
out = self.store.cleanup_attachment(ready, keys, RETENTION, barrier)
self.assertEqual(sorted(out["delete"]), sorted([(f"staging/{ready}", "staging_after_ready"),
(f"variants/{ready}/v1/stale.jpg", "orphan")]))
self.assertEqual({why for _key, why in out["keep"]}, {"published"})
dry = self.store.cleanup_attachment(orphan, [f"staging/{orphan}"], RETENTION, None) # пробный прогон
self.assertEqual(dry["delete"], [(f"staging/{orphan}", "orphan")])
self.assertEqual(len(barrier.deleted), 4) # dry-run ничего не удалил
print("\n orphan, staging after ready, objects of a failed attachment after retention -> deleted; "
"published original and variants, a failed attachment within retention -> kept; dry-run deletes nothing")
if __name__ == "__main__":
unittest.main(verbosity=2)