forked from mtusi2026/mtusi2026lab5
250 lines
15 KiB
Python
250 lines
15 KiB
Python
"""Замечание к MR: уборщик S3 не удаляет текущую работу — решение под блокировками в настоящей PostgreSQL.
|
|
|
|
Выполняется ВНУТРИ контейнера приложения вместе с остальными тестами БД:
|
|
make db-test == docker compose --profile tools run --rm db-test
|
|
|
|
Уборщик проверяется без S3: вместо удаления объекта ему передаётся функция-«барьер», которая
|
|
запоминает ключ и, если нужно, ждёт сигнала теста. Так порядок событий задаётся точно, как
|
|
в воспроизведении из замечания (барьер в тестовом драйвере), а не надеждой на тайминги:
|
|
1. снимок ссылок -> replay коммитит queued -> уборщик продолжает: staging НЕ удаляется
|
|
(решение перепроверяется по текущему состоянию под блокировкой);
|
|
2. уборщик под блокировкой уже пометил staging истёкшего failed-вложения удалённым, COMMIT ещё
|
|
нет -> replay ждёт и безопасно отказывает source_missing, задание остаётся failed, в журнале
|
|
обе записи; сам объект удаляется после COMMIT;
|
|
3. replay держит блокировку (проверяет исходник) -> уборщик ждёт и видит queued: не удаляет;
|
|
4. загрузка записала staging, COMMIT строки ещё не было -> уборщик вложение пропускает;
|
|
5. задание в processing (worker пишет оригинал и варианты) -> ничего не удаляется;
|
|
6. обычная уборка не сломана: «сирота» без строки, staging готового и объекты failed-вложения
|
|
после retention удаляются; опубликованное и всё failed-вложения до конца retention — нет.
|
|
"""
|
|
import threading
|
|
import time
|
|
import unittest
|
|
import uuid
|
|
|
|
from messenger import config, db, faults, jobs
|
|
from messenger.store import PostgresStore
|
|
|
|
RETENTION = 3600
|
|
|
|
|
|
class Barrier:
|
|
"""«Удаление объекта» для уборщика: запоминает ключи; может остановиться и ждать сигнала."""
|
|
|
|
def __init__(self, hold: bool = False):
|
|
self.deleted, self.entered, self.release = [], threading.Event(), threading.Event()
|
|
if not hold:
|
|
self.release.set()
|
|
|
|
def __call__(self, key):
|
|
self.deleted.append(key)
|
|
self.entered.set()
|
|
assert self.release.wait(20), "the test never released the barrier"
|
|
|
|
|
|
class CleanupRaceTest(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
config.validate()
|
|
db.migrate()
|
|
cls.pool = db.create_pool(max_size=6)
|
|
cls.pool.open(wait=True)
|
|
cls.store = PostgresStore(cls.pool)
|
|
cls.jobs = jobs.JobStore(cls.pool, "operator:db-test")
|
|
cls.worker = jobs.JobStore(cls.pool, "worker-db-test")
|
|
cls.user = cls.store.create_user("cl_" + uuid.uuid4().hex[:12], "Cleanup test", "not-a-real-hash")
|
|
cls.chat = cls.store.create_chat(cls.user.id, "cleanup", [cls.user.id])
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.pool.close()
|
|
|
|
def scalar(self, sql, *params):
|
|
with self.pool.connection() as conn:
|
|
return conn.execute(sql, params).fetchone()[0]
|
|
|
|
def execute(self, sql, *params):
|
|
with self.pool.connection() as conn:
|
|
conn.execute(sql, params)
|
|
|
|
def queued(self):
|
|
attachment_id = str(uuid.uuid4())
|
|
self.store.create_queued_attachment(
|
|
attachment_id, self.user.id, self.chat.id, "c.png", "image/png", 10, "0" * 64, 2, 2,
|
|
staging_key="staging/" + attachment_id, storage_key="originals/" + attachment_id, actor="db-test")
|
|
return attachment_id
|
|
|
|
def failed(self, expired: bool):
|
|
"""failed-вложение; expired — retention истёк (как в замечании: возраст через updated_at)."""
|
|
attachment_id = self.queued()
|
|
job_id = str(self.scalar("SELECT job_id FROM attachment_jobs WHERE attachment_id = %s", uuid.UUID(attachment_id)))
|
|
message = {"job_id": job_id, "attachment_id": attachment_id, "pipeline_version": 1, "attempt": 1}
|
|
lease = self.worker.claim(message, "host/1").lease
|
|
self.worker.fail(lease, "invalid_image", "test")
|
|
if expired:
|
|
self.execute("UPDATE attachments SET updated_at = now() - interval '2 days' WHERE id = %s",
|
|
uuid.UUID(attachment_id))
|
|
return attachment_id, job_id
|
|
|
|
def state(self, attachment_id):
|
|
with self.pool.connection() as conn:
|
|
return conn.execute("""SELECT a.status, a.staging_key IS NOT NULL, j.status FROM attachments a
|
|
JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id = %s""",
|
|
(uuid.UUID(attachment_id),)).fetchone()
|
|
|
|
def events(self, job_id):
|
|
with self.pool.connection() as conn:
|
|
return [r[0] for r in conn.execute("SELECT event FROM job_events WHERE job_id = %s ORDER BY id",
|
|
(uuid.UUID(job_id),))]
|
|
|
|
def test_1_replay_after_snapshot_keeps_staging(self):
|
|
attachment_id, job_id = self.failed(expired=True)
|
|
staging = "staging/" + attachment_id
|
|
refs = self.store.storage_references(RETENTION) # уборщик прочитал ссылки...
|
|
self.assertNotIn(staging, refs["needed"]) # ...по снимку staging лишний
|
|
self.assertEqual(self.jobs.replay(attachment_id, lambda key: True)["result"], "queued") # барьер: replay
|
|
barrier = Barrier()
|
|
outcome = self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier) # уборщик продолжает
|
|
self.assertEqual((outcome["delete"], outcome["keep"]), ([], [(staging, "current_work")]))
|
|
self.assertEqual(barrier.deleted, [])
|
|
self.assertEqual(self.state(attachment_id), ("queued", True, "queued"))
|
|
print(f"\n snapshot -> replay (queued) -> cleanup: kept {outcome['keep']}, nothing deleted")
|
|
|
|
def test_2_cleanup_first_then_replay_waits_and_refuses(self):
|
|
attachment_id, job_id = self.failed(expired=True)
|
|
staging = "staging/" + attachment_id
|
|
locked, release = threading.Event(), threading.Event()
|
|
real_pause = faults.pause
|
|
|
|
def pause(name): # барьер внутри транзакции уборщика (точка cleanup_pause_locked)
|
|
if name == "cleanup_pause_locked":
|
|
locked.set()
|
|
release.wait(20)
|
|
faults.pause = pause
|
|
try:
|
|
barrier, cleanup = Barrier(), {}
|
|
janitor = threading.Thread(target=lambda: cleanup.update(
|
|
self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier)))
|
|
janitor.start()
|
|
self.assertTrue(locked.wait(10), "cleanup did not reach its decision") # держит блокировки
|
|
replay = {}
|
|
operator = threading.Thread(target=lambda: replay.update(self.jobs.replay(attachment_id, lambda key: True)))
|
|
operator.start()
|
|
time.sleep(1.0)
|
|
self.assertTrue(operator.is_alive(), "replay must wait for the cleanup's lock")
|
|
self.assertEqual(barrier.deleted, []) # до COMMIT в S3 ничего не удаляется
|
|
release.set()
|
|
janitor.join(10)
|
|
operator.join(10)
|
|
finally:
|
|
faults.pause = real_pause
|
|
self.assertEqual(cleanup["delete"], [(staging, "staging_failed_expired")])
|
|
self.assertEqual(barrier.deleted, [staging]) # удалён после COMMIT
|
|
self.assertEqual((replay["result"], replay["reason"]), ("refused", "source_missing"))
|
|
self.assertEqual(self.state(attachment_id), ("failed", False, "failed"))
|
|
# Порядок в журнале: сначала уборщик, потом отказ replay (записи worker стенда, если он
|
|
# успел увидеть тестовое задание, сюда не относятся).
|
|
ours = [e for e in self.events(job_id) if e in ("staging_expired", "replay_refused", "replay_requested")]
|
|
self.assertEqual(ours, ["staging_expired", "replay_refused"])
|
|
print(f"\n cleanup marked the staging deleted, holds the lock -> replay waited 1 s+ -> {replay['result']} "
|
|
f"({replay['reason']}); job stays failed; journal: staging_expired, replay_refused")
|
|
|
|
def test_3_replay_first_then_cleanup_waits_and_keeps(self):
|
|
attachment_id, job_id = self.failed(expired=True)
|
|
staging = "staging/" + attachment_id
|
|
checking, release = threading.Event(), threading.Event()
|
|
|
|
def slow_head(key): # replay проверяет исходник в S3, держа блокировки
|
|
checking.set()
|
|
return release.wait(20)
|
|
replay = {}
|
|
operator = threading.Thread(target=lambda: replay.update(self.jobs.replay(attachment_id, slow_head)))
|
|
operator.start()
|
|
self.assertTrue(checking.wait(10))
|
|
cleanup, barrier = {}, Barrier()
|
|
janitor = threading.Thread(target=lambda: cleanup.update(
|
|
self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier) or {"busy": True}))
|
|
janitor.start()
|
|
time.sleep(1.0)
|
|
self.assertTrue(janitor.is_alive(), "cleanup must wait for the replay's lock")
|
|
release.set()
|
|
operator.join(10)
|
|
janitor.join(10)
|
|
self.assertEqual(replay["result"], "queued")
|
|
self.assertEqual(cleanup.get("keep"), [(staging, "current_work")])
|
|
self.assertEqual(barrier.deleted, [])
|
|
print(f"\n replay holds the lock -> cleanup waited -> kept {cleanup['keep']}")
|
|
|
|
def test_4_upload_in_progress_is_skipped(self):
|
|
attachment_id = str(uuid.uuid4())
|
|
staging = "staging/" + attachment_id
|
|
inside, finish = threading.Event(), threading.Event()
|
|
|
|
def upload(): # как attachments.accept: PUT в staging, потом строка, COMMIT
|
|
with self.store.receiving(attachment_id) as conn:
|
|
inside.set() # «PUT в staging прошёл»
|
|
finish.wait(20)
|
|
self.store.create_queued_attachment(
|
|
attachment_id, self.user.id, self.chat.id, "u.png", "image/png", 10, "0" * 64, 2, 2,
|
|
staging_key=staging, storage_key="originals/" + attachment_id, actor="db-test", conn=conn)
|
|
api = threading.Thread(target=upload)
|
|
api.start()
|
|
self.assertTrue(inside.wait(10))
|
|
barrier = Barrier()
|
|
self.assertIsNone(self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier)) # занято
|
|
finish.set()
|
|
api.join(10)
|
|
after = self.store.cleanup_attachment(attachment_id, [staging], RETENTION, barrier)
|
|
self.assertEqual((after["keep"], barrier.deleted), ([(staging, "current_work")], []))
|
|
self.assertEqual(self.state(attachment_id), ("queued", True, "queued"))
|
|
print("\n upload between PUT and COMMIT -> cleanup skipped it (busy); after COMMIT -> current work, kept")
|
|
|
|
def test_5_objects_of_processing_job_are_kept(self):
|
|
attachment_id = self.queued()
|
|
job_id = str(self.scalar("SELECT job_id FROM attachment_jobs WHERE attachment_id = %s", uuid.UUID(attachment_id)))
|
|
self.worker.claim({"job_id": job_id, "attachment_id": attachment_id, "pipeline_version": 1, "attempt": 1},
|
|
"host/1")
|
|
keys = [f"originals/{attachment_id}", f"variants/{attachment_id}/v1/thumbnail.jpg", f"staging/{attachment_id}"]
|
|
barrier = Barrier()
|
|
outcome = self.store.cleanup_attachment(attachment_id, keys, RETENTION, barrier)
|
|
self.assertEqual((outcome["delete"], barrier.deleted), ([], []))
|
|
self.assertEqual({why for _key, why in outcome["keep"]}, {"current_work"})
|
|
print("\n processing job (worker writes the original and variants before COMMIT) -> all 3 objects kept")
|
|
|
|
def test_6_regular_cleanup_still_works(self):
|
|
barrier = Barrier()
|
|
orphan = str(uuid.uuid4()) # строки нет, загрузка не идёт
|
|
out = self.store.cleanup_attachment(orphan, [f"staging/{orphan}"], RETENTION, barrier)
|
|
self.assertEqual(out["delete"], [(f"staging/{orphan}", "orphan")])
|
|
kept_id, _job = self.failed(expired=False) # retention не истёк: нужен для replay
|
|
out = self.store.cleanup_attachment(kept_id, [f"staging/{kept_id}", f"originals/{kept_id}"], RETENTION, barrier)
|
|
self.assertEqual(out["keep"], [(f"staging/{kept_id}", "staging_kept_for_replay"),
|
|
(f"originals/{kept_id}", "failed_kept_for_replay")]) # replay перезапишет
|
|
dead_id, dead_job = self.failed(expired=True) # retention истёк: удаляется всё
|
|
out = self.store.cleanup_attachment(dead_id, [f"originals/{dead_id}"], RETENTION, barrier)
|
|
self.assertEqual(out["delete"], [(f"originals/{dead_id}", "orphan")])
|
|
self.assertEqual(self.state(dead_id)[1], False) # staging помечен удалённым до удаления
|
|
self.assertEqual(self.jobs.replay(dead_id, lambda key: True)["reason"], "source_missing")
|
|
ready = self.queued()
|
|
job_id = str(self.scalar("SELECT job_id FROM attachment_jobs WHERE attachment_id = %s", uuid.UUID(ready)))
|
|
lease = self.worker.claim({"job_id": job_id, "attachment_id": ready, "pipeline_version": 1, "attempt": 1},
|
|
"host/1").lease
|
|
variants = [{"name": n, "storage_key": f"variants/{ready}/v1/{n}.jpg", "content_type": "image/jpeg",
|
|
"size_bytes": 5, "width": 1, "height": 1, "sha256": "a" * 64} for n in ("thumbnail", "crop")]
|
|
self.assertTrue(self.worker.complete(lease, variants))
|
|
keys = [f"staging/{ready}", f"originals/{ready}", f"variants/{ready}/v1/thumbnail.jpg",
|
|
f"variants/{ready}/v1/stale.jpg"]
|
|
out = self.store.cleanup_attachment(ready, keys, RETENTION, barrier)
|
|
self.assertEqual(sorted(out["delete"]), sorted([(f"staging/{ready}", "staging_after_ready"),
|
|
(f"variants/{ready}/v1/stale.jpg", "orphan")]))
|
|
self.assertEqual({why for _key, why in out["keep"]}, {"published"})
|
|
dry = self.store.cleanup_attachment(orphan, [f"staging/{orphan}"], RETENTION, None) # пробный прогон
|
|
self.assertEqual(dry["delete"], [(f"staging/{orphan}", "orphan")])
|
|
self.assertEqual(len(barrier.deleted), 4) # dry-run ничего не удалил
|
|
print("\n orphan, staging after ready, objects of a failed attachment after retention -> deleted; "
|
|
"published original and variants, a failed attachment within retention -> kept; dry-run deletes nothing")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|