Files
mtusi2026lab5/student-tests/staging_cleanup.py
T
2026-10-25 21:43:03 +03:00

113 lines
6.7 KiB
Python

"""lab5, уровень 4: очистка staging после безопасной фиксации и уборка объектов без ссылок из БД.
make staging-cleanup (= python student-tests/staging_cleanup.py)
1. Обычный путь: после COMMIT результата worker сам удаляет staging/<id>; остаются originals/<id>
и три варианта.
2. Состояния, которые оставляют сбои (создаются напрямую в S3 и через API):
- staging готового вложения (worker упал между COMMIT и удалением staging);
- staging без строки в БД (API упал между PUT в staging и COMMIT);
- вариант без строки в БД (worker записал объекты, а задание потом ушло в failed);
- staging failed-вложения (битый JPEG) — хранится STAGING_RETENTION_SECONDS для replay;
- staging задания в очереди (worker остановлен) — нужен, его трогать нельзя.
3. python -m messenger.cleanup:
--dry-run все найдены, ничего не удалено;
обычный запуск (grace 900 с) ничего не удалено: объекты моложе grace (это может быть
идущая прямо сейчас работа);
--grace-seconds 2 (только тест) удалены «сироты» и staging готового; staging задания в очереди,
staging failed (в пределах retention), оригиналы и варианты на месте;
--retention-seconds 0 удалён staging failed; replay такого вложения отказывается
(source_missing) вместо падения worker;
повторный запуск удалять нечего.
4. worker запущен: задание из очереди стало ready, его staging удалён.
"""
import json
import time
import uuid
import ops
from filegen import garbage_scan_jpeg, upload_when_ready, wait_status
from helpers import Client, check, run
PUT = """
import io
from messenger import objects
for key in %s:
objects.put(key, io.BytesIO(b"left-behind"), 11, "image/png", "0" * 64)
print("ok")
"""
def exists(keys):
present = set(ops.objects("staging/")) | set(ops.objects("originals/")) | set(ops.objects("variants/"))
return {name: key in present for name, key in keys.items()}
def cleanup(*args):
return ops.cli("api", "messenger.cleanup", *args)
def main():
api = Client(timeout=15)
ops.wait_healthy("worker", 1)
alice, bob, _eve, chat = api.bootstrap()
ready = upload_when_ready(api, alice, chat, name="keep.png")["json"]
wait_status(api, bob, ready["id"])
rid = ready["id"]
state = exists({"staging": f"staging/{rid}", "original": f"originals/{rid}",
"thumbnail": f"variants/{rid}/v1/thumbnail.jpg"})
check(state == {"staging": False, "original": True, "thumbnail": True}, f"unexpected objects: {state}")
print(f"1. ready {rid}: staging removed by the worker right after COMMIT; original and variants kept")
failed = upload_when_ready(api, alice, chat, data=garbage_scan_jpeg(), name="bad.jpg", ctype="image/jpeg")["json"]
wait_status(api, bob, failed["id"], ("failed",))
try:
ops.stop("worker")
queued = upload_when_ready(api, alice, chat, name="queued.png")["json"]
keys = {"ready_original": f"originals/{rid}", "ready_variant": f"variants/{rid}/v1/crop.jpg",
"ready_staging_leftover": f"staging/{rid}", "orphan_staging": f"staging/{uuid.uuid4()}",
"orphan_variant": f"variants/{uuid.uuid4()}/v1/thumbnail.jpg",
"failed_staging": f"staging/{failed['id']}", "queued_staging": f"staging/{queued['id']}"}
ops.python("api", PUT % json.dumps([keys["ready_staging_leftover"], keys["orphan_staging"],
keys["orphan_variant"]]))
check(all(exists(keys).values()), "setup failed")
print("2. prepared: staging leftover of a ready attachment, orphan staging, orphan variant, "
"staging of a failed attachment, staging of a queued job (worker stopped)")
dry = cleanup("--dry-run", "--grace-seconds", "0")
check(dry["deleted"] == 0 and all(exists(keys).values()), "dry-run deleted something")
print(f"3. dry-run: {json.dumps(dry['to_delete'])} found, deleted {dry['deleted']}")
normal = cleanup()
check(all(exists(keys).values()), "grace 900 s must protect young objects")
print(f" grace 900 s: deleted {normal['deleted']} — everything is younger than the grace period")
time.sleep(3)
short = cleanup("--grace-seconds", "2")
state = exists(keys)
check(not state["ready_staging_leftover"] and not state["orphan_staging"] and not state["orphan_variant"],
f"leftovers must be deleted: {state}")
check(state["ready_original"] and state["ready_variant"] and state["failed_staging"] and state["queued_staging"],
f"needed objects were deleted: {state}")
print(f" grace 2 s: {json.dumps(short['to_delete'])}, deleted {short['deleted']}; kept: ready original "
f"and variants, staging of the queued job, staging of the failed attachment (retention)")
expired = cleanup("--grace-seconds", "2", "--retention-seconds", "0")
check(not exists(keys)["failed_staging"], "failed staging past retention must be deleted")
refused = ops.replay(failed["id"])
check(refused.get("result") == "refused" and refused.get("reason") == "source_missing", f"{refused}")
print(f" retention 0 s: failed staging deleted ({json.dumps(expired['to_delete'])}); replay now: {refused}")
again = cleanup("--grace-seconds", "2", "--retention-seconds", "0")
check(again["deleted"] == 0, "repeated run is not a no-op")
print(f" repeated run: deleted {again['deleted']}")
finally:
ops.start("worker")
wait_status(api, bob, queued["id"], seconds=90)
check(not exists({"q": f"staging/{queued['id']}"})["q"], "staging of the processed job must be removed")
print(f"4. worker started: {queued['id']} ready, its staging removed")
print("PASS: staging is removed after the result is committed; objects without a reference in the database are "
"removed only after the grace period; the staging of failed uploads is kept for replay until retention "
"ends; dry-run and repeated runs are safe")
if __name__ == "__main__":
run(main)