forked from mtusi2026/mtusi2026lab5
113 lines
6.7 KiB
Python
113 lines
6.7 KiB
Python
"""lab5, уровень 4: очистка staging после безопасной фиксации и уборка объектов без ссылок из БД.
|
|
|
|
make staging-cleanup (= python student-tests/staging_cleanup.py)
|
|
|
|
1. Обычный путь: после COMMIT результата worker сам удаляет staging/<id>; остаются originals/<id>
|
|
и три варианта.
|
|
2. Состояния, которые оставляют сбои (создаются напрямую в S3 и через API):
|
|
- staging готового вложения (worker упал между COMMIT и удалением staging);
|
|
- staging без строки в БД (API упал между PUT в staging и COMMIT);
|
|
- вариант без строки в БД (worker записал объекты, а задание потом ушло в failed);
|
|
- staging failed-вложения (битый JPEG) — хранится STAGING_RETENTION_SECONDS для replay;
|
|
- staging задания в очереди (worker остановлен) — нужен, его трогать нельзя.
|
|
3. python -m messenger.cleanup:
|
|
--dry-run все найдены, ничего не удалено;
|
|
обычный запуск (grace 900 с) ничего не удалено: объекты моложе grace (это может быть
|
|
идущая прямо сейчас работа);
|
|
--grace-seconds 2 (только тест) удалены «сироты» и staging готового; staging задания в очереди,
|
|
staging failed (в пределах retention), оригиналы и варианты на месте;
|
|
--retention-seconds 0 удалён staging failed; replay такого вложения отказывается
|
|
(source_missing) вместо падения worker;
|
|
повторный запуск удалять нечего.
|
|
4. worker запущен: задание из очереди стало ready, его staging удалён.
|
|
"""
|
|
import json
|
|
import time
|
|
import uuid
|
|
|
|
import ops
|
|
from filegen import garbage_scan_jpeg, upload_when_ready, wait_status
|
|
from helpers import Client, check, run
|
|
|
|
PUT = """
|
|
import io
|
|
from messenger import objects
|
|
for key in %s:
|
|
objects.put(key, io.BytesIO(b"left-behind"), 11, "image/png", "0" * 64)
|
|
print("ok")
|
|
"""
|
|
|
|
|
|
def exists(keys):
|
|
present = set(ops.objects("staging/")) | set(ops.objects("originals/")) | set(ops.objects("variants/"))
|
|
return {name: key in present for name, key in keys.items()}
|
|
|
|
|
|
def cleanup(*args):
|
|
return ops.cli("api", "messenger.cleanup", *args)
|
|
|
|
|
|
def main():
|
|
api = Client(timeout=15)
|
|
ops.wait_healthy("worker", 1)
|
|
alice, bob, _eve, chat = api.bootstrap()
|
|
|
|
ready = upload_when_ready(api, alice, chat, name="keep.png")["json"]
|
|
wait_status(api, bob, ready["id"])
|
|
rid = ready["id"]
|
|
state = exists({"staging": f"staging/{rid}", "original": f"originals/{rid}",
|
|
"thumbnail": f"variants/{rid}/v1/thumbnail.jpg"})
|
|
check(state == {"staging": False, "original": True, "thumbnail": True}, f"unexpected objects: {state}")
|
|
print(f"1. ready {rid}: staging removed by the worker right after COMMIT; original and variants kept")
|
|
|
|
failed = upload_when_ready(api, alice, chat, data=garbage_scan_jpeg(), name="bad.jpg", ctype="image/jpeg")["json"]
|
|
wait_status(api, bob, failed["id"], ("failed",))
|
|
try:
|
|
ops.stop("worker")
|
|
queued = upload_when_ready(api, alice, chat, name="queued.png")["json"]
|
|
keys = {"ready_original": f"originals/{rid}", "ready_variant": f"variants/{rid}/v1/crop.jpg",
|
|
"ready_staging_leftover": f"staging/{rid}", "orphan_staging": f"staging/{uuid.uuid4()}",
|
|
"orphan_variant": f"variants/{uuid.uuid4()}/v1/thumbnail.jpg",
|
|
"failed_staging": f"staging/{failed['id']}", "queued_staging": f"staging/{queued['id']}"}
|
|
ops.python("api", PUT % json.dumps([keys["ready_staging_leftover"], keys["orphan_staging"],
|
|
keys["orphan_variant"]]))
|
|
check(all(exists(keys).values()), "setup failed")
|
|
print("2. prepared: staging leftover of a ready attachment, orphan staging, orphan variant, "
|
|
"staging of a failed attachment, staging of a queued job (worker stopped)")
|
|
|
|
dry = cleanup("--dry-run", "--grace-seconds", "0")
|
|
check(dry["deleted"] == 0 and all(exists(keys).values()), "dry-run deleted something")
|
|
print(f"3. dry-run: {json.dumps(dry['to_delete'])} found, deleted {dry['deleted']}")
|
|
normal = cleanup()
|
|
check(all(exists(keys).values()), "grace 900 s must protect young objects")
|
|
print(f" grace 900 s: deleted {normal['deleted']} — everything is younger than the grace period")
|
|
time.sleep(3)
|
|
short = cleanup("--grace-seconds", "2")
|
|
state = exists(keys)
|
|
check(not state["ready_staging_leftover"] and not state["orphan_staging"] and not state["orphan_variant"],
|
|
f"leftovers must be deleted: {state}")
|
|
check(state["ready_original"] and state["ready_variant"] and state["failed_staging"] and state["queued_staging"],
|
|
f"needed objects were deleted: {state}")
|
|
print(f" grace 2 s: {json.dumps(short['to_delete'])}, deleted {short['deleted']}; kept: ready original "
|
|
f"and variants, staging of the queued job, staging of the failed attachment (retention)")
|
|
expired = cleanup("--grace-seconds", "2", "--retention-seconds", "0")
|
|
check(not exists(keys)["failed_staging"], "failed staging past retention must be deleted")
|
|
refused = ops.replay(failed["id"])
|
|
check(refused.get("result") == "refused" and refused.get("reason") == "source_missing", f"{refused}")
|
|
print(f" retention 0 s: failed staging deleted ({json.dumps(expired['to_delete'])}); replay now: {refused}")
|
|
again = cleanup("--grace-seconds", "2", "--retention-seconds", "0")
|
|
check(again["deleted"] == 0, "repeated run is not a no-op")
|
|
print(f" repeated run: deleted {again['deleted']}")
|
|
finally:
|
|
ops.start("worker")
|
|
wait_status(api, bob, queued["id"], seconds=90)
|
|
check(not exists({"q": f"staging/{queued['id']}"})["q"], "staging of the processed job must be removed")
|
|
print(f"4. worker started: {queued['id']} ready, its staging removed")
|
|
print("PASS: staging is removed after the result is committed; objects without a reference in the database are "
|
|
"removed only after the grace period; the staging of failed uploads is kept for replay until retention "
|
|
"ends; dry-run and repeated runs are safe")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|