Files
mtusi2026lab5/student-tests/orphans.py
T

128 lines
7.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
Согласованность S3 ↔ PostgreSQL при сбоях (lab4, уровень «4»).
Сам управляет контейнерами через docker compose; запускать из корня репозитория:
docker compose --env-file .env --profile fault up -d api-fault
python student-tests/orphans.py --base-url http://localhost:8080 --fault-url http://localhost:8082
Сценарии:
1. Прерванная загрузка (клиент оборвал тело) → в S3 и журнале ничего не появилось.
2. PUT в S3 прошёл, запись в БД упала (db-error-after-s3-put) → 503, API сразу
компенсирует: объект и строка журнала удалены, вложения нет.
3. PUT прошёл, процесс убит до COMMIT (crash-after-s3-put, os._exit) → остаётся
объект + строка upload_intents (известная сирота). Janitor:
- с рабочим grace period (900 c) её НЕ трогает (может быть живая загрузка);
- dry-run показывает, что удалит, и ничего не меняет;
- после истечения grace удаляет объект и строку; повтор — без изменений.
4. Объект без журнала и без метаданных (страховка) → находится обходом bucket
(--scan-bucket) и удаляется только после grace.
5. Готовые вложения не удаляются никогда: после всех проходов скачиваются, sha256 совпадает.
"""
import argparse
import time
from common import SAMPLE_PNG, Api, Report, compose, janitor, raw_http, sha256, tool, upload, wait_ready as att_ready
def wait_ready(api: Api, seconds: int = 60) -> bool:
deadline = time.time() + seconds
while time.time() < deadline:
try:
if api.raw("GET", "/health/ready", timeout=3)[0] == 200:
return True
except OSError:
pass
time.sleep(1)
return False
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--fault-url", default="http://localhost:8082")
ap.add_argument("--grace", type=int, default=5, help="маленький grace для теста (с)")
args = ap.parse_args()
api, fault, r = Api(args.base_url), Api(args.fault_url), Report()
r.check(wait_ready(fault, 60), "fault replica (profile fault) is ready on --fault-url")
user = api.new_user("orphan")
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "orphans", "member_ids": [user["id"]]},
token=user["token"])
st, keep, _ = upload(api, user["token"], chat["id"], SAMPLE_PNG, "keep.png")
r.check(st == 202, "committed attachment that must survive every cleanup")
r.check(att_ready(api, user["token"], keep["id"]).get("status") == "ready", "it is processed to ready (lab5)")
# Сначала дочищаем всё старое, чтобы считать объекты от нуля.
janitor("--grace-seconds", "0", "--scan-bucket")
base = tool("stats")
print(" baseline:", base)
print("== 1. прерванная загрузка ==")
path = f"/api/v1/chats/{chat['id']}/attachments?filename=cut.png"
raw_http(args.base_url, "POST", path,
{"Authorization": "Bearer " + user["token"], "Content-Type": "image/png",
"Content-Length": str(len(SAMPLE_PNG))},
chunks=[SAMPLE_PNG[:20000]], close_after_send=True)
time.sleep(2)
now = tool("stats")
r.check(now == base, f"no object, no metadata, no journal row after interrupted upload ({now})")
print("== 2. отказ БД после PUT: немедленная компенсация ==")
st, body, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "dbfail.png",
headers={"X-Fault-Inject": "db-error-after-s3-put"})
r.check(st == 503, f"upload answered 503, not 202 (got {st})")
now = tool("stats")
r.check(now == base, f"object deleted by compensation, journal clean ({now})")
print("== 3. процесс убит между PUT и COMMIT ==")
try:
st, _, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "crash.png",
headers={"X-Fault-Inject": "crash-after-s3-put"}, timeout=20)
except OSError:
st = None
r.check(st != 202, f"client did NOT get 202 (got {st})")
time.sleep(1)
now = tool("stats")
r.check(now["objects"] == base["objects"] + 1 and now["upload_intents"] == base["upload_intents"] + 1
and now["attachments"] == base["attachments"],
f"orphan object + journal row left behind, no metadata ({now})")
s = janitor() # рабочий grace period
r.check(s["objects_deleted"] == 0 and tool("stats") == now,
f"default grace {s['grace_seconds']} s: fresh upload is NOT touched")
time.sleep(args.grace + 1)
s = janitor("--grace-seconds", str(args.grace), "--dry-run")
r.check(s["would_delete_objects"] == 1 and tool("stats") == now, "dry-run reports 1 object, changes nothing")
s = janitor("--grace-seconds", str(args.grace))
after = tool("stats")
r.check(s["objects_deleted"] == 1 and after == base, f"janitor removed orphan + journal row ({s})")
s = janitor("--grace-seconds", str(args.grace))
r.check(s["objects_deleted"] == 0 and tool("stats") == base, "second run is a no-op (idempotent)")
print("== 4. объект без журнала (страховочный обход bucket) ==")
compose("exec", "-T", "api", "python", "-c",
"import uuid; from app import storage; storage.init(); "
"storage.client().put_object(Bucket=storage.settings.s3_bucket, "
"Key=storage.object_key(uuid.uuid4()), Body=b'lost')")
r.check(tool("stats")["objects"] == base["objects"] + 1, "untracked object planted")
s = janitor("--scan-bucket", "--grace-seconds", "3600")
r.check(s["orphans_found"] == 0, "scan with grace 1 h ignores a fresh object")
time.sleep(args.grace + 1)
s = janitor("--scan-bucket", "--grace-seconds", str(args.grace), "--dry-run")
r.check(s["orphans_found"] == 1 and tool("stats")["objects"] == base["objects"] + 1, "scan dry-run finds 1")
s = janitor("--scan-bucket", "--grace-seconds", str(args.grace))
r.check(s["objects_deleted"] == 1 and tool("stats") == base, f"scan deletes the untracked object ({s})")
print("== 5. готовые вложения не тронуты ==")
st, raw, _ = api.raw("GET", f"/api/v1/attachments/{keep['id']}/content",
headers={"Authorization": "Bearer " + user["token"]})
r.check(st == 200 and sha256(raw) == keep["sha256"], "committed attachment still downloads, sha256 matches")
state = tool("state", keep["id"])
r.check(state == {"staging_object_exists": False, "original_object_exists": True,
"attachment_row": True, "attachment_status": "ready", "upload_intent": False},
f"published original + metadata, staging cleaned, no journal row ({state})")
r.finish()
if __name__ == "__main__":
main()