forked from mtusi2026/mtusi2026lab5
128 lines
7.3 KiB
Python
128 lines
7.3 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Согласованность S3 ↔ PostgreSQL при сбоях (lab4, уровень «4»).
|
||
Сам управляет контейнерами через docker compose; запускать из корня репозитория:
|
||
|
||
docker compose --env-file .env --profile fault up -d api-fault
|
||
python student-tests/orphans.py --base-url http://localhost:8080 --fault-url http://localhost:8082
|
||
|
||
Сценарии:
|
||
1. Прерванная загрузка (клиент оборвал тело) → в S3 и журнале ничего не появилось.
|
||
2. PUT в S3 прошёл, запись в БД упала (db-error-after-s3-put) → 503, API сразу
|
||
компенсирует: объект и строка журнала удалены, вложения нет.
|
||
3. PUT прошёл, процесс убит до COMMIT (crash-after-s3-put, os._exit) → остаётся
|
||
объект + строка upload_intents (известная сирота). Janitor:
|
||
- с рабочим grace period (900 c) её НЕ трогает (может быть живая загрузка);
|
||
- dry-run показывает, что удалит, и ничего не меняет;
|
||
- после истечения grace удаляет объект и строку; повтор — без изменений.
|
||
4. Объект без журнала и без метаданных (страховка) → находится обходом bucket
|
||
(--scan-bucket) и удаляется только после grace.
|
||
5. Готовые вложения не удаляются никогда: после всех проходов скачиваются, sha256 совпадает.
|
||
"""
|
||
import argparse
|
||
import time
|
||
|
||
from common import SAMPLE_PNG, Api, Report, compose, janitor, raw_http, sha256, tool, upload, wait_ready as att_ready
|
||
|
||
|
||
def wait_ready(api: Api, seconds: int = 60) -> bool:
|
||
deadline = time.time() + seconds
|
||
while time.time() < deadline:
|
||
try:
|
||
if api.raw("GET", "/health/ready", timeout=3)[0] == 200:
|
||
return True
|
||
except OSError:
|
||
pass
|
||
time.sleep(1)
|
||
return False
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||
ap.add_argument("--fault-url", default="http://localhost:8082")
|
||
ap.add_argument("--grace", type=int, default=5, help="маленький grace для теста (с)")
|
||
args = ap.parse_args()
|
||
api, fault, r = Api(args.base_url), Api(args.fault_url), Report()
|
||
r.check(wait_ready(fault, 60), "fault replica (profile fault) is ready on --fault-url")
|
||
|
||
user = api.new_user("orphan")
|
||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "orphans", "member_ids": [user["id"]]},
|
||
token=user["token"])
|
||
st, keep, _ = upload(api, user["token"], chat["id"], SAMPLE_PNG, "keep.png")
|
||
r.check(st == 202, "committed attachment that must survive every cleanup")
|
||
r.check(att_ready(api, user["token"], keep["id"]).get("status") == "ready", "it is processed to ready (lab5)")
|
||
# Сначала дочищаем всё старое, чтобы считать объекты от нуля.
|
||
janitor("--grace-seconds", "0", "--scan-bucket")
|
||
base = tool("stats")
|
||
print(" baseline:", base)
|
||
|
||
print("== 1. прерванная загрузка ==")
|
||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=cut.png"
|
||
raw_http(args.base_url, "POST", path,
|
||
{"Authorization": "Bearer " + user["token"], "Content-Type": "image/png",
|
||
"Content-Length": str(len(SAMPLE_PNG))},
|
||
chunks=[SAMPLE_PNG[:20000]], close_after_send=True)
|
||
time.sleep(2)
|
||
now = tool("stats")
|
||
r.check(now == base, f"no object, no metadata, no journal row after interrupted upload ({now})")
|
||
|
||
print("== 2. отказ БД после PUT: немедленная компенсация ==")
|
||
st, body, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "dbfail.png",
|
||
headers={"X-Fault-Inject": "db-error-after-s3-put"})
|
||
r.check(st == 503, f"upload answered 503, not 202 (got {st})")
|
||
now = tool("stats")
|
||
r.check(now == base, f"object deleted by compensation, journal clean ({now})")
|
||
|
||
print("== 3. процесс убит между PUT и COMMIT ==")
|
||
try:
|
||
st, _, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "crash.png",
|
||
headers={"X-Fault-Inject": "crash-after-s3-put"}, timeout=20)
|
||
except OSError:
|
||
st = None
|
||
r.check(st != 202, f"client did NOT get 202 (got {st})")
|
||
time.sleep(1)
|
||
now = tool("stats")
|
||
r.check(now["objects"] == base["objects"] + 1 and now["upload_intents"] == base["upload_intents"] + 1
|
||
and now["attachments"] == base["attachments"],
|
||
f"orphan object + journal row left behind, no metadata ({now})")
|
||
s = janitor() # рабочий grace period
|
||
r.check(s["objects_deleted"] == 0 and tool("stats") == now,
|
||
f"default grace {s['grace_seconds']} s: fresh upload is NOT touched")
|
||
time.sleep(args.grace + 1)
|
||
s = janitor("--grace-seconds", str(args.grace), "--dry-run")
|
||
r.check(s["would_delete_objects"] == 1 and tool("stats") == now, "dry-run reports 1 object, changes nothing")
|
||
s = janitor("--grace-seconds", str(args.grace))
|
||
after = tool("stats")
|
||
r.check(s["objects_deleted"] == 1 and after == base, f"janitor removed orphan + journal row ({s})")
|
||
s = janitor("--grace-seconds", str(args.grace))
|
||
r.check(s["objects_deleted"] == 0 and tool("stats") == base, "second run is a no-op (idempotent)")
|
||
|
||
print("== 4. объект без журнала (страховочный обход bucket) ==")
|
||
compose("exec", "-T", "api", "python", "-c",
|
||
"import uuid; from app import storage; storage.init(); "
|
||
"storage.client().put_object(Bucket=storage.settings.s3_bucket, "
|
||
"Key=storage.object_key(uuid.uuid4()), Body=b'lost')")
|
||
r.check(tool("stats")["objects"] == base["objects"] + 1, "untracked object planted")
|
||
s = janitor("--scan-bucket", "--grace-seconds", "3600")
|
||
r.check(s["orphans_found"] == 0, "scan with grace 1 h ignores a fresh object")
|
||
time.sleep(args.grace + 1)
|
||
s = janitor("--scan-bucket", "--grace-seconds", str(args.grace), "--dry-run")
|
||
r.check(s["orphans_found"] == 1 and tool("stats")["objects"] == base["objects"] + 1, "scan dry-run finds 1")
|
||
s = janitor("--scan-bucket", "--grace-seconds", str(args.grace))
|
||
r.check(s["objects_deleted"] == 1 and tool("stats") == base, f"scan deletes the untracked object ({s})")
|
||
|
||
print("== 5. готовые вложения не тронуты ==")
|
||
st, raw, _ = api.raw("GET", f"/api/v1/attachments/{keep['id']}/content",
|
||
headers={"Authorization": "Bearer " + user["token"]})
|
||
r.check(st == 200 and sha256(raw) == keep["sha256"], "committed attachment still downloads, sha256 matches")
|
||
state = tool("state", keep["id"])
|
||
r.check(state == {"staging_object_exists": False, "original_object_exists": True,
|
||
"attachment_row": True, "attachment_status": "ready", "upload_intent": False},
|
||
f"published original + metadata, staging cleaned, no journal row ({state})")
|
||
r.finish()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|