forked from mtusi2026/mtusi2026lab7
287 lines
15 KiB
Python
287 lines
15 KiB
Python
"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка).
|
|
|
|
Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`),
|
|
выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает
|
|
janitor вручную. Данные синтетические, volumes не удаляются.
|
|
|
|
python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]
|
|
"""
|
|
import json
|
|
import socket
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
|
|
import images
|
|
from common import (BASE_URL, tls_socket, app_python, call, check, compose, compose_output, docker, janitor, new_chat,
|
|
new_user, post_message, run_tests, sql_value, upload, wait_ready)
|
|
|
|
URL = urllib.parse.urlsplit(BASE_URL)
|
|
|
|
# lab7: у каждого сервиса свой S3-ключ с правами по prefix — проверки выполняются в контейнере с нужным правом
|
|
# (запись staging — api, чтение staging — worker, листинг bucket — janitor).
|
|
# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но enqueue_upload
|
|
# (lab5: UPDATE ... state='queued' + задание + outbox, COMMIT) падает — как если бы реплика умерла
|
|
# сразу после PUT в staging, до фиксации задания.
|
|
CRASH_AFTER_PUT = r'''
|
|
import json, struct, sys, tempfile, zlib
|
|
from uuid import UUID
|
|
import attachments, db, storage, uploads
|
|
chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2])
|
|
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
|
|
data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0))
|
|
+ chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b""))
|
|
f = tempfile.SpooledTemporaryFile(); f.write(data)
|
|
import hashlib
|
|
upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest())
|
|
image = uploads.inspect(upload, "image/png")
|
|
created = {}
|
|
real_intent = db.create_upload_intent
|
|
def remember(attachment_id, *args):
|
|
created["id"], created["key"] = str(attachment_id), args[-1]
|
|
return real_intent(attachment_id, *args)
|
|
def crash(attachment_id, *args):
|
|
raise RuntimeError("simulated crash between S3 PUT and DB COMMIT")
|
|
db.create_upload_intent = remember
|
|
db.enqueue_upload = crash
|
|
db.pool.open()
|
|
try:
|
|
attachments.store(chat_id, owner_id, "crash.png", upload, image)
|
|
outcome = "stored"
|
|
except RuntimeError as exc:
|
|
outcome = str(exc)
|
|
print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"]}))
|
|
'''
|
|
|
|
OBJECT_EXISTS = r'''
|
|
import json, sys, storage
|
|
print(json.dumps({"exists": storage.exists(sys.argv[1])}))
|
|
'''
|
|
|
|
PUT_ORPHAN = r'''
|
|
import hashlib, io, json, uuid, storage
|
|
data = b"orphan object without a row in PostgreSQL"
|
|
key = storage.new_key(uuid.uuid4().hex)
|
|
storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest())
|
|
print(json.dumps({"key": key}))
|
|
'''
|
|
|
|
DELETE_OBJECT = r'''
|
|
import json, sys, storage
|
|
storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]}))
|
|
'''
|
|
|
|
COUNT_OBJECTS = r'''
|
|
import json, storage
|
|
print(json.dumps({"objects": sum(1 for prefix in storage.ALL_PREFIXES for _ in storage.list_objects(prefix))}))
|
|
'''
|
|
|
|
|
|
def last_json(output):
|
|
return json.loads(output.splitlines()[-1])
|
|
|
|
|
|
def object_exists(key):
|
|
# lab7: ключ api не читает staging/ (только пишет) — проверяем изнутри worker, которому чтение staging/ нужно
|
|
return last_json(app_python(OBJECT_EXISTS, key, service="worker"))["exists"]
|
|
|
|
|
|
def backdate(where):
|
|
sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}")
|
|
|
|
|
|
def uploading_rows(chat_id):
|
|
return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'"))
|
|
|
|
|
|
def wait_download(user, att, seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"])
|
|
if reply.status == 200:
|
|
return reply
|
|
check(time.monotonic() < deadline, "download did not recover after S3 start")
|
|
time.sleep(1)
|
|
|
|
|
|
def s3_outage(stop_action, start_action, label):
|
|
alice, bob = new_user("alice"), new_user("bob")
|
|
chat = new_chat(alice, bob)
|
|
data = images.sample_png()
|
|
att = upload(alice, chat["id"], data).json
|
|
before = uploading_rows(chat["id"])
|
|
print(f"--- S3 {label}: docker compose {stop_action} s3")
|
|
compose(stop_action, "s3")
|
|
try:
|
|
started = time.monotonic()
|
|
reply = upload(alice, chat["id"], data, expected=503)
|
|
ms = (time.monotonic() - started) * 1000
|
|
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
|
|
body = reply.data.decode().lower()
|
|
check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")),
|
|
"error leaks storage details")
|
|
print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body")
|
|
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
|
|
print("download of an existing file -> 503")
|
|
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
|
|
check(meta["sha256"] == att["sha256"], "metadata changed")
|
|
post_message(alice, chat["id"], "текст без файла")
|
|
post_message(alice, chat["id"], "", [att["id"]])
|
|
call("GET", "/health/ready", 200)
|
|
print("metadata, messages (also with an already stored attachment) and /health/ready keep working: "
|
|
"S3 is needed only for file bytes")
|
|
after = uploading_rows(chat["id"])
|
|
print(f"uploading rows of this chat: before {before}, after the failed upload {after} "
|
|
"(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API")
|
|
finally:
|
|
print(f"docker compose {start_action} s3")
|
|
compose(start_action, "s3")
|
|
got = wait_download(bob, att)
|
|
check(got.data == data, "file changed after S3 recovery")
|
|
print("after S3 recovery the old file downloads byte for byte")
|
|
started = time.monotonic()
|
|
while True:
|
|
# SeaweedFS после рестарта отдаёт чтение раньше записи: мастер «прогревает» топологию
|
|
# (/dir/assign -> "master is warming up") ещё ~15 c после healthz. Ждём первую успешную запись.
|
|
reply = upload(alice, chat["id"], data, expected=(202, 503))
|
|
if reply.status == 202:
|
|
break
|
|
check(time.monotonic() - started < 90, "upload did not recover after S3 start")
|
|
time.sleep(1)
|
|
print(f"new upload -> 202 ({time.monotonic() - started:.1f} s after the old file became readable)")
|
|
leftover = uploading_rows(chat["id"])
|
|
if leftover:
|
|
backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'")
|
|
report = janitor()
|
|
print(f"janitor after grace period: {json.dumps(report)}")
|
|
check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain")
|
|
print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up")
|
|
|
|
|
|
def test_s3_stopped():
|
|
s3_outage("stop", "start", "stopped (connection refused / name not resolved)")
|
|
|
|
|
|
def test_s3_frozen():
|
|
s3_outage("pause", "unpause", "frozen (no answer, timeouts)")
|
|
|
|
|
|
def test_crash_between_s3_and_db():
|
|
alice, bob = new_user("alice"), new_user("bob")
|
|
chat = new_chat(alice, bob)
|
|
ready = upload(alice, chat["id"], images.sample_png()).json
|
|
result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"]))
|
|
check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}")
|
|
att_id, key = result["attachment_id"], result["object_key"]
|
|
state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'")
|
|
check(state == "uploading" and object_exists(result["object_key"]), "expected an uploading row and an object in S3")
|
|
print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3")
|
|
call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"])
|
|
call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"])
|
|
post_message(alice, chat["id"], "x", [att_id], expected=404)
|
|
print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404")
|
|
|
|
report = janitor()
|
|
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key),
|
|
"fresh upload was deleted inside the grace period")
|
|
print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> "
|
|
"fresh row and object are kept (the upload might still be running)")
|
|
backdate(f"id = '{att_id}'")
|
|
report = janitor("--dry-run")
|
|
check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report")
|
|
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row")
|
|
print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted")
|
|
report = janitor()
|
|
check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload")
|
|
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists")
|
|
check(not object_exists(key), "object still exists")
|
|
print(f"janitor: {json.dumps(report)} -> row and object deleted")
|
|
report = janitor()
|
|
print(f"janitor again: {json.dumps(report)} -> idempotent")
|
|
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
|
|
check(got.data == images.sample_png(), "ready attachment damaged by cleanup")
|
|
print("the ready attachment of the same chat is untouched")
|
|
print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period")
|
|
|
|
|
|
def test_orphan_objects():
|
|
alice, bob = new_user("alice"), new_user("bob")
|
|
chat = new_chat(alice, bob)
|
|
ready = upload(alice, chat["id"], images.sample_png()).json
|
|
key = last_json(app_python(PUT_ORPHAN))["key"]
|
|
print(f"object {key} put into the bucket without a row in PostgreSQL")
|
|
# Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с.
|
|
short = {"UPLOAD_TIMEOUT_SECONDS": "1"}
|
|
report = janitor("--grace-seconds", "3", env=short)
|
|
check(object_exists(key), "fresh orphan deleted inside grace period")
|
|
print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept")
|
|
time.sleep(4)
|
|
report = janitor("--grace-seconds", "3", "--dry-run", env=short)
|
|
check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key),
|
|
"dry run must report and keep the orphan")
|
|
print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept")
|
|
report = janitor("--grace-seconds", "3", env=short)
|
|
check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted")
|
|
print(f"janitor: {json.dumps(report)} -> orphan deleted")
|
|
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
|
|
check(got.data == images.sample_png(), "ready attachment was deleted with orphans")
|
|
print("ready attachments older than the grace period are untouched (they have rows)")
|
|
|
|
victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json
|
|
victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'")
|
|
app_python(DELETE_OBJECT, victim_key, service="janitor") # lab7: удалять готовые объекты может только janitor
|
|
report = janitor("--grace-seconds", "3", env=short)
|
|
check(report["missing_objects"] >= 1, "missing object not reported")
|
|
check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted")
|
|
reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"])
|
|
print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects="
|
|
f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)")
|
|
print("PASS: orphans are found and deleted only after the grace period; missing objects are reported")
|
|
|
|
|
|
def api_tmp_files():
|
|
counts = {}
|
|
for container in compose_output("ps", "-q", "api").split():
|
|
counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0)
|
|
return counts
|
|
|
|
|
|
def test_aborted_upload_leaves_nothing():
|
|
alice, bob = new_user("alice"), new_user("bob")
|
|
chat = new_chat(alice, bob)
|
|
objects = last_json(app_python(COUNT_OBJECTS, service="janitor"))["objects"]
|
|
rows = int(sql_value("SELECT count(*) FROM attachments"))
|
|
for sent in (100_000, 3_000_000):
|
|
sock = tls_socket(timeout=10)
|
|
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n"
|
|
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n"
|
|
f"Content-Length: 9000000\r\n\r\n")
|
|
sock.sendall(head.encode() + images.sample_png() + bytes(sent))
|
|
sock.close()
|
|
print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection")
|
|
time.sleep(2)
|
|
check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row")
|
|
check(last_json(app_python(COUNT_OBJECTS, service="janitor"))["objects"] == objects, "an aborted upload left an S3 object")
|
|
print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}")
|
|
tmp = api_tmp_files()
|
|
check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}")
|
|
print(f"temporary upload files in api containers: {tmp} (removed when the request ends)")
|
|
for container in compose_output("ps", "-q", "api").split():
|
|
mounts = docker("inspect", "-f", "{{json .Mounts}}", container)
|
|
# файловые secrets и (lab7) сертификаты монтируются read-only в /run/... — это не хранилище данных
|
|
storage = [m for m in json.loads(mounts)
|
|
if not (m.get("Destination", "").startswith("/run/") and m.get("RW") is False)]
|
|
check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}")
|
|
print("api containers have no volumes: the original is never stored locally")
|
|
print("PASS: an interrupted upload leaves no row, no object and no local file")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
target = sys.argv[1] if len(sys.argv) > 1 else "all"
|
|
groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db],
|
|
"orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]}
|
|
groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"]
|
|
check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]")
|
|
wait_ready()
|
|
run_tests(groups[target])
|