Files
mtusi2026lab7/student-tests/test_consistency.py
T

287 lines
15 KiB
Python

"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка).
Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`),
выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает
janitor вручную. Данные синтетические, volumes не удаляются.
python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]
"""
import json
import socket
import sys
import time
import urllib.parse
import images
from common import (BASE_URL, tls_socket, app_python, call, check, compose, compose_output, docker, janitor, new_chat,
new_user, post_message, run_tests, sql_value, upload, wait_ready)
URL = urllib.parse.urlsplit(BASE_URL)
# lab7: у каждого сервиса свой S3-ключ с правами по prefix — проверки выполняются в контейнере с нужным правом
# (запись staging — api, чтение staging — worker, листинг bucket — janitor).
# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но enqueue_upload
# (lab5: UPDATE ... state='queued' + задание + outbox, COMMIT) падает — как если бы реплика умерла
# сразу после PUT в staging, до фиксации задания.
CRASH_AFTER_PUT = r'''
import json, struct, sys, tempfile, zlib
from uuid import UUID
import attachments, db, storage, uploads
chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2])
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b""))
f = tempfile.SpooledTemporaryFile(); f.write(data)
import hashlib
upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest())
image = uploads.inspect(upload, "image/png")
created = {}
real_intent = db.create_upload_intent
def remember(attachment_id, *args):
created["id"], created["key"] = str(attachment_id), args[-1]
return real_intent(attachment_id, *args)
def crash(attachment_id, *args):
raise RuntimeError("simulated crash between S3 PUT and DB COMMIT")
db.create_upload_intent = remember
db.enqueue_upload = crash
db.pool.open()
try:
attachments.store(chat_id, owner_id, "crash.png", upload, image)
outcome = "stored"
except RuntimeError as exc:
outcome = str(exc)
print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"]}))
'''
OBJECT_EXISTS = r'''
import json, sys, storage
print(json.dumps({"exists": storage.exists(sys.argv[1])}))
'''
PUT_ORPHAN = r'''
import hashlib, io, json, uuid, storage
data = b"orphan object without a row in PostgreSQL"
key = storage.new_key(uuid.uuid4().hex)
storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest())
print(json.dumps({"key": key}))
'''
DELETE_OBJECT = r'''
import json, sys, storage
storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]}))
'''
COUNT_OBJECTS = r'''
import json, storage
print(json.dumps({"objects": sum(1 for prefix in storage.ALL_PREFIXES for _ in storage.list_objects(prefix))}))
'''
def last_json(output):
return json.loads(output.splitlines()[-1])
def object_exists(key):
# lab7: ключ api не читает staging/ (только пишет) — проверяем изнутри worker, которому чтение staging/ нужно
return last_json(app_python(OBJECT_EXISTS, key, service="worker"))["exists"]
def backdate(where):
sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}")
def uploading_rows(chat_id):
return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'"))
def wait_download(user, att, seconds=60):
deadline = time.monotonic() + seconds
while True:
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"])
if reply.status == 200:
return reply
check(time.monotonic() < deadline, "download did not recover after S3 start")
time.sleep(1)
def s3_outage(stop_action, start_action, label):
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
data = images.sample_png()
att = upload(alice, chat["id"], data).json
before = uploading_rows(chat["id"])
print(f"--- S3 {label}: docker compose {stop_action} s3")
compose(stop_action, "s3")
try:
started = time.monotonic()
reply = upload(alice, chat["id"], data, expected=503)
ms = (time.monotonic() - started) * 1000
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
body = reply.data.decode().lower()
check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")),
"error leaks storage details")
print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body")
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
print("download of an existing file -> 503")
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
check(meta["sha256"] == att["sha256"], "metadata changed")
post_message(alice, chat["id"], "текст без файла")
post_message(alice, chat["id"], "", [att["id"]])
call("GET", "/health/ready", 200)
print("metadata, messages (also with an already stored attachment) and /health/ready keep working: "
"S3 is needed only for file bytes")
after = uploading_rows(chat["id"])
print(f"uploading rows of this chat: before {before}, after the failed upload {after} "
"(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API")
finally:
print(f"docker compose {start_action} s3")
compose(start_action, "s3")
got = wait_download(bob, att)
check(got.data == data, "file changed after S3 recovery")
print("after S3 recovery the old file downloads byte for byte")
started = time.monotonic()
while True:
# SeaweedFS после рестарта отдаёт чтение раньше записи: мастер «прогревает» топологию
# (/dir/assign -> "master is warming up") ещё ~15 c после healthz. Ждём первую успешную запись.
reply = upload(alice, chat["id"], data, expected=(202, 503))
if reply.status == 202:
break
check(time.monotonic() - started < 90, "upload did not recover after S3 start")
time.sleep(1)
print(f"new upload -> 202 ({time.monotonic() - started:.1f} s after the old file became readable)")
leftover = uploading_rows(chat["id"])
if leftover:
backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'")
report = janitor()
print(f"janitor after grace period: {json.dumps(report)}")
check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain")
print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up")
def test_s3_stopped():
s3_outage("stop", "start", "stopped (connection refused / name not resolved)")
def test_s3_frozen():
s3_outage("pause", "unpause", "frozen (no answer, timeouts)")
def test_crash_between_s3_and_db():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
ready = upload(alice, chat["id"], images.sample_png()).json
result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"]))
check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}")
att_id, key = result["attachment_id"], result["object_key"]
state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'")
check(state == "uploading" and object_exists(result["object_key"]), "expected an uploading row and an object in S3")
print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3")
call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"])
call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"])
post_message(alice, chat["id"], "x", [att_id], expected=404)
print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404")
report = janitor()
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key),
"fresh upload was deleted inside the grace period")
print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> "
"fresh row and object are kept (the upload might still be running)")
backdate(f"id = '{att_id}'")
report = janitor("--dry-run")
check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report")
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row")
print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted")
report = janitor()
check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload")
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists")
check(not object_exists(key), "object still exists")
print(f"janitor: {json.dumps(report)} -> row and object deleted")
report = janitor()
print(f"janitor again: {json.dumps(report)} -> idempotent")
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
check(got.data == images.sample_png(), "ready attachment damaged by cleanup")
print("the ready attachment of the same chat is untouched")
print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period")
def test_orphan_objects():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
ready = upload(alice, chat["id"], images.sample_png()).json
key = last_json(app_python(PUT_ORPHAN))["key"]
print(f"object {key} put into the bucket without a row in PostgreSQL")
# Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с.
short = {"UPLOAD_TIMEOUT_SECONDS": "1"}
report = janitor("--grace-seconds", "3", env=short)
check(object_exists(key), "fresh orphan deleted inside grace period")
print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept")
time.sleep(4)
report = janitor("--grace-seconds", "3", "--dry-run", env=short)
check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key),
"dry run must report and keep the orphan")
print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept")
report = janitor("--grace-seconds", "3", env=short)
check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted")
print(f"janitor: {json.dumps(report)} -> orphan deleted")
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
check(got.data == images.sample_png(), "ready attachment was deleted with orphans")
print("ready attachments older than the grace period are untouched (they have rows)")
victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json
victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'")
app_python(DELETE_OBJECT, victim_key, service="janitor") # lab7: удалять готовые объекты может только janitor
report = janitor("--grace-seconds", "3", env=short)
check(report["missing_objects"] >= 1, "missing object not reported")
check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted")
reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"])
print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects="
f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)")
print("PASS: orphans are found and deleted only after the grace period; missing objects are reported")
def api_tmp_files():
counts = {}
for container in compose_output("ps", "-q", "api").split():
counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0)
return counts
def test_aborted_upload_leaves_nothing():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
objects = last_json(app_python(COUNT_OBJECTS, service="janitor"))["objects"]
rows = int(sql_value("SELECT count(*) FROM attachments"))
for sent in (100_000, 3_000_000):
sock = tls_socket(timeout=10)
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n"
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n"
f"Content-Length: 9000000\r\n\r\n")
sock.sendall(head.encode() + images.sample_png() + bytes(sent))
sock.close()
print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection")
time.sleep(2)
check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row")
check(last_json(app_python(COUNT_OBJECTS, service="janitor"))["objects"] == objects, "an aborted upload left an S3 object")
print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}")
tmp = api_tmp_files()
check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}")
print(f"temporary upload files in api containers: {tmp} (removed when the request ends)")
for container in compose_output("ps", "-q", "api").split():
mounts = docker("inspect", "-f", "{{json .Mounts}}", container)
# файловые secrets и (lab7) сертификаты монтируются read-only в /run/... — это не хранилище данных
storage = [m for m in json.loads(mounts)
if not (m.get("Destination", "").startswith("/run/") and m.get("RW") is False)]
check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}")
print("api containers have no volumes: the original is never stored locally")
print("PASS: an interrupted upload leaves no row, no object and no local file")
if __name__ == "__main__":
target = sys.argv[1] if len(sys.argv) > 1 else "all"
groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db],
"orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]}
groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"]
check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]")
wait_ready()
run_tests(groups[target])