Files
mtusi2026lab4/student-tests/test_reconcile.py
T
Alexey BodanyukandClaude Opus 5.5 6fb70de7ff fix(reconcile): карантин по контрольной сумме снимается только проходом с --verify-checksums
Найдено замером стоимости на чистом стенде: последующий --apply без проверки сумм видел
совпадающий размер и снимал карантин checksum_mismatch как restored_object.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 15:46:29 +03:00

218 lines
11 KiB
Python

"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
pairs from inside the storage network, PostgreSQL with psql):
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
other bytes (size / same-size checksum), object of a quarantined row restored;
DB side: stale intent with and without object, quarantine mark on a healthy row;
fresh: orphan and intent younger than --grace (an upload in progress).
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
the second run finds nothing to do; the safety valve and store outages stop the run without
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
from __future__ import annotations
import concurrent.futures
import hashlib
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
service_down)
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
GRACE = ["--grace", "8", "--allow-small-grace"]
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
def new_key() -> str:
return "o/" + uuid.uuid4().hex
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
attachment_id = str(uuid.uuid4())
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
"clock_timestamp() - make_interval(secs => %d))"
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
age_seconds))
return attachment_id
def snapshot() -> tuple[str, frozenset]:
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
return rows, frozenset(s3_keys())
def content_status(api, actor, attachment_id) -> int:
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
headers={"Authorization": "Bearer " + actor["token"]}).status_code
def test_detect_and_safely_fix_faults_on_both_sides(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for name, a in up.items()}
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
# ---- inject (old)
orphan_old = new_key()
s3_put(orphan_old, SAMPLE)
stale_with, stale_without = new_key(), new_key()
s3_put(stale_with, SAMPLE)
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"]) # ready + used, bytes lost
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
time.sleep(32) # everything above is now older than --grace
# ---- inject (fresh: an upload in progress)
fresh_orphan, fresh_intent = new_key(), new_key()
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
# ---- dry-run: report everything, change nothing
before = snapshot()
dry = reconcile(*LONG_GRACE)
assert snapshot() == before, "dry-run modified a store"
f = findings_by_key(dry)
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
key["D"]: "size_mismatch", key["E"]: "restored_object"}
assert {k: f[k]["kind"] for k in expected} == expected
assert key["A"] not in f and key["B"] not in f
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
# ---- apply
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
assert applied[k]["result"] == "applied", (k, applied[k])
keys = s3_keys()
assert orphan_old not in keys and stale_with not in keys
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
# fresh: untouched
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
# used, lost bytes: quarantined, NOT deleted; the message still references it
row_c = attachment_row(up["C"]["id"])
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
items, _ = api.history(bob, chat, 100)
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
assert content_status(api, bob, up["C"]["id"]) == 404
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
for name in "ABE":
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
assert got.content == data[name]
# ---- second run: idempotent, nothing left to do for these keys
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
assert k not in again, (k, again.get(k))
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
s3_put(key["C"], data["C"])
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
# tidy up the fresh items of this test once they are old enough
time.sleep(31)
reconcile("--apply", *LONG_GRACE)
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
def test_checksum_mode_finds_same_size_corruption(api):
alice = api.user("alice")
chat = api.chat(alice)
att = api.upload(alice, chat, SAMPLE).json()
k = attachment_row(att["id"])["object_key"]
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
time.sleep(9)
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
report = reconcile("--verify-checksums", *GRACE)
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays.
assert k not in findings_by_key(reconcile("--apply", *GRACE))
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# Restoring the original and verifying lifts it.
s3_put(k, SAMPLE)
lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]
assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied")
assert attachment_row(att["id"])["lost_reason"] is None
def test_safety_valve_refuses_mass_deletion():
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
--apply must stop before deleting anything."""
orphan = new_key()
s3_put(orphan, SAMPLE)
time.sleep(9)
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
assert orphan in s3_keys()
reconcile("--apply", *GRACE)
assert orphan not in s3_keys()
@pytest.mark.parametrize("service", ["s3", "postgres"])
def test_store_outage_stops_the_run_without_damage(api, service):
with service_down(service, "stop"):
report = reconcile(*GRACE, expect_code=1)
assert report["error"].startswith("store_unavailable")
assert reconcile(*GRACE)["error"] is None
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
ready and downloadable byte for byte, every message keeps its attachments."""
alice = api.user("alice")
chat = api.chat(alice)
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
done: list[tuple[dict, bytes]] = []
def worker(chunk: list[bytes]) -> None:
for p in chunk:
att = api.upload(alice, chat, p).json()
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
done.append((att, p))
with concurrent.futures.ThreadPoolExecutor(4) as pool:
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
runs = 0
while not all(x.done() for x in futures):
report = reconcile("--apply", *GRACE)
assert report["error"] is None
runs += 1
for x in futures:
x.result()
assert runs >= 1 and len(done) == len(payloads)
for att, p in done:
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
for att, p in done[::8]:
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p