forked from mtusi2026/mtusi2026lab4
Найдено замером стоимости на чистом стенде: последующий --apply без проверки сумм видел совпадающий размер и снимал карантин checksum_mismatch как restored_object. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
218 lines
11 KiB
Python
218 lines
11 KiB
Python
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
|
|
|
|
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
|
|
pairs from inside the storage network, PostgreSQL with psql):
|
|
|
|
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
|
|
other bytes (size / same-size checksum), object of a quarantined row restored;
|
|
DB side: stale intent with and without object, quarantine mark on a healthy row;
|
|
fresh: orphan and intent younger than --grace (an upload in progress).
|
|
|
|
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
|
|
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
|
|
the second run finds nothing to do; the safety valve and store outages stop the run without
|
|
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
|
|
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import hashlib
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import imagegen
|
|
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
|
|
service_down)
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
GRACE = ["--grace", "8", "--allow-small-grace"]
|
|
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
|
|
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
|
|
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
|
|
|
|
|
|
def new_key() -> str:
|
|
return "o/" + uuid.uuid4().hex
|
|
|
|
|
|
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
|
|
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
|
|
attachment_id = str(uuid.uuid4())
|
|
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
|
|
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
|
|
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
|
|
"clock_timestamp() - make_interval(secs => %d))"
|
|
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
|
|
age_seconds))
|
|
return attachment_id
|
|
|
|
|
|
def snapshot() -> tuple[str, frozenset]:
|
|
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
|
|
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
|
|
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
|
|
return rows, frozenset(s3_keys())
|
|
|
|
|
|
def content_status(api, actor, attachment_id) -> int:
|
|
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
|
|
headers={"Authorization": "Bearer " + actor["token"]}).status_code
|
|
|
|
|
|
def test_detect_and_safely_fix_faults_on_both_sides(api):
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
chat = api.chat(alice, bob)
|
|
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
|
|
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
|
|
for name, a in up.items()}
|
|
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
|
|
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
|
|
|
|
# ---- inject (old)
|
|
orphan_old = new_key()
|
|
s3_put(orphan_old, SAMPLE)
|
|
stale_with, stale_without = new_key(), new_key()
|
|
s3_put(stale_with, SAMPLE)
|
|
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
|
|
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
|
|
s3_delete(key["C"]) # ready + used, bytes lost
|
|
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
|
|
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
|
|
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
|
|
time.sleep(32) # everything above is now older than --grace
|
|
# ---- inject (fresh: an upload in progress)
|
|
fresh_orphan, fresh_intent = new_key(), new_key()
|
|
s3_put(fresh_orphan, SAMPLE)
|
|
s3_put(fresh_intent, SAMPLE)
|
|
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
|
|
|
|
# ---- dry-run: report everything, change nothing
|
|
before = snapshot()
|
|
dry = reconcile(*LONG_GRACE)
|
|
assert snapshot() == before, "dry-run modified a store"
|
|
f = findings_by_key(dry)
|
|
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
|
|
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
|
|
key["D"]: "size_mismatch", key["E"]: "restored_object"}
|
|
assert {k: f[k]["kind"] for k in expected} == expected
|
|
assert key["A"] not in f and key["B"] not in f
|
|
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
|
|
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
|
|
|
|
# ---- apply
|
|
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
|
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
|
|
assert applied[k]["result"] == "applied", (k, applied[k])
|
|
keys = s3_keys()
|
|
assert orphan_old not in keys and stale_with not in keys
|
|
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
|
|
# fresh: untouched
|
|
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
|
|
# used, lost bytes: quarantined, NOT deleted; the message still references it
|
|
row_c = attachment_row(up["C"]["id"])
|
|
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
|
|
items, _ = api.history(bob, chat, 100)
|
|
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
|
|
assert content_status(api, bob, up["C"]["id"]) == 404
|
|
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
|
|
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
|
|
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
|
|
for name in "ABE":
|
|
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
|
|
assert got.content == data[name]
|
|
|
|
# ---- second run: idempotent, nothing left to do for these keys
|
|
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
|
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
|
|
assert k not in again, (k, again.get(k))
|
|
|
|
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
|
|
s3_put(key["C"], data["C"])
|
|
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
|
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
|
|
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
|
|
# tidy up the fresh items of this test once they are old enough
|
|
time.sleep(31)
|
|
reconcile("--apply", *LONG_GRACE)
|
|
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
|
|
|
|
|
|
def test_checksum_mode_finds_same_size_corruption(api):
|
|
alice = api.user("alice")
|
|
chat = api.chat(alice)
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
k = attachment_row(att["id"])["object_key"]
|
|
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
|
|
time.sleep(9)
|
|
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
|
|
report = reconcile("--verify-checksums", *GRACE)
|
|
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
|
|
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
|
|
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
|
|
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
|
|
# A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays.
|
|
assert k not in findings_by_key(reconcile("--apply", *GRACE))
|
|
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
|
|
# Restoring the original and verifying lifts it.
|
|
s3_put(k, SAMPLE)
|
|
lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]
|
|
assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied")
|
|
assert attachment_row(att["id"])["lost_reason"] is None
|
|
|
|
|
|
def test_safety_valve_refuses_mass_deletion():
|
|
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
|
|
--apply must stop before deleting anything."""
|
|
orphan = new_key()
|
|
s3_put(orphan, SAMPLE)
|
|
time.sleep(9)
|
|
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
|
|
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
|
|
assert orphan in s3_keys()
|
|
reconcile("--apply", *GRACE)
|
|
assert orphan not in s3_keys()
|
|
|
|
|
|
@pytest.mark.parametrize("service", ["s3", "postgres"])
|
|
def test_store_outage_stops_the_run_without_damage(api, service):
|
|
with service_down(service, "stop"):
|
|
report = reconcile(*GRACE, expect_code=1)
|
|
assert report["error"].startswith("store_unavailable")
|
|
assert reconcile(*GRACE)["error"] is None
|
|
|
|
|
|
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
|
|
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
|
|
ready and downloadable byte for byte, every message keeps its attachments."""
|
|
alice = api.user("alice")
|
|
chat = api.chat(alice)
|
|
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
|
|
done: list[tuple[dict, bytes]] = []
|
|
|
|
def worker(chunk: list[bytes]) -> None:
|
|
for p in chunk:
|
|
att = api.upload(alice, chat, p).json()
|
|
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
|
|
done.append((att, p))
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(4) as pool:
|
|
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
|
|
runs = 0
|
|
while not all(x.done() for x in futures):
|
|
report = reconcile("--apply", *GRACE)
|
|
assert report["error"] is None
|
|
runs += 1
|
|
for x in futures:
|
|
x.result()
|
|
assert runs >= 1 and len(done) == len(payloads)
|
|
for att, p in done:
|
|
row = attachment_row(att["id"])
|
|
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
|
|
for att, p in done[::8]:
|
|
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p
|