Files
mtusi2026lab6/student-tests/reconcile.py
T
2026-11-05 21:17:50 +03:00

153 lines
7.5 KiB
Python

#!/usr/bin/env python3
"""Inject S3/DB disagreement and prove dry-run, grace period and safe repair."""
import json
import base64
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run
def compose(*args, input=None, env=None) -> str:
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True,
input=input, env=env)
def reconcile(*args) -> dict:
output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", *args)
return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{")))
def s3_code(action: str, key: str, data: bytes | None = None) -> None:
code = """from src.storage.s3 import S3Settings, S3Store
import base64, json, sys
s3 = S3Store(S3Settings.load())
payload = json.load(sys.stdin)
key = payload['key']
try:
if sys.argv[1] == 'put':
s3.client.put_object(Bucket=s3.settings.bucket, Key=key,
Body=base64.b64decode(payload.get('data', 'b3JwaGFu')))
else:
s3.client.delete_object(Bucket=s3.settings.bucket, Key=key)
finally:
s3.client.close()
"""
payload = {"key": key}
if data is not None:
payload["data"] = base64.b64encode(data).decode()
compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action,
input=json.dumps(payload))
def wait_ready(client: Client, env=None) -> None:
subprocess.run(["docker", "compose", "up", "-d", "--wait", "--no-deps", "api-a"],
cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 15
while True:
try:
client.call("GET", "/health/ready", schema="Health")
return
except Failure:
check(time.monotonic() < deadline, "stack did not recover")
time.sleep(.5)
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, _eve, chat = client.bootstrap()
image = (ROOT / "tests/fixtures/sample.png").read_bytes()
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename="
def upload(name, expected=201):
return client.call("POST", path + name, expected, actor=alice, raw=image,
headers={"Content-Type": "image/png"},
schema="Attachment" if expected == 201 else None)
# A reference object must survive an apply sweep, even with zero grace.
protected = upload("protected.png")["json"]
message = client.message(alice, chat, "protected", attachments=[protected["id"]])["json"]
orphan = "originals/test-orphan-" + uuid.uuid4().hex
s3_code("put", orphan)
report = reconcile("--grace-seconds", "3600")
check(orphan in report["fresh_unreferenced"], "fresh orphan missing from dry-run")
check(not report["deleted"], "dry-run deleted an object")
report = reconcile("--apply", "--grace-seconds", "3600", "--object-key", orphan)
check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored")
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan)
check(report["deleted"] == [orphan], "aged orphan was not deleted safely")
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan)
check(orphan not in report["deleted"] and orphan not in report["orphan_candidates"],
"second repair run was not idempotent")
check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image,
"reconciler deleted a referenced object")
# This models loss after metadata has committed: report it, do not delete metadata.
missing = upload("missing.png")["json"]
missing_key = "originals/" + missing["id"]
s3_code("delete", missing_key)
try:
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", missing_key)
expected = {"attachment_id": missing["id"], "object_key": missing_key}
check(expected in report["missing_objects"], "missing metadata/object pair not reported")
client.call("GET", "/api/v1/attachments/" + missing["id"], actor=bob, schema="Attachment")
client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob)
finally:
s3_code("put", missing_key, image)
check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image,
"test object was not restored")
report = reconcile("--grace-seconds", "0")
check(expected not in report["missing_objects"], "restored object remained missing on second scan")
print("PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair", flush=True)
# Inject a process-equivalent failure after PUT directly into the configured
# replica. The public load balancer may correctly select the healthy peer,
# which would not exercise this boundary.
environment = dict(__import__("os").environ, UPLOAD_FAULT_AFTER_S3_PUT="1")
before_fault = set(reconcile("--grace-seconds", "0")["orphan_candidates"])
try:
subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"],
cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL)
wait_ready(client, environment)
fault = """import base64, json, sys, urllib.error, urllib.request
payload = json.load(sys.stdin)
request = urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/' + payload['chat_id'] +
'/attachments?filename=fault-after-put.png', data=base64.b64decode(payload['image']),
method='POST', headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'})
try:
urllib.request.urlopen(request, timeout=30)
except urllib.error.HTTPError as error:
assert error.code == 503, error.code
assert json.load(error)['error']['code'] == 'unavailable'
else:
raise AssertionError('fault injection unexpectedly returned success')
"""
compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", fault,
input=json.dumps({"chat_id": chat["id"], "token": alice["token"],
"image": base64.b64encode(image).decode()}))
report = reconcile("--grace-seconds", "0")
injected = sorted(set(report["orphan_candidates"]) - before_fault)
check(len(injected) == 1, "fault must create exactly one new orphan; run without other writers")
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", injected[0])
check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan")
history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"]
check(message in history["items"], "message reference was lost")
check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image,
"used original was lost")
print("PASS: injected pre-INSERT failure repaired; message and used original survive", flush=True)
finally:
environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0"
subprocess.run(["docker", "compose", "up", "-d", "--wait", "--force-recreate", "--no-deps", "api-a"],
cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL)
if __name__ == "__main__":
run(main)