153 lines
7.5 KiB
Python
153 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Inject S3/DB disagreement and prove dry-run, grace period and safe repair."""
|
|
import json
|
|
import base64
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def compose(*args, input=None, env=None) -> str:
|
|
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True,
|
|
input=input, env=env)
|
|
|
|
|
|
def reconcile(*args) -> dict:
|
|
output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", *args)
|
|
return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{")))
|
|
|
|
|
|
def s3_code(action: str, key: str, data: bytes | None = None) -> None:
|
|
code = """from src.storage.s3 import S3Settings, S3Store
|
|
import base64, json, sys
|
|
s3 = S3Store(S3Settings.load())
|
|
payload = json.load(sys.stdin)
|
|
key = payload['key']
|
|
try:
|
|
if sys.argv[1] == 'put':
|
|
s3.client.put_object(Bucket=s3.settings.bucket, Key=key,
|
|
Body=base64.b64decode(payload.get('data', 'b3JwaGFu')))
|
|
else:
|
|
s3.client.delete_object(Bucket=s3.settings.bucket, Key=key)
|
|
finally:
|
|
s3.client.close()
|
|
"""
|
|
payload = {"key": key}
|
|
if data is not None:
|
|
payload["data"] = base64.b64encode(data).decode()
|
|
compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action,
|
|
input=json.dumps(payload))
|
|
|
|
|
|
def wait_ready(client: Client, env=None) -> None:
|
|
subprocess.run(["docker", "compose", "up", "-d", "--wait", "--no-deps", "api-a"],
|
|
cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL)
|
|
deadline = time.monotonic() + 15
|
|
while True:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure:
|
|
check(time.monotonic() < deadline, "stack did not recover")
|
|
time.sleep(.5)
|
|
|
|
|
|
def main():
|
|
args = arguments(__doc__)
|
|
client = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
image = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename="
|
|
|
|
def upload(name, expected=201):
|
|
return client.call("POST", path + name, expected, actor=alice, raw=image,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment" if expected == 201 else None)
|
|
|
|
# A reference object must survive an apply sweep, even with zero grace.
|
|
protected = upload("protected.png")["json"]
|
|
message = client.message(alice, chat, "protected", attachments=[protected["id"]])["json"]
|
|
orphan = "originals/test-orphan-" + uuid.uuid4().hex
|
|
s3_code("put", orphan)
|
|
|
|
report = reconcile("--grace-seconds", "3600")
|
|
check(orphan in report["fresh_unreferenced"], "fresh orphan missing from dry-run")
|
|
check(not report["deleted"], "dry-run deleted an object")
|
|
report = reconcile("--apply", "--grace-seconds", "3600", "--object-key", orphan)
|
|
check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored")
|
|
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan)
|
|
check(report["deleted"] == [orphan], "aged orphan was not deleted safely")
|
|
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan)
|
|
check(orphan not in report["deleted"] and orphan not in report["orphan_candidates"],
|
|
"second repair run was not idempotent")
|
|
check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image,
|
|
"reconciler deleted a referenced object")
|
|
|
|
# This models loss after metadata has committed: report it, do not delete metadata.
|
|
missing = upload("missing.png")["json"]
|
|
missing_key = "originals/" + missing["id"]
|
|
s3_code("delete", missing_key)
|
|
try:
|
|
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", missing_key)
|
|
expected = {"attachment_id": missing["id"], "object_key": missing_key}
|
|
check(expected in report["missing_objects"], "missing metadata/object pair not reported")
|
|
client.call("GET", "/api/v1/attachments/" + missing["id"], actor=bob, schema="Attachment")
|
|
client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob)
|
|
finally:
|
|
s3_code("put", missing_key, image)
|
|
check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image,
|
|
"test object was not restored")
|
|
report = reconcile("--grace-seconds", "0")
|
|
check(expected not in report["missing_objects"], "restored object remained missing on second scan")
|
|
print("PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair", flush=True)
|
|
|
|
# Inject a process-equivalent failure after PUT directly into the configured
|
|
# replica. The public load balancer may correctly select the healthy peer,
|
|
# which would not exercise this boundary.
|
|
environment = dict(__import__("os").environ, UPLOAD_FAULT_AFTER_S3_PUT="1")
|
|
before_fault = set(reconcile("--grace-seconds", "0")["orphan_candidates"])
|
|
try:
|
|
subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"],
|
|
cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL)
|
|
wait_ready(client, environment)
|
|
fault = """import base64, json, sys, urllib.error, urllib.request
|
|
payload = json.load(sys.stdin)
|
|
request = urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/' + payload['chat_id'] +
|
|
'/attachments?filename=fault-after-put.png', data=base64.b64decode(payload['image']),
|
|
method='POST', headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'})
|
|
try:
|
|
urllib.request.urlopen(request, timeout=30)
|
|
except urllib.error.HTTPError as error:
|
|
assert error.code == 503, error.code
|
|
assert json.load(error)['error']['code'] == 'unavailable'
|
|
else:
|
|
raise AssertionError('fault injection unexpectedly returned success')
|
|
"""
|
|
compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", fault,
|
|
input=json.dumps({"chat_id": chat["id"], "token": alice["token"],
|
|
"image": base64.b64encode(image).decode()}))
|
|
report = reconcile("--grace-seconds", "0")
|
|
injected = sorted(set(report["orphan_candidates"]) - before_fault)
|
|
check(len(injected) == 1, "fault must create exactly one new orphan; run without other writers")
|
|
report = reconcile("--apply", "--grace-seconds", "0", "--object-key", injected[0])
|
|
check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan")
|
|
history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"]
|
|
check(message in history["items"], "message reference was lost")
|
|
check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image,
|
|
"used original was lost")
|
|
print("PASS: injected pre-INSERT failure repaired; message and used original survive", flush=True)
|
|
finally:
|
|
environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0"
|
|
subprocess.run(["docker", "compose", "up", "-d", "--wait", "--force-recreate", "--no-deps", "api-a"],
|
|
cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|