Files
mtusi2026lab4/tests/reconcile_failure_check.py
2026-11-08 18:17:48 +03:00

492 lines
11 KiB
Python

import contextlib
import importlib.util
import io
import json
from pathlib import Path
import sys
import time
import uuid
from client import Client, Failure, arguments, check, run
import reconcile_check as helpers
ROOT = Path(__file__).resolve().parents[1]
def load_launcher():
spec = importlib.util.spec_from_file_location(
"reconciler_launcher",
ROOT / "scripts/reconcile.py",
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def state(container_id):
raw = helpers.command([
"docker",
"inspect",
"--format",
"{{json .State}}",
container_id,
])
return json.loads(raw)
def wait_container(container_id, seconds=90):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
current = state(container_id)
healthy = (
current.get("Running")
and current.get("Health", {}).get("Status")
== "healthy"
)
if healthy:
return
time.sleep(2)
raise Failure(
"Container did not become healthy: "
+ container_id[:12]
)
def wait_api(client, seconds=90):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
client.call(
"GET",
"/health/ready",
schema="Health",
)
return
except Failure:
time.sleep(2)
raise Failure(
"API did not recover after dependency restart"
)
def main():
args = arguments(
"Reconciler failure checks: "
"MinIO and PostgreSQL outage"
)
helpers.BUCKET = helpers.inspect_api()
launcher = load_launcher()
saved_services = {
name: launcher.service(name)
for name in ("api", "postgres", "minio")
}
launcher.service = lambda name: saved_services[name]
c = Client(
args.base_url,
args.ca_file,
timeout=10,
)
alice, bob, _, chat = c.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
attachment, content = c.attachment(alice, chat)
message = c.message(
alice,
chat,
"Reconciler dependency failure",
attachments=[attachment["id"]],
)["json"]
original_name = str(uuid.UUID(attachment["id"]))
orphan_key = (
"originals/"
+ chat_id
+ "/orphan-failure-"
+ uuid.uuid4().hex
)
orphan_name = orphan_key.rsplit("/", 1)[-1]
metadata_before = helpers.snapshot(chat_id)
directory = (
ROOT
/ "results"
/ ("reconcile-failure-" + chat_id)
)
backups = (
ROOT
/ "backups"
/ ("reconcile-failure-" + chat_id)
)
directory.mkdir(parents=True, exist_ok=False)
backups.mkdir(parents=True, exist_ok=False)
stopped = set()
summary = []
completed = False
def worker(label, apply, expected_code):
path = directory / (label + ".json")
previous_argv = sys.argv
try:
sys.argv = [
str(ROOT / "scripts/reconcile.py"),
"--apply" if apply else "--dry-run",
"--chat-id",
chat_id,
"--grace-seconds",
"60",
"--max-seconds",
"120",
"--backup-dir",
str(backups),
"--output",
str(path),
]
with contextlib.redirect_stdout(io.StringIO()):
code = launcher.main()
finally:
sys.argv = previous_argv
check(
code == expected_code,
"Unexpected return code at " + label,
)
report = json.loads(
path.read_text(encoding="utf-8")
)
summary.append({
"stage": label,
"status": report["status"],
"elapsed_seconds": report["elapsed_seconds"],
"error_type": report["error_type"],
})
print(
label
+ ": status="
+ report["status"]
+ ", error_type="
+ str(report["error_type"]),
flush=True,
)
return report
def verify_data(expect_orphan):
check(
helpers.snapshot(chat_id) == metadata_before,
"Attachment metadata changed",
)
names = helpers.object_names(chat_id)
expected = {original_name}
if expect_orphan:
expected.add(orphan_name)
check(
names == expected,
"Unexpected S3 object set",
)
result = c.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=bob,
)
check(
result["bytes"] == content,
"Confirmed original changed",
)
page = c.call(
"GET",
"/api/v1/chats/"
+ chat_id
+ "/messages?limit=100",
actor=bob,
schema="MessagePage",
)["json"]
check(
message in page["items"],
"Message or attachment references changed",
)
def restore(name):
container_id = saved_services[name]["Id"]
helpers.command(
[
"docker",
"start",
container_id,
],
timeout=60,
)
wait_container(container_id)
stopped.discard(name)
wait_api(c)
try:
helpers.mc(
"pipe",
helpers.target(orphan_key),
data=content,
)
print(
"Waiting 65 seconds "
"for the real orphan grace period...",
flush=True,
)
time.sleep(65)
baseline = worker(
"01-baseline",
apply=False,
expected_code=0,
)
check(
baseline["counts"].get(
"would_delete_orphans", 0
) == 1,
"No deletion candidate in baseline",
)
verify_data(expect_orphan=True)
for number, name in enumerate(
("minio", "postgres"),
start=2,
):
container_id = saved_services[name]["Id"]
print(
"Stopping "
+ name
+ " for the experiment",
flush=True,
)
stopped.add(name)
helpers.command(
[
"docker",
"stop",
"-t",
"10",
container_id,
],
timeout=60,
)
check(
not state(container_id).get("Running"),
"Dependency did not stop",
)
try:
report = worker(
f"{number:02d}-{name}-unavailable",
apply=True,
expected_code=1,
)
check(
report["status"] == "failed",
"Outage was not reported as failed",
)
check(
report["error_type"] is not None,
"Missing error type",
)
for operation in (
"delete_object",
"put_object",
):
check(
report["s3_operation_calls"].get(
operation, 0
) == 0,
"Mutation attempted "
"during dependency outage",
)
for counter in (
"missing_objects",
"unresolved_missing",
"deleted_orphans",
"restored",
):
check(
report["counts"].get(
counter, 0
) == 0,
"Outage was confused "
"with a data inconsistency",
)
finally:
restore(name)
verify_data(expect_orphan=True)
print(
"PASS: "
+ name
+ " outage aborted reconciliation; "
"data and session survived",
flush=True,
)
repaired = worker(
"04-recovery-apply",
apply=True,
expected_code=0,
)
check(
repaired["counts"].get(
"deleted_orphans", 0
) == 1,
"Orphan was not removed after recovery",
)
verify_data(expect_orphan=False)
repeated = worker(
"05-recovery-repeat",
apply=True,
expected_code=0,
)
for counter in (
"deleted_orphans",
"restored",
"missing_objects",
"unresolved_missing",
):
check(
repeated["counts"].get(
counter, 0
) == 0,
"Repeated run is not clean",
)
verify_data(expect_orphan=False)
completed = True
finally:
recovery_errors = []
for name in tuple(stopped):
try:
restore(name)
except Exception:
recovery_errors.append(name)
print(
"WARNING: could not restore service: "
+ name,
flush=True,
)
if not recovery_errors:
try:
if orphan_name in helpers.object_names(chat_id):
helpers.mc(
"rm",
helpers.target(orphan_key),
)
except Exception:
recovery_errors.append(
"test orphan cleanup"
)
print(
"WARNING: cannot remove synthetic orphan: "
+ orphan_key,
flush=True,
)
check(
not recovery_errors,
"Recovery incomplete: "
+ ", ".join(recovery_errors),
)
if completed:
report = {
"passed": True,
"chat_id": chat_id,
"stages": summary,
}
(directory / "summary.json").write_text(
json.dumps(
report,
ensure_ascii=False,
indent=2,
) + "\n",
encoding="utf-8",
)
print(
"PASS: reconciliation resumes after recovery "
"and repeated apply changes nothing"
)
print("Reports: " + str(directory))
print(
"PASS: reconciler dependency failure "
"scenario completed"
)
if __name__ == "__main__":
run(main)