forked from mtusi2026/mtusi2026lab4
492 lines
11 KiB
Python
492 lines
11 KiB
Python
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
import reconcile_check as helpers
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def load_launcher():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"reconciler_launcher",
|
|
ROOT / "scripts/reconcile.py",
|
|
)
|
|
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
|
|
return module
|
|
|
|
|
|
def state(container_id):
|
|
raw = helpers.command([
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{json .State}}",
|
|
container_id,
|
|
])
|
|
|
|
return json.loads(raw)
|
|
|
|
|
|
def wait_container(container_id, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
|
|
while time.monotonic() < deadline:
|
|
current = state(container_id)
|
|
|
|
healthy = (
|
|
current.get("Running")
|
|
and current.get("Health", {}).get("Status")
|
|
== "healthy"
|
|
)
|
|
|
|
if healthy:
|
|
return
|
|
|
|
time.sleep(2)
|
|
|
|
raise Failure(
|
|
"Container did not become healthy: "
|
|
+ container_id[:12]
|
|
)
|
|
|
|
|
|
def wait_api(client, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call(
|
|
"GET",
|
|
"/health/ready",
|
|
schema="Health",
|
|
)
|
|
|
|
return
|
|
|
|
except Failure:
|
|
time.sleep(2)
|
|
|
|
raise Failure(
|
|
"API did not recover after dependency restart"
|
|
)
|
|
|
|
|
|
def main():
|
|
args = arguments(
|
|
"Reconciler failure checks: "
|
|
"MinIO and PostgreSQL outage"
|
|
)
|
|
|
|
helpers.BUCKET = helpers.inspect_api()
|
|
|
|
launcher = load_launcher()
|
|
|
|
saved_services = {
|
|
name: launcher.service(name)
|
|
for name in ("api", "postgres", "minio")
|
|
}
|
|
|
|
launcher.service = lambda name: saved_services[name]
|
|
|
|
c = Client(
|
|
args.base_url,
|
|
args.ca_file,
|
|
timeout=10,
|
|
)
|
|
|
|
alice, bob, _, chat = c.bootstrap()
|
|
chat_id = str(uuid.UUID(chat["id"]))
|
|
|
|
attachment, content = c.attachment(alice, chat)
|
|
|
|
message = c.message(
|
|
alice,
|
|
chat,
|
|
"Reconciler dependency failure",
|
|
attachments=[attachment["id"]],
|
|
)["json"]
|
|
|
|
original_name = str(uuid.UUID(attachment["id"]))
|
|
|
|
orphan_key = (
|
|
"originals/"
|
|
+ chat_id
|
|
+ "/orphan-failure-"
|
|
+ uuid.uuid4().hex
|
|
)
|
|
|
|
orphan_name = orphan_key.rsplit("/", 1)[-1]
|
|
|
|
metadata_before = helpers.snapshot(chat_id)
|
|
|
|
directory = (
|
|
ROOT
|
|
/ "results"
|
|
/ ("reconcile-failure-" + chat_id)
|
|
)
|
|
|
|
backups = (
|
|
ROOT
|
|
/ "backups"
|
|
/ ("reconcile-failure-" + chat_id)
|
|
)
|
|
|
|
directory.mkdir(parents=True, exist_ok=False)
|
|
backups.mkdir(parents=True, exist_ok=False)
|
|
|
|
stopped = set()
|
|
summary = []
|
|
completed = False
|
|
|
|
def worker(label, apply, expected_code):
|
|
path = directory / (label + ".json")
|
|
previous_argv = sys.argv
|
|
|
|
try:
|
|
sys.argv = [
|
|
str(ROOT / "scripts/reconcile.py"),
|
|
"--apply" if apply else "--dry-run",
|
|
"--chat-id",
|
|
chat_id,
|
|
"--grace-seconds",
|
|
"60",
|
|
"--max-seconds",
|
|
"120",
|
|
"--backup-dir",
|
|
str(backups),
|
|
"--output",
|
|
str(path),
|
|
]
|
|
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
code = launcher.main()
|
|
|
|
finally:
|
|
sys.argv = previous_argv
|
|
|
|
check(
|
|
code == expected_code,
|
|
"Unexpected return code at " + label,
|
|
)
|
|
|
|
report = json.loads(
|
|
path.read_text(encoding="utf-8")
|
|
)
|
|
|
|
summary.append({
|
|
"stage": label,
|
|
"status": report["status"],
|
|
"elapsed_seconds": report["elapsed_seconds"],
|
|
"error_type": report["error_type"],
|
|
})
|
|
|
|
print(
|
|
label
|
|
+ ": status="
|
|
+ report["status"]
|
|
+ ", error_type="
|
|
+ str(report["error_type"]),
|
|
flush=True,
|
|
)
|
|
|
|
return report
|
|
|
|
def verify_data(expect_orphan):
|
|
check(
|
|
helpers.snapshot(chat_id) == metadata_before,
|
|
"Attachment metadata changed",
|
|
)
|
|
|
|
names = helpers.object_names(chat_id)
|
|
|
|
expected = {original_name}
|
|
|
|
if expect_orphan:
|
|
expected.add(orphan_name)
|
|
|
|
check(
|
|
names == expected,
|
|
"Unexpected S3 object set",
|
|
)
|
|
|
|
result = c.call(
|
|
"GET",
|
|
"/api/v1/attachments/"
|
|
+ attachment["id"]
|
|
+ "/content",
|
|
actor=bob,
|
|
)
|
|
|
|
check(
|
|
result["bytes"] == content,
|
|
"Confirmed original changed",
|
|
)
|
|
|
|
page = c.call(
|
|
"GET",
|
|
"/api/v1/chats/"
|
|
+ chat_id
|
|
+ "/messages?limit=100",
|
|
actor=bob,
|
|
schema="MessagePage",
|
|
)["json"]
|
|
|
|
check(
|
|
message in page["items"],
|
|
"Message or attachment references changed",
|
|
)
|
|
|
|
def restore(name):
|
|
container_id = saved_services[name]["Id"]
|
|
|
|
helpers.command(
|
|
[
|
|
"docker",
|
|
"start",
|
|
container_id,
|
|
],
|
|
timeout=60,
|
|
)
|
|
|
|
wait_container(container_id)
|
|
stopped.discard(name)
|
|
|
|
wait_api(c)
|
|
|
|
try:
|
|
helpers.mc(
|
|
"pipe",
|
|
helpers.target(orphan_key),
|
|
data=content,
|
|
)
|
|
|
|
print(
|
|
"Waiting 65 seconds "
|
|
"for the real orphan grace period...",
|
|
flush=True,
|
|
)
|
|
|
|
time.sleep(65)
|
|
|
|
baseline = worker(
|
|
"01-baseline",
|
|
apply=False,
|
|
expected_code=0,
|
|
)
|
|
|
|
check(
|
|
baseline["counts"].get(
|
|
"would_delete_orphans", 0
|
|
) == 1,
|
|
"No deletion candidate in baseline",
|
|
)
|
|
|
|
verify_data(expect_orphan=True)
|
|
|
|
for number, name in enumerate(
|
|
("minio", "postgres"),
|
|
start=2,
|
|
):
|
|
container_id = saved_services[name]["Id"]
|
|
|
|
print(
|
|
"Stopping "
|
|
+ name
|
|
+ " for the experiment",
|
|
flush=True,
|
|
)
|
|
|
|
stopped.add(name)
|
|
|
|
helpers.command(
|
|
[
|
|
"docker",
|
|
"stop",
|
|
"-t",
|
|
"10",
|
|
container_id,
|
|
],
|
|
timeout=60,
|
|
)
|
|
|
|
check(
|
|
not state(container_id).get("Running"),
|
|
"Dependency did not stop",
|
|
)
|
|
|
|
try:
|
|
report = worker(
|
|
f"{number:02d}-{name}-unavailable",
|
|
apply=True,
|
|
expected_code=1,
|
|
)
|
|
|
|
check(
|
|
report["status"] == "failed",
|
|
"Outage was not reported as failed",
|
|
)
|
|
|
|
check(
|
|
report["error_type"] is not None,
|
|
"Missing error type",
|
|
)
|
|
|
|
for operation in (
|
|
"delete_object",
|
|
"put_object",
|
|
):
|
|
check(
|
|
report["s3_operation_calls"].get(
|
|
operation, 0
|
|
) == 0,
|
|
"Mutation attempted "
|
|
"during dependency outage",
|
|
)
|
|
|
|
for counter in (
|
|
"missing_objects",
|
|
"unresolved_missing",
|
|
"deleted_orphans",
|
|
"restored",
|
|
):
|
|
check(
|
|
report["counts"].get(
|
|
counter, 0
|
|
) == 0,
|
|
"Outage was confused "
|
|
"with a data inconsistency",
|
|
)
|
|
|
|
finally:
|
|
restore(name)
|
|
|
|
verify_data(expect_orphan=True)
|
|
|
|
print(
|
|
"PASS: "
|
|
+ name
|
|
+ " outage aborted reconciliation; "
|
|
"data and session survived",
|
|
flush=True,
|
|
)
|
|
|
|
repaired = worker(
|
|
"04-recovery-apply",
|
|
apply=True,
|
|
expected_code=0,
|
|
)
|
|
|
|
check(
|
|
repaired["counts"].get(
|
|
"deleted_orphans", 0
|
|
) == 1,
|
|
"Orphan was not removed after recovery",
|
|
)
|
|
|
|
verify_data(expect_orphan=False)
|
|
|
|
repeated = worker(
|
|
"05-recovery-repeat",
|
|
apply=True,
|
|
expected_code=0,
|
|
)
|
|
|
|
for counter in (
|
|
"deleted_orphans",
|
|
"restored",
|
|
"missing_objects",
|
|
"unresolved_missing",
|
|
):
|
|
check(
|
|
repeated["counts"].get(
|
|
counter, 0
|
|
) == 0,
|
|
"Repeated run is not clean",
|
|
)
|
|
|
|
verify_data(expect_orphan=False)
|
|
completed = True
|
|
|
|
finally:
|
|
recovery_errors = []
|
|
|
|
for name in tuple(stopped):
|
|
try:
|
|
restore(name)
|
|
|
|
except Exception:
|
|
recovery_errors.append(name)
|
|
|
|
print(
|
|
"WARNING: could not restore service: "
|
|
+ name,
|
|
flush=True,
|
|
)
|
|
|
|
if not recovery_errors:
|
|
try:
|
|
if orphan_name in helpers.object_names(chat_id):
|
|
helpers.mc(
|
|
"rm",
|
|
helpers.target(orphan_key),
|
|
)
|
|
|
|
except Exception:
|
|
recovery_errors.append(
|
|
"test orphan cleanup"
|
|
)
|
|
|
|
print(
|
|
"WARNING: cannot remove synthetic orphan: "
|
|
+ orphan_key,
|
|
flush=True,
|
|
)
|
|
|
|
check(
|
|
not recovery_errors,
|
|
"Recovery incomplete: "
|
|
+ ", ".join(recovery_errors),
|
|
)
|
|
|
|
if completed:
|
|
report = {
|
|
"passed": True,
|
|
"chat_id": chat_id,
|
|
"stages": summary,
|
|
}
|
|
|
|
(directory / "summary.json").write_text(
|
|
json.dumps(
|
|
report,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
print(
|
|
"PASS: reconciliation resumes after recovery "
|
|
"and repeated apply changes nothing"
|
|
)
|
|
|
|
print("Reports: " + str(directory))
|
|
|
|
print(
|
|
"PASS: reconciler dependency failure "
|
|
"scenario completed"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |