Files
mtusi2026lab4/tests/reconcile_check.py
T
2026-11-08 18:17:48 +03:00

686 lines
15 KiB
Python

import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
from client import Client, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
BUCKET = None
def command(args, data=None, timeout=30):
result = subprocess.run(
args,
cwd=ROOT,
input=data,
capture_output=True,
timeout=timeout,
)
check(
result.returncode == 0,
"Command failed: " + " ".join(args[:3]),
)
return result.stdout
def sql(statement):
result = command(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"sh",
"-ec",
(
'exec psql -X -A -t -v ON_ERROR_STOP=1 '
'-U "$POSTGRES_USER" '
'-d "${POSTGRES_DB:-$POSTGRES_USER}"'
),
],
statement.encode(),
)
return result.decode().strip()
def mc(*args, data=None):
return command(
[
"docker",
"compose",
"exec",
"-T",
"minio",
"sh",
"-ec",
(
"mc alias set reconcile_test "
"http://localhost:9000 "
'"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" '
'>/dev/null; exec mc "$@"'
),
"sh",
*args,
],
data,
)
def target(key):
return "reconcile_test/" + BUCKET + "/" + key
def object_names(chat_id):
raw = mc(
"ls",
"--recursive",
"--json",
target("originals/" + chat_id + "/"),
)
names = set()
for line in raw.decode().splitlines():
if not line.strip():
continue
item = json.loads(line)
check(
item.get("status") == "success",
"Cannot list test S3 prefix",
)
if item.get("type") == "file":
names.add(
item["key"].rstrip("/").rsplit("/", 1)[-1]
)
return names
def inspect_api():
ids = command([
"docker",
"compose",
"ps",
"-q",
"api",
]).decode().split()
check(
len(ids) == 2,
"Start exactly two API replicas",
)
containers = json.loads(command([
"docker",
"inspect",
*ids,
]))
buckets = set()
for container in containers:
env = dict(
value.split("=", 1)
for value in container["Config"]["Env"]
if "=" in value
)
check(
env.get("APP_CLEANUP_ENABLED", "").lower()
== "false",
"Disable AttachmentCleanupJob "
"on both replicas first",
)
check(
bool(env.get("S3_BUCKET")),
"S3_BUCKET is missing",
)
buckets.add(env["S3_BUCKET"])
check(
len(buckets) == 1,
"API replicas use different buckets",
)
return buckets.pop()
def snapshot(chat_id):
# chat_id предварительно проверен через uuid.UUID.
return sql(
"SELECT coalesce("
"json_agg(a ORDER BY id), '[]'::json"
") FROM attachments a "
"WHERE chat_id='" + chat_id + "'::uuid;"
)
def reconcile(
chat_id,
backups,
directory,
label,
apply=False,
):
path = directory / (label + ".json")
result = subprocess.run(
[
sys.executable,
"-X",
"utf8",
str(ROOT / "scripts/reconcile.py"),
"--apply" if apply else "--dry-run",
"--chat-id",
chat_id,
"--grace-seconds",
"60",
"--max-seconds",
"120",
"--backup-dir",
str(backups),
"--output",
str(path),
],
cwd=ROOT,
capture_output=True,
text=True,
timeout=180,
)
check(
result.returncode in (0, 2),
"Reconciler failed at "
+ label
+ "; inspect "
+ str(path),
)
check(
path.exists(),
"Reconciler did not save report",
)
report = json.loads(
path.read_text(encoding="utf-8")
)
check(
report["status"] in (
"completed",
"needs_attention",
),
"Scan failed",
)
check(
report["counts"].get("busy", 0) == 0,
"Object lock was busy; repeat experiment",
)
check(
report["counts"].get("size_mismatches", 0) == 0,
"Unexpected size mismatch",
)
print(
label + ": " + json.dumps(report["counts"]),
flush=True,
)
return report
def count(report, name):
return report["counts"].get(name, 0)
def main():
global BUCKET
args = arguments(
"Lab4 reconciler: dry-run, repair "
"and repeated execution"
)
BUCKET = inspect_api()
c = Client(
args.base_url,
args.ca_file,
timeout=30,
)
alice, bob, _, chat = c.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
stable, content = c.attachment(alice, chat)
missing, missing_bytes = c.attachment(alice, chat)
message = c.message(
alice,
chat,
"Reconciler experiment",
attachments=[
stable["id"],
missing["id"],
],
)["json"]
missing_id = str(uuid.UUID(missing["id"]))
key = sql(
"SELECT object_key FROM attachments "
"WHERE id='" + missing_id + "'::uuid;"
)
check(
key.startswith("originals/" + chat_id + "/"),
"Unexpected original key",
)
prefix = "originals/" + chat_id + "/"
old_key = (
prefix + "orphan-old-" + uuid.uuid4().hex
)
fresh_key = None
fresh_started = None
# Запоминаем все созданные orphan-ключи,
# чтобы попытаться убрать их при исключении.
synthetic_keys = {old_key}
backups = (
ROOT / "backups" / ("reconcile-" + chat_id)
)
directory = (
ROOT / "results" / ("reconcile-" + chat_id)
)
backups.mkdir(parents=True, exist_ok=False)
directory.mkdir(parents=True, exist_ok=False)
metadata_before = snapshot(chat_id)
needs_restore = False
completed = False
def verify_metadata():
check(
snapshot(chat_id) == metadata_before,
"Attachment metadata changed",
)
page = c.call(
"GET",
"/api/v1/chats/"
+ chat_id
+ "/messages?limit=100",
actor=bob,
schema="MessagePage",
)["json"]
found = next(
(
item
for item in page["items"]
if item["id"] == message["id"]
),
None,
)
check(
found == message,
"Message or attachment references changed",
)
def prepare_fresh():
nonlocal fresh_key, fresh_started
# Предыдущий свежий объект больше не нужен.
# Его сохранность проверена предыдущим этапом.
if fresh_key is not None:
previous_name = fresh_key.rsplit("/", 1)[-1]
if previous_name in object_names(chat_id):
mc("rm", target(fresh_key))
fresh_key = (
prefix + "orphan-fresh-" + uuid.uuid4().hex
)
synthetic_keys.add(fresh_key)
# Начинаем отсчёт до PUT: это консервативная
# верхняя оценка возраста объекта.
fresh_started = time.monotonic()
mc(
"pipe",
target(fresh_key),
data=content,
)
def verify_fresh():
check(
fresh_key is not None
and fresh_started is not None,
"Fresh test object was not created",
)
names = object_names(chat_id)
elapsed = time.monotonic() - fresh_started
check(
elapsed < 55,
"A single reconciliation stage exceeded "
"the fresh-object timing budget; "
"repeat on an idle stand",
)
check(
fresh_key.rsplit("/", 1)[-1] in names,
"Fresh orphan was removed",
)
def download(attachment, expected):
result = c.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=bob,
)
check(
result["bytes"] == expected,
"Original bytes changed",
)
try:
mc(
"pipe",
target(old_key),
data=content,
)
print(
"Old orphan created. Waiting 65 seconds "
"for the real grace period...",
flush=True,
)
time.sleep(65)
prepare_fresh()
# Байты пока доступны только тесту.
# Reconciler ещё не получает резервную копию.
needs_restore = True
mc("rm", target(key))
before = object_names(chat_id)
dry = reconcile(
chat_id,
backups,
directory,
"01-dry-run",
)
check(
count(dry, "would_delete_orphans") == 1,
"Old orphan not detected",
)
check(
count(dry, "unresolved_missing") == 1,
"Missing original not detected",
)
check(
object_names(chat_id) == before,
"Dry-run changed S3 objects",
)
verify_metadata()
verify_fresh()
download(stable, content)
print(
"PASS: dry-run found both faults "
"and changed no test data",
flush=True,
)
# Для следующего этапа создаём новый свежий объект.
prepare_fresh()
applied = reconcile(
chat_id,
backups,
directory,
"02-apply-no-backup",
apply=True,
)
check(
count(applied, "deleted_orphans") == 1,
"Old orphan not deleted",
)
check(
count(applied, "unresolved_missing") == 1,
"Missing original incorrectly repaired",
)
names = object_names(chat_id)
check(
old_key.rsplit("/", 1)[-1] not in names,
"Old orphan still exists",
)
check(
key.rsplit("/", 1)[-1] not in names,
"Missing original unexpectedly exists",
)
verify_metadata()
verify_fresh()
c.call(
"GET",
"/api/v1/attachments/"
+ missing_id
+ "/content",
expected=503,
actor=bob,
)
print(
"PASS: old orphan deleted; missing original "
"reported without metadata loss",
flush=True,
)
# Предоставляем reconciler достоверную копию.
backup_file = (
backups / (missing["sha256"] + ".bin")
)
backup_file.write_bytes(missing_bytes)
prepare_fresh()
before = object_names(chat_id)
planned = reconcile(
chat_id,
backups,
directory,
"03-dry-run-with-backup",
)
check(
count(planned, "would_restore") == 1,
"Valid backup not recognized",
)
check(
object_names(chat_id) == before,
"Dry-run restored an object",
)
verify_metadata()
verify_fresh()
print(
"PASS: dry-run plans restoration "
"without changing objects",
flush=True,
)
prepare_fresh()
restored = reconcile(
chat_id,
backups,
directory,
"04-restore",
apply=True,
)
check(
count(restored, "restored") == 1,
"Original not restored",
)
download(missing, missing_bytes)
needs_restore = False
download(stable, content)
verify_metadata()
verify_fresh()
print(
"PASS: original restored byte-for-byte; "
"used attachment and message preserved",
flush=True,
)
prepare_fresh()
repeated = reconcile(
chat_id,
backups,
directory,
"05-repeat",
apply=True,
)
for name in (
"deleted_orphans",
"restored",
"unresolved_missing",
"missing_objects",
):
check(
count(repeated, name) == 0,
"Repeated run is not clean: " + name,
)
verify_metadata()
verify_fresh()
download(stable, content)
download(missing, missing_bytes)
completed = True
print(
"PASS: repeated apply makes "
"no additional changes",
flush=True,
)
print(
"Reports: " + str(directory),
flush=True,
)
finally:
# При исключении пытаемся восстановить оригинал,
# намеренно удалённый этим тестом.
if needs_restore:
try:
mc(
"pipe",
target(key),
data=missing_bytes,
)
print(
"Test original restored "
"during final cleanup",
flush=True,
)
except Exception:
print(
"WARNING: restore test original manually "
"from backup; key=" + key,
flush=True,
)
(
backups / (missing["sha256"] + ".bin")
).write_bytes(missing_bytes)
cleanup_ok = True
# Удаляем только существующие синтетические объекты.
# Уже удалённый reconciler старый orphan — не ошибка.
for orphan in sorted(synthetic_keys):
try:
name = orphan.rsplit("/", 1)[-1]
if name in object_names(chat_id):
mc("rm", target(orphan))
except Exception:
cleanup_ok = False
print(
"WARNING: cannot remove synthetic orphan: "
+ orphan,
flush=True,
)
if completed:
check(
cleanup_ok,
"Scenario passed but final "
"test-object cleanup failed",
)
print("PASS: reconciler experiment completed")
if __name__ == "__main__":
run(main)