forked from mtusi2026/mtusi2026lab4
686 lines
15 KiB
Python
686 lines
15 KiB
Python
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
from client import Client, arguments, check, run
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BUCKET = None
|
|
|
|
|
|
def command(args, data=None, timeout=30):
|
|
result = subprocess.run(
|
|
args,
|
|
cwd=ROOT,
|
|
input=data,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
)
|
|
|
|
check(
|
|
result.returncode == 0,
|
|
"Command failed: " + " ".join(args[:3]),
|
|
)
|
|
|
|
return result.stdout
|
|
|
|
|
|
def sql(statement):
|
|
result = command(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"sh",
|
|
"-ec",
|
|
(
|
|
'exec psql -X -A -t -v ON_ERROR_STOP=1 '
|
|
'-U "$POSTGRES_USER" '
|
|
'-d "${POSTGRES_DB:-$POSTGRES_USER}"'
|
|
),
|
|
],
|
|
statement.encode(),
|
|
)
|
|
|
|
return result.decode().strip()
|
|
|
|
|
|
def mc(*args, data=None):
|
|
return command(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"minio",
|
|
"sh",
|
|
"-ec",
|
|
(
|
|
"mc alias set reconcile_test "
|
|
"http://localhost:9000 "
|
|
'"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" '
|
|
'>/dev/null; exec mc "$@"'
|
|
),
|
|
"sh",
|
|
*args,
|
|
],
|
|
data,
|
|
)
|
|
|
|
|
|
def target(key):
|
|
return "reconcile_test/" + BUCKET + "/" + key
|
|
|
|
|
|
def object_names(chat_id):
|
|
raw = mc(
|
|
"ls",
|
|
"--recursive",
|
|
"--json",
|
|
target("originals/" + chat_id + "/"),
|
|
)
|
|
|
|
names = set()
|
|
|
|
for line in raw.decode().splitlines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
item = json.loads(line)
|
|
|
|
check(
|
|
item.get("status") == "success",
|
|
"Cannot list test S3 prefix",
|
|
)
|
|
|
|
if item.get("type") == "file":
|
|
names.add(
|
|
item["key"].rstrip("/").rsplit("/", 1)[-1]
|
|
)
|
|
|
|
return names
|
|
|
|
|
|
def inspect_api():
|
|
ids = command([
|
|
"docker",
|
|
"compose",
|
|
"ps",
|
|
"-q",
|
|
"api",
|
|
]).decode().split()
|
|
|
|
check(
|
|
len(ids) == 2,
|
|
"Start exactly two API replicas",
|
|
)
|
|
|
|
containers = json.loads(command([
|
|
"docker",
|
|
"inspect",
|
|
*ids,
|
|
]))
|
|
|
|
buckets = set()
|
|
|
|
for container in containers:
|
|
env = dict(
|
|
value.split("=", 1)
|
|
for value in container["Config"]["Env"]
|
|
if "=" in value
|
|
)
|
|
|
|
check(
|
|
env.get("APP_CLEANUP_ENABLED", "").lower()
|
|
== "false",
|
|
"Disable AttachmentCleanupJob "
|
|
"on both replicas first",
|
|
)
|
|
|
|
check(
|
|
bool(env.get("S3_BUCKET")),
|
|
"S3_BUCKET is missing",
|
|
)
|
|
|
|
buckets.add(env["S3_BUCKET"])
|
|
|
|
check(
|
|
len(buckets) == 1,
|
|
"API replicas use different buckets",
|
|
)
|
|
|
|
return buckets.pop()
|
|
|
|
|
|
def snapshot(chat_id):
|
|
# chat_id предварительно проверен через uuid.UUID.
|
|
return sql(
|
|
"SELECT coalesce("
|
|
"json_agg(a ORDER BY id), '[]'::json"
|
|
") FROM attachments a "
|
|
"WHERE chat_id='" + chat_id + "'::uuid;"
|
|
)
|
|
|
|
|
|
def reconcile(
|
|
chat_id,
|
|
backups,
|
|
directory,
|
|
label,
|
|
apply=False,
|
|
):
|
|
path = directory / (label + ".json")
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-X",
|
|
"utf8",
|
|
str(ROOT / "scripts/reconcile.py"),
|
|
"--apply" if apply else "--dry-run",
|
|
"--chat-id",
|
|
chat_id,
|
|
"--grace-seconds",
|
|
"60",
|
|
"--max-seconds",
|
|
"120",
|
|
"--backup-dir",
|
|
str(backups),
|
|
"--output",
|
|
str(path),
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=180,
|
|
)
|
|
|
|
check(
|
|
result.returncode in (0, 2),
|
|
"Reconciler failed at "
|
|
+ label
|
|
+ "; inspect "
|
|
+ str(path),
|
|
)
|
|
|
|
check(
|
|
path.exists(),
|
|
"Reconciler did not save report",
|
|
)
|
|
|
|
report = json.loads(
|
|
path.read_text(encoding="utf-8")
|
|
)
|
|
|
|
check(
|
|
report["status"] in (
|
|
"completed",
|
|
"needs_attention",
|
|
),
|
|
"Scan failed",
|
|
)
|
|
|
|
check(
|
|
report["counts"].get("busy", 0) == 0,
|
|
"Object lock was busy; repeat experiment",
|
|
)
|
|
|
|
check(
|
|
report["counts"].get("size_mismatches", 0) == 0,
|
|
"Unexpected size mismatch",
|
|
)
|
|
|
|
print(
|
|
label + ": " + json.dumps(report["counts"]),
|
|
flush=True,
|
|
)
|
|
|
|
return report
|
|
|
|
|
|
def count(report, name):
|
|
return report["counts"].get(name, 0)
|
|
|
|
|
|
def main():
|
|
global BUCKET
|
|
|
|
args = arguments(
|
|
"Lab4 reconciler: dry-run, repair "
|
|
"and repeated execution"
|
|
)
|
|
|
|
BUCKET = inspect_api()
|
|
|
|
c = Client(
|
|
args.base_url,
|
|
args.ca_file,
|
|
timeout=30,
|
|
)
|
|
|
|
alice, bob, _, chat = c.bootstrap()
|
|
chat_id = str(uuid.UUID(chat["id"]))
|
|
|
|
stable, content = c.attachment(alice, chat)
|
|
missing, missing_bytes = c.attachment(alice, chat)
|
|
|
|
message = c.message(
|
|
alice,
|
|
chat,
|
|
"Reconciler experiment",
|
|
attachments=[
|
|
stable["id"],
|
|
missing["id"],
|
|
],
|
|
)["json"]
|
|
|
|
missing_id = str(uuid.UUID(missing["id"]))
|
|
|
|
key = sql(
|
|
"SELECT object_key FROM attachments "
|
|
"WHERE id='" + missing_id + "'::uuid;"
|
|
)
|
|
|
|
check(
|
|
key.startswith("originals/" + chat_id + "/"),
|
|
"Unexpected original key",
|
|
)
|
|
|
|
prefix = "originals/" + chat_id + "/"
|
|
|
|
old_key = (
|
|
prefix + "orphan-old-" + uuid.uuid4().hex
|
|
)
|
|
|
|
fresh_key = None
|
|
fresh_started = None
|
|
|
|
# Запоминаем все созданные orphan-ключи,
|
|
# чтобы попытаться убрать их при исключении.
|
|
synthetic_keys = {old_key}
|
|
|
|
backups = (
|
|
ROOT / "backups" / ("reconcile-" + chat_id)
|
|
)
|
|
|
|
directory = (
|
|
ROOT / "results" / ("reconcile-" + chat_id)
|
|
)
|
|
|
|
backups.mkdir(parents=True, exist_ok=False)
|
|
directory.mkdir(parents=True, exist_ok=False)
|
|
|
|
metadata_before = snapshot(chat_id)
|
|
|
|
needs_restore = False
|
|
completed = False
|
|
|
|
def verify_metadata():
|
|
check(
|
|
snapshot(chat_id) == metadata_before,
|
|
"Attachment metadata changed",
|
|
)
|
|
|
|
page = c.call(
|
|
"GET",
|
|
"/api/v1/chats/"
|
|
+ chat_id
|
|
+ "/messages?limit=100",
|
|
actor=bob,
|
|
schema="MessagePage",
|
|
)["json"]
|
|
|
|
found = next(
|
|
(
|
|
item
|
|
for item in page["items"]
|
|
if item["id"] == message["id"]
|
|
),
|
|
None,
|
|
)
|
|
|
|
check(
|
|
found == message,
|
|
"Message or attachment references changed",
|
|
)
|
|
|
|
def prepare_fresh():
|
|
nonlocal fresh_key, fresh_started
|
|
|
|
# Предыдущий свежий объект больше не нужен.
|
|
# Его сохранность проверена предыдущим этапом.
|
|
if fresh_key is not None:
|
|
previous_name = fresh_key.rsplit("/", 1)[-1]
|
|
|
|
if previous_name in object_names(chat_id):
|
|
mc("rm", target(fresh_key))
|
|
|
|
fresh_key = (
|
|
prefix + "orphan-fresh-" + uuid.uuid4().hex
|
|
)
|
|
|
|
synthetic_keys.add(fresh_key)
|
|
|
|
# Начинаем отсчёт до PUT: это консервативная
|
|
# верхняя оценка возраста объекта.
|
|
fresh_started = time.monotonic()
|
|
|
|
mc(
|
|
"pipe",
|
|
target(fresh_key),
|
|
data=content,
|
|
)
|
|
|
|
def verify_fresh():
|
|
check(
|
|
fresh_key is not None
|
|
and fresh_started is not None,
|
|
"Fresh test object was not created",
|
|
)
|
|
|
|
names = object_names(chat_id)
|
|
elapsed = time.monotonic() - fresh_started
|
|
|
|
check(
|
|
elapsed < 55,
|
|
"A single reconciliation stage exceeded "
|
|
"the fresh-object timing budget; "
|
|
"repeat on an idle stand",
|
|
)
|
|
|
|
check(
|
|
fresh_key.rsplit("/", 1)[-1] in names,
|
|
"Fresh orphan was removed",
|
|
)
|
|
|
|
def download(attachment, expected):
|
|
result = c.call(
|
|
"GET",
|
|
"/api/v1/attachments/"
|
|
+ attachment["id"]
|
|
+ "/content",
|
|
actor=bob,
|
|
)
|
|
|
|
check(
|
|
result["bytes"] == expected,
|
|
"Original bytes changed",
|
|
)
|
|
|
|
try:
|
|
mc(
|
|
"pipe",
|
|
target(old_key),
|
|
data=content,
|
|
)
|
|
|
|
print(
|
|
"Old orphan created. Waiting 65 seconds "
|
|
"for the real grace period...",
|
|
flush=True,
|
|
)
|
|
|
|
time.sleep(65)
|
|
|
|
prepare_fresh()
|
|
|
|
# Байты пока доступны только тесту.
|
|
# Reconciler ещё не получает резервную копию.
|
|
needs_restore = True
|
|
mc("rm", target(key))
|
|
|
|
before = object_names(chat_id)
|
|
|
|
dry = reconcile(
|
|
chat_id,
|
|
backups,
|
|
directory,
|
|
"01-dry-run",
|
|
)
|
|
|
|
check(
|
|
count(dry, "would_delete_orphans") == 1,
|
|
"Old orphan not detected",
|
|
)
|
|
|
|
check(
|
|
count(dry, "unresolved_missing") == 1,
|
|
"Missing original not detected",
|
|
)
|
|
|
|
check(
|
|
object_names(chat_id) == before,
|
|
"Dry-run changed S3 objects",
|
|
)
|
|
|
|
verify_metadata()
|
|
verify_fresh()
|
|
download(stable, content)
|
|
|
|
print(
|
|
"PASS: dry-run found both faults "
|
|
"and changed no test data",
|
|
flush=True,
|
|
)
|
|
|
|
# Для следующего этапа создаём новый свежий объект.
|
|
prepare_fresh()
|
|
|
|
applied = reconcile(
|
|
chat_id,
|
|
backups,
|
|
directory,
|
|
"02-apply-no-backup",
|
|
apply=True,
|
|
)
|
|
|
|
check(
|
|
count(applied, "deleted_orphans") == 1,
|
|
"Old orphan not deleted",
|
|
)
|
|
|
|
check(
|
|
count(applied, "unresolved_missing") == 1,
|
|
"Missing original incorrectly repaired",
|
|
)
|
|
|
|
names = object_names(chat_id)
|
|
|
|
check(
|
|
old_key.rsplit("/", 1)[-1] not in names,
|
|
"Old orphan still exists",
|
|
)
|
|
|
|
check(
|
|
key.rsplit("/", 1)[-1] not in names,
|
|
"Missing original unexpectedly exists",
|
|
)
|
|
|
|
verify_metadata()
|
|
verify_fresh()
|
|
|
|
c.call(
|
|
"GET",
|
|
"/api/v1/attachments/"
|
|
+ missing_id
|
|
+ "/content",
|
|
expected=503,
|
|
actor=bob,
|
|
)
|
|
|
|
print(
|
|
"PASS: old orphan deleted; missing original "
|
|
"reported without metadata loss",
|
|
flush=True,
|
|
)
|
|
|
|
# Предоставляем reconciler достоверную копию.
|
|
backup_file = (
|
|
backups / (missing["sha256"] + ".bin")
|
|
)
|
|
|
|
backup_file.write_bytes(missing_bytes)
|
|
|
|
prepare_fresh()
|
|
before = object_names(chat_id)
|
|
|
|
planned = reconcile(
|
|
chat_id,
|
|
backups,
|
|
directory,
|
|
"03-dry-run-with-backup",
|
|
)
|
|
|
|
check(
|
|
count(planned, "would_restore") == 1,
|
|
"Valid backup not recognized",
|
|
)
|
|
|
|
check(
|
|
object_names(chat_id) == before,
|
|
"Dry-run restored an object",
|
|
)
|
|
|
|
verify_metadata()
|
|
verify_fresh()
|
|
|
|
print(
|
|
"PASS: dry-run plans restoration "
|
|
"without changing objects",
|
|
flush=True,
|
|
)
|
|
|
|
prepare_fresh()
|
|
|
|
restored = reconcile(
|
|
chat_id,
|
|
backups,
|
|
directory,
|
|
"04-restore",
|
|
apply=True,
|
|
)
|
|
|
|
check(
|
|
count(restored, "restored") == 1,
|
|
"Original not restored",
|
|
)
|
|
|
|
download(missing, missing_bytes)
|
|
needs_restore = False
|
|
|
|
download(stable, content)
|
|
verify_metadata()
|
|
verify_fresh()
|
|
|
|
print(
|
|
"PASS: original restored byte-for-byte; "
|
|
"used attachment and message preserved",
|
|
flush=True,
|
|
)
|
|
|
|
prepare_fresh()
|
|
|
|
repeated = reconcile(
|
|
chat_id,
|
|
backups,
|
|
directory,
|
|
"05-repeat",
|
|
apply=True,
|
|
)
|
|
|
|
for name in (
|
|
"deleted_orphans",
|
|
"restored",
|
|
"unresolved_missing",
|
|
"missing_objects",
|
|
):
|
|
check(
|
|
count(repeated, name) == 0,
|
|
"Repeated run is not clean: " + name,
|
|
)
|
|
|
|
verify_metadata()
|
|
verify_fresh()
|
|
|
|
download(stable, content)
|
|
download(missing, missing_bytes)
|
|
|
|
completed = True
|
|
|
|
print(
|
|
"PASS: repeated apply makes "
|
|
"no additional changes",
|
|
flush=True,
|
|
)
|
|
|
|
print(
|
|
"Reports: " + str(directory),
|
|
flush=True,
|
|
)
|
|
|
|
finally:
|
|
# При исключении пытаемся восстановить оригинал,
|
|
# намеренно удалённый этим тестом.
|
|
if needs_restore:
|
|
try:
|
|
mc(
|
|
"pipe",
|
|
target(key),
|
|
data=missing_bytes,
|
|
)
|
|
|
|
print(
|
|
"Test original restored "
|
|
"during final cleanup",
|
|
flush=True,
|
|
)
|
|
|
|
except Exception:
|
|
print(
|
|
"WARNING: restore test original manually "
|
|
"from backup; key=" + key,
|
|
flush=True,
|
|
)
|
|
|
|
(
|
|
backups / (missing["sha256"] + ".bin")
|
|
).write_bytes(missing_bytes)
|
|
|
|
cleanup_ok = True
|
|
|
|
# Удаляем только существующие синтетические объекты.
|
|
# Уже удалённый reconciler старый orphan — не ошибка.
|
|
for orphan in sorted(synthetic_keys):
|
|
try:
|
|
name = orphan.rsplit("/", 1)[-1]
|
|
|
|
if name in object_names(chat_id):
|
|
mc("rm", target(orphan))
|
|
|
|
except Exception:
|
|
cleanup_ok = False
|
|
|
|
print(
|
|
"WARNING: cannot remove synthetic orphan: "
|
|
+ orphan,
|
|
flush=True,
|
|
)
|
|
|
|
if completed:
|
|
check(
|
|
cleanup_ok,
|
|
"Scenario passed but final "
|
|
"test-object cleanup failed",
|
|
)
|
|
|
|
print("PASS: reconciler experiment completed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |