Files
mtusi2026lab5/tests/reconcile_integration.py
T
2026-11-10 16:21:46 +03:00

193 lines
7.1 KiB
Python

#!/usr/bin/env python3
"""Inject isolated DB/S3 inconsistencies and prove reconciler dry-run/apply/repeat."""
import json
import os
import subprocess
import time
USER_ID = "90000000-0000-4000-8000-000000000001"
CHAT_ID = "90000000-0000-4000-8000-000000000002"
MISSING_ID = "90000000-0000-4000-8000-000000000011"
PROTECTED_ID = "90000000-0000-4000-8000-000000000012"
FAILED_ID = "90000000-0000-4000-8000-000000000013"
MESSAGE_ID = "90000000-0000-4000-8000-000000000021"
PREFIX = "attachments/reconcile-integration"
ORPHAN_KEY = PREFIX + "/orphan"
FAILED_KEY = PREFIX + "/failed"
def run(command, *, input_text=None, capture=False):
result = subprocess.run(
command,
input=input_text,
text=True,
check=False,
capture_output=capture,
env=os.environ.copy(),
)
if result.returncode != 0:
details = result.stderr.strip() if capture else "see command output"
raise RuntimeError("command failed: %s: %s" % (command[0], details))
return result.stdout if capture else ""
def compose(*arguments, capture=False, input_text=None):
return run(
["docker", "compose", *arguments],
input_text=input_text,
capture=capture,
)
def database(sql):
compose(
"exec", "-T", "postgres",
"psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger",
input_text=sql,
)
def object_command(script):
compose(
"run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap",
"-ec", script,
)
def clean_objects():
object_command(
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'mc rm --force "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null 2>&1 || true\n'
'mc rm --force "local/${S3_BUCKET}/' + FAILED_KEY + '" >/dev/null 2>&1 || true\n'
)
def clean_database():
database(f"""
BEGIN;
DELETE FROM messages WHERE id = '{MESSAGE_ID}';
DELETE FROM attachments WHERE id IN ('{MISSING_ID}', '{PROTECTED_ID}', '{FAILED_ID}');
DELETE FROM chat_members WHERE chat_id = '{CHAT_ID}';
DELETE FROM chats WHERE id = '{CHAT_ID}';
DELETE FROM users WHERE id = '{USER_ID}';
COMMIT;
""")
def inject_database():
database(f"""
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER_ID}', 'reconcile_fixture', 'Reconcile Fixture', 'not-used');
INSERT INTO chats (id, title) VALUES ('{CHAT_ID}', 'Reconcile Integration');
INSERT INTO chat_members (chat_id, user_id, position) VALUES ('{CHAT_ID}', '{USER_ID}', 0);
INSERT INTO attachments (
id, chat_id, owner_id, object_key, filename, content_type,
size_bytes, sha256, width, height, status, error_code, created_at, updated_at
) VALUES
('{MISSING_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/missing', 'missing.png', 'image/png',
7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', NULL, now() - interval '2 hours', now() - interval '2 hours'),
('{PROTECTED_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/protected-missing', 'protected.png', 'image/png',
7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', NULL, now() - interval '2 hours', now() - interval '2 hours'),
('{FAILED_ID}', '{CHAT_ID}', '{USER_ID}', '{FAILED_KEY}', 'failed.png', 'image/png',
7, decode(repeat('00', 32), 'hex'), 1, 1, 'failed', 'fixture_failed', now() - interval '2 hours', now() - interval '2 hours');
INSERT INTO messages (id, chat_id, sender_id, text)
VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{USER_ID}', 'protected attachment reference');
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{PROTECTED_ID}', 0);
COMMIT;
""")
def inject_objects():
object_command(
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'printf orphan | mc pipe "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null\n'
'printf failed | mc pipe "local/${S3_BUCKET}/' + FAILED_KEY + '" >/dev/null\n'
)
def reconcile(apply=False):
command = [
"run", "--rm", "--entrypoint", "/app/reconciler", "api",
"--grace-period", "1s",
]
if apply:
command.append("--apply")
output = compose(*command, capture=True)
return json.loads(output)
def kinds(result):
return {issue["kind"] for issue in result["issues"]}
def fixture_issues(result):
return [issue for issue in result["issues"] if issue["object_key"].startswith(PREFIX)]
def main():
clean_objects()
clean_database()
try:
inject_database()
inject_objects()
time.sleep(2)
dry_run = reconcile(False)
expected = {
"orphan_object", "missing_object", "referenced_missing_object", "failed_object",
}
if kinds({"issues": fixture_issues(dry_run)}) != expected:
raise RuntimeError("dry-run did not detect all injected inconsistencies")
if dry_run["deleted_objects"] or dry_run["failed_attachments"]:
raise RuntimeError("dry-run performed mutations")
applied = reconcile(True)
applied_fixture = fixture_issues(applied)
applied_actions = {issue["kind"]: issue["action"] for issue in applied_fixture}
expected_actions = {
"orphan_object": "deleted",
"failed_object": "deleted",
"missing_object": "marked_failed",
"referenced_missing_object": "manual_repair_required",
}
if applied_actions != expected_actions:
raise RuntimeError("fixture apply actions differ: %s" % applied_actions)
repeated = reconcile(True)
if repeated["deleted_objects"] or repeated["failed_attachments"]:
raise RuntimeError("repeated apply was not mutation-idempotent")
protected = [issue for issue in fixture_issues(repeated) if issue.get("protected")]
if len(protected) != 1 or protected[0]["action"] != "manual_repair_required":
raise RuntimeError("referenced missing attachment was not protected")
print(json.dumps({
"dry_run_issues": len(dry_run["issues"]),
"apply_fixture_deleted_objects": sum(
issue["action"] == "deleted" for issue in applied_fixture
),
"apply_fixture_failed_attachments": sum(
issue["action"] == "marked_failed" for issue in applied_fixture
),
"repeat_mutations": repeated["deleted_objects"] + repeated["failed_attachments"],
"protected_manual_issues": len(protected),
"scanned_attachments": applied["scanned_attachments"],
"scanned_objects": applied["scanned_objects"],
"duration_ms": applied["duration_ms"],
}, ensure_ascii=False))
print("PASS: real PostgreSQL/MinIO reconciliation is dry-run safe, idempotent and reference-aware.")
finally:
clean_objects()
clean_database()
if __name__ == "__main__":
main()