Files
mtusi2026lab4/tests/upload_db_failure_check.py
T
2026-11-08 18:17:48 +03:00

278 lines
6.2 KiB
Python

#!/usr/bin/env python3
import json
import re
import subprocess
import time
import uuid
from urllib.parse import quote
from client import Client, arguments, check, run
def execute(command, input_text=None):
result = subprocess.run(
command,
input=input_text,
capture_output=True,
text=True,
timeout=30,
)
check(
result.returncode == 0,
"Infrastructure command failed. "
"Check PostgreSQL, MinIO and the configured bucket",
)
return result.stdout.strip()
def sql(query):
return execute(
[
"docker", "compose", "exec", "-T",
"postgres", "sh", "-ec",
'''
exec psql -X \
-v ON_ERROR_STOP=1 \
-U "$POSTGRES_USER" \
-d "$POSTGRES_DB" \
-At
''',
],
input_text=query,
)
def minio(script, *arguments):
return execute([
"docker", "compose", "exec", "-T",
"minio", "sh", "-ec",
script,
"failure-check",
*arguments,
])
def configure_minio():
minio(
'''
mc alias set failurecheck http://localhost:9000 \
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
'''
)
def list_objects(bucket, prefix):
output = minio(
'mc ls --json "failurecheck/$1/$2"',
bucket,
prefix,
)
objects = set()
for line in output.splitlines():
if not line.strip():
continue
item = json.loads(line)
check(
item.get("status") != "error",
"MinIO failed to list test objects",
)
if item.get("key"):
objects.add(item["key"])
return objects
def wait_until(deadline):
remaining = deadline - time.monotonic()
if remaining > 0:
time.sleep(remaining)
def main():
def configure(parser):
parser.add_argument(
"--bucket",
default="messenger-attachments",
)
args = arguments(
"Failure between S3 upload and metadata INSERT",
configure,
)
check(
re.fullmatch(
r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]",
args.bucket,
) is not None,
"Invalid bucket name",
)
client = Client(
args.base_url,
args.ca_file,
timeout=40,
)
alice, bob, eve, chat = client.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
configure_minio()
good_attachment, original = client.attachment(alice, chat)
prefix = f"originals/{chat_id}/"
before = list_objects(args.bucket, prefix)
check(
len(before) == 1,
"Expected one control object in the new test chat",
)
marker = uuid.uuid4().hex
filename = f"db-failure-{marker}.png"
function_name = f"test_attachment_failure_{marker}"
trigger_name = f"test_attachment_failure_{marker}"
cleanup_sql = f"""
DROP TRIGGER IF EXISTS {trigger_name}
ON public.attachments;
DROP FUNCTION IF EXISTS public.{function_name}();
"""
try:
sql(f"""
CREATE FUNCTION public.{function_name}()
RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
RAISE EXCEPTION
'Synthetic attachment metadata failure';
END;
$$;
CREATE TRIGGER {trigger_name}
BEFORE INSERT ON public.attachments
FOR EACH ROW
WHEN (NEW.filename = '{filename}')
EXECUTE FUNCTION public.{function_name}();
""")
print(
"Installed a DB failure for one synthetic filename",
flush=True,
)
response = client.call(
"POST",
"/api/v1/chats/" + chat_id
+ "/attachments?filename="
+ quote(filename, safe=""),
503,
actor=alice,
raw=original,
headers={"Content-Type": "image/png"},
)
check(
response["json"]["error"]["code"] == "unavailable",
"Expected unavailable error code",
)
print("PASS: failed metadata INSERT returns 503", flush=True)
sql(cleanup_sql)
count = sql(f"""
SELECT count(*)
FROM public.attachments
WHERE chat_id = '{chat_id}'::uuid
AND filename = '{filename}';
""")
check(
count == "0",
"Metadata was unexpectedly committed",
)
after = list_objects(args.bucket, prefix)
orphan_objects = after - before
check(
len(orphan_objects) == 1,
"Expected exactly one S3 object without metadata",
)
orphan = next(iter(orphan_objects))
observed_at = time.monotonic()
print(
"PASS: S3 object exists, but metadata was rolled back",
flush=True,
)
wait_until(observed_at + 25)
check(
orphan in list_objects(args.bucket, prefix),
"Orphan was deleted before the grace period",
)
print("PASS: grace period protects the recent object", flush=True)
print("Waiting for scheduled cleanup...", flush=True)
deadline = observed_at + 100
while orphan in list_objects(args.bucket, prefix):
check(
time.monotonic() < deadline,
"Orphan was not deleted. Check cleanup settings and logs",
)
time.sleep(2)
print(
"PASS: orphan created by a real failed INSERT was deleted",
flush=True,
)
downloaded = client.call(
"GET",
"/api/v1/attachments/"
+ good_attachment["id"]
+ "/content",
actor=bob,
)
check(
downloaded["bytes"] == original,
"Control attachment was lost or modified",
)
print("PASS: control attachment is intact", flush=True)
client.attachment(alice, chat)
print("PASS: normal uploads work after the failure", flush=True)
print("PASS: S3/DB failure scenario completed", flush=True)
finally:
sql(cleanup_sql)
if __name__ == "__main__":
run(main)