#!/usr/bin/env python3 import json import re import subprocess import time import uuid from urllib.parse import quote from client import Client, arguments, check, run def execute(command, input_text=None): result = subprocess.run( command, input=input_text, capture_output=True, text=True, timeout=30, ) check( result.returncode == 0, "Infrastructure command failed. " "Check PostgreSQL, MinIO and the configured bucket", ) return result.stdout.strip() def sql(query): return execute( [ "docker", "compose", "exec", "-T", "postgres", "sh", "-ec", ''' exec psql -X \ -v ON_ERROR_STOP=1 \ -U "$POSTGRES_USER" \ -d "$POSTGRES_DB" \ -At ''', ], input_text=query, ) def minio(script, *arguments): return execute([ "docker", "compose", "exec", "-T", "minio", "sh", "-ec", script, "failure-check", *arguments, ]) def configure_minio(): minio( ''' mc alias set failurecheck http://localhost:9000 \ "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null ''' ) def list_objects(bucket, prefix): output = minio( 'mc ls --json "failurecheck/$1/$2"', bucket, prefix, ) objects = set() for line in output.splitlines(): if not line.strip(): continue item = json.loads(line) check( item.get("status") != "error", "MinIO failed to list test objects", ) if item.get("key"): objects.add(item["key"]) return objects def wait_until(deadline): remaining = deadline - time.monotonic() if remaining > 0: time.sleep(remaining) def main(): def configure(parser): parser.add_argument( "--bucket", default="messenger-attachments", ) args = arguments( "Failure between S3 upload and metadata INSERT", configure, ) check( re.fullmatch( r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", args.bucket, ) is not None, "Invalid bucket name", ) client = Client( args.base_url, args.ca_file, timeout=40, ) alice, bob, eve, chat = client.bootstrap() chat_id = str(uuid.UUID(chat["id"])) configure_minio() good_attachment, original = client.attachment(alice, chat) prefix = f"originals/{chat_id}/" before = list_objects(args.bucket, prefix) check( len(before) == 1, "Expected one control object in the new test chat", ) marker = uuid.uuid4().hex filename = f"db-failure-{marker}.png" function_name = f"test_attachment_failure_{marker}" trigger_name = f"test_attachment_failure_{marker}" cleanup_sql = f""" DROP TRIGGER IF EXISTS {trigger_name} ON public.attachments; DROP FUNCTION IF EXISTS public.{function_name}(); """ try: sql(f""" CREATE FUNCTION public.{function_name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'Synthetic attachment metadata failure'; END; $$; CREATE TRIGGER {trigger_name} BEFORE INSERT ON public.attachments FOR EACH ROW WHEN (NEW.filename = '{filename}') EXECUTE FUNCTION public.{function_name}(); """) print( "Installed a DB failure for one synthetic filename", flush=True, ) response = client.call( "POST", "/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""), 503, actor=alice, raw=original, headers={"Content-Type": "image/png"}, ) check( response["json"]["error"]["code"] == "unavailable", "Expected unavailable error code", ) print("PASS: failed metadata INSERT returns 503", flush=True) sql(cleanup_sql) count = sql(f""" SELECT count(*) FROM public.attachments WHERE chat_id = '{chat_id}'::uuid AND filename = '{filename}'; """) check( count == "0", "Metadata was unexpectedly committed", ) after = list_objects(args.bucket, prefix) orphan_objects = after - before check( len(orphan_objects) == 1, "Expected exactly one S3 object without metadata", ) orphan = next(iter(orphan_objects)) observed_at = time.monotonic() print( "PASS: S3 object exists, but metadata was rolled back", flush=True, ) wait_until(observed_at + 25) check( orphan in list_objects(args.bucket, prefix), "Orphan was deleted before the grace period", ) print("PASS: grace period protects the recent object", flush=True) print("Waiting for scheduled cleanup...", flush=True) deadline = observed_at + 100 while orphan in list_objects(args.bucket, prefix): check( time.monotonic() < deadline, "Orphan was not deleted. Check cleanup settings and logs", ) time.sleep(2) print( "PASS: orphan created by a real failed INSERT was deleted", flush=True, ) downloaded = client.call( "GET", "/api/v1/attachments/" + good_attachment["id"] + "/content", actor=bob, ) check( downloaded["bytes"] == original, "Control attachment was lost or modified", ) print("PASS: control attachment is intact", flush=True) client.attachment(alice, chat) print("PASS: normal uploads work after the failure", flush=True) print("PASS: S3/DB failure scenario completed", flush=True) finally: sql(cleanup_sql) if __name__ == "__main__": run(main)