forked from mtusi2026/mtusi2026lab4
278 lines
6.2 KiB
Python
278 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
from client import Client, arguments, check, run
|
|
|
|
|
|
def execute(command, input_text=None):
|
|
result = subprocess.run(
|
|
command,
|
|
input=input_text,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
|
|
check(
|
|
result.returncode == 0,
|
|
"Infrastructure command failed. "
|
|
"Check PostgreSQL, MinIO and the configured bucket",
|
|
)
|
|
|
|
return result.stdout.strip()
|
|
|
|
|
|
def sql(query):
|
|
return execute(
|
|
[
|
|
"docker", "compose", "exec", "-T",
|
|
"postgres", "sh", "-ec",
|
|
'''
|
|
exec psql -X \
|
|
-v ON_ERROR_STOP=1 \
|
|
-U "$POSTGRES_USER" \
|
|
-d "$POSTGRES_DB" \
|
|
-At
|
|
''',
|
|
],
|
|
input_text=query,
|
|
)
|
|
|
|
|
|
def minio(script, *arguments):
|
|
return execute([
|
|
"docker", "compose", "exec", "-T",
|
|
"minio", "sh", "-ec",
|
|
script,
|
|
"failure-check",
|
|
*arguments,
|
|
])
|
|
|
|
|
|
def configure_minio():
|
|
minio(
|
|
'''
|
|
mc alias set failurecheck http://localhost:9000 \
|
|
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
|
'''
|
|
)
|
|
|
|
|
|
def list_objects(bucket, prefix):
|
|
output = minio(
|
|
'mc ls --json "failurecheck/$1/$2"',
|
|
bucket,
|
|
prefix,
|
|
)
|
|
|
|
objects = set()
|
|
|
|
for line in output.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
item = json.loads(line)
|
|
|
|
check(
|
|
item.get("status") != "error",
|
|
"MinIO failed to list test objects",
|
|
)
|
|
|
|
if item.get("key"):
|
|
objects.add(item["key"])
|
|
|
|
return objects
|
|
|
|
|
|
def wait_until(deadline):
|
|
remaining = deadline - time.monotonic()
|
|
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument(
|
|
"--bucket",
|
|
default="messenger-attachments",
|
|
)
|
|
|
|
args = arguments(
|
|
"Failure between S3 upload and metadata INSERT",
|
|
configure,
|
|
)
|
|
|
|
check(
|
|
re.fullmatch(
|
|
r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]",
|
|
args.bucket,
|
|
) is not None,
|
|
"Invalid bucket name",
|
|
)
|
|
|
|
client = Client(
|
|
args.base_url,
|
|
args.ca_file,
|
|
timeout=40,
|
|
)
|
|
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
|
|
chat_id = str(uuid.UUID(chat["id"]))
|
|
|
|
configure_minio()
|
|
|
|
good_attachment, original = client.attachment(alice, chat)
|
|
|
|
prefix = f"originals/{chat_id}/"
|
|
|
|
before = list_objects(args.bucket, prefix)
|
|
|
|
check(
|
|
len(before) == 1,
|
|
"Expected one control object in the new test chat",
|
|
)
|
|
|
|
marker = uuid.uuid4().hex
|
|
filename = f"db-failure-{marker}.png"
|
|
|
|
function_name = f"test_attachment_failure_{marker}"
|
|
trigger_name = f"test_attachment_failure_{marker}"
|
|
|
|
cleanup_sql = f"""
|
|
DROP TRIGGER IF EXISTS {trigger_name}
|
|
ON public.attachments;
|
|
|
|
DROP FUNCTION IF EXISTS public.{function_name}();
|
|
"""
|
|
|
|
try:
|
|
sql(f"""
|
|
CREATE FUNCTION public.{function_name}()
|
|
RETURNS trigger
|
|
LANGUAGE plpgsql
|
|
AS $$
|
|
BEGIN
|
|
RAISE EXCEPTION
|
|
'Synthetic attachment metadata failure';
|
|
END;
|
|
$$;
|
|
|
|
CREATE TRIGGER {trigger_name}
|
|
BEFORE INSERT ON public.attachments
|
|
FOR EACH ROW
|
|
WHEN (NEW.filename = '{filename}')
|
|
EXECUTE FUNCTION public.{function_name}();
|
|
""")
|
|
|
|
print(
|
|
"Installed a DB failure for one synthetic filename",
|
|
flush=True,
|
|
)
|
|
|
|
response = client.call(
|
|
"POST",
|
|
"/api/v1/chats/" + chat_id
|
|
+ "/attachments?filename="
|
|
+ quote(filename, safe=""),
|
|
503,
|
|
actor=alice,
|
|
raw=original,
|
|
headers={"Content-Type": "image/png"},
|
|
)
|
|
|
|
check(
|
|
response["json"]["error"]["code"] == "unavailable",
|
|
"Expected unavailable error code",
|
|
)
|
|
|
|
print("PASS: failed metadata INSERT returns 503", flush=True)
|
|
|
|
sql(cleanup_sql)
|
|
|
|
count = sql(f"""
|
|
SELECT count(*)
|
|
FROM public.attachments
|
|
WHERE chat_id = '{chat_id}'::uuid
|
|
AND filename = '{filename}';
|
|
""")
|
|
|
|
check(
|
|
count == "0",
|
|
"Metadata was unexpectedly committed",
|
|
)
|
|
|
|
after = list_objects(args.bucket, prefix)
|
|
orphan_objects = after - before
|
|
|
|
check(
|
|
len(orphan_objects) == 1,
|
|
"Expected exactly one S3 object without metadata",
|
|
)
|
|
|
|
orphan = next(iter(orphan_objects))
|
|
observed_at = time.monotonic()
|
|
|
|
print(
|
|
"PASS: S3 object exists, but metadata was rolled back",
|
|
flush=True,
|
|
)
|
|
|
|
wait_until(observed_at + 25)
|
|
|
|
check(
|
|
orphan in list_objects(args.bucket, prefix),
|
|
"Orphan was deleted before the grace period",
|
|
)
|
|
|
|
print("PASS: grace period protects the recent object", flush=True)
|
|
print("Waiting for scheduled cleanup...", flush=True)
|
|
|
|
deadline = observed_at + 100
|
|
|
|
while orphan in list_objects(args.bucket, prefix):
|
|
check(
|
|
time.monotonic() < deadline,
|
|
"Orphan was not deleted. Check cleanup settings and logs",
|
|
)
|
|
|
|
time.sleep(2)
|
|
|
|
print(
|
|
"PASS: orphan created by a real failed INSERT was deleted",
|
|
flush=True,
|
|
)
|
|
|
|
downloaded = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/"
|
|
+ good_attachment["id"]
|
|
+ "/content",
|
|
actor=bob,
|
|
)
|
|
|
|
check(
|
|
downloaded["bytes"] == original,
|
|
"Control attachment was lost or modified",
|
|
)
|
|
|
|
print("PASS: control attachment is intact", flush=True)
|
|
|
|
client.attachment(alice, chat)
|
|
|
|
print("PASS: normal uploads work after the failure", flush=True)
|
|
print("PASS: S3/DB failure scenario completed", flush=True)
|
|
|
|
finally:
|
|
sql(cleanup_sql)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |