Files
mtusi2026lab4/research/run.py
T

285 lines
12 KiB
Python

"""Real PostgreSQL/S3 fault injection in disposable DB and bucket, never production data."""
import hashlib
import json
import os
import time
from datetime import UTC, datetime
from io import BytesIO
from pathlib import Path
from uuid import uuid4
import psycopg
from PIL import Image
from psycopg import sql
from psycopg.conninfo import make_conninfo
from messenger.attachments import AttachmentService
from messenger.config import secret
from messenger.domain import Failure
from messenger.migrate import migrate
from messenger.postgres import PostgresStore
from messenger.reconcile import reconcile
from messenger.storage import ObjectStorage
def main():
original_dsn = secret("DATABASE_URL")
suffix = uuid4().hex[:12]
database, bucket = "research_" + suffix, "research-" + suffix
admin = psycopg.connect(original_dsn, autocommit=True)
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
os.environ.pop("DATABASE_URL_FILE", None)
os.environ["DATABASE_URL"] = make_conninfo(original_dsn, dbname=database)
os.environ["S3_BUCKET"] = bucket
storage = ObjectStorage(admin=True)
storage.client.create_bucket(Bucket=bucket)
store = None
result = {
"hypothesis": "Durable intent, row locks, grace and live HEAD make reconciliation repeatable and protect fresh/used data",
"started_at": datetime.now(UTC).isoformat(),
"database": database,
"bucket": bucket,
"grace_seconds": 2,
"checks": [],
"scan_cost": [],
"source": {},
}
for path in sorted(Path("src/messenger").glob("*.py")):
result["source"][str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
try:
migrate()
store = PostgresStore()
service = AttachmentService(store, storage)
user = store.register(
{
"username": "research_" + suffix,
"display_name": "Research",
"password": "Synthetic-research-password",
}
)
chat = store.create_chat(user["id"], {"title": "Research", "member_ids": [user["id"]]})
stream = BytesIO()
Image.new("RGB", (16, 10), "blue").save(stream, "PNG")
data = stream.getvalue()
def upload(name):
return service.upload(
user["id"], chat["id"], name, "image/png", BytesIO(data), len(data)
)
def record(name, condition):
if not condition:
raise AssertionError(name)
result["checks"].append({"name": name, "passed": True})
good = upload("used.png")
store.send(user["id"], chat["id"], "", uuid4().hex, [good["id"]])
missing = upload("missing.png")
store.send(user["id"], chat["id"], "", uuid4().hex, [missing["id"]])
storage.delete("original/" + missing["id"])
# A real PostgreSQL INSERT failure after a successful real S3 PUT.
with store.connection() as conn:
conn.execute(
"CREATE FUNCTION reject_research_upload() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='db-fault.png' THEN RAISE EXCEPTION 'injected DB write failure'; END IF; RETURN NEW; END $$"
)
conn.execute(
"CREATE TRIGGER research_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_research_upload()"
)
try:
upload("db-fault.png")
raise AssertionError("DB fault did not fire")
except psycopg.errors.RaiseException:
record("real_db_insert_failure_after_s3_put", True)
finally:
with store.connection() as conn:
conn.execute("DROP TRIGGER research_reject ON attachments")
conn.execute("DROP FUNCTION reject_research_upload()")
# Real connection failure to a closed S3 endpoint; no final attachment row.
os.environ["S3_ENDPOINT"] = "http://s3:1"
broken_storage = ObjectStorage(admin=True)
try:
AttachmentService(store, broken_storage).upload(
user["id"], chat["id"], "s3-fault.png", "image/png", BytesIO(data), len(data)
)
raise AssertionError("S3 fault did not fire")
except Failure as exc:
record("real_s3_connect_failure_is_503", exc.status == 503)
finally:
broken_storage.close()
os.environ["S3_ENDPOINT"] = "http://s3:8333"
# A successful COMMIT followed by an injected loss of its acknowledgement.
os.environ["UPLOAD_FAULT"] = "after_commit"
try:
upload("commit-ack-lost.png")
raise AssertionError("post-COMMIT fault did not fire")
except Failure:
record("commit_ack_loss_injected", True)
finally:
os.environ.pop("UPLOAD_FAULT")
with store.connection() as conn:
committed = conn.execute(
"SELECT * FROM attachments WHERE filename='commit-ack-lost.png'"
).fetchone()
record(
"uncertain_response_keeps_committed_original",
committed["status"] == "ready" and storage.exists(committed["object_key"]),
)
orphan_key = "original/orphan-" + suffix
storage.put(orphan_key, BytesIO(data), len(data), "image/png")
# All initial rows and objects become older than the real grace period.
time.sleep(2.2)
fresh = upload("fresh.png")
fresh_orphan = "original/fresh-orphan-" + suffix
storage.put(fresh_orphan, BytesIO(data), len(data), "image/png")
fresh_intent_id = str(uuid4())
fresh_intent_key = "original/" + fresh_intent_id
with store.connection() as conn:
conn.execute(
"INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')",
(fresh_intent_id, fresh_intent_key),
)
storage.put(fresh_intent_key, BytesIO(data), len(data), "image/png")
# A live writer holds an old intent; neither its row nor object may vanish.
active_id, active_key = str(uuid4()), "original/active-" + suffix
with store.connection() as conn:
conn.execute(
"INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')",
(active_id, active_key),
)
storage.put(active_key, BytesIO(data), len(data), "image/png")
locked = psycopg.connect(os.environ["DATABASE_URL"], autocommit=False)
locked.execute("SELECT * FROM upload_intents WHERE id=%s FOR UPDATE", (active_id,))
with store.connection() as conn:
before = {
"attachments": conn.execute("SELECT count(*) AS n FROM attachments").fetchone()[
"n"
],
"intents": conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"],
"audit": conn.execute(
"SELECT count(*) AS n FROM attachment_reconcile_audit"
).fetchone()["n"],
}
before_keys = sorted(x["Key"] for x in storage.objects())
dry = reconcile(store, storage, grace_seconds=2, page_size=1)
with store.connection() as conn:
after = {
"attachments": conn.execute("SELECT count(*) AS n FROM attachments").fetchone()[
"n"
],
"intents": conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"],
"audit": conn.execute(
"SELECT count(*) AS n FROM attachment_reconcile_audit"
).fetchone()["n"],
}
record(
"dry_run_changes_neither_db_nor_s3",
before == after and before_keys == sorted(x["Key"] for x in storage.objects()),
)
applied = reconcile(store, storage, apply=True, grace_seconds=2, page_size=1)
repeated = reconcile(store, storage, apply=True, grace_seconds=2, page_size=1)
result["dry_run"], result["apply"], result["repeat"] = dry, applied, repeated
record(
"all_reconcile_passes_have_no_errors",
not any(r["errors"] for r in [dry, applied, repeated]),
)
record("orphan_removed", not storage.exists(orphan_key))
record(
"missing_marked_failed_metadata_kept",
service.metadata(user["id"], missing["id"])["error_code"] == "storage_missing",
)
record("used_original_kept", storage.exists("original/" + good["id"]))
record(
"fresh_metadata_and_original_kept",
service.metadata(user["id"], fresh["id"])["status"] == "ready"
and storage.exists("original/" + fresh["id"]),
)
record("fresh_orphan_kept", storage.exists(fresh_orphan))
record("old_intent_fresh_object_kept", storage.exists(fresh_intent_key))
record("active_writer_object_kept", storage.exists(active_key))
record(
"committed_original_survives_reconciliation", storage.exists(committed["object_key"])
)
record(
"repeat_has_no_duplicate_mutations",
repeated["deleted_objects"]
== repeated["deleted_intents"]
== repeated["marked_failed"]
== 0,
)
locked.rollback()
locked.close()
# A failed S3 delete leaves the intent retryable. A dry run afterwards must
# not resume deletion or append audit records.
time.sleep(2.2)
class DeleteFailure:
def __getattr__(self, name):
return getattr(storage, name)
def delete(self, key):
raise Failure(503, "injected S3 delete failure")
failed_cleanup = reconcile(store, DeleteFailure(), apply=True, grace_seconds=2)
record("s3_delete_failure_observed", failed_cleanup["errors"] > 0)
with store.connection() as conn:
audit_before = conn.execute(
"SELECT count(*) AS n FROM attachment_reconcile_audit"
).fetchone()["n"]
recovery_dry = reconcile(store, storage, grace_seconds=2)
with store.connection() as conn:
record(
"dry_run_after_failed_apply_still_read_only",
conn.execute("SELECT count(*) AS n FROM attachment_reconcile_audit").fetchone()["n"]
== audit_before,
)
recovery_apply = reconcile(store, storage, apply=True, grace_seconds=2)
result["failed_cleanup"], result["recovery_dry"], result["recovery_apply"] = (
failed_cleanup,
recovery_dry,
recovery_apply,
)
record("failed_cleanup_recovers", recovery_apply["errors"] == 0)
# Measure an increasing population of intact, aged metadata/objects. Rows
# share no client names or object paths, and all are in this disposable DB.
current = 0
for target in (10, 100, 1000):
for _ in range(target - current):
upload("scan.png")
current = target
with store.connection() as conn:
conn.execute("UPDATE attachments SET created_at=now()-interval '1 day'")
for repeat in range(3):
run = reconcile(store, storage, grace_seconds=2, page_size=100)
record("scan_has_no_errors_" + str(target) + "_" + str(repeat), run["errors"] == 0)
result["scan_cost"].append({"seeded_intact": target, "repeat": repeat + 1, **run})
result["limitations"] = [
"One local host and warm services; scan measures HEAD/SQL round trips, not production SLA",
"Missing originals are marked failed; reconciliation cannot recreate lost bytes without backup",
"Valid unattached ready attachments are retained; only abandoned intents and untracked old original objects are removed",
"No atomic transaction spans S3 and PostgreSQL; retries plus durable intent recover the selected windows",
"Cost dataset is in disposable isolated DB/bucket, includes a few preserved fault fixtures beyond the seeded intact count",
]
finally:
if store:
store.close()
for obj in list(storage.objects()):
storage.delete(obj["Key"])
storage.client.delete_bucket(Bucket=bucket)
storage.close()
admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
admin.close()
result["temporary_resources_deleted"] = True
print("RESULT " + json.dumps(result, ensure_ascii=False, default=str))
if __name__ == "__main__":
main()