285 lines
12 KiB
Python
285 lines
12 KiB
Python
"""Real PostgreSQL/S3 fault injection in disposable DB and bucket, never production data."""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from PIL import Image
|
|
from psycopg import sql
|
|
from psycopg.conninfo import make_conninfo
|
|
|
|
from messenger.attachments import AttachmentService
|
|
from messenger.config import secret
|
|
from messenger.domain import Failure
|
|
from messenger.migrate import migrate
|
|
from messenger.postgres import PostgresStore
|
|
from messenger.reconcile import reconcile
|
|
from messenger.storage import ObjectStorage
|
|
|
|
|
|
def main():
|
|
original_dsn = secret("DATABASE_URL")
|
|
suffix = uuid4().hex[:12]
|
|
database, bucket = "research_" + suffix, "research-" + suffix
|
|
admin = psycopg.connect(original_dsn, autocommit=True)
|
|
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
|
|
os.environ.pop("DATABASE_URL_FILE", None)
|
|
os.environ["DATABASE_URL"] = make_conninfo(original_dsn, dbname=database)
|
|
os.environ["S3_BUCKET"] = bucket
|
|
storage = ObjectStorage(admin=True)
|
|
storage.client.create_bucket(Bucket=bucket)
|
|
store = None
|
|
result = {
|
|
"hypothesis": "Durable intent, row locks, grace and live HEAD make reconciliation repeatable and protect fresh/used data",
|
|
"started_at": datetime.now(UTC).isoformat(),
|
|
"database": database,
|
|
"bucket": bucket,
|
|
"grace_seconds": 2,
|
|
"checks": [],
|
|
"scan_cost": [],
|
|
"source": {},
|
|
}
|
|
for path in sorted(Path("src/messenger").glob("*.py")):
|
|
result["source"][str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
try:
|
|
migrate()
|
|
store = PostgresStore()
|
|
service = AttachmentService(store, storage)
|
|
user = store.register(
|
|
{
|
|
"username": "research_" + suffix,
|
|
"display_name": "Research",
|
|
"password": "Synthetic-research-password",
|
|
}
|
|
)
|
|
chat = store.create_chat(user["id"], {"title": "Research", "member_ids": [user["id"]]})
|
|
stream = BytesIO()
|
|
Image.new("RGB", (16, 10), "blue").save(stream, "PNG")
|
|
data = stream.getvalue()
|
|
|
|
def upload(name):
|
|
return service.upload(
|
|
user["id"], chat["id"], name, "image/png", BytesIO(data), len(data)
|
|
)
|
|
|
|
def record(name, condition):
|
|
if not condition:
|
|
raise AssertionError(name)
|
|
result["checks"].append({"name": name, "passed": True})
|
|
|
|
good = upload("used.png")
|
|
store.send(user["id"], chat["id"], "", uuid4().hex, [good["id"]])
|
|
missing = upload("missing.png")
|
|
store.send(user["id"], chat["id"], "", uuid4().hex, [missing["id"]])
|
|
storage.delete("original/" + missing["id"])
|
|
|
|
# A real PostgreSQL INSERT failure after a successful real S3 PUT.
|
|
with store.connection() as conn:
|
|
conn.execute(
|
|
"CREATE FUNCTION reject_research_upload() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='db-fault.png' THEN RAISE EXCEPTION 'injected DB write failure'; END IF; RETURN NEW; END $$"
|
|
)
|
|
conn.execute(
|
|
"CREATE TRIGGER research_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_research_upload()"
|
|
)
|
|
try:
|
|
upload("db-fault.png")
|
|
raise AssertionError("DB fault did not fire")
|
|
except psycopg.errors.RaiseException:
|
|
record("real_db_insert_failure_after_s3_put", True)
|
|
finally:
|
|
with store.connection() as conn:
|
|
conn.execute("DROP TRIGGER research_reject ON attachments")
|
|
conn.execute("DROP FUNCTION reject_research_upload()")
|
|
|
|
# Real connection failure to a closed S3 endpoint; no final attachment row.
|
|
os.environ["S3_ENDPOINT"] = "http://s3:1"
|
|
broken_storage = ObjectStorage(admin=True)
|
|
try:
|
|
AttachmentService(store, broken_storage).upload(
|
|
user["id"], chat["id"], "s3-fault.png", "image/png", BytesIO(data), len(data)
|
|
)
|
|
raise AssertionError("S3 fault did not fire")
|
|
except Failure as exc:
|
|
record("real_s3_connect_failure_is_503", exc.status == 503)
|
|
finally:
|
|
broken_storage.close()
|
|
os.environ["S3_ENDPOINT"] = "http://s3:8333"
|
|
|
|
# A successful COMMIT followed by an injected loss of its acknowledgement.
|
|
os.environ["UPLOAD_FAULT"] = "after_commit"
|
|
try:
|
|
upload("commit-ack-lost.png")
|
|
raise AssertionError("post-COMMIT fault did not fire")
|
|
except Failure:
|
|
record("commit_ack_loss_injected", True)
|
|
finally:
|
|
os.environ.pop("UPLOAD_FAULT")
|
|
with store.connection() as conn:
|
|
committed = conn.execute(
|
|
"SELECT * FROM attachments WHERE filename='commit-ack-lost.png'"
|
|
).fetchone()
|
|
record(
|
|
"uncertain_response_keeps_committed_original",
|
|
committed["status"] == "ready" and storage.exists(committed["object_key"]),
|
|
)
|
|
|
|
orphan_key = "original/orphan-" + suffix
|
|
storage.put(orphan_key, BytesIO(data), len(data), "image/png")
|
|
# All initial rows and objects become older than the real grace period.
|
|
time.sleep(2.2)
|
|
fresh = upload("fresh.png")
|
|
fresh_orphan = "original/fresh-orphan-" + suffix
|
|
storage.put(fresh_orphan, BytesIO(data), len(data), "image/png")
|
|
fresh_intent_id = str(uuid4())
|
|
fresh_intent_key = "original/" + fresh_intent_id
|
|
with store.connection() as conn:
|
|
conn.execute(
|
|
"INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')",
|
|
(fresh_intent_id, fresh_intent_key),
|
|
)
|
|
storage.put(fresh_intent_key, BytesIO(data), len(data), "image/png")
|
|
|
|
# A live writer holds an old intent; neither its row nor object may vanish.
|
|
active_id, active_key = str(uuid4()), "original/active-" + suffix
|
|
with store.connection() as conn:
|
|
conn.execute(
|
|
"INSERT INTO upload_intents VALUES (%s,%s,now()-interval '1 day')",
|
|
(active_id, active_key),
|
|
)
|
|
storage.put(active_key, BytesIO(data), len(data), "image/png")
|
|
locked = psycopg.connect(os.environ["DATABASE_URL"], autocommit=False)
|
|
locked.execute("SELECT * FROM upload_intents WHERE id=%s FOR UPDATE", (active_id,))
|
|
with store.connection() as conn:
|
|
before = {
|
|
"attachments": conn.execute("SELECT count(*) AS n FROM attachments").fetchone()[
|
|
"n"
|
|
],
|
|
"intents": conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"],
|
|
"audit": conn.execute(
|
|
"SELECT count(*) AS n FROM attachment_reconcile_audit"
|
|
).fetchone()["n"],
|
|
}
|
|
before_keys = sorted(x["Key"] for x in storage.objects())
|
|
dry = reconcile(store, storage, grace_seconds=2, page_size=1)
|
|
with store.connection() as conn:
|
|
after = {
|
|
"attachments": conn.execute("SELECT count(*) AS n FROM attachments").fetchone()[
|
|
"n"
|
|
],
|
|
"intents": conn.execute("SELECT count(*) AS n FROM upload_intents").fetchone()["n"],
|
|
"audit": conn.execute(
|
|
"SELECT count(*) AS n FROM attachment_reconcile_audit"
|
|
).fetchone()["n"],
|
|
}
|
|
record(
|
|
"dry_run_changes_neither_db_nor_s3",
|
|
before == after and before_keys == sorted(x["Key"] for x in storage.objects()),
|
|
)
|
|
applied = reconcile(store, storage, apply=True, grace_seconds=2, page_size=1)
|
|
repeated = reconcile(store, storage, apply=True, grace_seconds=2, page_size=1)
|
|
result["dry_run"], result["apply"], result["repeat"] = dry, applied, repeated
|
|
record(
|
|
"all_reconcile_passes_have_no_errors",
|
|
not any(r["errors"] for r in [dry, applied, repeated]),
|
|
)
|
|
record("orphan_removed", not storage.exists(orphan_key))
|
|
record(
|
|
"missing_marked_failed_metadata_kept",
|
|
service.metadata(user["id"], missing["id"])["error_code"] == "storage_missing",
|
|
)
|
|
record("used_original_kept", storage.exists("original/" + good["id"]))
|
|
record(
|
|
"fresh_metadata_and_original_kept",
|
|
service.metadata(user["id"], fresh["id"])["status"] == "ready"
|
|
and storage.exists("original/" + fresh["id"]),
|
|
)
|
|
record("fresh_orphan_kept", storage.exists(fresh_orphan))
|
|
record("old_intent_fresh_object_kept", storage.exists(fresh_intent_key))
|
|
record("active_writer_object_kept", storage.exists(active_key))
|
|
record(
|
|
"committed_original_survives_reconciliation", storage.exists(committed["object_key"])
|
|
)
|
|
record(
|
|
"repeat_has_no_duplicate_mutations",
|
|
repeated["deleted_objects"]
|
|
== repeated["deleted_intents"]
|
|
== repeated["marked_failed"]
|
|
== 0,
|
|
)
|
|
locked.rollback()
|
|
locked.close()
|
|
|
|
# A failed S3 delete leaves the intent retryable. A dry run afterwards must
|
|
# not resume deletion or append audit records.
|
|
time.sleep(2.2)
|
|
|
|
class DeleteFailure:
|
|
def __getattr__(self, name):
|
|
return getattr(storage, name)
|
|
|
|
def delete(self, key):
|
|
raise Failure(503, "injected S3 delete failure")
|
|
|
|
failed_cleanup = reconcile(store, DeleteFailure(), apply=True, grace_seconds=2)
|
|
record("s3_delete_failure_observed", failed_cleanup["errors"] > 0)
|
|
with store.connection() as conn:
|
|
audit_before = conn.execute(
|
|
"SELECT count(*) AS n FROM attachment_reconcile_audit"
|
|
).fetchone()["n"]
|
|
recovery_dry = reconcile(store, storage, grace_seconds=2)
|
|
with store.connection() as conn:
|
|
record(
|
|
"dry_run_after_failed_apply_still_read_only",
|
|
conn.execute("SELECT count(*) AS n FROM attachment_reconcile_audit").fetchone()["n"]
|
|
== audit_before,
|
|
)
|
|
recovery_apply = reconcile(store, storage, apply=True, grace_seconds=2)
|
|
result["failed_cleanup"], result["recovery_dry"], result["recovery_apply"] = (
|
|
failed_cleanup,
|
|
recovery_dry,
|
|
recovery_apply,
|
|
)
|
|
record("failed_cleanup_recovers", recovery_apply["errors"] == 0)
|
|
|
|
# Measure an increasing population of intact, aged metadata/objects. Rows
|
|
# share no client names or object paths, and all are in this disposable DB.
|
|
current = 0
|
|
for target in (10, 100, 1000):
|
|
for _ in range(target - current):
|
|
upload("scan.png")
|
|
current = target
|
|
with store.connection() as conn:
|
|
conn.execute("UPDATE attachments SET created_at=now()-interval '1 day'")
|
|
for repeat in range(3):
|
|
run = reconcile(store, storage, grace_seconds=2, page_size=100)
|
|
record("scan_has_no_errors_" + str(target) + "_" + str(repeat), run["errors"] == 0)
|
|
result["scan_cost"].append({"seeded_intact": target, "repeat": repeat + 1, **run})
|
|
result["limitations"] = [
|
|
"One local host and warm services; scan measures HEAD/SQL round trips, not production SLA",
|
|
"Missing originals are marked failed; reconciliation cannot recreate lost bytes without backup",
|
|
"Valid unattached ready attachments are retained; only abandoned intents and untracked old original objects are removed",
|
|
"No atomic transaction spans S3 and PostgreSQL; retries plus durable intent recover the selected windows",
|
|
"Cost dataset is in disposable isolated DB/bucket, includes a few preserved fault fixtures beyond the seeded intact count",
|
|
]
|
|
finally:
|
|
if store:
|
|
store.close()
|
|
for obj in list(storage.objects()):
|
|
storage.delete(obj["Key"])
|
|
storage.client.delete_bucket(Bucket=bucket)
|
|
storage.close()
|
|
admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
|
|
admin.close()
|
|
result["temporary_resources_deleted"] = True
|
|
print("RESULT " + json.dumps(result, ensure_ascii=False, default=str))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|