Files
mtusi2026lab4/app/reconcile.py
T

236 lines
8.5 KiB
Python

"""Reconcile PostgreSQL metadata with the private bucket.
Grace period: objects and rows younger than ORPHAN_GRACE_SECONDS are left alone,
because an upload may have written S3 and not committed metadata yet.
Attachments referenced by a message are never deleted.
A dry-run reports the same actions apply would perform, then apply is safe to repeat.
"""
import time
from datetime import datetime, timezone
from app.config import Settings
from app.db import pool
from app import s3client
def _aware(value: datetime) -> datetime:
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _age(moment: datetime, now: datetime) -> float:
return (_aware(now) - _aware(moment)).total_seconds()
async def reconcile(settings: Settings, apply: bool) -> dict:
started = time.perf_counter()
db_started = time.perf_counter()
async with pool().acquire() as conn:
now = await conn.fetchval("SELECT clock_timestamp()")
attachments = await conn.fetch(
"SELECT id, object_key, filename, created_at FROM attachments"
)
ledger_rows = await conn.fetch(
"SELECT object_key, attachment_id, state, note, created_at FROM object_ledger"
)
referenced = {
str(row["attachment_id"])
for row in await conn.fetch("SELECT attachment_id FROM message_attachments")
}
db_ms = (time.perf_counter() - db_started) * 1000
list_started = time.perf_counter()
objects, pages = s3client.list_objects(settings, "o/")
list_ms = (time.perf_counter() - list_started) * 1000
by_key = {row["object_key"]: row for row in attachments}
ledger = {row["object_key"]: row for row in ledger_rows}
object_keys = {item["Key"] for item in objects}
grace = settings.orphan_grace_seconds
actions = []
skipped: dict[str, int] = {}
def skip(reason: str) -> None:
skipped[reason] = skipped.get(reason, 0) + 1
for item in objects:
key = item["Key"]
led = ledger.get(key)
if key in by_key:
skip("accounted")
continue
moment = led["created_at"] if led is not None else item["LastModified"]
note = (led["note"] if led is not None and led["note"] else None) or (
led["state"] if led is not None else "untracked"
)
age = _age(moment, now)
attachment_id = str(led["attachment_id"]) if led is not None and led["attachment_id"] else None
if led is not None and led["state"] in {"aborted", "purged"} and age >= grace:
actions.append(
{
"op": "delete_orphan_object",
"note": note,
"object_key": key,
"attachment_id": attachment_id,
"age_seconds": int(age),
}
)
continue
if age < grace:
skip("fresh_orphan")
continue
actions.append(
{
"op": "delete_orphan_object",
"note": note,
"object_key": key,
"attachment_id": attachment_id,
"age_seconds": int(age),
}
)
for row in attachments:
if row["object_key"] in object_keys:
continue
attachment_id = str(row["id"])
age = _age(row["created_at"], now)
if attachment_id in referenced:
skip("in_use_missing")
continue
if age < grace:
skip("fresh_missing")
continue
actions.append(
{
"op": "delete_stale_metadata",
"note": row["filename"],
"object_key": row["object_key"],
"attachment_id": attachment_id,
"age_seconds": int(age),
}
)
for row in ledger_rows:
if row["object_key"] in object_keys or row["object_key"] in by_key:
continue
if row["state"] in {"aborted", "purged"}:
skip("terminal_ledger")
continue
age = _age(row["created_at"], now)
if age < grace:
skip("fresh_intent")
continue
actions.append(
{
"op": "abort_stale_intent",
"note": row["note"] or row["state"],
"object_key": row["object_key"],
"attachment_id": str(row["attachment_id"]) if row["attachment_id"] else None,
"age_seconds": int(age),
}
)
applied = []
if apply:
for action in actions:
result = await _apply_one(settings, action, grace)
if result is not None:
applied.append(result)
public_actions = [_public(item) for item in (applied if apply else actions)]
return {
"mode": "apply" if apply else "dry-run",
"duration_ms": round((time.perf_counter() - started) * 1000, 3),
"db_ms": round(db_ms, 3),
"list_ms": round(list_ms, 3),
"list_pages": pages,
"objects_seen": len(objects),
"attachment_rows": len(attachments),
"ledger_rows": len(ledger_rows),
"referenced_attachments": len(referenced),
"grace_seconds": grace,
"skipped": skipped,
"actions": public_actions,
"action_count": len(public_actions),
}
def _public(action: dict) -> dict:
return {
"op": action["op"],
"note": action.get("note"),
"attachment_id": action.get("attachment_id"),
"age_seconds": action.get("age_seconds"),
}
async def _apply_one(settings: Settings, action: dict, grace: int) -> dict | None:
key = action["object_key"]
op = action["op"]
async with pool().acquire() as conn:
now = await conn.fetchval("SELECT clock_timestamp()")
if op == "delete_orphan_object":
owner = await conn.fetchrow("SELECT id, created_at FROM attachments WHERE object_key = $1", key)
if owner is not None:
return None
led = await conn.fetchrow("SELECT created_at, state FROM object_ledger WHERE object_key = $1", key)
moment = led["created_at"] if led is not None else now
if led is not None and _age(moment, now) < grace and led["state"] not in {"aborted", "purged"}:
return None
s3client.delete_object(settings, key)
if led is not None:
await conn.execute(
"""
UPDATE object_ledger
SET state = 'purged', updated_at = clock_timestamp()
WHERE object_key = $1
""",
key,
)
return action
if op == "delete_stale_metadata":
attachment_id = action["attachment_id"]
row = await conn.fetchrow(
"SELECT object_key, created_at FROM attachments WHERE id = $1::uuid",
attachment_id,
)
if row is None or _age(row["created_at"], now) < grace:
return None
used = await conn.fetchval(
"SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid",
attachment_id,
)
if used is not None or s3client.object_exists(settings, row["object_key"]):
return None
async with conn.transaction():
await conn.execute("DELETE FROM attachments WHERE id = $1::uuid", attachment_id)
await conn.execute(
"""
UPDATE object_ledger
SET state = 'purged', updated_at = clock_timestamp()
WHERE object_key = $1
""",
row["object_key"],
)
return action
if op == "abort_stale_intent":
led = await conn.fetchrow(
"SELECT state, created_at FROM object_ledger WHERE object_key = $1",
key,
)
if led is None or led["state"] in {"aborted", "purged"} or _age(led["created_at"], now) < grace:
return None
if s3client.object_exists(settings, key):
s3client.delete_object(settings, key)
await conn.execute(
"""
UPDATE object_ledger
SET state = 'aborted', updated_at = clock_timestamp()
WHERE object_key = $1 AND state IN ('staging', 'committed')
""",
key,
)
return action
return None