forked from mtusi2026/mtusi2026lab4
236 lines
8.5 KiB
Python
236 lines
8.5 KiB
Python
"""Reconcile PostgreSQL metadata with the private bucket.
|
|
|
|
Grace period: objects and rows younger than ORPHAN_GRACE_SECONDS are left alone,
|
|
because an upload may have written S3 and not committed metadata yet.
|
|
Attachments referenced by a message are never deleted.
|
|
A dry-run reports the same actions apply would perform, then apply is safe to repeat.
|
|
"""
|
|
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
from app.config import Settings
|
|
from app.db import pool
|
|
from app import s3client
|
|
|
|
|
|
def _aware(value: datetime) -> datetime:
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc)
|
|
|
|
|
|
def _age(moment: datetime, now: datetime) -> float:
|
|
return (_aware(now) - _aware(moment)).total_seconds()
|
|
|
|
|
|
async def reconcile(settings: Settings, apply: bool) -> dict:
|
|
started = time.perf_counter()
|
|
db_started = time.perf_counter()
|
|
async with pool().acquire() as conn:
|
|
now = await conn.fetchval("SELECT clock_timestamp()")
|
|
attachments = await conn.fetch(
|
|
"SELECT id, object_key, filename, created_at FROM attachments"
|
|
)
|
|
ledger_rows = await conn.fetch(
|
|
"SELECT object_key, attachment_id, state, note, created_at FROM object_ledger"
|
|
)
|
|
referenced = {
|
|
str(row["attachment_id"])
|
|
for row in await conn.fetch("SELECT attachment_id FROM message_attachments")
|
|
}
|
|
db_ms = (time.perf_counter() - db_started) * 1000
|
|
list_started = time.perf_counter()
|
|
objects, pages = s3client.list_objects(settings, "o/")
|
|
list_ms = (time.perf_counter() - list_started) * 1000
|
|
|
|
by_key = {row["object_key"]: row for row in attachments}
|
|
ledger = {row["object_key"]: row for row in ledger_rows}
|
|
object_keys = {item["Key"] for item in objects}
|
|
grace = settings.orphan_grace_seconds
|
|
actions = []
|
|
skipped: dict[str, int] = {}
|
|
|
|
def skip(reason: str) -> None:
|
|
skipped[reason] = skipped.get(reason, 0) + 1
|
|
|
|
for item in objects:
|
|
key = item["Key"]
|
|
led = ledger.get(key)
|
|
if key in by_key:
|
|
skip("accounted")
|
|
continue
|
|
moment = led["created_at"] if led is not None else item["LastModified"]
|
|
note = (led["note"] if led is not None and led["note"] else None) or (
|
|
led["state"] if led is not None else "untracked"
|
|
)
|
|
age = _age(moment, now)
|
|
attachment_id = str(led["attachment_id"]) if led is not None and led["attachment_id"] else None
|
|
if led is not None and led["state"] in {"aborted", "purged"} and age >= grace:
|
|
actions.append(
|
|
{
|
|
"op": "delete_orphan_object",
|
|
"note": note,
|
|
"object_key": key,
|
|
"attachment_id": attachment_id,
|
|
"age_seconds": int(age),
|
|
}
|
|
)
|
|
continue
|
|
if age < grace:
|
|
skip("fresh_orphan")
|
|
continue
|
|
actions.append(
|
|
{
|
|
"op": "delete_orphan_object",
|
|
"note": note,
|
|
"object_key": key,
|
|
"attachment_id": attachment_id,
|
|
"age_seconds": int(age),
|
|
}
|
|
)
|
|
|
|
for row in attachments:
|
|
if row["object_key"] in object_keys:
|
|
continue
|
|
attachment_id = str(row["id"])
|
|
age = _age(row["created_at"], now)
|
|
if attachment_id in referenced:
|
|
skip("in_use_missing")
|
|
continue
|
|
if age < grace:
|
|
skip("fresh_missing")
|
|
continue
|
|
actions.append(
|
|
{
|
|
"op": "delete_stale_metadata",
|
|
"note": row["filename"],
|
|
"object_key": row["object_key"],
|
|
"attachment_id": attachment_id,
|
|
"age_seconds": int(age),
|
|
}
|
|
)
|
|
|
|
for row in ledger_rows:
|
|
if row["object_key"] in object_keys or row["object_key"] in by_key:
|
|
continue
|
|
if row["state"] in {"aborted", "purged"}:
|
|
skip("terminal_ledger")
|
|
continue
|
|
age = _age(row["created_at"], now)
|
|
if age < grace:
|
|
skip("fresh_intent")
|
|
continue
|
|
actions.append(
|
|
{
|
|
"op": "abort_stale_intent",
|
|
"note": row["note"] or row["state"],
|
|
"object_key": row["object_key"],
|
|
"attachment_id": str(row["attachment_id"]) if row["attachment_id"] else None,
|
|
"age_seconds": int(age),
|
|
}
|
|
)
|
|
|
|
applied = []
|
|
if apply:
|
|
for action in actions:
|
|
result = await _apply_one(settings, action, grace)
|
|
if result is not None:
|
|
applied.append(result)
|
|
|
|
public_actions = [_public(item) for item in (applied if apply else actions)]
|
|
return {
|
|
"mode": "apply" if apply else "dry-run",
|
|
"duration_ms": round((time.perf_counter() - started) * 1000, 3),
|
|
"db_ms": round(db_ms, 3),
|
|
"list_ms": round(list_ms, 3),
|
|
"list_pages": pages,
|
|
"objects_seen": len(objects),
|
|
"attachment_rows": len(attachments),
|
|
"ledger_rows": len(ledger_rows),
|
|
"referenced_attachments": len(referenced),
|
|
"grace_seconds": grace,
|
|
"skipped": skipped,
|
|
"actions": public_actions,
|
|
"action_count": len(public_actions),
|
|
}
|
|
|
|
|
|
def _public(action: dict) -> dict:
|
|
return {
|
|
"op": action["op"],
|
|
"note": action.get("note"),
|
|
"attachment_id": action.get("attachment_id"),
|
|
"age_seconds": action.get("age_seconds"),
|
|
}
|
|
|
|
|
|
async def _apply_one(settings: Settings, action: dict, grace: int) -> dict | None:
|
|
key = action["object_key"]
|
|
op = action["op"]
|
|
async with pool().acquire() as conn:
|
|
now = await conn.fetchval("SELECT clock_timestamp()")
|
|
if op == "delete_orphan_object":
|
|
owner = await conn.fetchrow("SELECT id, created_at FROM attachments WHERE object_key = $1", key)
|
|
if owner is not None:
|
|
return None
|
|
led = await conn.fetchrow("SELECT created_at, state FROM object_ledger WHERE object_key = $1", key)
|
|
moment = led["created_at"] if led is not None else now
|
|
if led is not None and _age(moment, now) < grace and led["state"] not in {"aborted", "purged"}:
|
|
return None
|
|
s3client.delete_object(settings, key)
|
|
if led is not None:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE object_ledger
|
|
SET state = 'purged', updated_at = clock_timestamp()
|
|
WHERE object_key = $1
|
|
""",
|
|
key,
|
|
)
|
|
return action
|
|
if op == "delete_stale_metadata":
|
|
attachment_id = action["attachment_id"]
|
|
row = await conn.fetchrow(
|
|
"SELECT object_key, created_at FROM attachments WHERE id = $1::uuid",
|
|
attachment_id,
|
|
)
|
|
if row is None or _age(row["created_at"], now) < grace:
|
|
return None
|
|
used = await conn.fetchval(
|
|
"SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid",
|
|
attachment_id,
|
|
)
|
|
if used is not None or s3client.object_exists(settings, row["object_key"]):
|
|
return None
|
|
async with conn.transaction():
|
|
await conn.execute("DELETE FROM attachments WHERE id = $1::uuid", attachment_id)
|
|
await conn.execute(
|
|
"""
|
|
UPDATE object_ledger
|
|
SET state = 'purged', updated_at = clock_timestamp()
|
|
WHERE object_key = $1
|
|
""",
|
|
row["object_key"],
|
|
)
|
|
return action
|
|
if op == "abort_stale_intent":
|
|
led = await conn.fetchrow(
|
|
"SELECT state, created_at FROM object_ledger WHERE object_key = $1",
|
|
key,
|
|
)
|
|
if led is None or led["state"] in {"aborted", "purged"} or _age(led["created_at"], now) < grace:
|
|
return None
|
|
if s3client.object_exists(settings, key):
|
|
s3client.delete_object(settings, key)
|
|
await conn.execute(
|
|
"""
|
|
UPDATE object_ledger
|
|
SET state = 'aborted', updated_at = clock_timestamp()
|
|
WHERE object_key = $1 AND state IN ('staging', 'committed')
|
|
""",
|
|
key,
|
|
)
|
|
return action
|
|
return None
|