Files
mtusi2026lab4/app/tools/reconcile_attachments.py

251 lines
8.5 KiB
Python

from __future__ import annotations
import argparse
import asyncio
import json
import time
import tracemalloc
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Sequence
from uuid import UUID
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage, StoredObject
MISSING_ERROR_CODE = 'original_missing'
@dataclass(frozen=True, slots=True)
class AttachmentSnapshot:
id: UUID
object_key: str
status: str
updated_at: datetime
@dataclass(frozen=True, slots=True)
class ReconcilePlan:
orphan: tuple[StoredObject, ...]
fresh_orphan: tuple[StoredObject, ...]
missing: tuple[AttachmentSnapshot, ...]
fresh_missing: tuple[AttachmentSnapshot, ...]
correct: int
def build_plan(
attachments: Sequence[AttachmentSnapshot],
objects: Sequence[StoredObject],
cutoff: datetime,
) -> ReconcilePlan:
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
known_keys = {attachment.object_key for attachment in attachments}
object_keys = {item.key for item in objects}
unknown_objects = [item for item in objects if item.key not in known_keys]
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
ready_without_object = [
attachment
for attachment in attachments
if attachment.status == 'ready' and attachment.object_key not in object_keys
]
missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
)
fresh_missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
)
correct = sum(attachment.object_key in object_keys for attachment in attachments)
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
async def _load_attachments(
sessions: async_sessionmaker[AsyncSession],
) -> tuple[AttachmentSnapshot, ...]:
async with sessions() as session:
rows = (
await session.execute(
select(
AttachmentRow.id,
AttachmentRow.object_key,
AttachmentRow.status,
AttachmentRow.updated_at,
)
)
).all()
return tuple(AttachmentSnapshot(*row) for row in rows)
async def _metadata_key_exists(
sessions: async_sessionmaker[AsyncSession], object_key: str
) -> bool:
async with sessions() as session:
attachment_id = await session.scalar(
select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1)
)
return attachment_id is not None
async def reconcile(
sessions: async_sessionmaker[AsyncSession],
storage: S3Storage,
*,
apply: bool,
grace_seconds: int,
now: datetime | None = None,
output: Callable[[str], None] = print,
) -> dict[str, int | float | str]:
started = time.perf_counter()
tracemalloc.start()
current_time = now or datetime.now(timezone.utc)
cutoff = current_time - timedelta(seconds=grace_seconds)
attachments = await _load_attachments(sessions)
objects = await storage.list('originals/')
plan = build_plan(attachments, objects, cutoff)
for item in plan.fresh_orphan:
output(
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
)
for attachment in plan.fresh_missing:
output(
'SKIP_FRESH_MISSING '
f'attachment_id={attachment.id} key={attachment.object_key} '
f'updated={attachment.updated_at.isoformat()}'
)
removed = 0
race_protected = 0
for item in plan.orphan:
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
if not apply:
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
continue
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
# metadata immediately before a destructive action so a row created after
# the initial snapshot protects its object.
if await _metadata_key_exists(sessions, item.key):
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
race_protected += 1
continue
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
await storage.remove(item.key)
removed += 1
marked_failed = 0
for attachment in plan.missing:
if not apply:
output(
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
continue
# The object may have been restored after list_objects(). Do not turn
# healthy metadata into failed based only on the earlier snapshot.
if await storage.exists(attachment.object_key):
output(
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
continue
async with sessions() as session:
async with session.begin():
result = await session.execute(
update(AttachmentRow)
.where(
AttachmentRow.id == attachment.id,
AttachmentRow.object_key == attachment.object_key,
AttachmentRow.status == 'ready',
AttachmentRow.updated_at == attachment.updated_at,
)
.values(
status='failed',
error_code=MISSING_ERROR_CODE,
updated_at=current_time,
)
)
changed = result.rowcount or 0
if changed:
output(
f'MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
marked_failed += changed
else:
output(
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
_, peak_bytes = tracemalloc.get_traced_memory()
tracemalloc.stop()
summary: dict[str, int | float | str] = {
'mode': 'apply' if apply else 'dry-run',
'metadata': len(attachments),
'objects': len(objects),
'correct': plan.correct,
'orphan': len(plan.orphan),
'missing': len(plan.missing),
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
'removed': removed,
'marked_failed': marked_failed,
'race_protected': race_protected,
'grace_seconds': grace_seconds,
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
'peak_memory_bytes': peak_bytes,
}
output(f'summary {json.dumps(summary, sort_keys=True)}')
return summary
async def run(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
await reconcile(
create_session_factory(engine),
storage,
apply=apply,
grace_seconds=grace_seconds,
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()