251 lines
8.5 KiB
Python
251 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
import tracemalloc
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Callable, Sequence
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
|
|
from ..config import load_settings
|
|
from ..db import create_engine, create_session_factory
|
|
from ..models.attachment import AttachmentRow
|
|
from ..storage import S3Storage, StoredObject
|
|
|
|
MISSING_ERROR_CODE = 'original_missing'
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class AttachmentSnapshot:
|
|
id: UUID
|
|
object_key: str
|
|
status: str
|
|
updated_at: datetime
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ReconcilePlan:
|
|
orphan: tuple[StoredObject, ...]
|
|
fresh_orphan: tuple[StoredObject, ...]
|
|
missing: tuple[AttachmentSnapshot, ...]
|
|
fresh_missing: tuple[AttachmentSnapshot, ...]
|
|
correct: int
|
|
|
|
|
|
def build_plan(
|
|
attachments: Sequence[AttachmentSnapshot],
|
|
objects: Sequence[StoredObject],
|
|
cutoff: datetime,
|
|
) -> ReconcilePlan:
|
|
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
|
|
known_keys = {attachment.object_key for attachment in attachments}
|
|
object_keys = {item.key for item in objects}
|
|
|
|
unknown_objects = [item for item in objects if item.key not in known_keys]
|
|
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
|
|
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
|
|
|
|
ready_without_object = [
|
|
attachment
|
|
for attachment in attachments
|
|
if attachment.status == 'ready' and attachment.object_key not in object_keys
|
|
]
|
|
missing = tuple(
|
|
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
|
|
)
|
|
fresh_missing = tuple(
|
|
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
|
|
)
|
|
correct = sum(attachment.object_key in object_keys for attachment in attachments)
|
|
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
|
|
|
|
|
|
async def _load_attachments(
|
|
sessions: async_sessionmaker[AsyncSession],
|
|
) -> tuple[AttachmentSnapshot, ...]:
|
|
async with sessions() as session:
|
|
rows = (
|
|
await session.execute(
|
|
select(
|
|
AttachmentRow.id,
|
|
AttachmentRow.object_key,
|
|
AttachmentRow.status,
|
|
AttachmentRow.updated_at,
|
|
)
|
|
)
|
|
).all()
|
|
return tuple(AttachmentSnapshot(*row) for row in rows)
|
|
|
|
|
|
async def _metadata_key_exists(
|
|
sessions: async_sessionmaker[AsyncSession], object_key: str
|
|
) -> bool:
|
|
async with sessions() as session:
|
|
attachment_id = await session.scalar(
|
|
select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1)
|
|
)
|
|
return attachment_id is not None
|
|
|
|
|
|
async def reconcile(
|
|
sessions: async_sessionmaker[AsyncSession],
|
|
storage: S3Storage,
|
|
*,
|
|
apply: bool,
|
|
grace_seconds: int,
|
|
now: datetime | None = None,
|
|
output: Callable[[str], None] = print,
|
|
) -> dict[str, int | float | str]:
|
|
started = time.perf_counter()
|
|
tracemalloc.start()
|
|
current_time = now or datetime.now(timezone.utc)
|
|
cutoff = current_time - timedelta(seconds=grace_seconds)
|
|
|
|
attachments = await _load_attachments(sessions)
|
|
objects = await storage.list('originals/')
|
|
plan = build_plan(attachments, objects, cutoff)
|
|
|
|
for item in plan.fresh_orphan:
|
|
output(
|
|
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
|
|
)
|
|
for attachment in plan.fresh_missing:
|
|
output(
|
|
'SKIP_FRESH_MISSING '
|
|
f'attachment_id={attachment.id} key={attachment.object_key} '
|
|
f'updated={attachment.updated_at.isoformat()}'
|
|
)
|
|
|
|
removed = 0
|
|
race_protected = 0
|
|
for item in plan.orphan:
|
|
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
|
|
if not apply:
|
|
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
|
continue
|
|
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
|
|
# metadata immediately before a destructive action so a row created after
|
|
# the initial snapshot protects its object.
|
|
if await _metadata_key_exists(sessions, item.key):
|
|
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
|
|
race_protected += 1
|
|
continue
|
|
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
|
await storage.remove(item.key)
|
|
removed += 1
|
|
|
|
marked_failed = 0
|
|
for attachment in plan.missing:
|
|
if not apply:
|
|
output(
|
|
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
|
|
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
|
)
|
|
continue
|
|
|
|
# The object may have been restored after list_objects(). Do not turn
|
|
# healthy metadata into failed based only on the earlier snapshot.
|
|
if await storage.exists(attachment.object_key):
|
|
output(
|
|
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
|
|
f'key={attachment.object_key}'
|
|
)
|
|
race_protected += 1
|
|
continue
|
|
|
|
async with sessions() as session:
|
|
async with session.begin():
|
|
result = await session.execute(
|
|
update(AttachmentRow)
|
|
.where(
|
|
AttachmentRow.id == attachment.id,
|
|
AttachmentRow.object_key == attachment.object_key,
|
|
AttachmentRow.status == 'ready',
|
|
AttachmentRow.updated_at == attachment.updated_at,
|
|
)
|
|
.values(
|
|
status='failed',
|
|
error_code=MISSING_ERROR_CODE,
|
|
updated_at=current_time,
|
|
)
|
|
)
|
|
changed = result.rowcount or 0
|
|
if changed:
|
|
output(
|
|
f'MARK_MISSING attachment_id={attachment.id} '
|
|
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
|
)
|
|
marked_failed += changed
|
|
else:
|
|
output(
|
|
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
|
|
f'key={attachment.object_key}'
|
|
)
|
|
race_protected += 1
|
|
|
|
_, peak_bytes = tracemalloc.get_traced_memory()
|
|
tracemalloc.stop()
|
|
summary: dict[str, int | float | str] = {
|
|
'mode': 'apply' if apply else 'dry-run',
|
|
'metadata': len(attachments),
|
|
'objects': len(objects),
|
|
'correct': plan.correct,
|
|
'orphan': len(plan.orphan),
|
|
'missing': len(plan.missing),
|
|
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
|
|
'removed': removed,
|
|
'marked_failed': marked_failed,
|
|
'race_protected': race_protected,
|
|
'grace_seconds': grace_seconds,
|
|
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
|
|
'peak_memory_bytes': peak_bytes,
|
|
}
|
|
output(f'summary {json.dumps(summary, sort_keys=True)}')
|
|
return summary
|
|
|
|
|
|
async def run(apply: bool, grace_seconds: int) -> int:
|
|
settings = load_settings()
|
|
engine = create_engine(settings.database_url)
|
|
storage = S3Storage(
|
|
settings.s3_endpoint,
|
|
settings.s3_access_key_id,
|
|
settings.s3_secret_access_key,
|
|
settings.s3_bucket,
|
|
settings.s3_region,
|
|
)
|
|
try:
|
|
await reconcile(
|
|
create_session_factory(engine),
|
|
storage,
|
|
apply=apply,
|
|
grace_seconds=grace_seconds,
|
|
)
|
|
return 0
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
|
|
)
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument('--dry-run', action='store_true')
|
|
mode.add_argument('--apply', action='store_true')
|
|
parser.add_argument('--grace-seconds', type=int, default=3600)
|
|
args = parser.parse_args()
|
|
if args.grace_seconds < 60:
|
|
parser.error('--grace-seconds must be at least 60')
|
|
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|