Files
mtusi2026lab4/scripts/reconcile.py
2026-11-17 02:09:04 +03:00

178 lines
6.1 KiB
Python
Executable File

#!/usr/bin/env python3
"""Reconcile attachment object keys and metadata; safe by default."""
import argparse
from datetime import datetime, timedelta, timezone
import json
import logging
import os
from pathlib import Path
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.settings import database_connection, read_setting
LOGGER = logging.getLogger("attachment-reconciler")
def plan_reconciliation(metadata, objects, now, grace):
cutoff = now - grace
object_keys = {item["Key"]: item for item in objects}
metadata_by_key = {row["object_key"]: row for row in metadata}
return {
"orphan_old": sorted(
key
for key, item in object_keys.items()
if key.startswith("attachments/")
and key not in metadata_by_key
and item.get("LastModified") is not None
and item["LastModified"] <= cutoff
),
"orphan_fresh": sorted(
key
for key, item in object_keys.items()
if key.startswith("attachments/")
and key not in metadata_by_key
and (
item.get("LastModified") is None
or item["LastModified"] > cutoff
)
),
"missing_unreferenced_old": sorted(
row["id"]
for row in metadata
if row["object_key"] not in object_keys
and not row["referenced"]
and row["created_at"] <= cutoff
),
"missing_protected": sorted(
row["id"]
for row in metadata
if row["object_key"] not in object_keys
and (row["referenced"] or row["created_at"] > cutoff)
),
"object_count": len(object_keys),
"metadata_count": len(metadata_by_key),
}
def load_state(connection, s3, bucket):
metadata = connection.execute(
"""
SELECT a.id::text AS id, a.object_key, a.created_at,
EXISTS (
SELECT 1 FROM message_attachments ma
WHERE ma.attachment_id = a.id
) AS referenced
FROM attachments a
"""
).fetchall()
paginator = s3.get_paginator("list_objects_v2")
objects = []
for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"):
objects.extend(page.get("Contents", []))
return metadata, objects
def apply_plan(connection, s3, bucket, plan):
deleted_objects = 0
deleted_metadata = 0
for key in plan["orphan_old"]:
s3.delete_object(Bucket=bucket, Key=key)
deleted_objects += 1
for attachment_id in plan["missing_unreferenced_old"]:
with connection.transaction():
row = connection.execute(
"""
DELETE FROM attachments AS a
WHERE a.id = %s
AND NOT EXISTS (
SELECT 1 FROM message_attachments ma
WHERE ma.attachment_id = a.id
)
RETURNING a.id
""",
(attachment_id,),
).fetchone()
if row:
deleted_metadata += 1
return deleted_objects, deleted_metadata
def main():
import boto3
import botocore
import psycopg
from botocore.config import Config
from psycopg.rows import dict_row
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--apply",
action="store_true",
help="delete stale orphan objects and stale unreferenced missing metadata",
)
parser.add_argument(
"--grace-hours",
type=float,
default=24,
help="minimum age before repair; defaults to 24 hours",
)
args = parser.parse_args()
if args.grace_hours < 0:
parser.error("--grace-hours cannot be negative")
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
started = time.monotonic()
s3 = boto3.client(
"s3",
endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"),
region_name=os.environ.get("S3_REGION", "us-east-1"),
aws_access_key_id=read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"),
aws_secret_access_key=read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"),
config=Config(
connect_timeout=2, read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
),
)
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
try:
with psycopg.connect(**database_connection(), row_factory=dict_row) as connection:
metadata, objects = load_state(connection, s3, bucket)
plan = plan_reconciliation(
metadata,
objects,
datetime.now(timezone.utc),
timedelta(hours=args.grace_hours),
)
changes = {"objects_deleted": 0, "metadata_deleted": 0}
if args.apply:
(
changes["objects_deleted"],
changes["metadata_deleted"],
) = apply_plan(connection, s3, bucket, plan)
except (psycopg.Error, botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError):
LOGGER.exception("reconciliation failed; no success result was produced")
raise SystemExit(1) from None
summary = {
"mode": "apply" if args.apply else "dry-run",
"grace_hours": args.grace_hours,
"objects_scanned": plan["object_count"],
"metadata_scanned": plan["metadata_count"],
"orphan_objects": len(plan["orphan_old"]) + len(plan["orphan_fresh"]),
"orphan_old_candidates": len(plan["orphan_old"]),
"orphan_fresh_preserved": len(plan["orphan_fresh"]),
"missing_metadata_rows": len(plan["missing_unreferenced_old"])
+ len(plan["missing_protected"]),
"missing_unreferenced_old_candidates": len(plan["missing_unreferenced_old"]),
"missing_referenced_or_fresh_preserved": len(plan["missing_protected"]),
**changes,
"elapsed_seconds": round(time.monotonic() - started, 3),
}
print(json.dumps(summary, sort_keys=True))
if __name__ == "__main__":
main()