forked from mtusi2026/mtusi2026lab4
178 lines
6.1 KiB
Python
Executable File
178 lines
6.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reconcile attachment object keys and metadata; safe by default."""
|
|
import argparse
|
|
from datetime import datetime, timedelta, timezone
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app.settings import database_connection, read_setting
|
|
|
|
LOGGER = logging.getLogger("attachment-reconciler")
|
|
|
|
|
|
def plan_reconciliation(metadata, objects, now, grace):
|
|
cutoff = now - grace
|
|
object_keys = {item["Key"]: item for item in objects}
|
|
metadata_by_key = {row["object_key"]: row for row in metadata}
|
|
return {
|
|
"orphan_old": sorted(
|
|
key
|
|
for key, item in object_keys.items()
|
|
if key.startswith("attachments/")
|
|
and key not in metadata_by_key
|
|
and item.get("LastModified") is not None
|
|
and item["LastModified"] <= cutoff
|
|
),
|
|
"orphan_fresh": sorted(
|
|
key
|
|
for key, item in object_keys.items()
|
|
if key.startswith("attachments/")
|
|
and key not in metadata_by_key
|
|
and (
|
|
item.get("LastModified") is None
|
|
or item["LastModified"] > cutoff
|
|
)
|
|
),
|
|
"missing_unreferenced_old": sorted(
|
|
row["id"]
|
|
for row in metadata
|
|
if row["object_key"] not in object_keys
|
|
and not row["referenced"]
|
|
and row["created_at"] <= cutoff
|
|
),
|
|
"missing_protected": sorted(
|
|
row["id"]
|
|
for row in metadata
|
|
if row["object_key"] not in object_keys
|
|
and (row["referenced"] or row["created_at"] > cutoff)
|
|
),
|
|
"object_count": len(object_keys),
|
|
"metadata_count": len(metadata_by_key),
|
|
}
|
|
|
|
|
|
def load_state(connection, s3, bucket):
|
|
metadata = connection.execute(
|
|
"""
|
|
SELECT a.id::text AS id, a.object_key, a.created_at,
|
|
EXISTS (
|
|
SELECT 1 FROM message_attachments ma
|
|
WHERE ma.attachment_id = a.id
|
|
) AS referenced
|
|
FROM attachments a
|
|
"""
|
|
).fetchall()
|
|
paginator = s3.get_paginator("list_objects_v2")
|
|
objects = []
|
|
for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"):
|
|
objects.extend(page.get("Contents", []))
|
|
return metadata, objects
|
|
|
|
|
|
def apply_plan(connection, s3, bucket, plan):
|
|
deleted_objects = 0
|
|
deleted_metadata = 0
|
|
for key in plan["orphan_old"]:
|
|
s3.delete_object(Bucket=bucket, Key=key)
|
|
deleted_objects += 1
|
|
for attachment_id in plan["missing_unreferenced_old"]:
|
|
with connection.transaction():
|
|
row = connection.execute(
|
|
"""
|
|
DELETE FROM attachments AS a
|
|
WHERE a.id = %s
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM message_attachments ma
|
|
WHERE ma.attachment_id = a.id
|
|
)
|
|
RETURNING a.id
|
|
""",
|
|
(attachment_id,),
|
|
).fetchone()
|
|
if row:
|
|
deleted_metadata += 1
|
|
return deleted_objects, deleted_metadata
|
|
|
|
|
|
def main():
|
|
import boto3
|
|
import botocore
|
|
import psycopg
|
|
from botocore.config import Config
|
|
from psycopg.rows import dict_row
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--apply",
|
|
action="store_true",
|
|
help="delete stale orphan objects and stale unreferenced missing metadata",
|
|
)
|
|
parser.add_argument(
|
|
"--grace-hours",
|
|
type=float,
|
|
default=24,
|
|
help="minimum age before repair; defaults to 24 hours",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.grace_hours < 0:
|
|
parser.error("--grace-hours cannot be negative")
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
started = time.monotonic()
|
|
s3 = boto3.client(
|
|
"s3",
|
|
endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"),
|
|
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
|
aws_access_key_id=read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"),
|
|
aws_secret_access_key=read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"),
|
|
config=Config(
|
|
connect_timeout=2, read_timeout=10,
|
|
retries={"max_attempts": 2, "mode": "standard"},
|
|
),
|
|
)
|
|
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
try:
|
|
with psycopg.connect(**database_connection(), row_factory=dict_row) as connection:
|
|
metadata, objects = load_state(connection, s3, bucket)
|
|
plan = plan_reconciliation(
|
|
metadata,
|
|
objects,
|
|
datetime.now(timezone.utc),
|
|
timedelta(hours=args.grace_hours),
|
|
)
|
|
changes = {"objects_deleted": 0, "metadata_deleted": 0}
|
|
if args.apply:
|
|
(
|
|
changes["objects_deleted"],
|
|
changes["metadata_deleted"],
|
|
) = apply_plan(connection, s3, bucket, plan)
|
|
except (psycopg.Error, botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError):
|
|
LOGGER.exception("reconciliation failed; no success result was produced")
|
|
raise SystemExit(1) from None
|
|
summary = {
|
|
"mode": "apply" if args.apply else "dry-run",
|
|
"grace_hours": args.grace_hours,
|
|
"objects_scanned": plan["object_count"],
|
|
"metadata_scanned": plan["metadata_count"],
|
|
"orphan_objects": len(plan["orphan_old"]) + len(plan["orphan_fresh"]),
|
|
"orphan_old_candidates": len(plan["orphan_old"]),
|
|
"orphan_fresh_preserved": len(plan["orphan_fresh"]),
|
|
"missing_metadata_rows": len(plan["missing_unreferenced_old"])
|
|
+ len(plan["missing_protected"]),
|
|
"missing_unreferenced_old_candidates": len(plan["missing_unreferenced_old"]),
|
|
"missing_referenced_or_fresh_preserved": len(plan["missing_protected"]),
|
|
**changes,
|
|
"elapsed_seconds": round(time.monotonic() - started, 3),
|
|
}
|
|
print(json.dumps(summary, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|