174 lines
7.6 KiB
Python
174 lines
7.6 KiB
Python
"""Compare PostgreSQL attachment metadata with private S3 originals.
|
|
|
|
Dry-run is the default. Run in the Compose maintenance profile with its separate S3 identity.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
import psycopg
|
|
from minio import Minio
|
|
from minio.error import S3Error
|
|
from psycopg.rows import dict_row
|
|
from urllib3 import PoolManager, Retry, Timeout
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from src.db import database_url # noqa: E402
|
|
|
|
BUCKET = "messenger-attachments"
|
|
ORIGINAL = re.compile(r"originals/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\Z")
|
|
MAX_SOURCE = 10 * 1024 * 1024
|
|
|
|
|
|
def secret(name: str) -> str:
|
|
return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip()
|
|
|
|
|
|
def storage() -> Minio:
|
|
return Minio(
|
|
os.environ.get("S3_ENDPOINT", "s3:9000"),
|
|
access_key=secret("S3_ACCESS_KEY_ID"), secret_key=secret("S3_SECRET_ACCESS_KEY"),
|
|
secure=False, region="us-east-1",
|
|
http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)),
|
|
)
|
|
|
|
|
|
def age_seconds(last_modified: datetime, now: datetime) -> float:
|
|
if last_modified is None or last_modified.tzinfo is None:
|
|
return 0 # Unknown age is never safe to delete.
|
|
return max(0, (now - last_modified).total_seconds())
|
|
|
|
|
|
def absent(client: Minio, key: str) -> bool:
|
|
try:
|
|
client.stat_object(BUCKET, key)
|
|
return False
|
|
except S3Error as exc:
|
|
if exc.code == "NoSuchKey":
|
|
return True
|
|
raise
|
|
|
|
|
|
def source_hash(path: Path) -> tuple[int, str]:
|
|
size, digest = 0, hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
while chunk := stream.read(1024 * 1024):
|
|
size += len(chunk)
|
|
if size > MAX_SOURCE:
|
|
raise ValueError("restore source exceeds 10 MiB")
|
|
digest.update(chunk)
|
|
return size, digest.hexdigest()
|
|
|
|
|
|
def delete_orphan(client: Minio, conn: psycopg.Connection, item, grace: int) -> bool:
|
|
key = item.object_name
|
|
if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone():
|
|
return False
|
|
try:
|
|
current = client.stat_object(BUCKET, key)
|
|
except S3Error as exc:
|
|
if exc.code == "NoSuchKey":
|
|
return False
|
|
raise
|
|
if (
|
|
current.etag != item.etag or current.last_modified.replace(microsecond=0) != item.last_modified.replace(microsecond=0)
|
|
or current.size != item.size or age_seconds(current.last_modified, datetime.now(timezone.utc)) < grace
|
|
):
|
|
return False
|
|
# Upload keys are random and S3 PUT + PostgreSQL INSERT is bounded; grace and recheck protect live uploads.
|
|
client.remove_object(BUCKET, key)
|
|
return True
|
|
|
|
|
|
def restore_missing(client: Minio, conn: psycopg.Connection, attachment_id: UUID, source: Path, apply: bool) -> str:
|
|
size, digest = source_hash(source)
|
|
row = conn.execute(
|
|
"SELECT id,object_key,size_bytes,sha256,content_type,status FROM attachments WHERE id=%s FOR UPDATE",
|
|
(attachment_id,),
|
|
).fetchone()
|
|
if row is None or row["status"] != "ready":
|
|
raise ValueError("ready attachment not found")
|
|
if not ORIGINAL.fullmatch(row["object_key"]):
|
|
raise ValueError("invalid original object key")
|
|
if (size, digest) != (row["size_bytes"], row["sha256"]):
|
|
raise ValueError("restore source size/SHA-256 mismatch")
|
|
if not absent(client, row["object_key"]):
|
|
return "already_present"
|
|
if not apply:
|
|
return "would_restore"
|
|
with source.open("rb") as stream:
|
|
client.put_object(BUCKET, row["object_key"], stream, size, content_type=row["content_type"])
|
|
if client.stat_object(BUCKET, row["object_key"]).size != size:
|
|
raise RuntimeError("restored object size mismatch")
|
|
return "restored"
|
|
|
|
|
|
def reconcile(client: Minio, conn: psycopg.Connection, apply: bool, grace: int, restore_id: UUID | None, source: Path | None) -> dict:
|
|
start = time.monotonic()
|
|
metadata = conn.execute(
|
|
"SELECT id,object_key,size_bytes,sha256,content_type,status FROM attachments"
|
|
).fetchall()
|
|
objects = list(client.list_objects(BUCKET, prefix="originals/", recursive=True))
|
|
by_key = {row["object_key"]: row for row in metadata}
|
|
by_object = {item.object_name: item for item in objects}
|
|
now = datetime.now(timezone.utc)
|
|
old = [item for item in objects if item.object_name not in by_key and ORIGINAL.fullmatch(item.object_name)
|
|
and age_seconds(item.last_modified, now) >= grace]
|
|
fresh = [item.object_name for item in objects if item.object_name not in by_key and ORIGINAL.fullmatch(item.object_name)
|
|
and age_seconds(item.last_modified, now) < grace]
|
|
unknown = [item.object_name for item in objects if item.object_name not in by_key and not ORIGINAL.fullmatch(item.object_name)]
|
|
missing = [row for row in metadata if row["status"] == "ready" and row["object_key"] not in by_object
|
|
and absent(client, row["object_key"])]
|
|
|
|
result = {
|
|
"mode": "apply" if apply else "dry_run", "grace_seconds": grace,
|
|
"inventory": {"objects": len(objects), "metadata": len(metadata)},
|
|
"old_orphans": [item.object_name for item in old], "fresh_orphans": fresh,
|
|
"unknown_keys_preserved": unknown, "missing_attachment_ids": [str(row["id"]) for row in missing],
|
|
"manual_incidents": [str(row["id"]) for row in missing],
|
|
"deleted": [], "preserved_after_recheck": [], "restore": None,
|
|
}
|
|
if restore_id is not None and apply:
|
|
# Reject an invalid source before any orphan deletion in this apply run.
|
|
restore_missing(client, conn, restore_id, source, False)
|
|
if apply:
|
|
for item in old:
|
|
(result["deleted"] if delete_orphan(client, conn, item, grace) else result["preserved_after_recheck"]).append(item.object_name)
|
|
if restore_id is not None:
|
|
result["restore"] = restore_missing(client, conn, restore_id, source, apply)
|
|
if result["restore"] in {"restored", "already_present"}:
|
|
result["manual_incidents"] = [item for item in result["manual_incidents"] if item != str(restore_id)]
|
|
result["duration_ms"] = round((time.monotonic() - start) * 1000, 3)
|
|
return result
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--apply", action="store_true", help="perform safe deletions and selected verified restore")
|
|
parser.add_argument("--grace-seconds", type=int, default=3600, help="minimum age before deleting an orphan (default 3600)")
|
|
parser.add_argument("--restore-id", type=UUID, help="one missing attachment to restore")
|
|
parser.add_argument("--source", type=Path, help="read-only original fixture with matching size and SHA-256")
|
|
args = parser.parse_args()
|
|
if args.grace_seconds < 1 or (args.restore_id is None) != (args.source is None):
|
|
parser.error("grace must be positive; restore-id and source must be supplied together")
|
|
try:
|
|
with psycopg.connect(database_url(), row_factory=dict_row, connect_timeout=5, options="-c statement_timeout=30000") as conn:
|
|
result = reconcile(storage(), conn, args.apply, args.grace_seconds, args.restore_id, args.source)
|
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
return 0
|
|
except (psycopg.Error, S3Error, OSError, ValueError, RuntimeError) as exc:
|
|
print(json.dumps({"error": type(exc).__name__, "message": "reconciliation stopped without further mutation"}))
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|