Files
mtusi2026lab4/scripts/reconcile_s3.py
T
2026-10-30 22:16:54 +03:00

115 lines
4.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Сверка S3 и метаданных PostgreSQL.
Классы проблем:
orphan_object — объект в S3 без строки attachments (или с pending > grace)
missing_object — attachments.status=ready, но объекта в S3 нет
failed_referenced — attachments.status=failed, объект всё ещё лежит в S3
Действия:
orphan_object → DELETE объекта (только старше grace)
missing_object → UPDATE status='failed', error_code='object_missing'
failed_referenced → DELETE объекта
Использование:
python scripts/reconcile_s3.py --dry-run
python scripts/reconcile_s3.py --apply
"""
import argparse
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import config, db, repo, s3 # noqa: E402
def _paginate(client, prefix):
"""Возвращает {key: last_modified}."""
token = None
out = {}
while True:
kwargs = {"Bucket": config.S3_BUCKET, "Prefix": prefix}
if token:
kwargs["ContinuationToken"] = token
resp = client.list_objects_v2(**kwargs)
for obj in resp.get("Contents", []):
out[obj["Key"]] = obj["LastModified"]
if not resp.get("IsTruncated"):
return out
token = resp["NextContinuationToken"]
def reconcile(dry_run: bool, grace_seconds: int):
client = s3._get_client()
now = time.time()
report = {"orphans_deleted": [], "orphans_skipped_fresh": [],
"missing_marked_failed": [], "failed_objects_deleted": []}
with db.connection() as conn:
rows = conn.execute(
"""SELECT id, object_key, status, created_at,
extract(epoch FROM created_at) AS created_epoch
FROM attachments""").fetchall()
by_key = {r["object_key"]: r for r in rows}
referenced = set(by_key)
for key, last_modified in _paginate(client, "originals/").items():
if key in referenced:
continue
age = now - last_modified.timestamp()
if age < grace_seconds:
report["orphans_skipped_fresh"].append(key)
continue
report["orphans_deleted"].append(key)
if not dry_run:
s3.delete_object(key)
for row in rows:
key = row["object_key"]
if row["status"] == "ready" and key not in _paginate(client, key):
report["missing_marked_failed"].append(key)
if not dry_run:
with db.connection() as conn:
conn.execute(
"""UPDATE attachments
SET status='failed', error_code='object_missing',
updated_at=clock_timestamp()
WHERE id=%s AND status='ready'""", (row["id"],))
elif row["status"] == "failed":
if key in _paginate(client, key):
report["failed_objects_deleted"].append(key)
if not dry_run:
s3.delete_object(key)
elif row["status"] == "pending":
age = now - float(row["created_epoch"])
if age >= grace_seconds and key in _paginate(client, key):
# pending дольше grace, а объект есть → API упал между S3 PUT и COMMIT
report["orphans_deleted"].append(key)
if not dry_run:
with db.connection() as conn:
repo.mark_attachment_failed(conn, row["id"], "orphaned_pending")
s3.delete_object(key)
import json
print(json.dumps(report, ensure_ascii=False, indent=2, default=str))
print("dry-run" if dry_run else "applied")
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--dry-run", action="store_true")
p.add_argument("--grace-seconds", type=int, default=config.S3_ORPHAN_GRACE_SECONDS)
args = p.parse_args()
if args.apply and args.dry_run:
sys.exit("use --apply or --dry-run, not both")
if not (args.apply or args.dry_run):
sys.exit("choose --dry-run or --apply")
reconcile(dry_run=not args.apply, grace_seconds=args.grace_seconds)
if __name__ == "__main__":
main()