Files
mtusi2026lab6/src/reconcile.py
T

175 lines
6.1 KiB
Python

"""Сверка строк attachments и объектов bucket.
Свежий объект моложе grace не удаляется: загрузка могла записать файл
раньше, чем строка в базе. Строка, на которую ссылается сообщение, тоже
не удаляется, даже если файла уже нет. По умолчанию только план, без удалений.
"""
import argparse
import json
import os
import sys
import time
from datetime import datetime, timezone
import psycopg
from psycopg.rows import dict_row
from blobs import BUCKET, delete_quiet, list_objects
def grace_seconds():
raw = os.environ.get("RECONCILE_GRACE_SECONDS", "120")
value = int(raw)
if value < 1:
raise RuntimeError("RECONCILE_GRACE_SECONDS")
return value
def database_url():
path = os.environ.get("DATABASE_URL_FILE")
if not path:
raise RuntimeError("DATABASE_URL_FILE")
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
def aware(value):
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def plan(grace):
started = time.perf_counter()
objects = list_objects()
listed = time.perf_counter()
now = datetime.now(timezone.utc)
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
rows = conn.execute(
"""
SELECT a.id, a.object_key, a.staging_key, a.created_at, a.status,
EXISTS (
SELECT 1 FROM message_attachments m
WHERE m.attachment_id = a.id
) AS in_use
FROM attachments a
"""
).fetchall()
variant_rows = conn.execute("SELECT object_key FROM attachment_variants").fetchall()
by_key = {}
for row in rows:
by_key[row["object_key"]] = row
if row["staging_key"]:
by_key[row["staging_key"]] = row
for item in variant_rows:
by_key.setdefault(item["object_key"], {"status": "variant"})
object_keys = {item["key"] for item in objects}
actions = []
preserved = []
for item in objects:
row = by_key.get(item["key"])
if row is not None:
continue
created = aware(item["created_at"]) or now
age = (now - created).total_seconds()
entry = {"key": item["key"], "age_seconds": round(age, 3)}
if age < grace:
entry["reason"] = "fresh_object"
preserved.append(entry)
continue
entry["action"] = "delete_orphan_object"
actions.append(entry)
for row in rows:
if row["object_key"] in object_keys:
continue
created = aware(row["created_at"]) or now
age = (now - created).total_seconds()
entry = {
"attachment_id": str(row["id"]),
"key": row["object_key"],
"age_seconds": round(age, 3),
}
if row["status"] != "ready":
entry["reason"] = "not_ready"
preserved.append(entry)
continue
if row["in_use"]:
entry["reason"] = "missing_but_in_use"
preserved.append(entry)
continue
if age < grace:
entry["reason"] = "fresh_row"
preserved.append(entry)
continue
entry["action"] = "delete_orphan_row"
actions.append(entry)
return {
"bucket": BUCKET,
"grace_seconds": grace,
"objects": len(objects),
"rows": len(rows),
"list_seconds": round(listed - started, 6),
"plan_seconds": round(time.perf_counter() - started, 6),
"actions": actions,
"preserved": preserved,
}
def apply_plan(report):
deleted_objects = 0
deleted_rows = 0
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
for action in report["actions"]:
if action["action"] == "delete_orphan_object":
if delete_quiet(action["key"]):
deleted_objects += 1
elif action["action"] == "delete_orphan_row":
with conn.transaction():
result = conn.execute(
"""
DELETE FROM attachments
WHERE id = %s
AND status = 'ready'
AND NOT EXISTS (
SELECT 1 FROM message_attachments m
WHERE m.attachment_id = attachments.id
)
AND NOT EXISTS (
SELECT 1 FROM attachment_jobs j
WHERE j.attachment_id = attachments.id
)
AND NOT EXISTS (
SELECT 1 FROM attachment_variants v
WHERE v.attachment_id = attachments.id
)
""",
(action["attachment_id"],),
)
deleted_rows += result.rowcount
report["deleted_objects"] = deleted_objects
report["deleted_rows"] = deleted_rows
return report
def main():
parser = argparse.ArgumentParser(description="Сверить вложения в базе и в bucket")
parser.add_argument("--apply", action="store_true", help="выполнить удаления; без флага только план")
parser.add_argument("--grace", type=int, default=None)
args = parser.parse_args()
grace = args.grace if args.grace is not None else grace_seconds()
if grace < 1:
print("grace должен быть не меньше 1 секунды", file=sys.stderr)
return 2
report = plan(grace)
report["apply"] = bool(args.apply)
if args.apply:
apply_plan(report)
print(json.dumps(report, ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())