175 lines
6.1 KiB
Python
175 lines
6.1 KiB
Python
"""Сверка строк attachments и объектов bucket.
|
|
|
|
Свежий объект моложе grace не удаляется: загрузка могла записать файл
|
|
раньше, чем строка в базе. Строка, на которую ссылается сообщение, тоже
|
|
не удаляется, даже если файла уже нет. По умолчанию только план, без удалений.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
|
|
from blobs import BUCKET, delete_quiet, list_objects
|
|
|
|
|
|
def grace_seconds():
|
|
raw = os.environ.get("RECONCILE_GRACE_SECONDS", "120")
|
|
value = int(raw)
|
|
if value < 1:
|
|
raise RuntimeError("RECONCILE_GRACE_SECONDS")
|
|
return value
|
|
|
|
|
|
def database_url():
|
|
path = os.environ.get("DATABASE_URL_FILE")
|
|
if not path:
|
|
raise RuntimeError("DATABASE_URL_FILE")
|
|
with open(path, encoding="utf-8") as handle:
|
|
return handle.read().strip()
|
|
|
|
|
|
def aware(value):
|
|
if value is None:
|
|
return None
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc)
|
|
|
|
|
|
def plan(grace):
|
|
started = time.perf_counter()
|
|
objects = list_objects()
|
|
listed = time.perf_counter()
|
|
now = datetime.now(timezone.utc)
|
|
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT a.id, a.object_key, a.staging_key, a.created_at, a.status,
|
|
EXISTS (
|
|
SELECT 1 FROM message_attachments m
|
|
WHERE m.attachment_id = a.id
|
|
) AS in_use
|
|
FROM attachments a
|
|
"""
|
|
).fetchall()
|
|
variant_rows = conn.execute("SELECT object_key FROM attachment_variants").fetchall()
|
|
by_key = {}
|
|
for row in rows:
|
|
by_key[row["object_key"]] = row
|
|
if row["staging_key"]:
|
|
by_key[row["staging_key"]] = row
|
|
for item in variant_rows:
|
|
by_key.setdefault(item["object_key"], {"status": "variant"})
|
|
object_keys = {item["key"] for item in objects}
|
|
actions = []
|
|
preserved = []
|
|
for item in objects:
|
|
row = by_key.get(item["key"])
|
|
if row is not None:
|
|
continue
|
|
created = aware(item["created_at"]) or now
|
|
age = (now - created).total_seconds()
|
|
entry = {"key": item["key"], "age_seconds": round(age, 3)}
|
|
if age < grace:
|
|
entry["reason"] = "fresh_object"
|
|
preserved.append(entry)
|
|
continue
|
|
entry["action"] = "delete_orphan_object"
|
|
actions.append(entry)
|
|
for row in rows:
|
|
if row["object_key"] in object_keys:
|
|
continue
|
|
created = aware(row["created_at"]) or now
|
|
age = (now - created).total_seconds()
|
|
entry = {
|
|
"attachment_id": str(row["id"]),
|
|
"key": row["object_key"],
|
|
"age_seconds": round(age, 3),
|
|
}
|
|
if row["status"] != "ready":
|
|
entry["reason"] = "not_ready"
|
|
preserved.append(entry)
|
|
continue
|
|
if row["in_use"]:
|
|
entry["reason"] = "missing_but_in_use"
|
|
preserved.append(entry)
|
|
continue
|
|
if age < grace:
|
|
entry["reason"] = "fresh_row"
|
|
preserved.append(entry)
|
|
continue
|
|
entry["action"] = "delete_orphan_row"
|
|
actions.append(entry)
|
|
return {
|
|
"bucket": BUCKET,
|
|
"grace_seconds": grace,
|
|
"objects": len(objects),
|
|
"rows": len(rows),
|
|
"list_seconds": round(listed - started, 6),
|
|
"plan_seconds": round(time.perf_counter() - started, 6),
|
|
"actions": actions,
|
|
"preserved": preserved,
|
|
}
|
|
|
|
|
|
def apply_plan(report):
|
|
deleted_objects = 0
|
|
deleted_rows = 0
|
|
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
|
|
for action in report["actions"]:
|
|
if action["action"] == "delete_orphan_object":
|
|
if delete_quiet(action["key"]):
|
|
deleted_objects += 1
|
|
elif action["action"] == "delete_orphan_row":
|
|
with conn.transaction():
|
|
result = conn.execute(
|
|
"""
|
|
DELETE FROM attachments
|
|
WHERE id = %s
|
|
AND status = 'ready'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM message_attachments m
|
|
WHERE m.attachment_id = attachments.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM attachment_jobs j
|
|
WHERE j.attachment_id = attachments.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM attachment_variants v
|
|
WHERE v.attachment_id = attachments.id
|
|
)
|
|
""",
|
|
(action["attachment_id"],),
|
|
)
|
|
deleted_rows += result.rowcount
|
|
report["deleted_objects"] = deleted_objects
|
|
report["deleted_rows"] = deleted_rows
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Сверить вложения в базе и в bucket")
|
|
parser.add_argument("--apply", action="store_true", help="выполнить удаления; без флага только план")
|
|
parser.add_argument("--grace", type=int, default=None)
|
|
args = parser.parse_args()
|
|
grace = args.grace if args.grace is not None else grace_seconds()
|
|
if grace < 1:
|
|
print("grace должен быть не меньше 1 секунды", file=sys.stderr)
|
|
return 2
|
|
report = plan(grace)
|
|
report["apply"] = bool(args.apply)
|
|
if args.apply:
|
|
apply_plan(report)
|
|
print(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|