146 lines
5.3 KiB
Python
146 lines
5.3 KiB
Python
import argparse
|
|
import asyncio
|
|
from datetime import datetime, timedelta, timezone
|
|
from minio import Minio
|
|
from sqlalchemy import select, cast, Text
|
|
from app.models.chat import Chat, ChatMember
|
|
from app.models.user import User
|
|
from app.models.message import Message
|
|
from app.database import async_session_maker
|
|
from app.models.attachment import Attachment
|
|
from app.config import (
|
|
S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
|
|
)
|
|
|
|
GRACE_PERIOD = timedelta(seconds=1)
|
|
|
|
|
|
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
|
|
minio_client = Minio(
|
|
endpoint,
|
|
access_key=S3_ACCESS_KEY_ID,
|
|
secret_key=S3_SECRET_ACCESS_KEY,
|
|
secure=False,
|
|
)
|
|
BUCKET = S3_BUCKET
|
|
|
|
|
|
async def find_orphan_s3_objects(dry_run: bool) -> int:
|
|
print("Поиск orphan объектов в S3, файлы без записи в БД")
|
|
|
|
async with async_session_maker() as session:
|
|
result = await session.execute(select(Attachment.object_key))
|
|
db_keys = {row[0] for row in result.all()}
|
|
|
|
print(f"Файлов в БД: {len(db_keys)}")
|
|
|
|
objects = list(minio_client.list_objects(BUCKET, recursive=True))
|
|
print(f"Объектов в MinIO: {len(objects)}")
|
|
|
|
now = datetime.now(timezone.utc)
|
|
orphans = []
|
|
|
|
for obj in objects:
|
|
if obj.object_name not in db_keys:
|
|
age = now - obj.last_modified if obj.last_modified else timedelta(0)
|
|
if age > GRACE_PERIOD:
|
|
orphans.append(obj.object_name)
|
|
print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)")
|
|
else:
|
|
print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)")
|
|
|
|
if not orphans:
|
|
print("Orphan-объектов не найдено.")
|
|
return 0
|
|
|
|
if dry_run:
|
|
print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.")
|
|
else:
|
|
for key in orphans:
|
|
minio_client.remove_object(BUCKET, key)
|
|
print(f" [DELETED] {key}")
|
|
print(f"Удалено {len(orphans)} orphan-объектов.")
|
|
|
|
return len(orphans)
|
|
|
|
|
|
async def find_missing_s3_objects(dry_run: bool) -> int:
|
|
print("\nПоиск missing объектов, записи в БД без файла в MinIO")
|
|
|
|
async with async_session_maker() as session:
|
|
result = await session.execute(select(Attachment.id, Attachment.object_key))
|
|
attachments = result.all()
|
|
|
|
print(f"Записей в БД: {len(attachments)}")
|
|
|
|
missing = []
|
|
used_missing = []
|
|
|
|
for att_id, key in attachments:
|
|
try:
|
|
minio_client.stat_object(BUCKET, key)
|
|
except Exception:
|
|
|
|
async with async_session_maker() as session:
|
|
result_msg = await session.execute(
|
|
select(Message).where(
|
|
cast(Message.attachment_ids, Text).contains(f'"{att_id}"')
|
|
)
|
|
)
|
|
is_used = result_msg.scalar_one_or_none() is not None
|
|
|
|
if is_used:
|
|
used_missing.append((att_id, key))
|
|
print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем")
|
|
else:
|
|
missing.append((att_id, key))
|
|
print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить")
|
|
|
|
total = len(missing) + len(used_missing)
|
|
if total == 0:
|
|
print("Missing-записей не найдено.")
|
|
return 0
|
|
|
|
if dry_run:
|
|
print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.")
|
|
else:
|
|
if missing:
|
|
async with async_session_maker() as session:
|
|
for att_id, _ in missing:
|
|
att = await session.get(Attachment, att_id)
|
|
if att:
|
|
await session.delete(att)
|
|
await session.commit()
|
|
print(f"Удалено {len(missing)} неиспользуемых missing из БД.")
|
|
if used_missing:
|
|
print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).")
|
|
|
|
return len(missing)
|
|
|
|
|
|
async def main():
|
|
parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД")
|
|
parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять")
|
|
parser.add_argument("--execute", action="store_true", help="Удалить найденное")
|
|
args = parser.parse_args()
|
|
|
|
if not args.dry_run and not args.execute:
|
|
print("Укажите --dry-run или --execute")
|
|
return
|
|
|
|
dry_run = args.dry_run and not args.execute
|
|
|
|
orphans = await find_orphan_s3_objects(dry_run)
|
|
missing = await find_missing_s3_objects(dry_run)
|
|
|
|
print("\nИтог")
|
|
print(f"Orphan S3 без БД: {orphans}")
|
|
print(f"Missing БД без S3 (удалено): {missing}")
|
|
if dry_run:
|
|
print("Режим: DRY-RUN (ничего не удалено)")
|
|
else:
|
|
print("Режим: EXECUTE (изменения применены)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |