Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
291 lines
10 KiB
Python
291 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Reconciler для согласованности S3 и PostgreSQL (Трек 5).
|
|
|
|
Функции:
|
|
- Обнаружение orphan объектов (в S3, но нет в БД)
|
|
- Обнаружение missing файлов (в БД, но нет в S3)
|
|
- Dry-run режим для безопасной проверки
|
|
- Grace period для защиты свежих файлов
|
|
- Метрики и отчет
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
import psycopg2
|
|
import boto3
|
|
from botocore.client import Config
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description='S3/DB reconciler for attachments')
|
|
parser.add_argument('--db-host', default='localhost', help='PostgreSQL host')
|
|
parser.add_argument('--db-port', type=int, default=5432, help='PostgreSQL port')
|
|
parser.add_argument('--db-name', default='messenger', help='Database name')
|
|
parser.add_argument('--db-user', default='messenger', help='Database user')
|
|
parser.add_argument('--db-password', default='messenger', help='Database password')
|
|
parser.add_argument('--s3-endpoint', default='http://localhost:9000', help='S3 endpoint')
|
|
parser.add_argument('--s3-access-key', default='minioadmin', help='S3 access key')
|
|
parser.add_argument('--s3-secret-key', default='minioadmin', help='S3 secret key')
|
|
parser.add_argument('--s3-bucket', default='messenger-attachments', help='S3 bucket name')
|
|
parser.add_argument('--grace-period-hours', type=int, default=1, help='Grace period in hours for fresh files')
|
|
parser.add_argument('--dry-run', action='store_true', help='Dry run mode (no deletions)')
|
|
parser.add_argument('--output', default='reconciler_report.json', help='Output report file')
|
|
return parser.parse_args()
|
|
|
|
def connect_db(args):
|
|
"""Подключение к PostgreSQL."""
|
|
return psycopg2.connect(
|
|
host=args.db_host,
|
|
port=args.db_port,
|
|
dbname=args.db_name,
|
|
user=args.db_user,
|
|
password=args.db_password
|
|
)
|
|
|
|
def connect_s3(args):
|
|
"""Подключение к S3 (MinIO)."""
|
|
return boto3.client(
|
|
's3',
|
|
endpoint_url=args.s3_endpoint,
|
|
aws_access_key_id=args.s3_access_key,
|
|
aws_secret_access_key=args.s3_secret_key,
|
|
config=Config(signature_version='s3v4')
|
|
)
|
|
|
|
def get_db_attachments(conn):
|
|
"""Получить список всех s3_key из БД с метаданными."""
|
|
cur = conn.cursor()
|
|
cur.execute("""
|
|
SELECT id, s3_key, created_at, status, chat_id
|
|
FROM attachments
|
|
ORDER BY created_at DESC
|
|
""")
|
|
|
|
attachments = {}
|
|
for row in cur.fetchall():
|
|
att_id, s3_key, created_at, status, chat_id = row
|
|
attachments[s3_key] = {
|
|
'id': str(att_id),
|
|
'created_at': created_at,
|
|
'status': status,
|
|
'chat_id': str(chat_id)
|
|
}
|
|
|
|
cur.close()
|
|
return attachments
|
|
|
|
def get_s3_objects(s3_client, bucket):
|
|
"""Получить список всех объектов из S3."""
|
|
objects = {}
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
|
|
|
for page in paginator.paginate(Bucket=bucket, Prefix='attachments/'):
|
|
if 'Contents' not in page:
|
|
continue
|
|
|
|
for obj in page['Contents']:
|
|
key = obj['Key']
|
|
objects[key] = {
|
|
'size': obj['Size'],
|
|
'last_modified': obj['LastModified']
|
|
}
|
|
|
|
return objects
|
|
|
|
def find_orphans(s3_objects, db_attachments, grace_period_hours):
|
|
"""Найти orphan объекты в S3 (нет в БД)."""
|
|
orphans = []
|
|
grace_cutoff = datetime.now(timezone.utc) - timedelta(hours=grace_period_hours)
|
|
|
|
for s3_key, s3_meta in s3_objects.items():
|
|
if s3_key not in db_attachments:
|
|
# Проверяем grace period.
|
|
if s3_meta['last_modified'] > grace_cutoff:
|
|
continue
|
|
|
|
orphans.append({
|
|
's3_key': s3_key,
|
|
'size': s3_meta['size'],
|
|
'last_modified': s3_meta['last_modified'].isoformat(),
|
|
'age_hours': (datetime.now(timezone.utc) - s3_meta['last_modified']).total_seconds() / 3600
|
|
})
|
|
|
|
return orphans
|
|
|
|
def find_missing(db_attachments, s3_objects, grace_period_hours):
|
|
"""Найти missing файлы (в БД, но нет в S3)."""
|
|
missing = []
|
|
grace_cutoff = datetime.now(timezone.utc) - timedelta(hours=grace_period_hours)
|
|
|
|
for s3_key, db_meta in db_attachments.items():
|
|
if s3_key not in s3_objects:
|
|
# Проверяем grace period.
|
|
created_at_utc = db_meta['created_at'].replace(tzinfo=timezone.utc)
|
|
if created_at_utc > grace_cutoff:
|
|
continue
|
|
|
|
missing.append({
|
|
's3_key': s3_key,
|
|
'attachment_id': db_meta['id'],
|
|
'created_at': db_meta['created_at'].isoformat(),
|
|
'status': db_meta['status'],
|
|
'chat_id': db_meta['chat_id'],
|
|
'age_hours': (datetime.now(timezone.utc) - created_at_utc).total_seconds() / 3600
|
|
})
|
|
|
|
return missing
|
|
|
|
def delete_orphans(s3_client, bucket, orphans, dry_run):
|
|
"""Удалить orphan объекты из S3."""
|
|
deleted = 0
|
|
|
|
for orphan in orphans:
|
|
s3_key = orphan['s3_key']
|
|
if dry_run:
|
|
print(f"[DRY-RUN] Would delete S3 object: {s3_key}")
|
|
else:
|
|
try:
|
|
s3_client.delete_object(Bucket=bucket, Key=s3_key)
|
|
print(f"Deleted S3 orphan: {s3_key}")
|
|
deleted += 1
|
|
except Exception as e:
|
|
print(f"Failed to delete {s3_key}: {e}", file=sys.stderr)
|
|
|
|
return deleted
|
|
|
|
def mark_missing(conn, missing, dry_run):
|
|
"""Отметить missing файлы в БД (status=failed, error_code)."""
|
|
marked = 0
|
|
cur = conn.cursor()
|
|
|
|
for miss in missing:
|
|
att_id = miss['attachment_id']
|
|
if dry_run:
|
|
print(f"[DRY-RUN] Would mark attachment {att_id} as failed (s3_object_missing)")
|
|
else:
|
|
try:
|
|
cur.execute("""
|
|
UPDATE attachments
|
|
SET status = 'failed', error_code = 's3_object_missing'
|
|
WHERE id = %s AND status != 'failed'
|
|
""", (att_id,))
|
|
if cur.rowcount > 0:
|
|
print(f"Marked attachment {att_id} as failed")
|
|
marked += 1
|
|
except Exception as e:
|
|
print(f"Failed to mark {att_id}: {e}", file=sys.stderr)
|
|
|
|
if not dry_run:
|
|
conn.commit()
|
|
cur.close()
|
|
|
|
return marked
|
|
|
|
def generate_report(args, db_attachments, s3_objects, orphans, missing, deleted_count, marked_count):
|
|
"""Сгенерировать JSON-отчет."""
|
|
total_db = len(db_attachments)
|
|
total_s3 = len(s3_objects)
|
|
total_orphans = len(orphans)
|
|
total_missing = len(missing)
|
|
|
|
# Размеры.
|
|
orphan_bytes = sum(o['size'] for o in orphans)
|
|
|
|
report = {
|
|
'timestamp': datetime.now(timezone.utc).isoformat(),
|
|
'parameters': {
|
|
'grace_period_hours': args.grace_period_hours,
|
|
'dry_run': args.dry_run
|
|
},
|
|
'summary': {
|
|
'total_db_attachments': total_db,
|
|
'total_s3_objects': total_s3,
|
|
'orphan_objects': total_orphans,
|
|
'orphan_bytes': orphan_bytes,
|
|
'missing_files': total_missing,
|
|
'deleted_orphans': deleted_count,
|
|
'marked_missing': marked_count
|
|
},
|
|
'orphans': orphans[:100], # Первые 100 для отчета.
|
|
'missing': missing[:100]
|
|
}
|
|
|
|
with open(args.output, 'w') as f:
|
|
json.dump(report, f, indent=2, default=str)
|
|
|
|
return report
|
|
|
|
def main():
|
|
args = parse_args()
|
|
|
|
print("=== S3/DB Reconciler ===")
|
|
print(f"Mode: {'DRY-RUN' if args.dry_run else 'LIVE'}")
|
|
print(f"Grace period: {args.grace_period_hours} hours")
|
|
print()
|
|
|
|
# Подключение.
|
|
print("Connecting to PostgreSQL...")
|
|
conn = connect_db(args)
|
|
|
|
print("Connecting to S3...")
|
|
s3 = connect_s3(args)
|
|
|
|
# Сканирование.
|
|
print("Scanning database...")
|
|
db_attachments = get_db_attachments(conn)
|
|
print(f"Found {len(db_attachments)} attachments in database")
|
|
|
|
print("Scanning S3 bucket...")
|
|
s3_objects = get_s3_objects(s3, args.s3_bucket)
|
|
print(f"Found {len(s3_objects)} objects in S3")
|
|
print()
|
|
|
|
# Поиск несоответствий.
|
|
print("Finding orphan objects...")
|
|
orphans = find_orphans(s3_objects, db_attachments, args.grace_period_hours)
|
|
print(f"Found {len(orphans)} orphan objects (older than {args.grace_period_hours}h)")
|
|
|
|
print("Finding missing files...")
|
|
missing = find_missing(db_attachments, s3_objects, args.grace_period_hours)
|
|
print(f"Found {len(missing)} missing files (older than {args.grace_period_hours}h)")
|
|
print()
|
|
|
|
# Очистка.
|
|
deleted_count = 0
|
|
marked_count = 0
|
|
|
|
if orphans:
|
|
print("Processing orphan objects...")
|
|
deleted_count = delete_orphans(s3, args.s3_bucket, orphans, args.dry_run)
|
|
print()
|
|
|
|
if missing:
|
|
print("Processing missing files...")
|
|
marked_count = mark_missing(conn, missing, args.dry_run)
|
|
print()
|
|
|
|
# Отчет.
|
|
print("Generating report...")
|
|
report = generate_report(args, db_attachments, s3_objects, orphans, missing, deleted_count, marked_count)
|
|
print(f"Report saved to: {args.output}")
|
|
print()
|
|
|
|
# Итоги.
|
|
print("=== Summary ===")
|
|
print(f"Database attachments: {report['summary']['total_db_attachments']}")
|
|
print(f"S3 objects: {report['summary']['total_s3_objects']}")
|
|
print(f"Orphan objects: {report['summary']['orphan_objects']} ({report['summary']['orphan_bytes']} bytes)")
|
|
print(f"Missing files: {report['summary']['missing_files']}")
|
|
if not args.dry_run:
|
|
print(f"Deleted orphans: {deleted_count}")
|
|
print(f"Marked missing: {marked_count}")
|
|
else:
|
|
print("(DRY-RUN: no changes made)")
|
|
|
|
conn.close()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|