Files
mtusi2026lab4/app/reconcile.py
T
alenchha 078a8341b5 ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API
с потоковой проверкой целостности, метаданные в PostgreSQL и байты в
приватном bucket, отдельный S3-пользователь с ограниченными правами,
идемпотентный bootstrap bucket, компенсация и reconciler (трек 2),
тесты отказов, эксперименты по памяти и согласованности.
2026-11-02 16:59:17 +03:00

116 lines
5.0 KiB
Python

"""DB <-> S3 reconciliation for attachments.
Invariant: a row is inserted as 'uploading' BEFORE the object is written, and becomes 'ready' only AFTER the
PUT succeeded. So every object has a row, and a client only ever sees 'ready' rows.
Cases (all gated by a grace period so in-flight uploads are never touched):
stale_uploading row 'uploading' older than grace AND its object (if any) older than grace -> delete both;
a fresh object under an old row is left alone (re-checked via LastModified)
orphan_object object without any row, older than grace -> delete object
missing_object 'ready' row confirmed longer than grace ago whose object is gone (re-checked by HEAD) -> mark 'failed'/object_missing, never delete the row
Rows referenced by messages are never deleted: only 'uploading' rows can be removed, and those cannot be linked.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
from starlette.concurrency import run_in_threadpool
log = logging.getLogger("messenger.reconcile")
LOCK_ID = 2026_0104
PREFIX = "attachments/"
BATCH = 500
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
started = time.monotonic()
result = {
"dry_run": dry_run, "grace_seconds": grace_seconds,
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0,
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
}
async with db.connection() as conn:
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
if not (await cur.fetchone())["got"]:
return {"skipped": "another reconciler is running"}
try:
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
await _stale_uploading(conn, s3, cutoff, dry_run, result)
await _orphan_objects(conn, s3, cutoff, dry_run, result)
await _missing_objects(conn, s3, cutoff, dry_run, result)
finally:
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
result["seconds"] = round(time.monotonic() - started, 3)
return result
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'uploading' AND created_at < %s", (cutoff,),
)
for row in await cur.fetchall():
modified = await run_in_threadpool(s3.head, row["object_key"])
if modified is not None and modified >= cutoff:
result["protected_fresh_objects"] += 1
continue
result["stale_uploading"] += 1
if dry_run:
continue
if modified is not None:
await run_in_threadpool(s3.delete, row["object_key"])
result["deleted_objects"] += 1
done = await conn.execute(
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
)
result["removed_rows"] += done.rowcount
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
pages = await run_in_threadpool(lambda: iter(s3.list_pages(PREFIX)))
while True:
page = await run_in_threadpool(next, pages, None)
if page is None:
break
result["objects_scanned"] += len(page)
keys = [o["Key"] for o in page]
if not keys:
continue
cur = await conn.execute("SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,))
known = {row["object_key"] for row in await cur.fetchall()}
for obj in page:
if obj["Key"] in known or obj["LastModified"] >= cutoff:
continue
result["orphan_objects"] += 1
if not dry_run:
again = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (obj["Key"],))
if await again.fetchone() is None:
await run_in_threadpool(s3.delete, obj["Key"])
result["deleted_objects"] += 1
async def _missing_objects(conn, s3, cutoff, dry_run, result):
last = None
while True:
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
(cutoff, last, last, BATCH),
)
rows = await cur.fetchall()
if not rows:
return
for row in rows:
result["rows_scanned"] += 1
if await run_in_threadpool(s3.exists, row["object_key"]):
continue
result["missing_objects"] += 1
if not dry_run:
done = await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
)
result["marked_failed"] += done.rowcount
last = str(rows[-1]["id"])