forked from mtusi2026/mtusi2026lab4
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
116 lines
5.0 KiB
Python
116 lines
5.0 KiB
Python
"""DB <-> S3 reconciliation for attachments.
|
|
|
|
Invariant: a row is inserted as 'uploading' BEFORE the object is written, and becomes 'ready' only AFTER the
|
|
PUT succeeded. So every object has a row, and a client only ever sees 'ready' rows.
|
|
|
|
Cases (all gated by a grace period so in-flight uploads are never touched):
|
|
stale_uploading row 'uploading' older than grace AND its object (if any) older than grace -> delete both;
|
|
a fresh object under an old row is left alone (re-checked via LastModified)
|
|
orphan_object object without any row, older than grace -> delete object
|
|
missing_object 'ready' row confirmed longer than grace ago whose object is gone (re-checked by HEAD) -> mark 'failed'/object_missing, never delete the row
|
|
Rows referenced by messages are never deleted: only 'uploading' rows can be removed, and those cannot be linked.
|
|
"""
|
|
import logging
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
log = logging.getLogger("messenger.reconcile")
|
|
|
|
LOCK_ID = 2026_0104
|
|
PREFIX = "attachments/"
|
|
BATCH = 500
|
|
|
|
|
|
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
|
|
started = time.monotonic()
|
|
result = {
|
|
"dry_run": dry_run, "grace_seconds": grace_seconds,
|
|
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0,
|
|
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
|
|
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
|
|
}
|
|
async with db.connection() as conn:
|
|
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
|
|
if not (await cur.fetchone())["got"]:
|
|
return {"skipped": "another reconciler is running"}
|
|
try:
|
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
|
await _stale_uploading(conn, s3, cutoff, dry_run, result)
|
|
await _orphan_objects(conn, s3, cutoff, dry_run, result)
|
|
await _missing_objects(conn, s3, cutoff, dry_run, result)
|
|
finally:
|
|
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
|
result["seconds"] = round(time.monotonic() - started, 3)
|
|
return result
|
|
|
|
|
|
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
|
|
cur = await conn.execute(
|
|
"SELECT id, object_key FROM attachments WHERE status = 'uploading' AND created_at < %s", (cutoff,),
|
|
)
|
|
for row in await cur.fetchall():
|
|
modified = await run_in_threadpool(s3.head, row["object_key"])
|
|
if modified is not None and modified >= cutoff:
|
|
result["protected_fresh_objects"] += 1
|
|
continue
|
|
result["stale_uploading"] += 1
|
|
if dry_run:
|
|
continue
|
|
if modified is not None:
|
|
await run_in_threadpool(s3.delete, row["object_key"])
|
|
result["deleted_objects"] += 1
|
|
done = await conn.execute(
|
|
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
|
|
)
|
|
result["removed_rows"] += done.rowcount
|
|
|
|
|
|
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
|
|
pages = await run_in_threadpool(lambda: iter(s3.list_pages(PREFIX)))
|
|
while True:
|
|
page = await run_in_threadpool(next, pages, None)
|
|
if page is None:
|
|
break
|
|
result["objects_scanned"] += len(page)
|
|
keys = [o["Key"] for o in page]
|
|
if not keys:
|
|
continue
|
|
cur = await conn.execute("SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,))
|
|
known = {row["object_key"] for row in await cur.fetchall()}
|
|
for obj in page:
|
|
if obj["Key"] in known or obj["LastModified"] >= cutoff:
|
|
continue
|
|
result["orphan_objects"] += 1
|
|
if not dry_run:
|
|
again = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (obj["Key"],))
|
|
if await again.fetchone() is None:
|
|
await run_in_threadpool(s3.delete, obj["Key"])
|
|
result["deleted_objects"] += 1
|
|
|
|
|
|
async def _missing_objects(conn, s3, cutoff, dry_run, result):
|
|
last = None
|
|
while True:
|
|
cur = await conn.execute(
|
|
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
|
|
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
|
|
(cutoff, last, last, BATCH),
|
|
)
|
|
rows = await cur.fetchall()
|
|
if not rows:
|
|
return
|
|
for row in rows:
|
|
result["rows_scanned"] += 1
|
|
if await run_in_threadpool(s3.exists, row["object_key"]):
|
|
continue
|
|
result["missing_objects"] += 1
|
|
if not dry_run:
|
|
done = await conn.execute(
|
|
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
|
|
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
|
|
)
|
|
result["marked_failed"] += done.rowcount
|
|
last = str(rows[-1]["id"])
|