forked from mtusi2026/mtusi2026lab4
- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket, отдельные ограниченные ключи API, служебные порты закрыты; идемпотентный s3-init с проверкой приватности и прогревом записи - вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс, тип по содержимому, проверка filename, скачивание только участникам - согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper с grace и dry-run; attachment_ids в сообщениях, FK по чату - back-pressure на S3 PUT после найденного OOM SeaweedFS - тесты student-tests, evidence, REPORT.md, DEMO.md
131 lines
5.2 KiB
Python
131 lines
5.2 KiB
Python
"""Cleanup of attachments that never became ready.
|
|
|
|
Invariant kept by the upload path: a metadata row (status=pending) is committed BEFORE the S3 PUT and
|
|
switched to ready AFTER it. Therefore, after a crash/abort/failed COMMIT there can only be
|
|
* a stale pending row (object may or may not exist), or
|
|
* an object without any row (row already removed, compensating delete failed).
|
|
Both are deleted only when older than ORPHAN_GRACE_SECONDS, far longer than any live upload
|
|
(UPLOAD_TIMEOUT_SECONDS + S3 timeouts), so an in-flight upload is never touched. Ready rows and objects
|
|
referenced by any row are never deleted. Every step is idempotent; repeating a run is safe.
|
|
|
|
python -m app.sweeper --once --dry-run # report only
|
|
python -m app.sweeper --once # one real pass
|
|
python -m app.sweeper # loop every SWEEP_INTERVAL_SECONDS
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import asyncpg
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from app import config
|
|
from app.objects import ObjectStore, S3Unavailable
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
|
logger = logging.getLogger("sweeper")
|
|
|
|
LOCK_ID = 2026_0004
|
|
HEARTBEAT_FILE = "/tmp/sweeper-heartbeat"
|
|
BATCH = 1000
|
|
|
|
|
|
async def stale_pending(conn: asyncpg.Connection, grace: int):
|
|
return await conn.fetch(
|
|
"""
|
|
SELECT id, object_key FROM attachments
|
|
WHERE status = 'pending' AND created_at < clock_timestamp() - make_interval(secs => $1)
|
|
ORDER BY created_at
|
|
""",
|
|
float(grace),
|
|
)
|
|
|
|
|
|
async def sweep(conn: asyncpg.Connection, objects: ObjectStore, grace: int, dry_run: bool) -> dict:
|
|
started = time.monotonic()
|
|
report = {"dry_run": dry_run, "grace_seconds": grace, "pending_found": 0, "pending_deleted": 0,
|
|
"objects_scanned": 0, "orphans_found": 0, "orphans_deleted": 0, "orphan_keys_sample": []}
|
|
|
|
rows = await stale_pending(conn, grace)
|
|
report["pending_found"] = len(rows)
|
|
if not dry_run:
|
|
for row in rows:
|
|
# Object first, then the row: if we crash in between, the row remains and the next pass repeats.
|
|
await run_in_threadpool(objects.delete_sync, row["object_key"])
|
|
deleted = await conn.execute(
|
|
"""
|
|
DELETE FROM attachments
|
|
WHERE id = $1 AND status = 'pending' AND created_at < clock_timestamp() - make_interval(secs => $2)
|
|
""",
|
|
row["id"], float(grace),
|
|
)
|
|
report["pending_deleted"] += int(deleted.split()[-1])
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace)
|
|
|
|
def old_keys():
|
|
for key, modified in objects.list_originals():
|
|
report["objects_scanned"] += 1
|
|
if modified < cutoff:
|
|
yield key
|
|
|
|
keys = await run_in_threadpool(lambda: list(old_keys()))
|
|
for i in range(0, len(keys), BATCH):
|
|
batch = keys[i:i + BATCH]
|
|
known = {r["object_key"] for r in await conn.fetch(
|
|
"SELECT object_key FROM attachments WHERE object_key = ANY($1::text[])", batch)}
|
|
orphans = [k for k in batch if k not in known]
|
|
report["orphans_found"] += len(orphans)
|
|
report["orphan_keys_sample"].extend(orphans[:5 - len(report["orphan_keys_sample"])])
|
|
if orphans and not dry_run:
|
|
report["orphans_deleted"] += await run_in_threadpool(objects.delete_many, orphans)
|
|
|
|
report["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
|
|
return report
|
|
|
|
|
|
async def run_once(grace: int, dry_run: bool) -> dict:
|
|
conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
|
|
try:
|
|
if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_ID):
|
|
return {"skipped": "another sweeper holds the lock"}
|
|
try:
|
|
return await sweep(conn, ObjectStore(), grace, dry_run)
|
|
finally:
|
|
await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID)
|
|
finally:
|
|
await conn.close()
|
|
|
|
|
|
async def main() -> int:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--once", action="store_true")
|
|
p.add_argument("--dry-run", action="store_true")
|
|
p.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
|
|
p.add_argument("--interval", type=int, default=config.SWEEP_INTERVAL_SECONDS)
|
|
args = p.parse_args()
|
|
if args.grace < 60:
|
|
p.error("grace below 60 s could remove an upload that is still in progress")
|
|
while True:
|
|
try:
|
|
report = await run_once(args.grace, args.dry_run)
|
|
print(json.dumps(report, ensure_ascii=False), flush=True)
|
|
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
|
|
fh.write(str(time.time()))
|
|
except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError, S3Unavailable) as exc:
|
|
logger.warning("sweep failed (%s); will retry", type(exc).__name__)
|
|
if args.once:
|
|
return 1
|
|
if args.once:
|
|
return 0
|
|
await asyncio.sleep(args.interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(asyncio.run(main()))
|