Files
mtusi2026lab4/app/sweeper.py
Ilya a97c35fc12 выполнение ЛР4 на 4
- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket,
  отдельные ограниченные ключи API, служебные порты закрыты;
  идемпотентный s3-init с проверкой приватности и прогревом записи
- вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс,
  тип по содержимому, проверка filename, скачивание только участникам
- согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper
  с grace и dry-run; attachment_ids в сообщениях, FK по чату
- back-pressure на S3 PUT после найденного OOM SeaweedFS
- тесты student-tests, evidence, REPORT.md, DEMO.md
2026-10-28 18:06:07 +03:00

131 lines
5.2 KiB
Python

"""Cleanup of attachments that never became ready.
Invariant kept by the upload path: a metadata row (status=pending) is committed BEFORE the S3 PUT and
switched to ready AFTER it. Therefore, after a crash/abort/failed COMMIT there can only be
* a stale pending row (object may or may not exist), or
* an object without any row (row already removed, compensating delete failed).
Both are deleted only when older than ORPHAN_GRACE_SECONDS, far longer than any live upload
(UPLOAD_TIMEOUT_SECONDS + S3 timeouts), so an in-flight upload is never touched. Ready rows and objects
referenced by any row are never deleted. Every step is idempotent; repeating a run is safe.
python -m app.sweeper --once --dry-run # report only
python -m app.sweeper --once # one real pass
python -m app.sweeper # loop every SWEEP_INTERVAL_SECONDS
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import time
from datetime import datetime, timedelta, timezone
import asyncpg
from starlette.concurrency import run_in_threadpool
from app import config
from app.objects import ObjectStore, S3Unavailable
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("sweeper")
LOCK_ID = 2026_0004
HEARTBEAT_FILE = "/tmp/sweeper-heartbeat"
BATCH = 1000
async def stale_pending(conn: asyncpg.Connection, grace: int):
return await conn.fetch(
"""
SELECT id, object_key FROM attachments
WHERE status = 'pending' AND created_at < clock_timestamp() - make_interval(secs => $1)
ORDER BY created_at
""",
float(grace),
)
async def sweep(conn: asyncpg.Connection, objects: ObjectStore, grace: int, dry_run: bool) -> dict:
started = time.monotonic()
report = {"dry_run": dry_run, "grace_seconds": grace, "pending_found": 0, "pending_deleted": 0,
"objects_scanned": 0, "orphans_found": 0, "orphans_deleted": 0, "orphan_keys_sample": []}
rows = await stale_pending(conn, grace)
report["pending_found"] = len(rows)
if not dry_run:
for row in rows:
# Object first, then the row: if we crash in between, the row remains and the next pass repeats.
await run_in_threadpool(objects.delete_sync, row["object_key"])
deleted = await conn.execute(
"""
DELETE FROM attachments
WHERE id = $1 AND status = 'pending' AND created_at < clock_timestamp() - make_interval(secs => $2)
""",
row["id"], float(grace),
)
report["pending_deleted"] += int(deleted.split()[-1])
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace)
def old_keys():
for key, modified in objects.list_originals():
report["objects_scanned"] += 1
if modified < cutoff:
yield key
keys = await run_in_threadpool(lambda: list(old_keys()))
for i in range(0, len(keys), BATCH):
batch = keys[i:i + BATCH]
known = {r["object_key"] for r in await conn.fetch(
"SELECT object_key FROM attachments WHERE object_key = ANY($1::text[])", batch)}
orphans = [k for k in batch if k not in known]
report["orphans_found"] += len(orphans)
report["orphan_keys_sample"].extend(orphans[:5 - len(report["orphan_keys_sample"])])
if orphans and not dry_run:
report["orphans_deleted"] += await run_in_threadpool(objects.delete_many, orphans)
report["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
return report
async def run_once(grace: int, dry_run: bool) -> dict:
conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
try:
if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_ID):
return {"skipped": "another sweeper holds the lock"}
try:
return await sweep(conn, ObjectStore(), grace, dry_run)
finally:
await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID)
finally:
await conn.close()
async def main() -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--once", action="store_true")
p.add_argument("--dry-run", action="store_true")
p.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
p.add_argument("--interval", type=int, default=config.SWEEP_INTERVAL_SECONDS)
args = p.parse_args()
if args.grace < 60:
p.error("grace below 60 s could remove an upload that is still in progress")
while True:
try:
report = await run_once(args.grace, args.dry_run)
print(json.dumps(report, ensure_ascii=False), flush=True)
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
fh.write(str(time.time()))
except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError, S3Unavailable) as exc:
logger.warning("sweep failed (%s); will retry", type(exc).__name__)
if args.once:
return 1
if args.once:
return 0
await asyncio.sleep(args.interval)
if __name__ == "__main__":
raise SystemExit(asyncio.run(main()))