forked from mtusi2026/mtusi2026lab4
Байты лежат в SeaweedFS, метаданные в Postgres. Другая реплика скачивает тот же объект, потому что локальный диск API не является источником файла.
215 lines
6.5 KiB
Python
215 lines
6.5 KiB
Python
"""Upload path: validate bytes, PUT, then commit metadata. See migration 003."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from uuid import UUID, uuid4
|
|
|
|
from app import config, db, s3
|
|
from app.errors import AppError
|
|
from app.store import format_ts
|
|
|
|
logger = logging.getLogger("messenger.attachments")
|
|
_sweep_task: asyncio.Task | None = None
|
|
|
|
|
|
def _public(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"owner_id": str(row["owner_id"]),
|
|
"filename": row["filename"],
|
|
"content_type": row["content_type"],
|
|
"size_bytes": row["size_bytes"],
|
|
"sha256": row["sha256"],
|
|
"status": row["status"],
|
|
"variants": [],
|
|
"error_code": None,
|
|
"created_at": format_ts(row["created_at"]),
|
|
"updated_at": format_ts(row["updated_at"]),
|
|
}
|
|
|
|
|
|
async def _drop_intent(key: str) -> None:
|
|
async with db.require().acquire() as conn:
|
|
await conn.execute(
|
|
"DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed",
|
|
key,
|
|
)
|
|
|
|
|
|
async def _compensate(key: str) -> None:
|
|
try:
|
|
await s3.delete_key(key)
|
|
except AppError:
|
|
logger.warning("orphan object left for sweeper key=%s", key)
|
|
return
|
|
try:
|
|
await _drop_intent(key)
|
|
except Exception:
|
|
logger.warning("orphan intent left for sweeper key=%s", key)
|
|
|
|
|
|
async def create_attachment(
|
|
chat_id: str,
|
|
owner_id: str,
|
|
filename: str,
|
|
content_type: str,
|
|
data: bytes,
|
|
) -> dict:
|
|
attachment_id = uuid4()
|
|
key = f"originals/{attachment_id}"
|
|
created = datetime.now(timezone.utc)
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
async with db.require().acquire() as conn:
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO blob_writes (object_key, committed, created_at)
|
|
VALUES ($1, FALSE, $2)
|
|
""",
|
|
key,
|
|
created,
|
|
)
|
|
try:
|
|
await s3.put_bytes(key, data, content_type)
|
|
except Exception:
|
|
await _compensate(key)
|
|
raise
|
|
try:
|
|
async with db.require().acquire() as conn:
|
|
async with conn.transaction():
|
|
intent = await conn.fetchrow(
|
|
"SELECT committed FROM blob_writes WHERE object_key = $1 FOR UPDATE",
|
|
key,
|
|
)
|
|
if intent is None or intent["committed"]:
|
|
raise AppError(503, "service unavailable")
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO attachments (
|
|
id, chat_id, owner_id, filename, content_type, size_bytes,
|
|
sha256, object_key, status, error_code, created_at, updated_at
|
|
)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ready', NULL, $9, $9)
|
|
""",
|
|
attachment_id,
|
|
UUID(chat_id),
|
|
UUID(owner_id),
|
|
filename,
|
|
content_type,
|
|
len(data),
|
|
digest,
|
|
key,
|
|
created,
|
|
)
|
|
await conn.execute(
|
|
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
|
key,
|
|
)
|
|
except AppError:
|
|
await _compensate(key)
|
|
raise
|
|
except Exception:
|
|
await _compensate(key)
|
|
raise AppError(503, "service unavailable") from None
|
|
return _public(
|
|
{
|
|
"id": attachment_id,
|
|
"chat_id": UUID(chat_id),
|
|
"owner_id": UUID(owner_id),
|
|
"filename": filename,
|
|
"content_type": content_type,
|
|
"size_bytes": len(data),
|
|
"sha256": digest,
|
|
"status": "ready",
|
|
"created_at": created,
|
|
"updated_at": created,
|
|
}
|
|
)
|
|
|
|
|
|
async def get_for_member(attachment_id: str, user_id: str):
|
|
async with db.require().acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
|
a.size_bytes, a.sha256, a.object_key, a.status, a.created_at, a.updated_at
|
|
FROM attachments a
|
|
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2
|
|
WHERE a.id = $1
|
|
""",
|
|
UUID(attachment_id),
|
|
UUID(user_id),
|
|
)
|
|
return row
|
|
|
|
|
|
async def sweep_orphans() -> int:
|
|
"""Delete uncommitted objects older than the grace period. In-use rows stay."""
|
|
deleted = 0
|
|
async with db.require().acquire() as conn:
|
|
async with conn.transaction():
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT object_key
|
|
FROM blob_writes
|
|
WHERE NOT committed
|
|
AND created_at < now() - ($1::int * interval '1 second')
|
|
ORDER BY created_at
|
|
LIMIT 50
|
|
FOR UPDATE SKIP LOCKED
|
|
""",
|
|
config.ORPHAN_GRACE_SECONDS,
|
|
)
|
|
for row in rows:
|
|
key = row["object_key"]
|
|
linked = await conn.fetchval(
|
|
"SELECT 1 FROM attachments WHERE object_key = $1",
|
|
key,
|
|
)
|
|
if linked:
|
|
await conn.execute(
|
|
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
|
key,
|
|
)
|
|
continue
|
|
await s3.delete_key(key)
|
|
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", key)
|
|
deleted += 1
|
|
if deleted:
|
|
logger.info("removed %s uncommitted objects past grace", deleted)
|
|
return deleted
|
|
|
|
|
|
async def _sweep_loop() -> None:
|
|
while True:
|
|
try:
|
|
await sweep_orphans()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.warning("orphan sweep failed")
|
|
await asyncio.sleep(config.S3_SWEEP_INTERVAL_SECONDS)
|
|
|
|
|
|
def start_sweeper() -> None:
|
|
global _sweep_task
|
|
if _sweep_task is None or _sweep_task.done():
|
|
_sweep_task = asyncio.create_task(_sweep_loop())
|
|
|
|
|
|
async def stop_sweeper() -> None:
|
|
global _sweep_task
|
|
task = _sweep_task
|
|
_sweep_task = None
|
|
if task is None:
|
|
return
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|