Files
mtusi2026lab6/app/attachments.py
T
dawuduk 14ab803529 Перенести рабочую реализацию lab5 в skeleton лабораторной 6.
API, worker, брокер и стенд остаются базой, контракт lab6 не менялся.
2026-11-17 15:15:42 +03:00

311 lines
9.9 KiB
Python

"""Upload path: validate bytes, PUT, then commit metadata. See migration 003."""
from __future__ import annotations
import asyncio
import hashlib
import logging
from datetime import datetime, timezone
from uuid import UUID, uuid4
from app import config, db, jobs, s3
from app.errors import AppError
from app.store import format_ts
logger = logging.getLogger("messenger.attachments")
_sweep_task: asyncio.Task | None = None
def _public(row, variants=None) -> dict:
error_code = row["error_code"] if "error_code" in row.keys() else None
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": variants or [],
"error_code": error_code if row["status"] == "failed" else None,
"created_at": format_ts(row["created_at"]),
"updated_at": format_ts(row["updated_at"]),
}
async def _variants(attachment_id, pipeline_version: int) -> list:
async with db.require().acquire() as conn:
rows = await conn.fetch(
"""
SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2
ORDER BY CASE name
WHEN 'thumbnail' THEN 1
WHEN 'crop' THEN 2
WHEN 'optimized' THEN 3
ELSE 4
END
""",
attachment_id,
pipeline_version,
)
return [
{
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}
for row in rows
]
async def present(row) -> dict:
version = int(row["applied_pipeline_version"]) if "applied_pipeline_version" in row.keys() else 1
variants = await _variants(row["id"], version) if row["status"] == "ready" else []
return _public(row, variants)
async def _drop_intent(key: str) -> None:
async with db.require().acquire() as conn:
await conn.execute(
"DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed",
key,
)
async def _compensate(key: str) -> None:
try:
await s3.delete_key(key)
except AppError:
logger.warning("orphan object left for sweeper key=%s", key)
return
try:
await _drop_intent(key)
except Exception:
logger.warning("orphan intent left for sweeper key=%s", key)
async def create_attachment(
chat_id: str,
owner_id: str,
filename: str,
content_type: str,
data: bytes,
) -> dict:
attachment_id = uuid4()
key = f"staging/{attachment_id}"
created = datetime.now(timezone.utc)
digest = hashlib.sha256(data).hexdigest()
async with db.require().acquire() as conn:
await conn.execute(
"""
INSERT INTO blob_writes (object_key, committed, created_at)
VALUES ($1, FALSE, $2)
""",
key,
created,
)
try:
await s3.put_bytes(key, data, content_type)
except Exception:
await _compensate(key)
raise
try:
async with db.require().acquire() as conn:
async with conn.transaction():
intent = await conn.fetchrow(
"SELECT committed FROM blob_writes WHERE object_key = $1 FOR UPDATE",
key,
)
if intent is None or intent["committed"]:
raise AppError(503, "service unavailable")
await conn.execute(
"""
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, status, error_code, created_at, updated_at
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NULL, $9, $9)
""",
attachment_id,
UUID(chat_id),
UUID(owner_id),
filename,
content_type,
len(data),
digest,
key,
created,
)
await jobs.insert_job(conn, attachment_id, created)
await conn.execute(
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
key,
)
except AppError:
await _compensate(key)
raise
except Exception:
await _compensate(key)
raise AppError(503, "service unavailable") from None
jobs.kick_publish()
return _public(
{
"id": attachment_id,
"chat_id": UUID(chat_id),
"owner_id": UUID(owner_id),
"filename": filename,
"content_type": content_type,
"size_bytes": len(data),
"sha256": digest,
"status": "queued",
"error_code": None,
"created_at": created,
"updated_at": created,
}
)
async def get_for_member(attachment_id: str, user_id: str):
async with db.require().acquire() as conn:
row = await conn.fetchrow(
"""
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.object_key, a.status, a.error_code,
a.applied_pipeline_version, a.created_at, a.updated_at
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2
WHERE a.id = $1
""",
UUID(attachment_id),
UUID(user_id),
)
return row
async def sweep_orphans() -> int:
"""Delete uncommitted objects older than the grace period. In-use rows stay."""
deleted = 0
async with db.require().acquire() as conn:
async with conn.transaction():
rows = await conn.fetch(
"""
SELECT object_key
FROM blob_writes
WHERE NOT committed
AND created_at < now() - ($1::int * interval '1 second')
ORDER BY created_at
LIMIT 50
FOR UPDATE SKIP LOCKED
""",
config.ORPHAN_GRACE_SECONDS,
)
for row in rows:
key = row["object_key"]
linked = await conn.fetchval(
"""
SELECT 1 FROM attachments WHERE object_key = $1
UNION ALL
SELECT 1 FROM attachment_variants WHERE object_key = $1
LIMIT 1
""",
key,
)
busy = False
parts = key.split("/")
if len(parts) >= 2:
try:
owner = UUID(parts[1])
except ValueError:
owner = None
if owner is not None:
busy = await conn.fetchval(
"""
SELECT 1 FROM attachment_jobs
WHERE attachment_id = $1 AND status IN ('queued', 'processing')
""",
owner,
)
if busy:
continue
if linked:
await conn.execute(
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
key,
)
continue
await s3.delete_key(key)
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", key)
await conn.execute(
"""
UPDATE direct_uploads
SET status = 'aborted'
WHERE object_key = $1 AND status = 'pending'
""",
key,
)
deleted += 1
if deleted:
logger.info("removed %s uncommitted objects past grace", deleted)
return deleted
async def sweep_failed_staging() -> int:
"""Drop staging bytes of failed jobs after the retention window. Metadata stays."""
deleted = 0
async with db.require().acquire() as conn:
rows = await conn.fetch(
"""
SELECT id, object_key
FROM attachments
WHERE status = 'failed'
AND object_key LIKE 'staging/%'
AND updated_at < now() - ($1::int * interval '1 second')
ORDER BY updated_at
LIMIT 20
""",
config.STAGING_RETENTION_SECONDS,
)
for row in rows:
await s3.delete_key(row["object_key"])
async with db.require().acquire() as conn:
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", row["object_key"])
deleted += 1
return deleted
async def _sweep_loop() -> None:
while True:
try:
await sweep_orphans()
await sweep_failed_staging()
except asyncio.CancelledError:
raise
except Exception:
logger.warning("orphan sweep failed")
await asyncio.sleep(config.S3_SWEEP_INTERVAL_SECONDS)
def start_sweeper() -> None:
global _sweep_task
if _sweep_task is None or _sweep_task.done():
_sweep_task = asyncio.create_task(_sweep_loop())
async def stop_sweeper() -> None:
global _sweep_task
task = _sweep_task
_sweep_task = None
if task is None:
return
task.cancel()
try:
await task
except asyncio.CancelledError:
pass