diff --git a/src/app/storage/db.py b/src/app/storage/db.py index 4285debe..7de88c25 100644 --- a/src/app/storage/db.py +++ b/src/app/storage/db.py @@ -115,8 +115,9 @@ CREATE TABLE IF NOT EXISTS attachments ( size_bytes BIGINT NOT NULL, sha256 TEXT NOT NULL, object_key TEXT NOT NULL UNIQUE, - status TEXT NOT NULL, -- ready (lab4), queued/processing/failed (lab5) + status TEXT NOT NULL, -- queued/processing/ready/failed error_code TEXT, + processing_until TIMESTAMPTZ, -- lease для worker (lab5) created_at TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ NOT NULL ); @@ -132,6 +133,32 @@ CREATE TABLE IF NOT EXISTS message_attachments ( ); CREATE INDEX IF NOT EXISTS idx_msg_att_message ON message_attachments (message_id, position); + +-- lab5: варианты обработки (идемпотентность по (attachment_id, pipeline_version, variant)) +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INT NOT NULL, + variant TEXT NOT NULL, + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INT NOT NULL, + height INT NOT NULL, + sha256 TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); + +-- lab5 level 4: transactional outbox (разрыв COMMIT → publish) +CREATE TABLE IF NOT EXISTS outbox ( + id BIGSERIAL PRIMARY KEY, + job_id UUID NOT NULL UNIQUE, + payload JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ +); +CREATE INDEX IF NOT EXISTS idx_outbox_unpublished + ON outbox (id) WHERE published_at IS NULL; """ diff --git a/src/app/storage/s3.py b/src/app/storage/s3.py index 29a03c15..a0c9a7cf 100644 --- a/src/app/storage/s3.py +++ b/src/app/storage/s3.py @@ -1,6 +1,9 @@ -"""S3-клиент (MinIO) для lab4. +"""S3-клиент (MinIO) для lab4/lab5. -Object key генерирует сервер: attachments///original. +Object key генерирует сервер: + staging: staging///original + final: attachments///original + variant: attachments/// filename используется только для отображения, не в ключе. """ import os @@ -42,11 +45,20 @@ def client(): ) +def staging_key(chat_id: UUID, attachment_id: UUID) -> str: + """Staging — приватный prefix, worker читает отсюда.""" + return "staging/%s/%s/original" % (chat_id, attachment_id) + + def original_key(chat_id: UUID, attachment_id: UUID) -> str: - """Единственный ключ оригинала. Не зависит от filename.""" + """Финальный ключ оригинала. Не зависит от filename.""" return "attachments/%s/%s/original" % (chat_id, attachment_id) +def variant_key(chat_id: UUID, attachment_id: UUID, variant: str) -> str: + return "attachments/%s/%s/%s" % (chat_id, attachment_id, variant) + + def put_bytes(key: str, data: bytes, content_type: str) -> None: client().put_object( Bucket=_bucket(), diff --git a/src/app/storage/s3_cleanup.py b/src/app/storage/s3_cleanup.py index 7f548e1d..61e3c2e4 100644 --- a/src/app/storage/s3_cleanup.py +++ b/src/app/storage/s3_cleanup.py @@ -1,22 +1,20 @@ -"""Фоновая сверка S3 и PostgreSQL (уровень 4). +"""Фоновая сверка S3 и PostgreSQL (уровень 4 lab4/lab5). -- Orphan: объект в S3 есть, а метаданных в attachments нет. +- Orphan: объект в S3 есть, а метаданных/вариантов в БД нет. Удаляем только если объект старше ORPHAN_GRACE_SECONDS. -- Missing: метаданные есть, объекта в S3 нет. - Пишем warning в лог (не удаляем метаданные — учебная политика). -- Object key кодирует chat_id и attachment_id, поэтому сверка по метаданным - attachments. -- Каждый цикл независим; падение логируется и не роняет процесс. +- Missing: метаданные есть, объекта в S3 нет — warning. +- Staging: объекты staging/ старше grace удаляются (не привязаны к текущей работе). +- Также периодически возвращает зависшие processing в queued. """ import logging import os import sys import time from datetime import datetime, timedelta, timezone -from uuid import UUID from . import s3 from .db import connection, init_pool, close_pool +from .postgres import storage logging.basicConfig( @@ -42,12 +40,23 @@ def _grace() -> int: def _known_keys() -> set[str]: - """Все ожидаемые object key из таблицы attachments.""" + """Все ожидаемые object key: оригиналы + варианты.""" keys: set[str] = set() with connection() as conn, conn.cursor() as cur: - cur.execute("SELECT id, chat_id FROM attachments") - for aid, cid in cur.fetchall(): + cur.execute("SELECT id, chat_id, object_key FROM attachments") + for aid, cid, okey in cur.fetchall(): + keys.add(okey) keys.add(s3.original_key(cid, aid)) + cur.execute("SELECT attachment_id, variant FROM attachment_variants") + # нужен chat_id для variant_key + # variants: отдельным запросом с join + with connection() as conn, conn.cursor() as cur: + cur.execute( + "SELECT a.chat_id, v.attachment_id, v.variant" + " FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id" + ) + for cid, aid, variant in cur.fetchall(): + keys.add(s3.variant_key(cid, aid, variant)) return keys @@ -59,7 +68,9 @@ def _run_once() -> dict: orphan_deleted = 0 missing = 0 scanned = 0 + staging_deleted = 0 + # attachments/ — orphan по known for obj in s3.list_objects("attachments/"): scanned += 1 key = obj["Key"] @@ -79,19 +90,44 @@ def _run_once() -> dict: orphan_deleted += 1 log.info("event=orphan_deleted key=%s", key) except Exception as exc: # noqa: BLE001 - log.error("event=orphan_delete_failed key=%s type=%s", key, type(exc).__name__) + log.error("event=orphan_delete_failed key=%s type=%s", + key, type(exc).__name__) - # Проверяем обратную сторону: метаданные без объекта. + # staging/ — удаляем только старое (не in-flight). + for obj in s3.list_objects("staging/"): + lm = obj.get("LastModified") + if lm is None: + continue + if lm.tzinfo is None: + lm = lm.replace(tzinfo=timezone.utc) + if lm > grace_cutoff: + continue + try: + s3.delete_object(obj["Key"]) + staging_deleted += 1 + except Exception: + pass + + # missing for key in known: if s3.head_object(key) is None: missing += 1 log.warning("event=missing_object key=%s", key) + # зависшие processing → queued + try: + recovered = storage.recover_stuck_processing() + if recovered: + log.info("event=stuck_processing_recovered count=%s", recovered) + except Exception as exc: # noqa: BLE001 + log.error("event=recover_stuck_failed type=%s", type(exc).__name__) + return { "scanned": scanned, "orphan_deleted": orphan_deleted, "missing": missing, "known": len(known), + "staging_deleted": staging_deleted, } @@ -104,9 +140,10 @@ def main() -> None: try: stats = _run_once() log.info( - "event=cleanup_pass scanned=%s known=%s orphan_deleted=%s missing=%s", - stats["scanned"], stats["known"], - stats["orphan_deleted"], stats["missing"], + "event=cleanup_pass scanned=%s known=%s orphan_deleted=%s" + " missing=%s staging_deleted=%s", + stats["scanned"], stats["known"], stats["orphan_deleted"], + stats["missing"], stats["staging_deleted"], ) except Exception as exc: # noqa: BLE001 log.error("event=cleanup_failed type=%s", type(exc).__name__)