Обновила файлы s3 и db.py

This commit is contained in:
vasyukova
2026-10-23 00:06:07 +03:00
parent 1637e30dba
commit 7d39c2dab6
3 changed files with 96 additions and 20 deletions
+28 -1
View File
@@ -115,8 +115,9 @@ CREATE TABLE IF NOT EXISTS attachments (
size_bytes BIGINT NOT NULL,
sha256 TEXT NOT NULL,
object_key TEXT NOT NULL UNIQUE,
status TEXT NOT NULL, -- ready (lab4), queued/processing/failed (lab5)
status TEXT NOT NULL, -- queued/processing/ready/failed
error_code TEXT,
processing_until TIMESTAMPTZ, -- lease для worker (lab5)
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
@@ -132,6 +133,32 @@ CREATE TABLE IF NOT EXISTS message_attachments (
);
CREATE INDEX IF NOT EXISTS idx_msg_att_message
ON message_attachments (message_id, position);
-- lab5: варианты обработки (идемпотентность по (attachment_id, pipeline_version, variant))
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INT NOT NULL,
variant TEXT NOT NULL,
object_key TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
width INT NOT NULL,
height INT NOT NULL,
sha256 TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
-- lab5 level 4: transactional outbox (разрыв COMMIT → publish)
CREATE TABLE IF NOT EXISTS outbox (
id BIGSERIAL PRIMARY KEY,
job_id UUID NOT NULL UNIQUE,
payload JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished
ON outbox (id) WHERE published_at IS NULL;
"""
+15 -3
View File
@@ -1,6 +1,9 @@
"""S3-клиент (MinIO) для lab4.
"""S3-клиент (MinIO) для lab4/lab5.
Object key генерирует сервер: attachments/<chat_id>/<attachment_id>/original.
Object key генерирует сервер:
staging: staging/<chat_id>/<attachment_id>/original
final: attachments/<chat_id>/<attachment_id>/original
variant: attachments/<chat_id>/<attachment_id>/<variant>
filename используется только для отображения, не в ключе.
"""
import os
@@ -42,11 +45,20 @@ def client():
)
def staging_key(chat_id: UUID, attachment_id: UUID) -> str:
"""Staging — приватный prefix, worker читает отсюда."""
return "staging/%s/%s/original" % (chat_id, attachment_id)
def original_key(chat_id: UUID, attachment_id: UUID) -> str:
"""Единственный ключ оригинала. Не зависит от filename."""
"""Финальный ключ оригинала. Не зависит от filename."""
return "attachments/%s/%s/original" % (chat_id, attachment_id)
def variant_key(chat_id: UUID, attachment_id: UUID, variant: str) -> str:
return "attachments/%s/%s/%s" % (chat_id, attachment_id, variant)
def put_bytes(key: str, data: bytes, content_type: str) -> None:
client().put_object(
Bucket=_bucket(),
+53 -16
View File
@@ -1,22 +1,20 @@
"""Фоновая сверка S3 и PostgreSQL (уровень 4).
"""Фоновая сверка S3 и PostgreSQL (уровень 4 lab4/lab5).
- Orphan: объект в S3 есть, а метаданных в attachments нет.
- Orphan: объект в S3 есть, а метаданных/вариантов в БД нет.
Удаляем только если объект старше ORPHAN_GRACE_SECONDS.
- Missing: метаданные есть, объекта в S3 нет.
Пишем warning в лог (не удаляем метаданные — учебная политика).
- Object key кодирует chat_id и attachment_id, поэтому сверка по метаданным
attachments.
- Каждый цикл независим; падение логируется и не роняет процесс.
- Missing: метаданные есть, объекта в S3 нет — warning.
- Staging: объекты staging/ старше grace удаляются (не привязаны к текущей работе).
- Также периодически возвращает зависшие processing в queued.
"""
import logging
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from uuid import UUID
from . import s3
from .db import connection, init_pool, close_pool
from .postgres import storage
logging.basicConfig(
@@ -42,12 +40,23 @@ def _grace() -> int:
def _known_keys() -> set[str]:
"""Все ожидаемые object key из таблицы attachments."""
"""Все ожидаемые object key: оригиналы + варианты."""
keys: set[str] = set()
with connection() as conn, conn.cursor() as cur:
cur.execute("SELECT id, chat_id FROM attachments")
for aid, cid in cur.fetchall():
cur.execute("SELECT id, chat_id, object_key FROM attachments")
for aid, cid, okey in cur.fetchall():
keys.add(okey)
keys.add(s3.original_key(cid, aid))
cur.execute("SELECT attachment_id, variant FROM attachment_variants")
# нужен chat_id для variant_key
# variants: отдельным запросом с join
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT a.chat_id, v.attachment_id, v.variant"
" FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id"
)
for cid, aid, variant in cur.fetchall():
keys.add(s3.variant_key(cid, aid, variant))
return keys
@@ -59,7 +68,9 @@ def _run_once() -> dict:
orphan_deleted = 0
missing = 0
scanned = 0
staging_deleted = 0
# attachments/ — orphan по known
for obj in s3.list_objects("attachments/"):
scanned += 1
key = obj["Key"]
@@ -79,19 +90,44 @@ def _run_once() -> dict:
orphan_deleted += 1
log.info("event=orphan_deleted key=%s", key)
except Exception as exc: # noqa: BLE001
log.error("event=orphan_delete_failed key=%s type=%s", key, type(exc).__name__)
log.error("event=orphan_delete_failed key=%s type=%s",
key, type(exc).__name__)
# Проверяем обратную сторону: метаданные без объекта.
# staging/ — удаляем только старое (не in-flight).
for obj in s3.list_objects("staging/"):
lm = obj.get("LastModified")
if lm is None:
continue
if lm.tzinfo is None:
lm = lm.replace(tzinfo=timezone.utc)
if lm > grace_cutoff:
continue
try:
s3.delete_object(obj["Key"])
staging_deleted += 1
except Exception:
pass
# missing
for key in known:
if s3.head_object(key) is None:
missing += 1
log.warning("event=missing_object key=%s", key)
# зависшие processing → queued
try:
recovered = storage.recover_stuck_processing()
if recovered:
log.info("event=stuck_processing_recovered count=%s", recovered)
except Exception as exc: # noqa: BLE001
log.error("event=recover_stuck_failed type=%s", type(exc).__name__)
return {
"scanned": scanned,
"orphan_deleted": orphan_deleted,
"missing": missing,
"known": len(known),
"staging_deleted": staging_deleted,
}
@@ -104,9 +140,10 @@ def main() -> None:
try:
stats = _run_once()
log.info(
"event=cleanup_pass scanned=%s known=%s orphan_deleted=%s missing=%s",
stats["scanned"], stats["known"],
stats["orphan_deleted"], stats["missing"],
"event=cleanup_pass scanned=%s known=%s orphan_deleted=%s"
" missing=%s staging_deleted=%s",
stats["scanned"], stats["known"], stats["orphan_deleted"],
stats["missing"], stats["staging_deleted"],
)
except Exception as exc: # noqa: BLE001
log.error("event=cleanup_failed type=%s", type(exc).__name__)