Обновила файлы s3 и db.py
This commit is contained in:
+28
-1
@@ -115,8 +115,9 @@ CREATE TABLE IF NOT EXISTS attachments (
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL, -- ready (lab4), queued/processing/failed (lab5)
|
||||
status TEXT NOT NULL, -- queued/processing/ready/failed
|
||||
error_code TEXT,
|
||||
processing_until TIMESTAMPTZ, -- lease для worker (lab5)
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
@@ -132,6 +133,32 @@ CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_msg_att_message
|
||||
ON message_attachments (message_id, position);
|
||||
|
||||
-- lab5: варианты обработки (идемпотентность по (attachment_id, pipeline_version, variant))
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version INT NOT NULL,
|
||||
variant TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
width INT NOT NULL,
|
||||
height INT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
-- lab5 level 4: transactional outbox (разрыв COMMIT → publish)
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID NOT NULL UNIQUE,
|
||||
payload JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
published_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished
|
||||
ON outbox (id) WHERE published_at IS NULL;
|
||||
"""
|
||||
|
||||
|
||||
|
||||
+15
-3
@@ -1,6 +1,9 @@
|
||||
"""S3-клиент (MinIO) для lab4.
|
||||
"""S3-клиент (MinIO) для lab4/lab5.
|
||||
|
||||
Object key генерирует сервер: attachments/<chat_id>/<attachment_id>/original.
|
||||
Object key генерирует сервер:
|
||||
staging: staging/<chat_id>/<attachment_id>/original
|
||||
final: attachments/<chat_id>/<attachment_id>/original
|
||||
variant: attachments/<chat_id>/<attachment_id>/<variant>
|
||||
filename используется только для отображения, не в ключе.
|
||||
"""
|
||||
import os
|
||||
@@ -42,11 +45,20 @@ def client():
|
||||
)
|
||||
|
||||
|
||||
def staging_key(chat_id: UUID, attachment_id: UUID) -> str:
|
||||
"""Staging — приватный prefix, worker читает отсюда."""
|
||||
return "staging/%s/%s/original" % (chat_id, attachment_id)
|
||||
|
||||
|
||||
def original_key(chat_id: UUID, attachment_id: UUID) -> str:
|
||||
"""Единственный ключ оригинала. Не зависит от filename."""
|
||||
"""Финальный ключ оригинала. Не зависит от filename."""
|
||||
return "attachments/%s/%s/original" % (chat_id, attachment_id)
|
||||
|
||||
|
||||
def variant_key(chat_id: UUID, attachment_id: UUID, variant: str) -> str:
|
||||
return "attachments/%s/%s/%s" % (chat_id, attachment_id, variant)
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str) -> None:
|
||||
client().put_object(
|
||||
Bucket=_bucket(),
|
||||
|
||||
@@ -1,22 +1,20 @@
|
||||
"""Фоновая сверка S3 и PostgreSQL (уровень 4).
|
||||
"""Фоновая сверка S3 и PostgreSQL (уровень 4 lab4/lab5).
|
||||
|
||||
- Orphan: объект в S3 есть, а метаданных в attachments нет.
|
||||
- Orphan: объект в S3 есть, а метаданных/вариантов в БД нет.
|
||||
Удаляем только если объект старше ORPHAN_GRACE_SECONDS.
|
||||
- Missing: метаданные есть, объекта в S3 нет.
|
||||
Пишем warning в лог (не удаляем метаданные — учебная политика).
|
||||
- Object key кодирует chat_id и attachment_id, поэтому сверка по метаданным
|
||||
attachments.
|
||||
- Каждый цикл независим; падение логируется и не роняет процесс.
|
||||
- Missing: метаданные есть, объекта в S3 нет — warning.
|
||||
- Staging: объекты staging/ старше grace удаляются (не привязаны к текущей работе).
|
||||
- Также периодически возвращает зависшие processing в queued.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from . import s3
|
||||
from .db import connection, init_pool, close_pool
|
||||
from .postgres import storage
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
@@ -42,12 +40,23 @@ def _grace() -> int:
|
||||
|
||||
|
||||
def _known_keys() -> set[str]:
|
||||
"""Все ожидаемые object key из таблицы attachments."""
|
||||
"""Все ожидаемые object key: оригиналы + варианты."""
|
||||
keys: set[str] = set()
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute("SELECT id, chat_id FROM attachments")
|
||||
for aid, cid in cur.fetchall():
|
||||
cur.execute("SELECT id, chat_id, object_key FROM attachments")
|
||||
for aid, cid, okey in cur.fetchall():
|
||||
keys.add(okey)
|
||||
keys.add(s3.original_key(cid, aid))
|
||||
cur.execute("SELECT attachment_id, variant FROM attachment_variants")
|
||||
# нужен chat_id для variant_key
|
||||
# variants: отдельным запросом с join
|
||||
with connection() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT a.chat_id, v.attachment_id, v.variant"
|
||||
" FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id"
|
||||
)
|
||||
for cid, aid, variant in cur.fetchall():
|
||||
keys.add(s3.variant_key(cid, aid, variant))
|
||||
return keys
|
||||
|
||||
|
||||
@@ -59,7 +68,9 @@ def _run_once() -> dict:
|
||||
orphan_deleted = 0
|
||||
missing = 0
|
||||
scanned = 0
|
||||
staging_deleted = 0
|
||||
|
||||
# attachments/ — orphan по known
|
||||
for obj in s3.list_objects("attachments/"):
|
||||
scanned += 1
|
||||
key = obj["Key"]
|
||||
@@ -79,19 +90,44 @@ def _run_once() -> dict:
|
||||
orphan_deleted += 1
|
||||
log.info("event=orphan_deleted key=%s", key)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=orphan_delete_failed key=%s type=%s", key, type(exc).__name__)
|
||||
log.error("event=orphan_delete_failed key=%s type=%s",
|
||||
key, type(exc).__name__)
|
||||
|
||||
# Проверяем обратную сторону: метаданные без объекта.
|
||||
# staging/ — удаляем только старое (не in-flight).
|
||||
for obj in s3.list_objects("staging/"):
|
||||
lm = obj.get("LastModified")
|
||||
if lm is None:
|
||||
continue
|
||||
if lm.tzinfo is None:
|
||||
lm = lm.replace(tzinfo=timezone.utc)
|
||||
if lm > grace_cutoff:
|
||||
continue
|
||||
try:
|
||||
s3.delete_object(obj["Key"])
|
||||
staging_deleted += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# missing
|
||||
for key in known:
|
||||
if s3.head_object(key) is None:
|
||||
missing += 1
|
||||
log.warning("event=missing_object key=%s", key)
|
||||
|
||||
# зависшие processing → queued
|
||||
try:
|
||||
recovered = storage.recover_stuck_processing()
|
||||
if recovered:
|
||||
log.info("event=stuck_processing_recovered count=%s", recovered)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=recover_stuck_failed type=%s", type(exc).__name__)
|
||||
|
||||
return {
|
||||
"scanned": scanned,
|
||||
"orphan_deleted": orphan_deleted,
|
||||
"missing": missing,
|
||||
"known": len(known),
|
||||
"staging_deleted": staging_deleted,
|
||||
}
|
||||
|
||||
|
||||
@@ -104,9 +140,10 @@ def main() -> None:
|
||||
try:
|
||||
stats = _run_once()
|
||||
log.info(
|
||||
"event=cleanup_pass scanned=%s known=%s orphan_deleted=%s missing=%s",
|
||||
stats["scanned"], stats["known"],
|
||||
stats["orphan_deleted"], stats["missing"],
|
||||
"event=cleanup_pass scanned=%s known=%s orphan_deleted=%s"
|
||||
" missing=%s staging_deleted=%s",
|
||||
stats["scanned"], stats["known"], stats["orphan_deleted"],
|
||||
stats["missing"], stats["staging_deleted"],
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=cleanup_failed type=%s", type(exc).__name__)
|
||||
|
||||
Reference in New Issue
Block a user