Files
mtusi2026lab5/app/outbox.py
T
Artu-RitoandClaude Sonnet 5 bdba93f022 Лаба 5: Redis Streams очередь + асинхронная обработка изображений (оценка 4)
Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь
202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized.
Transactional outbox закрывает разрыв COMMIT/publish; durable retry
(job_retry_schedule) переживает падение worker во время backoff; lease
recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator
replay (scripts/replay_failed.py). make test/test-persistence/test-failover/
validate зелёные; в процессе экспериментов найдены и исправлены два
реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов
asyncpg в построении interval) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:21:42 +03:00

92 lines
3.4 KiB
Python

"""Transactional outbox: bridges the PostgreSQL COMMIT and the broker publish.
enqueue() writes the job row in the SAME transaction as the attachments
INSERT, so either both happen or neither does. The caller then makes a
best-effort immediate publish; if the process crashes between COMMIT and
that publish, the row is still there with published_at IS NULL, and the
background relay (any API replica) picks it up and publishes it later --
no job is silently lost, and none is invented without a committed row.
"""
import asyncio
import datetime as dt
import logging
import uuid
import asyncpg
from . import broker
logger = logging.getLogger("app.outbox")
RELAY_INTERVAL_SECONDS = 2
RELAY_GRACE_SECONDS = 1 # don't race the enqueuing request's own immediate publish
async def enqueue(conn: asyncpg.Connection, attachment_id: uuid.UUID) -> dict:
job_id = uuid.uuid4()
await conn.execute(
"INSERT INTO job_outbox (id, attachment_id, job_id, pipeline_version, attempt) "
"VALUES (gen_random_uuid(), $1, $2, 1, 1)",
attachment_id, job_id,
)
return {
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"attempt": 1,
"created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
}
async def publish_best_effort(pool: asyncpg.Pool, job: dict) -> None:
"""Try to publish right away; on failure, leave it for the relay. Never
raises -- a broker hiccup here must not fail the already-committed upload."""
try:
await broker.publish(job)
async with pool.acquire() as conn:
await conn.execute(
"UPDATE job_outbox SET published_at=now() WHERE job_id=$1", uuid.UUID(job["job_id"])
)
except Exception: # noqa: BLE001
logger.warning("immediate publish failed for job %s; relay will retry", job["job_id"])
async def relay_once(pool: asyncpg.Pool) -> int:
async with pool.acquire() as conn:
rows = await conn.fetch(
"""
SELECT id, attachment_id, job_id, pipeline_version, attempt, created_at
FROM job_outbox
WHERE published_at IS NULL AND created_at < now() - ($1 * interval '1 second')
ORDER BY created_at
LIMIT 50
""",
RELAY_GRACE_SECONDS,
)
for row in rows:
job = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"created_at": row["created_at"].astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
}
await broker.publish(job)
await conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=$1", row["id"])
return len(rows)
async def relay_loop(pool: asyncpg.Pool) -> None:
while True:
try:
await relay_once(pool)
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - the relay must survive transient DB/broker hiccups
logger.exception("outbox relay tick failed")
await asyncio.sleep(RELAY_INTERVAL_SECONDS)