forked from mtusi2026/mtusi2026lab5
Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь 202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized. Transactional outbox закрывает разрыв COMMIT/publish; durable retry (job_retry_schedule) переживает падение worker во время backoff; lease recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator replay (scripts/replay_failed.py). make test/test-persistence/test-failover/ validate зелёные; в процессе экспериментов найдены и исправлены два реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов asyncpg в построении interval) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
92 lines
3.4 KiB
Python
92 lines
3.4 KiB
Python
"""Transactional outbox: bridges the PostgreSQL COMMIT and the broker publish.
|
|
|
|
enqueue() writes the job row in the SAME transaction as the attachments
|
|
INSERT, so either both happen or neither does. The caller then makes a
|
|
best-effort immediate publish; if the process crashes between COMMIT and
|
|
that publish, the row is still there with published_at IS NULL, and the
|
|
background relay (any API replica) picks it up and publishes it later --
|
|
no job is silently lost, and none is invented without a committed row.
|
|
"""
|
|
import asyncio
|
|
import datetime as dt
|
|
import logging
|
|
import uuid
|
|
|
|
import asyncpg
|
|
|
|
from . import broker
|
|
|
|
logger = logging.getLogger("app.outbox")
|
|
|
|
RELAY_INTERVAL_SECONDS = 2
|
|
RELAY_GRACE_SECONDS = 1 # don't race the enqueuing request's own immediate publish
|
|
|
|
|
|
async def enqueue(conn: asyncpg.Connection, attachment_id: uuid.UUID) -> dict:
|
|
job_id = uuid.uuid4()
|
|
await conn.execute(
|
|
"INSERT INTO job_outbox (id, attachment_id, job_id, pipeline_version, attempt) "
|
|
"VALUES (gen_random_uuid(), $1, $2, 1, 1)",
|
|
attachment_id, job_id,
|
|
)
|
|
return {
|
|
"schema_version": 1,
|
|
"job_id": str(job_id),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(attachment_id),
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
"created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
}
|
|
|
|
|
|
async def publish_best_effort(pool: asyncpg.Pool, job: dict) -> None:
|
|
"""Try to publish right away; on failure, leave it for the relay. Never
|
|
raises -- a broker hiccup here must not fail the already-committed upload."""
|
|
try:
|
|
await broker.publish(job)
|
|
async with pool.acquire() as conn:
|
|
await conn.execute(
|
|
"UPDATE job_outbox SET published_at=now() WHERE job_id=$1", uuid.UUID(job["job_id"])
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
logger.warning("immediate publish failed for job %s; relay will retry", job["job_id"])
|
|
|
|
|
|
async def relay_once(pool: asyncpg.Pool) -> int:
|
|
async with pool.acquire() as conn:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT id, attachment_id, job_id, pipeline_version, attempt, created_at
|
|
FROM job_outbox
|
|
WHERE published_at IS NULL AND created_at < now() - ($1 * interval '1 second')
|
|
ORDER BY created_at
|
|
LIMIT 50
|
|
""",
|
|
RELAY_GRACE_SECONDS,
|
|
)
|
|
for row in rows:
|
|
job = {
|
|
"schema_version": 1,
|
|
"job_id": str(row["job_id"]),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(row["attachment_id"]),
|
|
"pipeline_version": row["pipeline_version"],
|
|
"attempt": row["attempt"],
|
|
"created_at": row["created_at"].astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
}
|
|
await broker.publish(job)
|
|
await conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=$1", row["id"])
|
|
return len(rows)
|
|
|
|
|
|
async def relay_loop(pool: asyncpg.Pool) -> None:
|
|
while True:
|
|
try:
|
|
await relay_once(pool)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception: # noqa: BLE001 - the relay must survive transient DB/broker hiccups
|
|
logger.exception("outbox relay tick failed")
|
|
await asyncio.sleep(RELAY_INTERVAL_SECONDS)
|