Files
mtusi2026lab6/app/outbox.py
T
Artu-RitoandClaude Sonnet 5 6c50a63743 Лаба 6: OpenTelemetry + Grafana наблюдаемость (оценка 4)
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в
otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов,
дашборда и alert rules в Git. W3C trace context переносится через поле
traceparent в задании очереди (API producer span -> worker consumer span
в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых
инцидента прогнаны и задокументированы (worker down -> queue lag + alert,
DB down отличим от медленного handler по error-span, telemetry backend down
не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага
(middleware после старта приложения, detached span context, замерзающая
метрика очереди при падении её единственного источника, неатомарный
XACK+XDEL) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:59:33 +03:00

97 lines
3.6 KiB
Python

"""Transactional outbox: bridges the PostgreSQL COMMIT and the broker publish.
enqueue() writes the job row in the SAME transaction as the attachments
INSERT, so either both happen or neither does. The caller then makes a
best-effort immediate publish; if the process crashes between COMMIT and
that publish, the row is still there with published_at IS NULL, and the
background relay (any API replica) picks it up and publishes it later --
no job is silently lost, and none is invented without a committed row.
"""
import asyncio
import datetime as dt
import logging
import uuid
import asyncpg
from . import broker, telemetry
logger = logging.getLogger("app.outbox")
RELAY_INTERVAL_SECONDS = 2
RELAY_GRACE_SECONDS = 1 # don't race the enqueuing request's own immediate publish
async def enqueue(conn: asyncpg.Connection, attachment_id: uuid.UUID) -> dict:
job_id = uuid.uuid4()
await conn.execute(
"INSERT INTO job_outbox (id, attachment_id, job_id, pipeline_version, attempt) "
"VALUES (gen_random_uuid(), $1, $2, 1, 1)",
attachment_id, job_id,
)
job = {
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"attempt": 1,
"created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
}
# W3C trace context: lets the worker's consumer span link back to this
# request's producer span (contracts/observability.md: sequenced upload->
# publish->consume->S3/DB chain).
telemetry.inject_context(job)
return job
async def publish_best_effort(pool: asyncpg.Pool, job: dict) -> None:
"""Try to publish right away; on failure, leave it for the relay. Never
raises -- a broker hiccup here must not fail the already-committed upload."""
try:
await broker.publish(job)
async with pool.acquire() as conn:
await conn.execute(
"UPDATE job_outbox SET published_at=now() WHERE job_id=$1", uuid.UUID(job["job_id"])
)
except Exception: # noqa: BLE001
logger.warning("immediate publish failed for job %s; relay will retry", job["job_id"])
async def relay_once(pool: asyncpg.Pool) -> int:
async with pool.acquire() as conn:
rows = await conn.fetch(
"""
SELECT id, attachment_id, job_id, pipeline_version, attempt, created_at
FROM job_outbox
WHERE published_at IS NULL AND created_at < now() - ($1 * interval '1 second')
ORDER BY created_at
LIMIT 50
""",
RELAY_GRACE_SECONDS,
)
for row in rows:
job = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"created_at": row["created_at"].astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
}
await broker.publish(job)
await conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=$1", row["id"])
return len(rows)
async def relay_loop(pool: asyncpg.Pool) -> None:
while True:
try:
await relay_once(pool)
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - the relay must survive transient DB/broker hiccups
logger.exception("outbox relay tick failed")
await asyncio.sleep(RELAY_INTERVAL_SECONDS)