forked from mtusi2026/mtusi2026lab6
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов, дашборда и alert rules в Git. W3C trace context переносится через поле traceparent в задании очереди (API producer span -> worker consumer span в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых инцидента прогнаны и задокументированы (worker down -> queue lag + alert, DB down отличим от медленного handler по error-span, telemetry backend down не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага (middleware после старта приложения, detached span context, замерзающая метрика очереди при падении её единственного источника, неатомарный XACK+XDEL) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
67 lines
2.2 KiB
Python
Executable File
67 lines
2.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Explicit operator replay for a failed attachment: resets status to
|
|
'queued' and republishes a fresh job (new job_id, attempt=1). Never replays
|
|
an attachment that is already 'ready' -- the same protection the worker
|
|
itself applies against stale duplicates.
|
|
|
|
Usage: python3 scripts/replay_failed.py <attachment-id>
|
|
"""
|
|
import asyncio
|
|
import datetime as dt
|
|
import os
|
|
import sys
|
|
import uuid
|
|
|
|
import asyncpg
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from app import broker # noqa: E402
|
|
|
|
|
|
def read_secret(file_env, inline_env):
|
|
path = os.environ.get(file_env)
|
|
if path and os.path.isfile(path):
|
|
return open(path, encoding="utf-8").read().strip()
|
|
return os.environ[inline_env]
|
|
|
|
|
|
async def main():
|
|
if len(sys.argv) != 2:
|
|
print("usage: replay_failed.py <attachment-id>", file=sys.stderr)
|
|
return 2
|
|
attachment_id = uuid.UUID(sys.argv[1])
|
|
|
|
conn = await asyncpg.connect(read_secret("DATABASE_URL_FILE", "DATABASE_URL"))
|
|
record = await conn.fetchrow("SELECT status FROM attachments WHERE id=$1", attachment_id)
|
|
if record is None:
|
|
print("no such attachment", file=sys.stderr)
|
|
return 1
|
|
if record["status"] == "ready":
|
|
print("attachment is already ready; refusing to replay a finished job")
|
|
return 0
|
|
if record["status"] != "failed":
|
|
print(f"attachment status is '{record['status']}', not 'failed'; nothing to replay")
|
|
return 0
|
|
|
|
job_id = uuid.uuid4()
|
|
await conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=$1", attachment_id)
|
|
await conn.close()
|
|
|
|
broker.init(read_secret("CACHE_URL_FILE", "CACHE_URL"))
|
|
job = {
|
|
"schema_version": 1,
|
|
"job_id": str(job_id),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(attachment_id),
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
"created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
}
|
|
await broker.publish(job)
|
|
print(f"replayed attachment {attachment_id} as job {job_id}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main()))
|