Preserve distributed job context and expose grade 4 operational signals
This commit is contained in:
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
|
||||
RATE ?= 100
|
||||
FROM_REV ?=
|
||||
SHIFT ?= 0
|
||||
SCRIPTS_PY = scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
SCRIPTS_PY = scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
|
||||
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
SCENARIO ?= failover-upload
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision = "0007_trace_context"
|
||||
down_revision = "0006_attachment_quarantine"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table in ("jobs", "outbox"):
|
||||
op.add_column(
|
||||
table, sa.Column("trace_context", JSONB(), nullable=False, server_default="{}")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("Downgrade would delete trace correlation data")
|
||||
@@ -12,7 +12,7 @@
|
||||
"graphTooltip": 1,
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
@@ -302,6 +302,268 @@
|
||||
"w": 12,
|
||||
"h": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "timeseries",
|
||||
"title": "Durable queue depth (includes retry)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 20,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "timeseries",
|
||||
"title": "Oldest durable job age",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 20,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_age_seconds)",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool used / waiting / capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 28,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (__name__,job) ({__name__=~\"messenger_db_pool_(used|waiting|capacity)\"})",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"type": "timeseries",
|
||||
"title": "Worker busy / capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 28,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (__name__) ({__name__=~\"messenger_worker_(busy|capacity)\"})",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU cores",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 36,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (job) (rate(process_cpu_time_seconds_total{job=~\"messenger-.*\"}[1m]))",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "timeseries",
|
||||
"title": "Process resident memory",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 36,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (job) (process_memory_usage_bytes{job=~\"messenger-.*\"})",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "timeseries",
|
||||
"title": "Time to ready p50 / p95 / p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 44,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
|
||||
"legendFormat": "0.5"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
|
||||
"legendFormat": "0.95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
|
||||
"legendFormat": "0.99"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"type": "timeseries",
|
||||
"title": "Ready within 30s, SLO 95% / 1h",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 44,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger:ready_under_30s:ratio1h",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 24,
|
||||
"type": "timeseries",
|
||||
"title": "API availability, SLO 99% / 1h",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 52,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger:availability:ratio1h",
|
||||
"legendFormat": "{{job}} {{__name__}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- record: messenger:http_errors:ratio_rate1m
|
||||
expr: |
|
||||
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*",http_response_status_code=~"5.."}[1m])) or vector(0))
|
||||
/ clamp_min(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*"}[1m])), 0.001)
|
||||
- record: messenger:availability:ratio1h
|
||||
expr: 1 - messenger:http_errors:ratio_rate1h
|
||||
- record: messenger:ready_under_30s:ratio1h
|
||||
expr: sum(increase(messenger_job_time_to_ready_seconds_bucket{le="30"}[1h])) / clamp_min(sum(increase(messenger_job_time_to_ready_seconds_count[1h])), 1)
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: messenger:http_errors:ratio_rate1m > 0.05
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: More than 5% business HTTP 5xx over 1 minute
|
||||
description: Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.
|
||||
runbook: infra/prometheus/runbooks.md#messengerhigherrorrate
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) > 15 and max(messenger_queue_depth{queue_name="attachments.process"}) > 0
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Durable attachment backlog has waited more than 15 seconds
|
||||
description: Target is 95% ready within 30 seconds over 1 hour; depth includes retry and leased work.
|
||||
runbook: infra/prometheus/runbooks.md#messengerqueuelag
|
||||
@@ -0,0 +1,15 @@
|
||||
# Локальная диагностика
|
||||
|
||||
SLI availability = 1 − business HTTP 5xx / все business HTTP за 1 h, цель 99%; health исключены, 4xx остаются в знаменателе. SLI ready = доля завершённых jobs с time-to-ready <=30 s за 1 h, цель 95%. Незавершённые jobs не входят в histogram, поэтому возраст/depth контролируются отдельно. Учебные alerts реагируют раньше окна SLO: это пороговые предупреждения, не multi-window burn rates.
|
||||
|
||||
## MessengerHighErrorRate
|
||||
|
||||
Условие: 5xx/все >5% за 1 m непрерывно 15 s; вычисление каждые 5 s. В API dashboard проверить error ratio и p95. По request_id найти лог и trace. Длинный/ошибочный PostgreSQL client span указывает на зависимость; длинный server span без медленных client spans — на handler. Проверить used/waiting/capacity пула и ресурсы процесса. В своём стенде восстановить postgres, дождаться ready обеих API и успешных запросов. Alert должен исчезнуть после ухода ошибок из минутного окна.
|
||||
|
||||
## MessengerQueueLag
|
||||
|
||||
Условие: durable depth>0 и oldest age>15 s непрерывно 15 s. Проверить jobs outcomes, worker busy/capacity, CPU/RAM и наличие worker logs. Отсутствие worker signal при растущем backlog указывает на остановку процесса; retry с S3 ERROR spans — на хранилище. Проверить producer/consumer цепочку конкретного attachment. Восстановить только свой worker/зависимость; дождаться ready, depth=0, age=0 и отсутствия firing alert. Queue depth относится к PostgreSQL queued/processing, включая scheduled retry, а не только AMQP ready messages.
|
||||
|
||||
## Отказ telemetry backend
|
||||
|
||||
Readiness API не зависит от Collector/хранилищ. Сначала проверить бизнес-smoke. Collector: queue_size/capacity, send_failed и sent counters; Docker RAM против mem_limit. Retry ограничен 60 s, очереди ограничены; потеря диагностических данных при заполнении допустима, потеря сообщений — нет. Восстановить backend и проверить новый request trace/log и метрики. Не удалять volumes для восстановления.
|
||||
@@ -1,2 +1,4 @@
|
||||
schema: spec-driven
|
||||
created: 2026-10-29
|
||||
archived: 2026-11-06
|
||||
lab: 6
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
schema: spec-driven
|
||||
created: 2026-11-06
|
||||
@@ -0,0 +1,23 @@
|
||||
## Context
|
||||
|
||||
SQLAlchemy async, transactional outbox и manual ack после COMMIT уже работают. SDK экспортирует асинхронно, Collector имеет memory_limiter, batch, ограниченные очереди и retry 60 s.
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
Закрыть все пункты уровня 4; не реализовывать трек 5 в этом change и не менять контракт API.
|
||||
|
||||
## Decisions
|
||||
|
||||
W3C traceparent/tracestate хранятся отдельно от нормативного payload в JSONB job и outbox. При retry берётся активный consumer context, при восстановлении аренды — сохранённый origin. Миграция добавляет поля с пустым default; прежние jobs остаются пригодными. Relay извлекает контекст и FastStream создаёт producer spans; ack остаётся после бизнес-COMMIT. S3 spans содержат только операцию/сервис, без ключей и exception payload.
|
||||
|
||||
Queue depth/age измеряют durable backlog queued/processing PostgreSQL, включая retry: это не размер AMQP очереди. Pool used/waiting/capacity измеряются у SQLAlchemy engine и в ObservedPool._do_get, без идентификаторов. Worker busy/capacity отражают конкурентность; time-to-ready отсчитывается от created_at job до done. CPU — процессорные секунды, RAM — текущий RSS из /proc. Все gauges отдаются OTLP; статические labels и resource service instance. Histogram seconds использует границы 0.05…60, включая SLO 30 s.
|
||||
|
||||
Availability SLO: >=99% успешных бизнес-HTTP за 1 h; ready SLO: >=95% done за 30 s на окне 1 h. Alerts: error ratio >5% за 1 m, for 15 s; backlog age >15 s, for 15 s. Thresholds короткие для воспроизводимой учебной демонстрации, не производственные рекомендации. Runbooks лежат в Git; внешняя доставка не требуется.
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
ObservedPool оборачивает получение connection, сохраняя ленивые sessions, timeout, autocommit и границы UoW. Waiting включает получение/создание соединения; используется закреплённая версия SQLAlchemy. Poller очереди работает отдельно, ошибки измерений не останавливают relay. Нулевой backlog возвращает age 0. При telemetry outage потеря сигналов допустима и измеряется; успешность сообщений обязательна. Проверки последовательны на собственном project, состояние Docker записывается; постоянные volumes не удаляются.
|
||||
|
||||
## Validation
|
||||
|
||||
Миграции legacy/fresh, тесты durable context и retry, make check/validate/lint/student-test. Реальные API→S3/DB→producer→consumer traces, retry, строгие dashboards, leak checks. Два инцидента с временной шкалой, queries и trace IDs, firing/resolved. Отказ хранилищ: smoke, RAM samples, Collector counters до/после, восстановление. Финальный REPORT делает следующий агент.
|
||||
@@ -0,0 +1,18 @@
|
||||
## Why
|
||||
|
||||
Уровень 3 доказан. Сейчас outbox теряет контекст HTTP, relay не создаёт producer spans, а наблюдаемость не объясняет накопление очереди и зависание БД. Это обязательные критерии уровня 4.
|
||||
|
||||
## What Changes
|
||||
|
||||
- Добавочная Alembic-миграция сохраняет W3C контекст job/outbox, FastStream связывает API, producer и consumer; S3 получает безопасные client spans.
|
||||
- Добавляются queue age/depth, pool used/waiting, worker busy/capacity, time-to-ready и process CPU/RAM, dashboards и SLI/SLO с alerts.
|
||||
- Реальные инциденты worker stop и PostgreSQL outage подтверждают firing/resolved; отдельный отказ хранилищ проверяет доступность API и ограничение памяти.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- `lab6-observability-resilience`: сквозная диагностика и устойчивость наблюдаемости уровня 4.
|
||||
|
||||
## Impact
|
||||
|
||||
Меняются собственные адаптеры, миграции, инструменты и provisioning. API, нормативные файлы и REPORT не меняются. Следующему агенту передаются evidence и заметки. Данные существующих таблиц сохраняются.
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Durable distributed context
|
||||
The system SHALL preserve W3C context through API, transactional outbox, relay producer and worker consumer including retries, and expose safe database and S3 client spans.
|
||||
|
||||
#### Scenario: Deferred delivery and retry
|
||||
- **WHEN** an accepted upload is delivered after relay restart or retried after S3 failure
|
||||
- **THEN** its producer and consumer spans share the originating trace and causally valid parents, without adding fields to the normative job payload
|
||||
|
||||
### Requirement: Operational signals and SLO
|
||||
The system SHALL expose queue depth and oldest age, database pool usage and waiting, worker saturation, process CPU/RAM and durable time-to-ready; dashboards SHALL show histogram percentiles and error ratio and define availability and ready SLO windows.
|
||||
|
||||
#### Scenario: Stopped worker
|
||||
- **WHEN** queued uploads wait for a stopped worker
|
||||
- **THEN** depth and age increase, a queue-lag alert fires and resolves after recovery
|
||||
|
||||
#### Scenario: Unavailable database
|
||||
- **WHEN** PostgreSQL becomes unavailable
|
||||
- **THEN** API failure metrics and database client spans distinguish the dependency failure and an error-rate alert fires and resolves
|
||||
|
||||
### Requirement: Bounded observability failure
|
||||
Collector SHALL use memory limits, batching and bounded queues; unavailable telemetry storage SHALL NOT prevent business requests or cause unbounded memory growth.
|
||||
|
||||
#### Scenario: Storage outage
|
||||
- **WHEN** telemetry storage is stopped during requests
|
||||
- **THEN** business smoke passes, measured memory remains within container limits, losses are recorded, and exports resume after recovery
|
||||
@@ -0,0 +1,10 @@
|
||||
## Implementation
|
||||
- [ ] 1. Durable context migration, repository and publisher wiring; propagation/retry tests
|
||||
- [ ] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks
|
||||
- [ ] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation
|
||||
## Verification
|
||||
- [ ] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes
|
||||
- [ ] 5. End-to-end and retry traces, dashboards, leaks and cardinality
|
||||
- [ ] 6. Worker and database incidents: evidence, firing/resolved and diagnosis
|
||||
- [ ] 7. Storage outage: availability, bounded memory, lost signals and recovery
|
||||
- [ ] 8. Evidence notes, requirement sync and archive after all checks
|
||||
@@ -131,6 +131,7 @@ def tempo_spans(body):
|
||||
duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6
|
||||
spans.append({"service": service, "name": span["name"], "kind": span.get("kind"),
|
||||
"status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"),
|
||||
"span_id": span.get("spanId"), "parent_span_id": span.get("parentSpanId"),
|
||||
"duration_ms": round(duration, 3), "links": len(span.get("links", [])),
|
||||
"attributes": values})
|
||||
return spans
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for
|
||||
from obs_recovery import backends, compose
|
||||
|
||||
|
||||
def actor():
|
||||
api = Api(os.environ["BASE_URL"])
|
||||
user, username, password, _ = api.register()
|
||||
auth, _ = api.login(username, password)
|
||||
return api, auth, api.chat(auth, user)
|
||||
|
||||
|
||||
def upload(api, auth, chat):
|
||||
return api.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png",
|
||||
Path("tests/fixtures/sample.png").read_bytes(),
|
||||
dict(auth, **{"Content-Type": "image/png"}),
|
||||
expect=(202,),
|
||||
)
|
||||
|
||||
|
||||
def ready(api, auth, attachment):
|
||||
def check():
|
||||
item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0]
|
||||
if item["status"] == "failed":
|
||||
raise RuntimeError("attachment failed")
|
||||
return item if item["status"] == "ready" else None
|
||||
|
||||
return wait_for("ready", check, 120, 1)
|
||||
|
||||
|
||||
def value(b, expression):
|
||||
result = b.promql(expression)
|
||||
return float(result[0]["value"][1]) if result else None
|
||||
|
||||
|
||||
def alert(b, name, firing):
|
||||
def check():
|
||||
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
|
||||
found = [
|
||||
a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing"
|
||||
]
|
||||
if bool(found) == firing:
|
||||
return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found}
|
||||
return None
|
||||
|
||||
return wait_for(name, check, 150)
|
||||
|
||||
|
||||
def chain(b, retry):
|
||||
api, auth, chat = actor()
|
||||
started = time.time()
|
||||
if retry:
|
||||
compose("stop", "worker")
|
||||
attachment, rid = upload(api, auth, chat)
|
||||
try:
|
||||
if retry:
|
||||
compose("stop", "s3")
|
||||
compose("start", "worker")
|
||||
query = (
|
||||
'{service_name="messenger-worker"} | attachment_id="'
|
||||
+ attachment["id"]
|
||||
+ '" | outcome="retry_scheduled"'
|
||||
)
|
||||
wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60)
|
||||
finally:
|
||||
if retry:
|
||||
compose("up", "-d", "--no-deps", "--wait", "s3", "worker")
|
||||
ready(api, auth, attachment)
|
||||
logs = wait_for(
|
||||
"upload log",
|
||||
lambda: b.logs(
|
||||
'{service_name="messenger-api"} | request_id="' + rid + '"',
|
||||
started - 1,
|
||||
time.time() + 1,
|
||||
),
|
||||
60,
|
||||
)
|
||||
trace_id = logs[0][0]["trace_id"]
|
||||
|
||||
def complete():
|
||||
spans = tempo_spans(b.trace(trace_id)[0])
|
||||
return (
|
||||
spans
|
||||
if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans)
|
||||
else None
|
||||
)
|
||||
|
||||
spans = wait_for("complete trace", complete, 90)
|
||||
roots = [
|
||||
s
|
||||
for s in spans
|
||||
if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid
|
||||
]
|
||||
ids = {s["span_id"] for s in spans}
|
||||
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
|
||||
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
|
||||
clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"]
|
||||
if (
|
||||
not roots
|
||||
or not producers
|
||||
or not consumers
|
||||
or any(s["parent_span_id"] not in ids for s in producers + consumers)
|
||||
):
|
||||
raise Failure("broken distributed parent chain")
|
||||
if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any(
|
||||
s["attributes"].get("db.system") == "postgresql" for s in clients
|
||||
):
|
||||
raise Failure("missing dependency spans")
|
||||
outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers]
|
||||
if retry and "retry_scheduled" not in outcomes:
|
||||
raise Failure("retry left original trace")
|
||||
api.logout(auth)
|
||||
fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms")
|
||||
return {
|
||||
"upload_request_id": rid,
|
||||
"trace_id": trace_id,
|
||||
"retry": retry,
|
||||
"outcomes": outcomes,
|
||||
"spans": [
|
||||
{**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")}
|
||||
for s in spans
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def worker(b):
|
||||
api, auth, chat = actor()
|
||||
compose("stop", "worker")
|
||||
stopped = now()
|
||||
try:
|
||||
items = [upload(api, auth, chat)[0] for _ in range(8)]
|
||||
firing = alert(b, "MessengerQueueLag", True)
|
||||
observed = {
|
||||
"depth": value(b, "max(messenger_queue_depth)"),
|
||||
"age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"),
|
||||
}
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "worker")
|
||||
restored = now()
|
||||
for item in items:
|
||||
ready(api, auth, item)
|
||||
resolved = alert(b, "MessengerQueueLag", False)
|
||||
api.logout(auth)
|
||||
return {
|
||||
"stopped": stopped,
|
||||
"restored": restored,
|
||||
"jobs": len(items),
|
||||
"backlog": observed,
|
||||
"firing": firing,
|
||||
"resolved": resolved,
|
||||
"depth_after": value(b, "max(messenger_queue_depth)"),
|
||||
}
|
||||
|
||||
|
||||
def database(b):
|
||||
api, auth, _ = actor()
|
||||
started = time.time()
|
||||
prefix = "db-incident-" + uuid.uuid4().hex[:12]
|
||||
|
||||
def request(index):
|
||||
rid = prefix + "-" + str(index)
|
||||
status = http(
|
||||
"GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid})
|
||||
)[0]
|
||||
return {"request_id": rid, "status": status}
|
||||
|
||||
for index in range(10):
|
||||
request(index)
|
||||
time.sleep(12)
|
||||
compose("pause", "postgres")
|
||||
paused = now()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
responses = list(pool.map(request, range(10, 34)))
|
||||
firing = alert(b, "MessengerHighErrorRate", True)
|
||||
signals = {
|
||||
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
|
||||
"pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"),
|
||||
}
|
||||
finally:
|
||||
compose("unpause", "postgres")
|
||||
restored = now()
|
||||
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
|
||||
for index in range(34, 74):
|
||||
request(index)
|
||||
time.sleep(0.15)
|
||||
resolved = alert(b, "MessengerHighErrorRate", False)
|
||||
traces = b.traceql(
|
||||
'{ span.messenger.request_id =~ "' + prefix + '.*" }',
|
||||
started - 1,
|
||||
time.time() + 1,
|
||||
limit=100,
|
||||
)
|
||||
dependency = []
|
||||
for trace in traces:
|
||||
spans = tempo_spans(b.trace(trace["traceID"])[0])
|
||||
failed = [
|
||||
s
|
||||
for s in spans
|
||||
if s["attributes"].get("db.system") == "postgresql"
|
||||
and s["status"] == "STATUS_CODE_ERROR"
|
||||
]
|
||||
if failed:
|
||||
dependency.append(
|
||||
{
|
||||
"trace_id": trace["traceID"],
|
||||
"db_errors": [
|
||||
{k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed
|
||||
],
|
||||
}
|
||||
)
|
||||
if not dependency or not any(r["status"] == 503 for r in responses):
|
||||
raise Failure("database incident has no dependency failure trace")
|
||||
api.logout(auth)
|
||||
return {
|
||||
"paused": paused,
|
||||
"restored": restored,
|
||||
"requests": responses,
|
||||
"signals": signals,
|
||||
"firing": firing,
|
||||
"resolved": resolved,
|
||||
"dependency_traces": dependency,
|
||||
}
|
||||
|
||||
|
||||
def collector_metrics():
|
||||
source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())'
|
||||
raw = compose("exec", "-T", "api", "python", "-c", source)
|
||||
return {
|
||||
line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1])
|
||||
for line in raw.splitlines()
|
||||
if line.startswith(
|
||||
(
|
||||
"otelcol_receiver_accepted_",
|
||||
"otelcol_exporter_sent_",
|
||||
"otelcol_exporter_send_failed_",
|
||||
"otelcol_exporter_queue_",
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def memory():
|
||||
ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines()
|
||||
raw = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
|
||||
)
|
||||
return [
|
||||
{k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")}
|
||||
for row in map(json.loads, raw.splitlines())
|
||||
]
|
||||
|
||||
|
||||
def storage(b):
|
||||
api, auth, _ = actor()
|
||||
before = collector_metrics()
|
||||
samples = [{"utc": now(), "containers": memory()}]
|
||||
compose("stop", "prometheus", "loki", "tempo")
|
||||
stopped = now()
|
||||
requests = []
|
||||
try:
|
||||
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True)
|
||||
until = time.monotonic() + 85
|
||||
while time.monotonic() < until:
|
||||
rid = "storage-outage-" + uuid.uuid4().hex
|
||||
status = http(
|
||||
"GET",
|
||||
api.base + "/api/v1/users/me",
|
||||
headers=dict(auth, **{"X-Request-Id": rid}),
|
||||
expect=(200,),
|
||||
)[0]
|
||||
requests.append({"id": rid, "status": status})
|
||||
if len(requests) % 15 == 0:
|
||||
samples.append({"utc": now(), "containers": memory()})
|
||||
time.sleep(1)
|
||||
during = collector_metrics()
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo")
|
||||
restored = now()
|
||||
time.sleep(20)
|
||||
after = collector_metrics()
|
||||
samples.append({"utc": now(), "containers": memory()})
|
||||
found = 0
|
||||
for item in requests:
|
||||
found += bool(
|
||||
b.traceql(
|
||||
'{ span.messenger.request_id = "' + item["id"] + '" }',
|
||||
time.time() - 300,
|
||||
time.time() + 1,
|
||||
)
|
||||
)
|
||||
peak_percent = max(
|
||||
float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"]
|
||||
)
|
||||
if peak_percent >= 100:
|
||||
raise Failure("container memory limit reached")
|
||||
api.logout(auth)
|
||||
return {
|
||||
"stopped": stopped,
|
||||
"restored": restored,
|
||||
"peak_memory_percent_of_limit": peak_percent,
|
||||
"smoke": result.stdout,
|
||||
"successful_probes": len(requests),
|
||||
"stored_probe_traces": found,
|
||||
"lost_probe_traces": len(requests) - found,
|
||||
"memory_samples": samples,
|
||||
"collector_before": before,
|
||||
"collector_during": during,
|
||||
"collector_after": after,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage"))
|
||||
args = parser.parse_args()
|
||||
b = backends()
|
||||
started = now()
|
||||
report = (
|
||||
chain(b, args.scenario == "retry")
|
||||
if args.scenario in ("chain", "retry")
|
||||
else {"worker": worker, "database": database, "storage": storage}[args.scenario](b)
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"started": started,
|
||||
"finished": now(),
|
||||
"result": "PASS",
|
||||
"scenario": args.scenario,
|
||||
"observations": report,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -223,7 +223,7 @@ class PublishOutboxHandler:
|
||||
for entry in await self._uow.outbox.pending(command.limit):
|
||||
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
|
||||
try:
|
||||
await self._publisher.publish(entry.job_id, entry.body)
|
||||
await self._publisher.publish(entry.job_id, entry.body, entry.trace_context)
|
||||
except Unavailable as error:
|
||||
failure = error
|
||||
break
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Protocol, runtime_checkable
|
||||
from uuid import UUID
|
||||
@@ -33,6 +33,7 @@ class OutboxEntry:
|
||||
id: int
|
||||
job_id: UUID
|
||||
body: bytes
|
||||
trace_context: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -77,7 +78,9 @@ class VariantRenderer(Protocol):
|
||||
|
||||
@runtime_checkable
|
||||
class JobPublisher(Protocol):
|
||||
async def publish(self, job_id: UUID, body: bytes) -> None: ...
|
||||
async def publish(
|
||||
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
@@ -179,6 +182,8 @@ class AttachmentDirectory(Protocol):
|
||||
|
||||
@runtime_checkable
|
||||
class JobObserver(Protocol):
|
||||
def active(self, delta: int) -> None: ...
|
||||
|
||||
def observe(self, job: Job, outcome: str, seconds: float) -> None: ...
|
||||
|
||||
|
||||
|
||||
@@ -203,8 +203,8 @@ class WorkerProvider(Provider):
|
||||
return RandomJitter()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def observer(self) -> JobObserver:
|
||||
return OtelJobObserver()
|
||||
def observer(self, settings: PipelineSettings) -> JobObserver:
|
||||
return OtelJobObserver(capacity=settings.worker_concurrency)
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def command_bus(
|
||||
|
||||
@@ -35,6 +35,7 @@ async def handle_job(
|
||||
logger.warning("job.rejected", reason="message does not match jobs.schema.json")
|
||||
await message.reject(requeue=False)
|
||||
return
|
||||
observer.active(1)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
outcome = (await bus.execute(ProcessJob(job))).value
|
||||
@@ -49,6 +50,8 @@ async def handle_job(
|
||||
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
|
||||
await message.nack(requeue=True)
|
||||
return
|
||||
finally:
|
||||
observer.active(-1)
|
||||
observed(job, outcome, started, observer, logger)
|
||||
await message.ack()
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ from messenger.shared.infrastructure.postgres import (
|
||||
EventSource,
|
||||
PostgresUnitOfWork,
|
||||
)
|
||||
from messenger.shared.infrastructure.trace_context import capture_context
|
||||
|
||||
|
||||
class AttachmentRow(Base):
|
||||
@@ -153,6 +154,7 @@ class JobRow(Base):
|
||||
pipeline_version: Mapped[int]
|
||||
status: Mapped[str]
|
||||
attempt: Mapped[int]
|
||||
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
|
||||
claims: Mapped[int] = mapped_column(server_default="0")
|
||||
lease_token: Mapped[UUID | None]
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
@@ -168,6 +170,7 @@ class OutboxRow(Base):
|
||||
id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
|
||||
job_id: Mapped[UUID] = mapped_column(ForeignKey("jobs.job_id"))
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSONB)
|
||||
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
|
||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
@@ -472,10 +475,18 @@ class PostgresJobRepository:
|
||||
self._session = session
|
||||
|
||||
async def _enqueue(self, job: Job, available_at: datetime | None = None) -> None:
|
||||
context = (
|
||||
capture_context()
|
||||
or await self._session.scalar(
|
||||
select(JobRow.trace_context).where(JobRow.job_id == job.id)
|
||||
)
|
||||
or {}
|
||||
)
|
||||
await self._session.execute(
|
||||
insert(OutboxRow).values(
|
||||
job_id=job.id,
|
||||
payload=job.payload(),
|
||||
trace_context=context,
|
||||
available_at=func.now() if available_at is None else available_at,
|
||||
)
|
||||
)
|
||||
@@ -518,6 +529,7 @@ class PostgresJobRepository:
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
status="queued",
|
||||
trace_context=capture_context(),
|
||||
attempt=job.attempt,
|
||||
created_at=job.created_at,
|
||||
updated_at=job.created_at,
|
||||
@@ -824,7 +836,10 @@ class PostgresOutboxRepository:
|
||||
.limit(limit)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
return [OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode()) for row in rows]
|
||||
return [
|
||||
OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode(), row.trace_context)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
async def published(self, entry_ids: Sequence[int]) -> None:
|
||||
await self._session.execute(
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.metrics import Observation
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from messenger.attachments.infrastructure.postgres import JobRow
|
||||
from messenger.shared.application.ports import Logger
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
|
||||
class QueueMetrics:
|
||||
__slots__ = ("_database", "_depth", "_logger", "_oldest", "_task")
|
||||
|
||||
def __init__(self, database: Database, logger: Logger) -> None:
|
||||
self._database = database
|
||||
self._logger = logger
|
||||
self._depth = 0
|
||||
self._oldest: datetime | None = None
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
meter = metrics.get_meter("messenger.queue")
|
||||
labels = {"queue.name": "attachments.process"}
|
||||
meter.create_observable_gauge(
|
||||
"messenger.queue.depth",
|
||||
unit="{job}",
|
||||
callbacks=[lambda _: [Observation(self._depth, labels)]],
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
"messenger.queue.oldest_age",
|
||||
unit="s",
|
||||
callbacks=[lambda _: [Observation(self.age(), labels)]],
|
||||
)
|
||||
|
||||
def age(self) -> float:
|
||||
return max(0, (datetime.now(UTC) - self._oldest).total_seconds()) if self._oldest else 0
|
||||
|
||||
def start(self) -> None:
|
||||
self._task = asyncio.create_task(self._poll())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await self._task
|
||||
|
||||
async def _poll(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
async with self._database.session() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(func.count(), func.min(JobRow.created_at)).where(
|
||||
JobRow.status.in_(("queued", "processing"))
|
||||
)
|
||||
)
|
||||
).one()
|
||||
self._depth, self._oldest = row
|
||||
except Exception as error:
|
||||
self._logger.warning("queue.metrics_failed", error=type(error).__name__)
|
||||
await asyncio.sleep(5)
|
||||
@@ -5,6 +5,7 @@ from pamqp.commands import Basic
|
||||
|
||||
from messenger.attachments.domain.jobs import JOB_TYPE
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.infrastructure.trace_context import restored_context
|
||||
|
||||
EXCHANGE = RabbitExchange("attachments", declare=False)
|
||||
QUEUE = "attachments.process"
|
||||
@@ -18,19 +19,22 @@ class RabbitJobPublisher:
|
||||
self._broker = broker
|
||||
self._timeout = timeout
|
||||
|
||||
async def publish(self, job_id: UUID, body: bytes) -> None:
|
||||
async def publish(
|
||||
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
|
||||
) -> None:
|
||||
try:
|
||||
confirmation = await self._broker.publish(
|
||||
body,
|
||||
exchange=EXCHANGE,
|
||||
routing_key=JOB_TYPE,
|
||||
mandatory=True,
|
||||
persist=True,
|
||||
message_id=str(job_id),
|
||||
message_type=JOB_TYPE,
|
||||
content_type="application/json",
|
||||
timeout=self._timeout,
|
||||
)
|
||||
with restored_context(trace_context):
|
||||
confirmation = await self._broker.publish(
|
||||
body,
|
||||
exchange=EXCHANGE,
|
||||
routing_key=JOB_TYPE,
|
||||
mandatory=True,
|
||||
persist=True,
|
||||
message_id=str(job_id),
|
||||
message_type=JOB_TYPE,
|
||||
content_type="application/json",
|
||||
timeout=self._timeout,
|
||||
)
|
||||
except Exception as error:
|
||||
raise Unavailable("брокер не принял задание") from error
|
||||
if not isinstance(confirmation, Basic.Ack):
|
||||
|
||||
@@ -6,6 +6,8 @@ from aiobotocore.session import get_session
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from botocore.handlers import add_expect_header
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanKind, StatusCode
|
||||
|
||||
from messenger.attachments.application.errors import ObjectMissingError
|
||||
from messenger.attachments.application.ports import ObjectContent, StoredObject
|
||||
@@ -60,17 +62,31 @@ class S3ObjectStorage:
|
||||
self._client = client
|
||||
self._bucket = bucket
|
||||
|
||||
async def _call(self, operation: str, **arguments: Any) -> Any:
|
||||
with trace.get_tracer(__name__).start_as_current_span(
|
||||
f"S3 {operation}",
|
||||
kind=SpanKind.CLIENT,
|
||||
attributes={"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": operation},
|
||||
record_exception=False,
|
||||
set_status_on_exception=False,
|
||||
) as span:
|
||||
try:
|
||||
return await getattr(self._client, operation)(**arguments)
|
||||
except Exception as error:
|
||||
span.set_status(StatusCode.ERROR, type(error).__name__)
|
||||
raise
|
||||
|
||||
async def put(self, key: str, data: bytes, content_type: str) -> None:
|
||||
try:
|
||||
await self._client.put_object(
|
||||
Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
|
||||
await self._call(
|
||||
"put_object", Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
|
||||
)
|
||||
except FAILURES as error:
|
||||
raise Unavailable("S3 недоступно") from error
|
||||
|
||||
async def open(self, key: str) -> ObjectContent:
|
||||
try:
|
||||
response = await self._client.get_object(Bucket=self._bucket, Key=key)
|
||||
response = await self._call("get_object", Bucket=self._bucket, Key=key)
|
||||
except FAILURES as error:
|
||||
if missing(error):
|
||||
raise ObjectMissingError(key) from error
|
||||
@@ -86,15 +102,18 @@ class S3ObjectStorage:
|
||||
|
||||
async def copy(self, source: str, target: str) -> None:
|
||||
try:
|
||||
await self._client.copy_object(
|
||||
Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source}
|
||||
await self._call(
|
||||
"copy_object",
|
||||
Bucket=self._bucket,
|
||||
Key=target,
|
||||
CopySource={"Bucket": self._bucket, "Key": source},
|
||||
)
|
||||
except FAILURES as error:
|
||||
raise Unavailable("S3 недоступно") from error
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
try:
|
||||
await self._client.delete_object(Bucket=self._bucket, Key=key)
|
||||
await self._call("delete_object", Bucket=self._bucket, Key=key)
|
||||
except FAILURES as error:
|
||||
raise Unavailable("S3 недоступно") from error
|
||||
|
||||
@@ -113,7 +132,7 @@ class S3ObjectStorage:
|
||||
|
||||
async def stat(self, key: str) -> StoredObject | None:
|
||||
try:
|
||||
head = await self._client.head_object(Bucket=self._bucket, Key=key)
|
||||
head = await self._call("head_object", Bucket=self._bucket, Key=key)
|
||||
except FAILURES as error:
|
||||
if missing(error):
|
||||
return None
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.metrics import MeterProvider
|
||||
from opentelemetry.metrics import MeterProvider, Observation
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from messenger.attachments.domain.jobs import JOB_TYPE, Job
|
||||
@@ -9,10 +11,22 @@ FAILURES = frozenset({"retry_scheduled", "failed", "error"})
|
||||
|
||||
|
||||
class OtelJobObserver:
|
||||
__slots__ = ("_duration", "_processed")
|
||||
__slots__ = ("_busy", "_duration", "_processed", "_ready")
|
||||
|
||||
def __init__(self, provider: MeterProvider | None = None) -> None:
|
||||
def __init__(self, provider: MeterProvider | None = None, capacity: int = 1) -> None:
|
||||
meter = metrics.get_meter("messenger.attachments", meter_provider=provider)
|
||||
self._busy = 0
|
||||
meter.create_observable_gauge(
|
||||
"messenger.worker.busy", callbacks=[lambda _: [Observation(self._busy)]], unit="{job}"
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
"messenger.worker.capacity", callbacks=[lambda _: [Observation(capacity)]], unit="{job}"
|
||||
)
|
||||
self._ready = meter.create_histogram(
|
||||
"messenger.job.time_to_ready",
|
||||
unit="s",
|
||||
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
|
||||
)
|
||||
self._processed = meter.create_counter(
|
||||
"messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу"
|
||||
)
|
||||
@@ -23,7 +37,14 @@ class OtelJobObserver:
|
||||
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
|
||||
)
|
||||
|
||||
def active(self, delta: int) -> None:
|
||||
self._busy += delta
|
||||
|
||||
def observe(self, job: Job, outcome: str, seconds: float) -> None:
|
||||
if outcome == "done":
|
||||
self._ready.record(
|
||||
max(0, (datetime.now(UTC) - job.created_at).total_seconds()), {"job.type": JOB_TYPE}
|
||||
)
|
||||
labels = {"job.type": JOB_TYPE, "outcome": outcome}
|
||||
self._processed.add(1, labels)
|
||||
self._duration.record(seconds, labels)
|
||||
|
||||
@@ -3,16 +3,19 @@ import signal
|
||||
|
||||
from dishka import make_async_container
|
||||
from faststream.rabbit import Channel, RabbitBroker
|
||||
from faststream.rabbit.opentelemetry import RabbitTelemetryMiddleware
|
||||
|
||||
from messenger.attachments.config import PipelineSettings
|
||||
from messenger.attachments.entrypoints.heartbeat import Heartbeat
|
||||
from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider
|
||||
from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal
|
||||
from messenger.attachments.entrypoints.relay import Relay
|
||||
from messenger.attachments.infrastructure.queue_metrics import QueueMetrics
|
||||
from messenger.shared.application.messagebus import EventBus
|
||||
from messenger.shared.config import PlatformSettings
|
||||
from messenger.shared.entrypoints.ioc import PlatformProvider
|
||||
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
from messenger.shared.infrastructure.telemetry import start_telemetry
|
||||
|
||||
PING_TIMEOUT_SECONDS = 2.0
|
||||
@@ -28,6 +31,7 @@ async def serve() -> None:
|
||||
pipeline.broker_url.get_secret_value(),
|
||||
default_channel=Channel(publisher_confirms=True, on_return_raises=True),
|
||||
logger=None,
|
||||
middlewares=() if telemetry is None else (RabbitTelemetryMiddleware(),),
|
||||
)
|
||||
container = make_async_container(
|
||||
PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker)
|
||||
@@ -47,6 +51,8 @@ async def serve() -> None:
|
||||
pipeline.heartbeat_interval_seconds,
|
||||
logger,
|
||||
)
|
||||
queue_metrics = QueueMetrics(await container.get(Database), logger)
|
||||
queue_metrics.start()
|
||||
relay.start()
|
||||
heartbeat.start()
|
||||
logger.info("relay.started")
|
||||
@@ -54,6 +60,7 @@ async def serve() -> None:
|
||||
await stop.wait()
|
||||
finally:
|
||||
await heartbeat.stop()
|
||||
await queue_metrics.stop()
|
||||
await relay.stop()
|
||||
await bus.stop()
|
||||
await broker.stop()
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import ClassVar, Protocol
|
||||
from typing import ClassVar, Protocol, cast
|
||||
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.metrics import Observation
|
||||
from sqlalchemy import Text, make_url, select
|
||||
from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeout
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy.pool import AsyncAdaptedQueuePool, ConnectionPoolEntry
|
||||
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.domain.events import DomainEvent
|
||||
@@ -38,6 +41,17 @@ def unavailable(error: BaseException) -> bool:
|
||||
return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03")
|
||||
|
||||
|
||||
class ObservedPool(AsyncAdaptedQueuePool):
|
||||
waiting = 0
|
||||
|
||||
def _do_get(self) -> ConnectionPoolEntry:
|
||||
self.waiting += 1
|
||||
try:
|
||||
return super()._do_get()
|
||||
finally:
|
||||
self.waiting -= 1
|
||||
|
||||
|
||||
class Database:
|
||||
__slots__ = ("_engine", "_sessions")
|
||||
|
||||
@@ -52,6 +66,7 @@ class Database:
|
||||
) -> None:
|
||||
self._engine = create_async_engine(
|
||||
make_url(dsn).set(drivername="postgresql+asyncpg"),
|
||||
poolclass=ObservedPool,
|
||||
pool_size=max_size,
|
||||
max_overflow=0,
|
||||
pool_timeout=acquire_timeout,
|
||||
@@ -68,6 +83,22 @@ class Database:
|
||||
},
|
||||
},
|
||||
)
|
||||
meter = metrics.get_meter("messenger.database")
|
||||
meter.create_observable_gauge(
|
||||
"messenger.db.pool.used",
|
||||
callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).checkedout())]],
|
||||
unit="{connection}",
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
"messenger.db.pool.waiting",
|
||||
callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).waiting)]],
|
||||
unit="{request}",
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
"messenger.db.pool.capacity",
|
||||
callbacks=[lambda _: [Observation(max_size)]],
|
||||
unit="{connection}",
|
||||
)
|
||||
self._sessions = async_sessionmaker(self._engine, expire_on_commit=False)
|
||||
|
||||
async def open(self) -> None:
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import os
|
||||
import resource
|
||||
from pathlib import Path
|
||||
|
||||
from opentelemetry.metrics import MeterProvider, Observation
|
||||
|
||||
|
||||
def rss() -> int:
|
||||
try:
|
||||
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
|
||||
except OSError:
|
||||
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
|
||||
|
||||
def observe_process(provider: MeterProvider) -> None:
|
||||
meter = provider.get_meter("messenger.process")
|
||||
meter.create_observable_counter(
|
||||
"process.cpu.time",
|
||||
unit="s",
|
||||
callbacks=[lambda _: [Observation(sum(os.times()[:2]))]],
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
"process.memory.usage",
|
||||
unit="By",
|
||||
callbacks=[lambda _: [Observation(rss())]],
|
||||
)
|
||||
@@ -25,6 +25,7 @@ from opentelemetry.trace import Link, SpanKind, TraceState
|
||||
from opentelemetry.util.types import Attributes
|
||||
|
||||
from messenger.shared.config import PlatformSettings
|
||||
from messenger.shared.infrastructure.runtime_metrics import observe_process
|
||||
|
||||
SERVICE_VERSION = version("mtusi-messenger")
|
||||
REQUEST_ID = "messenger.request_id"
|
||||
@@ -127,6 +128,7 @@ def start_telemetry(
|
||||
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
observe_process(meter)
|
||||
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
||||
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
trace.set_tracer_provider(tracer)
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from opentelemetry.context import attach, detach
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
PROPAGATOR = TraceContextTextMapPropagator()
|
||||
|
||||
|
||||
def capture_context() -> dict[str, str]:
|
||||
carrier: dict[str, str] = {}
|
||||
PROPAGATOR.inject(carrier)
|
||||
return carrier
|
||||
|
||||
|
||||
@contextmanager
|
||||
def restored_context(carrier: dict[str, str] | None) -> Iterator[None]:
|
||||
token = attach(PROPAGATOR.extract(carrier or {}))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
detach(token)
|
||||
@@ -112,6 +112,8 @@ def test_shared_core_holds_only_platform() -> None:
|
||||
"logging.py",
|
||||
"postgres.py",
|
||||
"telemetry.py",
|
||||
"trace_context.py",
|
||||
"runtime_metrics.py",
|
||||
"readiness.py",
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import sys
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
@@ -15,6 +16,7 @@ from alembic.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
from alembic.util import CommandError
|
||||
from sqlalchemy import MetaData, Table, case, func, insert, inspect, literal, make_url, select, text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
@@ -44,6 +46,7 @@ VERSIONS = [
|
||||
"0004_jobs",
|
||||
"0005_failed_registry",
|
||||
"0006_attachment_quarantine",
|
||||
"0007_trace_context",
|
||||
]
|
||||
TABLES = [
|
||||
UserRow,
|
||||
@@ -247,13 +250,15 @@ async def contents(dsn: str) -> dict:
|
||||
table = await connection.run_sync(
|
||||
lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync)
|
||||
)
|
||||
missing = {}
|
||||
missing: dict[str, Any] = {}
|
||||
if model is AttachmentRow and "published_version" not in table.c:
|
||||
missing = {
|
||||
"original_key": case((table.c.status == "ready", table.c.object_key)),
|
||||
"staging_key": case((table.c.status == "pending", table.c.object_key)),
|
||||
"published_version": case((table.c.status == "ready", 1), else_=0),
|
||||
}
|
||||
if model in (JobRow, OutboxRow):
|
||||
missing["trace_context"] = literal({}, type_=JSONB)
|
||||
columns = [
|
||||
table.c[col.name]
|
||||
if col.name in table.c
|
||||
@@ -449,7 +454,7 @@ async def test_failed_revision_rolls_back_ddl_and_version(
|
||||
"from alembic import op\n"
|
||||
"import sqlalchemy as sa\n"
|
||||
"revision = '0003_failure'\n"
|
||||
"down_revision = '0006_attachment_quarantine'\n"
|
||||
"down_revision = '0007_trace_context'\n"
|
||||
"def upgrade():\n"
|
||||
" op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n"
|
||||
" raise RuntimeError('migration failed')\n"
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
from uuid import UUID
|
||||
|
||||
from httpx import AsyncClient
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.trace import SpanKind
|
||||
from sqlalchemy import select
|
||||
from test_pipeline import accepted, job_of
|
||||
|
||||
from conftest import sql_connection
|
||||
from messenger.attachments.infrastructure.postgres import JobRow, OutboxRow, PostgresJobRepository
|
||||
from messenger.shared.infrastructure.trace_context import capture_context, restored_context
|
||||
from pipeline import pipeline_of
|
||||
|
||||
|
||||
async def test_context_survives_outbox_delivery_and_lease_recovery(client: AsyncClient) -> None:
|
||||
provider = TracerProvider(shutdown_on_exit=False)
|
||||
tracer = provider.get_tracer("test")
|
||||
with tracer.start_as_current_span("upload", kind=SpanKind.SERVER):
|
||||
origin = capture_context()
|
||||
_, _, attachment_id = await accepted(client)
|
||||
job = await job_of(attachment_id)
|
||||
async with sql_connection() as connection:
|
||||
assert (
|
||||
await connection.scalar(select(JobRow.trace_context).where(JobRow.job_id == job.id))
|
||||
== origin
|
||||
)
|
||||
assert (
|
||||
await connection.scalar(
|
||||
select(OutboxRow.trace_context).where(OutboxRow.job_id == job.id)
|
||||
)
|
||||
== origin
|
||||
)
|
||||
pipeline = await pipeline_of(client)
|
||||
await pipeline.publish()
|
||||
assert pipeline.publisher.contexts == [origin]
|
||||
uow = pipeline.uow()
|
||||
async with uow.transaction():
|
||||
claim = await uow.jobs.claim(job, UUID(int=1), timedelta(seconds=-1))
|
||||
assert claim is not None
|
||||
await uow.commit()
|
||||
await pipeline.recover()
|
||||
await asyncio.sleep(0.02)
|
||||
await pipeline.publish()
|
||||
assert len(pipeline.publisher.contexts) == 2
|
||||
assert pipeline.publisher.contexts[-1] == origin
|
||||
with restored_context(origin), tracer.start_as_current_span("retry", kind=SpanKind.CONSUMER):
|
||||
retry = capture_context()
|
||||
async with uow.transaction():
|
||||
await PostgresJobRepository(uow.session)._enqueue(job)
|
||||
await uow.commit()
|
||||
async with sql_connection() as connection:
|
||||
assert (
|
||||
await connection.scalar(
|
||||
select(OutboxRow.trace_context)
|
||||
.where(OutboxRow.job_id == job.id)
|
||||
.order_by(OutboxRow.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
== retry
|
||||
)
|
||||
assert retry["traceparent"].split("-")[1] == origin["traceparent"].split("-")[1]
|
||||
assert retry != origin
|
||||
provider.shutdown()
|
||||
@@ -60,18 +60,22 @@ class InlineRenderer:
|
||||
|
||||
|
||||
class CapturingPublisher:
|
||||
__slots__ = ("bodies", "failing", "job_ids")
|
||||
__slots__ = ("bodies", "contexts", "failing", "job_ids")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.bodies: list[bytes] = []
|
||||
self.contexts: list[dict[str, str]] = []
|
||||
self.job_ids: list[UUID] = []
|
||||
self.failing = False
|
||||
|
||||
async def publish(self, job_id: UUID, body: bytes) -> None:
|
||||
async def publish(
|
||||
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
|
||||
) -> None:
|
||||
if self.failing:
|
||||
raise Unavailable("брокер недоступен")
|
||||
self.job_ids.append(job_id)
|
||||
self.bodies.append(body)
|
||||
self.contexts.append(trace_context or {})
|
||||
|
||||
|
||||
class InlinePipeline:
|
||||
|
||||
@@ -31,6 +31,7 @@ def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None:
|
||||
for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")):
|
||||
with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER):
|
||||
observer.observe(item, outcome, 0.3)
|
||||
observer.active(1)
|
||||
data = reader.get_metrics_data()
|
||||
assert data is not None
|
||||
found = {
|
||||
@@ -39,6 +40,18 @@ def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None:
|
||||
for scope in resource.scope_metrics
|
||||
for metric in scope.metrics
|
||||
}
|
||||
assert (
|
||||
sum(
|
||||
p.count
|
||||
for p in found["messenger.job.time_to_ready"].data.data_points
|
||||
if isinstance(p, HistogramDataPoint)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
busy = next(iter(found["messenger.worker.busy"].data.data_points))
|
||||
assert isinstance(busy, NumberDataPoint)
|
||||
assert busy.value == 1
|
||||
observer.active(-1)
|
||||
counted = {
|
||||
(point.attributes or {})["outcome"]: point.value
|
||||
for point in found["messenger.jobs.processed"].data.data_points
|
||||
|
||||
Reference in New Issue
Block a user