Preserve distributed job context and expose grade 4 operational signals

This commit is contained in:
Artyom
2026-11-06 12:57:10 +03:00
parent 1a171d0d0f
commit 44d81fd80e
32 changed files with 1098 additions and 35 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
RATE ?= 100
FROM_REV ?=
SHIFT ?= 0
SCRIPTS_PY = scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
SCRIPTS_PY = scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
SCENARIO ?= failover-upload
+19
View File
@@ -0,0 +1,19 @@
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.postgresql import JSONB
revision = "0007_trace_context"
down_revision = "0006_attachment_quarantine"
branch_labels = None
depends_on = None
def upgrade() -> None:
for table in ("jobs", "outbox"):
op.add_column(
table, sa.Column("trace_context", JSONB(), nullable=False, server_default="{}")
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete trace correlation data")
@@ -12,7 +12,7 @@
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 41,
"version": 1,
"version": 2,
"time": {
"from": "now-30m",
"to": "now"
@@ -302,6 +302,268 @@
"w": 12,
"h": 10
}
},
{
"id": 16,
"type": "timeseries",
"title": "Durable queue depth (includes retry)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 17,
"type": "timeseries",
"title": "Oldest durable job age",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 20,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_age_seconds)",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 18,
"type": "timeseries",
"title": "DB pool used / waiting / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 28,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (__name__,job) ({__name__=~\"messenger_db_pool_(used|waiting|capacity)\"})",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 19,
"type": "timeseries",
"title": "Worker busy / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 28,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (__name__) ({__name__=~\"messenger_worker_(busy|capacity)\"})",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 20,
"type": "timeseries",
"title": "Process CPU cores",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 36,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (job) (rate(process_cpu_time_seconds_total{job=~\"messenger-.*\"}[1m]))",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 21,
"type": "timeseries",
"title": "Process resident memory",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 36,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (job) (process_memory_usage_bytes{job=~\"messenger-.*\"})",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
},
{
"id": 22,
"type": "timeseries",
"title": "Time to ready p50 / p95 / p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 44,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "0.5"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "0.95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "0.99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 23,
"type": "timeseries",
"title": "Ready within 30s, SLO 95% / 1h",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 44,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:ready_under_30s:ratio1h",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 24,
"type": "timeseries",
"title": "API availability, SLO 99% / 1h",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 52,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:availability:ratio1h",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
}
]
}
@@ -0,0 +1,30 @@
groups:
- name: messenger-slo
interval: 5s
rules:
- record: messenger:http_errors:ratio_rate1m
expr: |
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*",http_response_status_code=~"5.."}[1m])) or vector(0))
/ clamp_min(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*"}[1m])), 0.001)
- record: messenger:availability:ratio1h
expr: 1 - messenger:http_errors:ratio_rate1h
- record: messenger:ready_under_30s:ratio1h
expr: sum(increase(messenger_job_time_to_ready_seconds_bucket{le="30"}[1h])) / clamp_min(sum(increase(messenger_job_time_to_ready_seconds_count[1h])), 1)
- alert: MessengerHighErrorRate
expr: messenger:http_errors:ratio_rate1m > 0.05
for: 15s
labels:
severity: warning
annotations:
summary: More than 5% business HTTP 5xx over 1 minute
description: Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.
runbook: infra/prometheus/runbooks.md#messengerhigherrorrate
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) > 15 and max(messenger_queue_depth{queue_name="attachments.process"}) > 0
for: 15s
labels:
severity: warning
annotations:
summary: Durable attachment backlog has waited more than 15 seconds
description: Target is 95% ready within 30 seconds over 1 hour; depth includes retry and leased work.
runbook: infra/prometheus/runbooks.md#messengerqueuelag
+15
View File
@@ -0,0 +1,15 @@
# Локальная диагностика
SLI availability = 1 − business HTTP 5xx / все business HTTP за 1 h, цель 99%; health исключены, 4xx остаются в знаменателе. SLI ready = доля завершённых jobs с time-to-ready <=30 s за 1 h, цель 95%. Незавершённые jobs не входят в histogram, поэтому возраст/depth контролируются отдельно. Учебные alerts реагируют раньше окна SLO: это пороговые предупреждения, не multi-window burn rates.
## MessengerHighErrorRate
Условие: 5xx/все >5% за 1 m непрерывно 15 s; вычисление каждые 5 s. В API dashboard проверить error ratio и p95. По request_id найти лог и trace. Длинный/ошибочный PostgreSQL client span указывает на зависимость; длинный server span без медленных client spans — на handler. Проверить used/waiting/capacity пула и ресурсы процесса. В своём стенде восстановить postgres, дождаться ready обеих API и успешных запросов. Alert должен исчезнуть после ухода ошибок из минутного окна.
## MessengerQueueLag
Условие: durable depth>0 и oldest age>15 s непрерывно 15 s. Проверить jobs outcomes, worker busy/capacity, CPU/RAM и наличие worker logs. Отсутствие worker signal при растущем backlog указывает на остановку процесса; retry с S3 ERROR spans — на хранилище. Проверить producer/consumer цепочку конкретного attachment. Восстановить только свой worker/зависимость; дождаться ready, depth=0, age=0 и отсутствия firing alert. Queue depth относится к PostgreSQL queued/processing, включая scheduled retry, а не только AMQP ready messages.
## Отказ telemetry backend
Readiness API не зависит от Collector/хранилищ. Сначала проверить бизнес-smoke. Collector: queue_size/capacity, send_failed и sent counters; Docker RAM против mem_limit. Retry ограничен 60 s, очереди ограничены; потеря диагностических данных при заполнении допустима, потеря сообщений — нет. Восстановить backend и проверить новый request trace/log и метрики. Не удалять volumes для восстановления.
@@ -1,2 +1,4 @@
schema: spec-driven
created: 2026-10-29
archived: 2026-11-06
lab: 6
@@ -0,0 +1,2 @@
schema: spec-driven
created: 2026-11-06
@@ -0,0 +1,23 @@
## Context
SQLAlchemy async, transactional outbox и manual ack после COMMIT уже работают. SDK экспортирует асинхронно, Collector имеет memory_limiter, batch, ограниченные очереди и retry 60 s.
## Goals / Non-Goals
Закрыть все пункты уровня 4; не реализовывать трек 5 в этом change и не менять контракт API.
## Decisions
W3C traceparent/tracestate хранятся отдельно от нормативного payload в JSONB job и outbox. При retry берётся активный consumer context, при восстановлении аренды — сохранённый origin. Миграция добавляет поля с пустым default; прежние jobs остаются пригодными. Relay извлекает контекст и FastStream создаёт producer spans; ack остаётся после бизнес-COMMIT. S3 spans содержат только операцию/сервис, без ключей и exception payload.
Queue depth/age измеряют durable backlog queued/processing PostgreSQL, включая retry: это не размер AMQP очереди. Pool used/waiting/capacity измеряются у SQLAlchemy engine и в ObservedPool._do_get, без идентификаторов. Worker busy/capacity отражают конкурентность; time-to-ready отсчитывается от created_at job до done. CPU — процессорные секунды, RAM — текущий RSS из /proc. Все gauges отдаются OTLP; статические labels и resource service instance. Histogram seconds использует границы 0.05…60, включая SLO 30 s.
Availability SLO: >=99% успешных бизнес-HTTP за 1 h; ready SLO: >=95% done за 30 s на окне 1 h. Alerts: error ratio >5% за 1 m, for 15 s; backlog age >15 s, for 15 s. Thresholds короткие для воспроизводимой учебной демонстрации, не производственные рекомендации. Runbooks лежат в Git; внешняя доставка не требуется.
## Risks / Trade-offs
ObservedPool оборачивает получение connection, сохраняя ленивые sessions, timeout, autocommit и границы UoW. Waiting включает получение/создание соединения; используется закреплённая версия SQLAlchemy. Poller очереди работает отдельно, ошибки измерений не останавливают relay. Нулевой backlog возвращает age 0. При telemetry outage потеря сигналов допустима и измеряется; успешность сообщений обязательна. Проверки последовательны на собственном project, состояние Docker записывается; постоянные volumes не удаляются.
## Validation
Миграции legacy/fresh, тесты durable context и retry, make check/validate/lint/student-test. Реальные API→S3/DB→producer→consumer traces, retry, строгие dashboards, leak checks. Два инцидента с временной шкалой, queries и trace IDs, firing/resolved. Отказ хранилищ: smoke, RAM samples, Collector counters до/после, восстановление. Финальный REPORT делает следующий агент.
@@ -0,0 +1,18 @@
## Why
Уровень 3 доказан. Сейчас outbox теряет контекст HTTP, relay не создаёт producer spans, а наблюдаемость не объясняет накопление очереди и зависание БД. Это обязательные критерии уровня 4.
## What Changes
- Добавочная Alembic-миграция сохраняет W3C контекст job/outbox, FastStream связывает API, producer и consumer; S3 получает безопасные client spans.
- Добавляются queue age/depth, pool used/waiting, worker busy/capacity, time-to-ready и process CPU/RAM, dashboards и SLI/SLO с alerts.
- Реальные инциденты worker stop и PostgreSQL outage подтверждают firing/resolved; отдельный отказ хранилищ проверяет доступность API и ограничение памяти.
## Capabilities
### New Capabilities
- `lab6-observability-resilience`: сквозная диагностика и устойчивость наблюдаемости уровня 4.
## Impact
Меняются собственные адаптеры, миграции, инструменты и provisioning. API, нормативные файлы и REPORT не меняются. Следующему агенту передаются evidence и заметки. Данные существующих таблиц сохраняются.
@@ -0,0 +1,26 @@
## ADDED Requirements
### Requirement: Durable distributed context
The system SHALL preserve W3C context through API, transactional outbox, relay producer and worker consumer including retries, and expose safe database and S3 client spans.
#### Scenario: Deferred delivery and retry
- **WHEN** an accepted upload is delivered after relay restart or retried after S3 failure
- **THEN** its producer and consumer spans share the originating trace and causally valid parents, without adding fields to the normative job payload
### Requirement: Operational signals and SLO
The system SHALL expose queue depth and oldest age, database pool usage and waiting, worker saturation, process CPU/RAM and durable time-to-ready; dashboards SHALL show histogram percentiles and error ratio and define availability and ready SLO windows.
#### Scenario: Stopped worker
- **WHEN** queued uploads wait for a stopped worker
- **THEN** depth and age increase, a queue-lag alert fires and resolves after recovery
#### Scenario: Unavailable database
- **WHEN** PostgreSQL becomes unavailable
- **THEN** API failure metrics and database client spans distinguish the dependency failure and an error-rate alert fires and resolves
### Requirement: Bounded observability failure
Collector SHALL use memory limits, batching and bounded queues; unavailable telemetry storage SHALL NOT prevent business requests or cause unbounded memory growth.
#### Scenario: Storage outage
- **WHEN** telemetry storage is stopped during requests
- **THEN** business smoke passes, measured memory remains within container limits, losses are recorded, and exports resume after recovery
@@ -0,0 +1,10 @@
## Implementation
- [ ] 1. Durable context migration, repository and publisher wiring; propagation/retry tests
- [ ] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks
- [ ] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation
## Verification
- [ ] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes
- [ ] 5. End-to-end and retry traces, dashboards, leaks and cardinality
- [ ] 6. Worker and database incidents: evidence, firing/resolved and diagnosis
- [ ] 7. Storage outage: availability, bounded memory, lost signals and recovery
- [ ] 8. Evidence notes, requirement sync and archive after all checks
+1
View File
@@ -131,6 +131,7 @@ def tempo_spans(body):
duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6
spans.append({"service": service, "name": span["name"], "kind": span.get("kind"),
"status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"),
"span_id": span.get("spanId"), "parent_span_id": span.get("parentSpanId"),
"duration_ms": round(duration, 3), "links": len(span.get("links", [])),
"attributes": values})
return spans
+349
View File
@@ -0,0 +1,349 @@
import argparse
import concurrent.futures
import json
import os
import subprocess
import time
import uuid
from pathlib import Path
from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for
from obs_recovery import backends, compose
def actor():
api = Api(os.environ["BASE_URL"])
user, username, password, _ = api.register()
auth, _ = api.login(username, password)
return api, auth, api.chat(auth, user)
def upload(api, auth, chat):
return api.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png",
Path("tests/fixtures/sample.png").read_bytes(),
dict(auth, **{"Content-Type": "image/png"}),
expect=(202,),
)
def ready(api, auth, attachment):
def check():
item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0]
if item["status"] == "failed":
raise RuntimeError("attachment failed")
return item if item["status"] == "ready" else None
return wait_for("ready", check, 120, 1)
def value(b, expression):
result = b.promql(expression)
return float(result[0]["value"][1]) if result else None
def alert(b, name, firing):
def check():
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
found = [
a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing"
]
if bool(found) == firing:
return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found}
return None
return wait_for(name, check, 150)
def chain(b, retry):
api, auth, chat = actor()
started = time.time()
if retry:
compose("stop", "worker")
attachment, rid = upload(api, auth, chat)
try:
if retry:
compose("stop", "s3")
compose("start", "worker")
query = (
'{service_name="messenger-worker"} | attachment_id="'
+ attachment["id"]
+ '" | outcome="retry_scheduled"'
)
wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60)
finally:
if retry:
compose("up", "-d", "--no-deps", "--wait", "s3", "worker")
ready(api, auth, attachment)
logs = wait_for(
"upload log",
lambda: b.logs(
'{service_name="messenger-api"} | request_id="' + rid + '"',
started - 1,
time.time() + 1,
),
60,
)
trace_id = logs[0][0]["trace_id"]
def complete():
spans = tempo_spans(b.trace(trace_id)[0])
return (
spans
if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans)
else None
)
spans = wait_for("complete trace", complete, 90)
roots = [
s
for s in spans
if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid
]
ids = {s["span_id"] for s in spans}
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"]
if (
not roots
or not producers
or not consumers
or any(s["parent_span_id"] not in ids for s in producers + consumers)
):
raise Failure("broken distributed parent chain")
if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any(
s["attributes"].get("db.system") == "postgresql" for s in clients
):
raise Failure("missing dependency spans")
outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers]
if retry and "retry_scheduled" not in outcomes:
raise Failure("retry left original trace")
api.logout(auth)
fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms")
return {
"upload_request_id": rid,
"trace_id": trace_id,
"retry": retry,
"outcomes": outcomes,
"spans": [
{**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")}
for s in spans
],
}
def worker(b):
api, auth, chat = actor()
compose("stop", "worker")
stopped = now()
try:
items = [upload(api, auth, chat)[0] for _ in range(8)]
firing = alert(b, "MessengerQueueLag", True)
observed = {
"depth": value(b, "max(messenger_queue_depth)"),
"age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"),
}
finally:
compose("up", "-d", "--no-deps", "--wait", "worker")
restored = now()
for item in items:
ready(api, auth, item)
resolved = alert(b, "MessengerQueueLag", False)
api.logout(auth)
return {
"stopped": stopped,
"restored": restored,
"jobs": len(items),
"backlog": observed,
"firing": firing,
"resolved": resolved,
"depth_after": value(b, "max(messenger_queue_depth)"),
}
def database(b):
api, auth, _ = actor()
started = time.time()
prefix = "db-incident-" + uuid.uuid4().hex[:12]
def request(index):
rid = prefix + "-" + str(index)
status = http(
"GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid})
)[0]
return {"request_id": rid, "status": status}
for index in range(10):
request(index)
time.sleep(12)
compose("pause", "postgres")
paused = now()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
responses = list(pool.map(request, range(10, 34)))
firing = alert(b, "MessengerHighErrorRate", True)
signals = {
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
"pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"),
}
finally:
compose("unpause", "postgres")
restored = now()
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
for index in range(34, 74):
request(index)
time.sleep(0.15)
resolved = alert(b, "MessengerHighErrorRate", False)
traces = b.traceql(
'{ span.messenger.request_id =~ "' + prefix + '.*" }',
started - 1,
time.time() + 1,
limit=100,
)
dependency = []
for trace in traces:
spans = tempo_spans(b.trace(trace["traceID"])[0])
failed = [
s
for s in spans
if s["attributes"].get("db.system") == "postgresql"
and s["status"] == "STATUS_CODE_ERROR"
]
if failed:
dependency.append(
{
"trace_id": trace["traceID"],
"db_errors": [
{k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed
],
}
)
if not dependency or not any(r["status"] == 503 for r in responses):
raise Failure("database incident has no dependency failure trace")
api.logout(auth)
return {
"paused": paused,
"restored": restored,
"requests": responses,
"signals": signals,
"firing": firing,
"resolved": resolved,
"dependency_traces": dependency,
}
def collector_metrics():
source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())'
raw = compose("exec", "-T", "api", "python", "-c", source)
return {
line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1])
for line in raw.splitlines()
if line.startswith(
(
"otelcol_receiver_accepted_",
"otelcol_exporter_sent_",
"otelcol_exporter_send_failed_",
"otelcol_exporter_queue_",
)
)
}
def memory():
ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines()
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
)
return [
{k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")}
for row in map(json.loads, raw.splitlines())
]
def storage(b):
api, auth, _ = actor()
before = collector_metrics()
samples = [{"utc": now(), "containers": memory()}]
compose("stop", "prometheus", "loki", "tempo")
stopped = now()
requests = []
try:
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True)
until = time.monotonic() + 85
while time.monotonic() < until:
rid = "storage-outage-" + uuid.uuid4().hex
status = http(
"GET",
api.base + "/api/v1/users/me",
headers=dict(auth, **{"X-Request-Id": rid}),
expect=(200,),
)[0]
requests.append({"id": rid, "status": status})
if len(requests) % 15 == 0:
samples.append({"utc": now(), "containers": memory()})
time.sleep(1)
during = collector_metrics()
finally:
compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo")
restored = now()
time.sleep(20)
after = collector_metrics()
samples.append({"utc": now(), "containers": memory()})
found = 0
for item in requests:
found += bool(
b.traceql(
'{ span.messenger.request_id = "' + item["id"] + '" }',
time.time() - 300,
time.time() + 1,
)
)
peak_percent = max(
float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"]
)
if peak_percent >= 100:
raise Failure("container memory limit reached")
api.logout(auth)
return {
"stopped": stopped,
"restored": restored,
"peak_memory_percent_of_limit": peak_percent,
"smoke": result.stdout,
"successful_probes": len(requests),
"stored_probe_traces": found,
"lost_probe_traces": len(requests) - found,
"memory_samples": samples,
"collector_before": before,
"collector_during": during,
"collector_after": after,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage"))
args = parser.parse_args()
b = backends()
started = now()
report = (
chain(b, args.scenario == "retry")
if args.scenario in ("chain", "retry")
else {"worker": worker, "database": database, "storage": storage}[args.scenario](b)
)
print(
json.dumps(
{
"started": started,
"finished": now(),
"result": "PASS",
"scenario": args.scenario,
"observations": report,
},
indent=2,
)
)
if __name__ == "__main__":
main()
@@ -223,7 +223,7 @@ class PublishOutboxHandler:
for entry in await self._uow.outbox.pending(command.limit):
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
try:
await self._publisher.publish(entry.job_id, entry.body)
await self._publisher.publish(entry.job_id, entry.body, entry.trace_context)
except Unavailable as error:
failure = error
break
@@ -1,6 +1,6 @@
from collections.abc import AsyncIterator, Sequence
from contextlib import AbstractAsyncContextManager
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from typing import Protocol, runtime_checkable
from uuid import UUID
@@ -33,6 +33,7 @@ class OutboxEntry:
id: int
job_id: UUID
body: bytes
trace_context: dict[str, str] = field(default_factory=dict)
@dataclass(frozen=True, slots=True)
@@ -77,7 +78,9 @@ class VariantRenderer(Protocol):
@runtime_checkable
class JobPublisher(Protocol):
async def publish(self, job_id: UUID, body: bytes) -> None: ...
async def publish(
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
) -> None: ...
@runtime_checkable
@@ -179,6 +182,8 @@ class AttachmentDirectory(Protocol):
@runtime_checkable
class JobObserver(Protocol):
def active(self, delta: int) -> None: ...
def observe(self, job: Job, outcome: str, seconds: float) -> None: ...
+2 -2
View File
@@ -203,8 +203,8 @@ class WorkerProvider(Provider):
return RandomJitter()
@provide(scope=Scope.APP)
def observer(self) -> JobObserver:
return OtelJobObserver()
def observer(self, settings: PipelineSettings) -> JobObserver:
return OtelJobObserver(capacity=settings.worker_concurrency)
@provide(scope=Scope.REQUEST)
def command_bus(
@@ -35,6 +35,7 @@ async def handle_job(
logger.warning("job.rejected", reason="message does not match jobs.schema.json")
await message.reject(requeue=False)
return
observer.active(1)
started = time.perf_counter()
try:
outcome = (await bus.execute(ProcessJob(job))).value
@@ -49,6 +50,8 @@ async def handle_job(
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
await message.nack(requeue=True)
return
finally:
observer.active(-1)
observed(job, outcome, started, observer, logger)
await message.ack()
@@ -45,6 +45,7 @@ from messenger.shared.infrastructure.postgres import (
EventSource,
PostgresUnitOfWork,
)
from messenger.shared.infrastructure.trace_context import capture_context
class AttachmentRow(Base):
@@ -153,6 +154,7 @@ class JobRow(Base):
pipeline_version: Mapped[int]
status: Mapped[str]
attempt: Mapped[int]
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
claims: Mapped[int] = mapped_column(server_default="0")
lease_token: Mapped[UUID | None]
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@@ -168,6 +170,7 @@ class OutboxRow(Base):
id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
job_id: Mapped[UUID] = mapped_column(ForeignKey("jobs.job_id"))
payload: Mapped[dict[str, Any]] = mapped_column(JSONB)
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@@ -472,10 +475,18 @@ class PostgresJobRepository:
self._session = session
async def _enqueue(self, job: Job, available_at: datetime | None = None) -> None:
context = (
capture_context()
or await self._session.scalar(
select(JobRow.trace_context).where(JobRow.job_id == job.id)
)
or {}
)
await self._session.execute(
insert(OutboxRow).values(
job_id=job.id,
payload=job.payload(),
trace_context=context,
available_at=func.now() if available_at is None else available_at,
)
)
@@ -518,6 +529,7 @@ class PostgresJobRepository:
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
status="queued",
trace_context=capture_context(),
attempt=job.attempt,
created_at=job.created_at,
updated_at=job.created_at,
@@ -824,7 +836,10 @@ class PostgresOutboxRepository:
.limit(limit)
.with_for_update(skip_locked=True)
)
return [OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode()) for row in rows]
return [
OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode(), row.trace_context)
for row in rows
]
async def published(self, entry_ids: Sequence[int]) -> None:
await self._session.execute(
@@ -0,0 +1,62 @@
import asyncio
from contextlib import suppress
from datetime import UTC, datetime
from opentelemetry import metrics
from opentelemetry.metrics import Observation
from sqlalchemy import func, select
from messenger.attachments.infrastructure.postgres import JobRow
from messenger.shared.application.ports import Logger
from messenger.shared.infrastructure.postgres import Database
class QueueMetrics:
__slots__ = ("_database", "_depth", "_logger", "_oldest", "_task")
def __init__(self, database: Database, logger: Logger) -> None:
self._database = database
self._logger = logger
self._depth = 0
self._oldest: datetime | None = None
self._task: asyncio.Task[None] | None = None
meter = metrics.get_meter("messenger.queue")
labels = {"queue.name": "attachments.process"}
meter.create_observable_gauge(
"messenger.queue.depth",
unit="{job}",
callbacks=[lambda _: [Observation(self._depth, labels)]],
)
meter.create_observable_gauge(
"messenger.queue.oldest_age",
unit="s",
callbacks=[lambda _: [Observation(self.age(), labels)]],
)
def age(self) -> float:
return max(0, (datetime.now(UTC) - self._oldest).total_seconds()) if self._oldest else 0
def start(self) -> None:
self._task = asyncio.create_task(self._poll())
async def stop(self) -> None:
if self._task:
self._task.cancel()
with suppress(asyncio.CancelledError):
await self._task
async def _poll(self) -> None:
while True:
try:
async with self._database.session() as session:
row = (
await session.execute(
select(func.count(), func.min(JobRow.created_at)).where(
JobRow.status.in_(("queued", "processing"))
)
)
).one()
self._depth, self._oldest = row
except Exception as error:
self._logger.warning("queue.metrics_failed", error=type(error).__name__)
await asyncio.sleep(5)
@@ -5,6 +5,7 @@ from pamqp.commands import Basic
from messenger.attachments.domain.jobs import JOB_TYPE
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.trace_context import restored_context
EXCHANGE = RabbitExchange("attachments", declare=False)
QUEUE = "attachments.process"
@@ -18,19 +19,22 @@ class RabbitJobPublisher:
self._broker = broker
self._timeout = timeout
async def publish(self, job_id: UUID, body: bytes) -> None:
async def publish(
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
) -> None:
try:
confirmation = await self._broker.publish(
body,
exchange=EXCHANGE,
routing_key=JOB_TYPE,
mandatory=True,
persist=True,
message_id=str(job_id),
message_type=JOB_TYPE,
content_type="application/json",
timeout=self._timeout,
)
with restored_context(trace_context):
confirmation = await self._broker.publish(
body,
exchange=EXCHANGE,
routing_key=JOB_TYPE,
mandatory=True,
persist=True,
message_id=str(job_id),
message_type=JOB_TYPE,
content_type="application/json",
timeout=self._timeout,
)
except Exception as error:
raise Unavailable("брокер не принял задание") from error
if not isinstance(confirmation, Basic.Ack):
+26 -7
View File
@@ -6,6 +6,8 @@ from aiobotocore.session import get_session
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from botocore.handlers import add_expect_header
from opentelemetry import trace
from opentelemetry.trace import SpanKind, StatusCode
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import ObjectContent, StoredObject
@@ -60,17 +62,31 @@ class S3ObjectStorage:
self._client = client
self._bucket = bucket
async def _call(self, operation: str, **arguments: Any) -> Any:
with trace.get_tracer(__name__).start_as_current_span(
f"S3 {operation}",
kind=SpanKind.CLIENT,
attributes={"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": operation},
record_exception=False,
set_status_on_exception=False,
) as span:
try:
return await getattr(self._client, operation)(**arguments)
except Exception as error:
span.set_status(StatusCode.ERROR, type(error).__name__)
raise
async def put(self, key: str, data: bytes, content_type: str) -> None:
try:
await self._client.put_object(
Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
await self._call(
"put_object", Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def open(self, key: str) -> ObjectContent:
try:
response = await self._client.get_object(Bucket=self._bucket, Key=key)
response = await self._call("get_object", Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
raise ObjectMissingError(key) from error
@@ -86,15 +102,18 @@ class S3ObjectStorage:
async def copy(self, source: str, target: str) -> None:
try:
await self._client.copy_object(
Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source}
await self._call(
"copy_object",
Bucket=self._bucket,
Key=target,
CopySource={"Bucket": self._bucket, "Key": source},
)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def delete(self, key: str) -> None:
try:
await self._client.delete_object(Bucket=self._bucket, Key=key)
await self._call("delete_object", Bucket=self._bucket, Key=key)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
@@ -113,7 +132,7 @@ class S3ObjectStorage:
async def stat(self, key: str) -> StoredObject | None:
try:
head = await self._client.head_object(Bucket=self._bucket, Key=key)
head = await self._call("head_object", Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
return None
@@ -1,5 +1,7 @@
from datetime import UTC, datetime
from opentelemetry import metrics, trace
from opentelemetry.metrics import MeterProvider
from opentelemetry.metrics import MeterProvider, Observation
from opentelemetry.trace import StatusCode
from messenger.attachments.domain.jobs import JOB_TYPE, Job
@@ -9,10 +11,22 @@ FAILURES = frozenset({"retry_scheduled", "failed", "error"})
class OtelJobObserver:
__slots__ = ("_duration", "_processed")
__slots__ = ("_busy", "_duration", "_processed", "_ready")
def __init__(self, provider: MeterProvider | None = None) -> None:
def __init__(self, provider: MeterProvider | None = None, capacity: int = 1) -> None:
meter = metrics.get_meter("messenger.attachments", meter_provider=provider)
self._busy = 0
meter.create_observable_gauge(
"messenger.worker.busy", callbacks=[lambda _: [Observation(self._busy)]], unit="{job}"
)
meter.create_observable_gauge(
"messenger.worker.capacity", callbacks=[lambda _: [Observation(capacity)]], unit="{job}"
)
self._ready = meter.create_histogram(
"messenger.job.time_to_ready",
unit="s",
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
)
self._processed = meter.create_counter(
"messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу"
)
@@ -23,7 +37,14 @@ class OtelJobObserver:
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
)
def active(self, delta: int) -> None:
self._busy += delta
def observe(self, job: Job, outcome: str, seconds: float) -> None:
if outcome == "done":
self._ready.record(
max(0, (datetime.now(UTC) - job.created_at).total_seconds()), {"job.type": JOB_TYPE}
)
labels = {"job.type": JOB_TYPE, "outcome": outcome}
self._processed.add(1, labels)
self._duration.record(seconds, labels)
+7
View File
@@ -3,16 +3,19 @@ import signal
from dishka import make_async_container
from faststream.rabbit import Channel, RabbitBroker
from faststream.rabbit.opentelemetry import RabbitTelemetryMiddleware
from messenger.attachments.config import PipelineSettings
from messenger.attachments.entrypoints.heartbeat import Heartbeat
from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal
from messenger.attachments.entrypoints.relay import Relay
from messenger.attachments.infrastructure.queue_metrics import QueueMetrics
from messenger.shared.application.messagebus import EventBus
from messenger.shared.config import PlatformSettings
from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
from messenger.shared.infrastructure.postgres import Database
from messenger.shared.infrastructure.telemetry import start_telemetry
PING_TIMEOUT_SECONDS = 2.0
@@ -28,6 +31,7 @@ async def serve() -> None:
pipeline.broker_url.get_secret_value(),
default_channel=Channel(publisher_confirms=True, on_return_raises=True),
logger=None,
middlewares=() if telemetry is None else (RabbitTelemetryMiddleware(),),
)
container = make_async_container(
PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker)
@@ -47,6 +51,8 @@ async def serve() -> None:
pipeline.heartbeat_interval_seconds,
logger,
)
queue_metrics = QueueMetrics(await container.get(Database), logger)
queue_metrics.start()
relay.start()
heartbeat.start()
logger.info("relay.started")
@@ -54,6 +60,7 @@ async def serve() -> None:
await stop.wait()
finally:
await heartbeat.stop()
await queue_metrics.stop()
await relay.stop()
await bus.stop()
await broker.stop()
@@ -1,13 +1,16 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import ClassVar, Protocol
from typing import ClassVar, Protocol, cast
from opentelemetry import metrics
from opentelemetry.metrics import Observation
from sqlalchemy import Text, make_url, select
from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeout
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.pool import AsyncAdaptedQueuePool, ConnectionPoolEntry
from messenger.shared.application.errors import Unavailable
from messenger.shared.domain.events import DomainEvent
@@ -38,6 +41,17 @@ def unavailable(error: BaseException) -> bool:
return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03")
class ObservedPool(AsyncAdaptedQueuePool):
waiting = 0
def _do_get(self) -> ConnectionPoolEntry:
self.waiting += 1
try:
return super()._do_get()
finally:
self.waiting -= 1
class Database:
__slots__ = ("_engine", "_sessions")
@@ -52,6 +66,7 @@ class Database:
) -> None:
self._engine = create_async_engine(
make_url(dsn).set(drivername="postgresql+asyncpg"),
poolclass=ObservedPool,
pool_size=max_size,
max_overflow=0,
pool_timeout=acquire_timeout,
@@ -68,6 +83,22 @@ class Database:
},
},
)
meter = metrics.get_meter("messenger.database")
meter.create_observable_gauge(
"messenger.db.pool.used",
callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).checkedout())]],
unit="{connection}",
)
meter.create_observable_gauge(
"messenger.db.pool.waiting",
callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).waiting)]],
unit="{request}",
)
meter.create_observable_gauge(
"messenger.db.pool.capacity",
callbacks=[lambda _: [Observation(max_size)]],
unit="{connection}",
)
self._sessions = async_sessionmaker(self._engine, expire_on_commit=False)
async def open(self) -> None:
@@ -0,0 +1,26 @@
import os
import resource
from pathlib import Path
from opentelemetry.metrics import MeterProvider, Observation
def rss() -> int:
try:
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
except OSError:
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
def observe_process(provider: MeterProvider) -> None:
meter = provider.get_meter("messenger.process")
meter.create_observable_counter(
"process.cpu.time",
unit="s",
callbacks=[lambda _: [Observation(sum(os.times()[:2]))]],
)
meter.create_observable_gauge(
"process.memory.usage",
unit="By",
callbacks=[lambda _: [Observation(rss())]],
)
@@ -25,6 +25,7 @@ from opentelemetry.trace import Link, SpanKind, TraceState
from opentelemetry.util.types import Attributes
from messenger.shared.config import PlatformSettings
from messenger.shared.infrastructure.runtime_metrics import observe_process
SERVICE_VERSION = version("mtusi-messenger")
REQUEST_ID = "messenger.request_id"
@@ -127,6 +128,7 @@ def start_telemetry(
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
shutdown_on_exit=False,
)
observe_process(meter)
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
trace.set_tracer_provider(tracer)
@@ -0,0 +1,22 @@
from collections.abc import Iterator
from contextlib import contextmanager
from opentelemetry.context import attach, detach
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
PROPAGATOR = TraceContextTextMapPropagator()
def capture_context() -> dict[str, str]:
carrier: dict[str, str] = {}
PROPAGATOR.inject(carrier)
return carrier
@contextmanager
def restored_context(carrier: dict[str, str] | None) -> Iterator[None]:
token = attach(PROPAGATOR.extract(carrier or {}))
try:
yield
finally:
detach(token)
@@ -112,6 +112,8 @@ def test_shared_core_holds_only_platform() -> None:
"logging.py",
"postgres.py",
"telemetry.py",
"trace_context.py",
"runtime_metrics.py",
"readiness.py",
}
+7 -2
View File
@@ -6,6 +6,7 @@ import sys
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import uuid4
import pytest
@@ -15,6 +16,7 @@ from alembic.migration import MigrationContext
from alembic.script import ScriptDirectory
from alembic.util import CommandError
from sqlalchemy import MetaData, Table, case, func, insert, inspect, literal, make_url, select, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import NullPool
@@ -44,6 +46,7 @@ VERSIONS = [
"0004_jobs",
"0005_failed_registry",
"0006_attachment_quarantine",
"0007_trace_context",
]
TABLES = [
UserRow,
@@ -247,13 +250,15 @@ async def contents(dsn: str) -> dict:
table = await connection.run_sync(
lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync)
)
missing = {}
missing: dict[str, Any] = {}
if model is AttachmentRow and "published_version" not in table.c:
missing = {
"original_key": case((table.c.status == "ready", table.c.object_key)),
"staging_key": case((table.c.status == "pending", table.c.object_key)),
"published_version": case((table.c.status == "ready", 1), else_=0),
}
if model in (JobRow, OutboxRow):
missing["trace_context"] = literal({}, type_=JSONB)
columns = [
table.c[col.name]
if col.name in table.c
@@ -449,7 +454,7 @@ async def test_failed_revision_rolls_back_ddl_and_version(
"from alembic import op\n"
"import sqlalchemy as sa\n"
"revision = '0003_failure'\n"
"down_revision = '0006_attachment_quarantine'\n"
"down_revision = '0007_trace_context'\n"
"def upgrade():\n"
" op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n"
" raise RuntimeError('migration failed')\n"
+65
View File
@@ -0,0 +1,65 @@
import asyncio
from datetime import timedelta
from uuid import UUID
from httpx import AsyncClient
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import SpanKind
from sqlalchemy import select
from test_pipeline import accepted, job_of
from conftest import sql_connection
from messenger.attachments.infrastructure.postgres import JobRow, OutboxRow, PostgresJobRepository
from messenger.shared.infrastructure.trace_context import capture_context, restored_context
from pipeline import pipeline_of
async def test_context_survives_outbox_delivery_and_lease_recovery(client: AsyncClient) -> None:
provider = TracerProvider(shutdown_on_exit=False)
tracer = provider.get_tracer("test")
with tracer.start_as_current_span("upload", kind=SpanKind.SERVER):
origin = capture_context()
_, _, attachment_id = await accepted(client)
job = await job_of(attachment_id)
async with sql_connection() as connection:
assert (
await connection.scalar(select(JobRow.trace_context).where(JobRow.job_id == job.id))
== origin
)
assert (
await connection.scalar(
select(OutboxRow.trace_context).where(OutboxRow.job_id == job.id)
)
== origin
)
pipeline = await pipeline_of(client)
await pipeline.publish()
assert pipeline.publisher.contexts == [origin]
uow = pipeline.uow()
async with uow.transaction():
claim = await uow.jobs.claim(job, UUID(int=1), timedelta(seconds=-1))
assert claim is not None
await uow.commit()
await pipeline.recover()
await asyncio.sleep(0.02)
await pipeline.publish()
assert len(pipeline.publisher.contexts) == 2
assert pipeline.publisher.contexts[-1] == origin
with restored_context(origin), tracer.start_as_current_span("retry", kind=SpanKind.CONSUMER):
retry = capture_context()
async with uow.transaction():
await PostgresJobRepository(uow.session)._enqueue(job)
await uow.commit()
async with sql_connection() as connection:
assert (
await connection.scalar(
select(OutboxRow.trace_context)
.where(OutboxRow.job_id == job.id)
.order_by(OutboxRow.id.desc())
.limit(1)
)
== retry
)
assert retry["traceparent"].split("-")[1] == origin["traceparent"].split("-")[1]
assert retry != origin
provider.shutdown()
+6 -2
View File
@@ -60,18 +60,22 @@ class InlineRenderer:
class CapturingPublisher:
__slots__ = ("bodies", "failing", "job_ids")
__slots__ = ("bodies", "contexts", "failing", "job_ids")
def __init__(self) -> None:
self.bodies: list[bytes] = []
self.contexts: list[dict[str, str]] = []
self.job_ids: list[UUID] = []
self.failing = False
async def publish(self, job_id: UUID, body: bytes) -> None:
async def publish(
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
) -> None:
if self.failing:
raise Unavailable("брокер недоступен")
self.job_ids.append(job_id)
self.bodies.append(body)
self.contexts.append(trace_context or {})
class InlinePipeline:
+13
View File
@@ -31,6 +31,7 @@ def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None:
for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")):
with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER):
observer.observe(item, outcome, 0.3)
observer.active(1)
data = reader.get_metrics_data()
assert data is not None
found = {
@@ -39,6 +40,18 @@ def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None:
for scope in resource.scope_metrics
for metric in scope.metrics
}
assert (
sum(
p.count
for p in found["messenger.job.time_to_ready"].data.data_points
if isinstance(p, HistogramDataPoint)
)
== 1
)
busy = next(iter(found["messenger.worker.busy"].data.data_points))
assert isinstance(busy, NumberDataPoint)
assert busy.value == 1
observer.active(-1)
counted = {
(point.attributes or {})["outcome"]: point.value
for point in found["messenger.jobs.processed"].data.data_points