forked from mtusi2026/mtusi2026lab6
ЛР6: OpenTelemetry, Collector, Prometheus/Loki/Tempo, Grafana, SLO и алерты, инциденты
This commit is contained in:
@@ -0,0 +1,265 @@
|
||||
"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git).
|
||||
|
||||
python scripts/build_dashboard.py
|
||||
|
||||
Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос
|
||||
против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой.
|
||||
Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "infra/grafana/dashboards/messenger.json"
|
||||
|
||||
PROM = {"type": "prometheus", "uid": "prometheus"}
|
||||
LOKI = {"type": "loki", "uid": "loki"}
|
||||
TEMPO = {"type": "tempo", "uid": "tempo"}
|
||||
|
||||
API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO)
|
||||
QUEUE = 'messaging_destination_name="attachments.process.v1"'
|
||||
|
||||
_panels = []
|
||||
_next_id = [1]
|
||||
_y = [0]
|
||||
|
||||
|
||||
def _id():
|
||||
_next_id[0] += 1
|
||||
return _next_id[0]
|
||||
|
||||
|
||||
def row(title):
|
||||
_panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []})
|
||||
_y[0] += 1
|
||||
|
||||
|
||||
class Line:
|
||||
"""Раскладка панелей в строку сетки Grafana (24 колонки)."""
|
||||
|
||||
def __init__(self, height):
|
||||
self.x, self.h = 0, height
|
||||
|
||||
def place(self, width):
|
||||
pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]}
|
||||
self.x += width
|
||||
return pos
|
||||
|
||||
def end(self):
|
||||
_y[0] += self.h
|
||||
|
||||
|
||||
def prom(expr, legend="", instant=False, fmt=None):
|
||||
target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "",
|
||||
"range": not instant, "instant": instant}
|
||||
if fmt:
|
||||
target["format"] = fmt
|
||||
return target
|
||||
|
||||
|
||||
def _with_refs(targets):
|
||||
for n, target in enumerate(targets):
|
||||
target["refId"] = chr(ord("A") + n)
|
||||
return targets
|
||||
|
||||
|
||||
def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None):
|
||||
defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never",
|
||||
"stacking": {"mode": "normal" if stack else "none"}}}
|
||||
if minimum is not None:
|
||||
defaults["min"] = minimum
|
||||
if thresholds:
|
||||
defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [
|
||||
{"color": color, "value": value} for value, color in thresholds]}
|
||||
defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"}
|
||||
_panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description,
|
||||
"datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets),
|
||||
"fieldConfig": {"defaults": defaults, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}}})
|
||||
|
||||
|
||||
def stat(line, width, title, description, expr, unit, steps, decimals=None):
|
||||
defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [
|
||||
{"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}}
|
||||
if decimals is not None:
|
||||
defaults["decimals"] = decimals
|
||||
_panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM,
|
||||
"gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]),
|
||||
"fieldConfig": {"defaults": defaults, "overrides": []},
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"colorMode": "background", "graphMode": "none", "textMode": "value"}})
|
||||
|
||||
|
||||
def quantiles(metric, selector, by="le"):
|
||||
return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))",
|
||||
f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)]
|
||||
|
||||
|
||||
def build():
|
||||
row("Обзор и SLO (окно SLO — 1 час)")
|
||||
line = Line(4)
|
||||
stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.",
|
||||
"messenger:http_requests:rate1m", "reqps", [(None, "blue")])
|
||||
stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.",
|
||||
"messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2)
|
||||
stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).",
|
||||
"messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")])
|
||||
stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.",
|
||||
f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")])
|
||||
stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.",
|
||||
"messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2)
|
||||
stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.",
|
||||
"messenger:slo_availability:error_budget_remaining_1h", "percentunit",
|
||||
[(None, "red"), (0, "orange"), (0.5, "green")], 0)
|
||||
stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.",
|
||||
"messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
|
||||
stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.",
|
||||
"messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
|
||||
line.end()
|
||||
line = Line(5)
|
||||
_panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)",
|
||||
"description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.",
|
||||
"datasource": PROM, "gridPos": line.place(24),
|
||||
"targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]),
|
||||
"transformations": [{"id": "organize", "options": {"excludeByName": {
|
||||
"Time": True, "Value": True, "__name__": True}}}],
|
||||
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
|
||||
line.end()
|
||||
|
||||
row("API: запросы, ошибки, задержка (RED)")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.",
|
||||
[prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
|
||||
"{{http_request_method}} {{http_route}}")], "reqps")
|
||||
timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.",
|
||||
[prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
|
||||
"{{http_response_status_class}}")], "reqps", stack=True)
|
||||
timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.",
|
||||
[prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) '
|
||||
f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")],
|
||||
"percentunit", thresholds=[(0.05, "red")], minimum=0)
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.",
|
||||
quantiles("http_server_request_duration_seconds", API), "s")
|
||||
timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.",
|
||||
[prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))",
|
||||
"{{http_route}}")], "s")
|
||||
timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.",
|
||||
[prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short")
|
||||
line.end()
|
||||
|
||||
row("Фоновая обработка: очередь, worker, время до ready")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).",
|
||||
[prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")],
|
||||
"ops", stack=True)
|
||||
timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.",
|
||||
quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s")
|
||||
timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).",
|
||||
quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s",
|
||||
thresholds=[(30, "red")])
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.",
|
||||
[prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"),
|
||||
prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short")
|
||||
timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.",
|
||||
[prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"),
|
||||
prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s",
|
||||
thresholds=[(30, "red")])
|
||||
timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.",
|
||||
[prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"),
|
||||
prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")],
|
||||
"ops")
|
||||
line.end()
|
||||
|
||||
row("Насыщение: пул БД, worker, CPU/RAM")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).",
|
||||
[prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"),
|
||||
prom("max(db_client_connection_max)", "max per replica"),
|
||||
prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short")
|
||||
timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).",
|
||||
[prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))",
|
||||
"p95 {{instance}}")], "s")
|
||||
timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.",
|
||||
[prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit",
|
||||
thresholds=[(0.8, "orange"), (1, "red")], minimum=0)
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.",
|
||||
[prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")],
|
||||
"short")
|
||||
timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.",
|
||||
[prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)",
|
||||
"{{job}} {{instance}}")], "percentunit")
|
||||
timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).",
|
||||
[prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"),
|
||||
prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes")
|
||||
line.end()
|
||||
|
||||
row("Конвейер телеметрии: Collector и SDK")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.",
|
||||
[prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"),
|
||||
prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"),
|
||||
prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")],
|
||||
"ops")
|
||||
timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.",
|
||||
[prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"),
|
||||
prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")],
|
||||
"ops")
|
||||
timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.",
|
||||
[prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")],
|
||||
"percentunit", minimum=0)
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.",
|
||||
[prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes")
|
||||
timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).",
|
||||
[prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"),
|
||||
prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"),
|
||||
prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"),
|
||||
prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")],
|
||||
"ops")
|
||||
line.end()
|
||||
|
||||
row("Логи и трассы")
|
||||
line = Line(9)
|
||||
_panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)",
|
||||
"description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.",
|
||||
"datasource": LOKI, "gridPos": line.place(12),
|
||||
"targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range",
|
||||
"expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]),
|
||||
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending",
|
||||
"enableLogDetails": True}})
|
||||
_panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)",
|
||||
"description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.",
|
||||
"datasource": TEMPO, "gridPos": line.place(12),
|
||||
"targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql",
|
||||
"query": "{ status = error }", "limit": 20, "tableType": "traces"}]),
|
||||
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
|
||||
line.end()
|
||||
|
||||
return {
|
||||
"uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"],
|
||||
"timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1,
|
||||
"refresh": "10s", "time": {"from": "now-30m", "to": "now"},
|
||||
"templating": {"list": []}, "annotations": {"list": [{
|
||||
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
|
||||
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
|
||||
"panels": _panels,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels")
|
||||
@@ -0,0 +1,157 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
with open(SECRETS / name, "w", encoding="utf-8", newline="") as file:
|
||||
file.write(value)
|
||||
(SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore
|
||||
|
||||
|
||||
def bucket_name() -> str:
|
||||
"""S3_BUCKET из .env (если задан), иначе значение по умолчанию."""
|
||||
env = ROOT / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("S3_BUCKET="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
return "messenger-attachments"
|
||||
|
||||
|
||||
def s3_config(bucket: str) -> str:
|
||||
"""Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket.
|
||||
|
||||
Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects)
|
||||
с суффиксом :<bucket> действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет.
|
||||
Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются.
|
||||
"""
|
||||
def read(name):
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
config = {"identities": [
|
||||
{"name": "admin",
|
||||
"credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "messenger-api",
|
||||
"credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
|
||||
]}
|
||||
return json.dumps(config, indent=2)
|
||||
|
||||
|
||||
BROKER_VHOST = "messenger"
|
||||
# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей).
|
||||
EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1"
|
||||
DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq"
|
||||
|
||||
|
||||
def rabbit_hash(password: str) -> str:
|
||||
"""Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль))."""
|
||||
salt = secrets.token_bytes(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii")
|
||||
|
||||
|
||||
def rabbit_definitions(passwords: dict) -> str:
|
||||
"""Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf).
|
||||
|
||||
Права (configure / write / read, регулярные выражения по именам ресурсов):
|
||||
dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет;
|
||||
worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет.
|
||||
Ни у кого из сервисов нет тега administrator и права создать/удалить очередь.
|
||||
"""
|
||||
queue_args = {
|
||||
"x-queue-type": "quorum",
|
||||
"x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-dead-letter-routing-key": DLQ_ROUTING_KEY,
|
||||
"x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish)
|
||||
"x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит
|
||||
"x-max-length": 100000,
|
||||
}
|
||||
definitions = {
|
||||
"users": [{"name": name, "password_hash": rabbit_hash(password),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []}
|
||||
for name, password in passwords.items()],
|
||||
"vhosts": [{"name": BROKER_VHOST}],
|
||||
"permissions": [
|
||||
{"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"},
|
||||
{"user": "worker", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"},
|
||||
],
|
||||
"exchanges": [
|
||||
{"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
{"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
],
|
||||
"queues": [
|
||||
{"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args},
|
||||
# DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный
|
||||
# просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет.
|
||||
{"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "classic"}},
|
||||
],
|
||||
"bindings": [
|
||||
{"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue",
|
||||
"routing_key": ROUTING_KEY, "arguments": {}},
|
||||
{"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue",
|
||||
"routing_key": DLQ_ROUTING_KEY, "arguments": {}},
|
||||
],
|
||||
}
|
||||
return json.dumps(definitions, indent=2)
|
||||
|
||||
|
||||
def broker_secrets(created: list, host: str = "rabbitmq") -> None:
|
||||
if (SECRETS / "rabbitmq_definitions.json").exists():
|
||||
return
|
||||
passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)}
|
||||
for name, password in passwords.items():
|
||||
write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}")
|
||||
write("rabbitmq_definitions.json", rabbit_definitions(passwords))
|
||||
created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes)."""
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
created = []
|
||||
if not (SECRETS / "postgres_password").exists():
|
||||
db_password = secrets.token_hex(24)
|
||||
redis_password = secrets.token_hex(24)
|
||||
write("postgres_password", db_password)
|
||||
write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{redis_password}@redis:6379/0")
|
||||
write("redis_auth", f"requirepass {redis_password}\n")
|
||||
created += ["postgres_password", "database_url", "cache_url", "redis_auth"]
|
||||
if not (SECRETS / "s3_root_password").exists():
|
||||
# admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init
|
||||
write("s3_root_user", "root-" + secrets.token_hex(6))
|
||||
write("s3_root_password", secrets.token_hex(24))
|
||||
created += ["s3_root_user", "s3_root_password"]
|
||||
if not (SECRETS / "s3_secret_access_key").exists():
|
||||
# отдельный ключ API с политикой только на свой bucket (создаёт s3-init)
|
||||
write("s3_access_key_id", "api-" + secrets.token_hex(8))
|
||||
write("s3_secret_access_key", secrets.token_hex(24))
|
||||
created += ["s3_access_key_id", "s3_secret_access_key"]
|
||||
if not (SECRETS / "s3_config.json").exists():
|
||||
write("s3_config.json", s3_config(bucket_name()))
|
||||
created.append("s3_config.json")
|
||||
broker_secrets(created)
|
||||
if not (SECRETS / "grafana_admin_password").exists():
|
||||
# lab6: пароль администратора Grafana (GF_SECURITY_ADMIN_PASSWORD__FILE), логин admin
|
||||
write("grafana_admin_password", secrets.token_urlsafe(18))
|
||||
created.append("grafana_admin_password")
|
||||
if created:
|
||||
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
|
||||
else:
|
||||
print("secrets/ is complete, nothing changed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
@echo off
|
||||
rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, Prometheus, Loki, Tempo), then otel-collector,
|
||||
rem api, janitor, dispatcher, worker, grafana,
|
||||
rem then the balancer. Volumes are kept.
|
||||
rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
|
||||
rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1
|
||||
docker compose restart otel-collector api janitor dispatcher worker grafana || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps otel-collector api lb dispatcher worker grafana || exit /b 1
|
||||
exit /b 0
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for tests/resilience.py --mode persistence (Linux/macOS).
|
||||
# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ; lab6: Prometheus, Loki, Tempo), then
|
||||
# otel-collector, api, janitor, dispatcher,
|
||||
# worker, then the balancer. Volumes are kept (no down -v), nothing is re-seeded.
|
||||
# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
|
||||
# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres redis s3 rabbitmq prometheus loki tempo
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq prometheus loki tempo
|
||||
docker compose restart otel-collector api janitor dispatcher worker grafana
|
||||
docker compose restart lb
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps otel-collector api lb dispatcher worker grafana
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py.
|
||||
python "%~dp0stop_one.py"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID.
|
||||
# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd.
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/stop_one.py"
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STOP_TIMEOUT_SECONDS = 8
|
||||
|
||||
|
||||
def main() -> int:
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{12,64}", target):
|
||||
print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr)
|
||||
return 2
|
||||
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout.split()
|
||||
matches = [container for container in running if container.startswith(target)]
|
||||
if len(matches) != 1:
|
||||
print(f"no running api container with id {target}", file=sys.stderr)
|
||||
return 1
|
||||
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
print(f"stopped api container {matches[0][:12]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept).
|
||||
cd /d "%~dp0.."
|
||||
if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% )
|
||||
if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% )
|
||||
if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% )
|
||||
echo usage: %0 stop^|start^|status
|
||||
exit /b 2
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Demo hook for the defense: stop or start the worker without touching volumes.
|
||||
# scripts/worker.sh stop | start | status
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${1:-}" in
|
||||
stop) docker compose stop worker ;;
|
||||
start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;;
|
||||
status) docker compose ps worker ;;
|
||||
*) echo "usage: $0 stop|start|status" >&2; exit 2 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user