forked from mtusi2026/mtusi2026lab6
lab6: EAI_AGAIN as unavailable, report, evidence, cold start
This commit is contained in:
@@ -79,7 +79,8 @@ Evidence — под `evidence/lab6/`; навигация — `evidence/lab6/summ
|
||||
| 4: два инцидента + диагностика | `make lab6-incident-worker lab6-incident-db` | PASS | `incident-*.json` |
|
||||
| 5: трек 1 | `make lab6-track1` | PASS, p25 выбран и провалидирован | `track1/` |
|
||||
| История/persistence backend | `make lab6-history` | старые sample/log/trace читаются после restart | `history.json` |
|
||||
| Регрессия lab1–5 | `make check validate test test-e2e test-dependencies` | см. `regression/` | `regression/` |
|
||||
| Регрессия lab1–5 | `make check validate test test-e2e test-dependencies test-persistence test-failover lab5-demo`; `npm test` | PASS: e2e 44 passed/9 skipped; dependencies 7/7; persistence (hook рестартует и postgres redis s3 broker api worker haproxy, и otel-collector prometheus loki tempo grafana); failover (0 failed attempts, recovery 3.3 s); lab5-demo; npm 89/89 | `regression/` |
|
||||
| Холодный старт | отдельный clone коммита `d558148`, проект `lab6-cold`, порты 18080/18404/15432/16379/19000/19091/13000 (19090 занят чужим процессом) | `make stand-up`, `test`, `lab6-ready`, `lab6-correlate`, `lab6-signals` PASS; dashboards/datasources появились автоматически; проект удалён `down -v` только для `lab6-cold` | `cold-start/` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
@@ -115,6 +116,10 @@ Evidence — под `evidence/lab6/`; навигация — `evidence/lab6/summ
|
||||
|
||||
**Выбор.** p25: критерии приёмки пройдены (`selected.json`: все 6 проверок True: valid-clean, ≥1 error и slow в каждом run, 0 missing, spans ≤50% full, p95 ≤110% full). Независимая проверка на свежем run (`track1/validation/p25`, `selected-validation.json`): PASS против **зафиксированного** full-reference (1.39 spans/op ≤ 50% от 4.9; p95 13.27 мс ≤ 110% от 13.38). Это компромисс, не гарантия: любую конкретную редкую ошибку p25 сохраняет с вероятностью ≈25%.
|
||||
|
||||
### Дефект, найденный регрессией
|
||||
|
||||
`make test-dependencies` (остановленный postgres/S3) сначала падал: из-за `dns_opt timeout:1 attempts:1` остановленный сервис давал `EAI_AGAIN`, который приложение отдавало 500 вместо 503. Исправлено добавлением `EAI_AGAIN` в `UNAVAILABLE_ERROR_CODES` (`api-exception.filter.ts`); после этого 7/7. Замеры трека 1 и инциденты сняты на образе до этой правки; она влияет только на путь недоступной зависимости, не на healthy-путь.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
- Один хост (8 ядер, 15 GiB, занят другими контейнерами), фиксированный синтетический workload 20 rps, короткие окна 180 с, три повтора; p95 ~13 мс близок к шуму, поэтому различия latency/CPU между режимами не интерпретируются как эффект телеметрии.
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
stand is ready (http://localhost:18080)
|
||||
python3 tests/smoke.py --base-url "http://localhost:18080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 scripts/lab6/verify.py ready --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000"
|
||||
PASS: ready
|
||||
python3 scripts/lab6/verify.py correlation --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000"
|
||||
PASS: correlation (http, upload, retry, failed)
|
||||
python3 scripts/lab6/verify.py signals --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000"
|
||||
PASS: signals (23 logical signals)
|
||||
@@ -0,0 +1,6 @@
|
||||
ports: app18080 stats18404 pg15432 redis16379 s3 19000 prometheus19091(19090 busy: mihomo) grafana13000
|
||||
Container lab6-cold-api-1 Healthy
|
||||
Container lab6-cold-s3-init-1 Exited
|
||||
Container lab6-cold-haproxy-1 Healthy
|
||||
Container lab6-cold-migrate-1 Exited
|
||||
Container lab6-cold-worker-1 Healthy
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"attachment_id": "5217f331-2025-4b40-8cb9-0f903815a963",
|
||||
"captured": "2026-11-09T23:00:57Z",
|
||||
"consumer_spans": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"instance": "54ab1af5724b",
|
||||
"kind": "CONSUMER",
|
||||
"name": "process attachments.process",
|
||||
"outcome": "failed",
|
||||
"parent_span_id": "d7c00750aa8307df",
|
||||
"service": "messenger-worker",
|
||||
"span_id": "496ff12ff42c7060",
|
||||
"status": "ERROR"
|
||||
}
|
||||
],
|
||||
"final_attachment_status": "failed",
|
||||
"final_job": {
|
||||
"attempt": 1,
|
||||
"last_error_code": "injected_fault",
|
||||
"status": "failed"
|
||||
},
|
||||
"job_id": "161ffedb-0a73-4834-b9cd-9e3720062dad",
|
||||
"kind": "permanent",
|
||||
"pipeline_version": 1,
|
||||
"request_id": "fd8ea75b-a7fd-46d7-bff0-75093abefe92",
|
||||
"trace_id": "0eeec1516eba748b30d404a147cc0033"
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"captured": "2026-11-09T23:00:57Z",
|
||||
"db_children": [
|
||||
{
|
||||
"instance": "b61243743d89",
|
||||
"kind": "CLIENT",
|
||||
"name": "SELECT users",
|
||||
"parent_span_id": "050698d2997e9641",
|
||||
"service": "messenger-api",
|
||||
"span_id": "c598408f5d0834f5",
|
||||
"status": "UNSET"
|
||||
},
|
||||
{
|
||||
"instance": "b61243743d89",
|
||||
"kind": "CLIENT",
|
||||
"name": "WITH idempotency_keys",
|
||||
"parent_span_id": "050698d2997e9641",
|
||||
"service": "messenger-api",
|
||||
"span_id": "e44ee4f9da9e18c7",
|
||||
"status": "UNSET"
|
||||
}
|
||||
],
|
||||
"log_span_id": "050698d2997e9641",
|
||||
"request_id": "1df5f3c3-d3e7-4c8f-87c0-7332393a5182",
|
||||
"server_span": {
|
||||
"instance": "b61243743d89",
|
||||
"kind": "SERVER",
|
||||
"name": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"parent_span_id": "",
|
||||
"service": "messenger-api",
|
||||
"span_id": "050698d2997e9641",
|
||||
"status": "UNSET"
|
||||
},
|
||||
"trace_id": "611265fa918e94d3606f7521e20a9627"
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"attachment_id": "ddd0a403-af0b-447a-b41a-4ea2d791db6c",
|
||||
"captured": "2026-11-09T23:00:57Z",
|
||||
"consumer_spans": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"instance": "54ab1af5724b",
|
||||
"kind": "CONSUMER",
|
||||
"name": "process attachments.process",
|
||||
"outcome": "retry",
|
||||
"parent_span_id": "be90f2e1ee549aa4",
|
||||
"service": "messenger-worker",
|
||||
"span_id": "33a7b57255493c51",
|
||||
"status": "ERROR"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"instance": "54ab1af5724b",
|
||||
"kind": "CONSUMER",
|
||||
"name": "process attachments.process",
|
||||
"outcome": "retry",
|
||||
"parent_span_id": "5b73c06a02d4f844",
|
||||
"service": "messenger-worker",
|
||||
"span_id": "486c12ef2231fc33",
|
||||
"status": "ERROR"
|
||||
},
|
||||
{
|
||||
"attempt": 3,
|
||||
"instance": "54ab1af5724b",
|
||||
"kind": "CONSUMER",
|
||||
"name": "process attachments.process",
|
||||
"outcome": "retry",
|
||||
"parent_span_id": "a320a0009b317475",
|
||||
"service": "messenger-worker",
|
||||
"span_id": "d9ef4f0d6e0193f3",
|
||||
"status": "ERROR"
|
||||
},
|
||||
{
|
||||
"attempt": 4,
|
||||
"instance": "54ab1af5724b",
|
||||
"kind": "CONSUMER",
|
||||
"name": "process attachments.process",
|
||||
"outcome": "retry",
|
||||
"parent_span_id": "ad953e5ca632b9eb",
|
||||
"service": "messenger-worker",
|
||||
"span_id": "9a8c4f22e3ec9bbe",
|
||||
"status": "ERROR"
|
||||
}
|
||||
],
|
||||
"final_attachment_status": "ready",
|
||||
"final_job": {
|
||||
"attempt": 5,
|
||||
"last_error_code": "injected_fault",
|
||||
"status": "succeeded"
|
||||
},
|
||||
"job_id": "a4c71b25-74ee-431b-a595-4cee3cb153f8",
|
||||
"kind": "transient",
|
||||
"pipeline_version": 1,
|
||||
"request_id": "06f85dce-b7a2-4991-bb90-8a70aa261ffb",
|
||||
"trace_id": "16e58ce6545b82e3d8258b35eca400c1"
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"attachment_id": "7ad73537-cb62-4c33-90cb-765509f22205",
|
||||
"captured": "2026-11-09T23:00:57Z",
|
||||
"chain_consumer_to_root": [
|
||||
{
|
||||
"instance": "54ab1af5724b",
|
||||
"kind": "CONSUMER",
|
||||
"name": "process attachments.process",
|
||||
"parent_span_id": "643f5038a4f4d54c",
|
||||
"service": "messenger-worker",
|
||||
"span_id": "2d619e1f288ee544",
|
||||
"status": "UNSET"
|
||||
},
|
||||
{
|
||||
"instance": "b61243743d89",
|
||||
"kind": "PRODUCER",
|
||||
"name": "publish attachments.jobs",
|
||||
"parent_span_id": "4307e07bf374f047",
|
||||
"service": "messenger-api",
|
||||
"span_id": "643f5038a4f4d54c",
|
||||
"status": "UNSET"
|
||||
},
|
||||
{
|
||||
"instance": "b61243743d89",
|
||||
"kind": "CLIENT",
|
||||
"name": "TRANSACTION jobs.accept_upload",
|
||||
"parent_span_id": "111c15c5e3d4880c",
|
||||
"service": "messenger-api",
|
||||
"span_id": "4307e07bf374f047",
|
||||
"status": "UNSET"
|
||||
},
|
||||
{
|
||||
"instance": "b61243743d89",
|
||||
"kind": "SERVER",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"parent_span_id": "",
|
||||
"service": "messenger-api",
|
||||
"span_id": "111c15c5e3d4880c",
|
||||
"status": "UNSET"
|
||||
}
|
||||
],
|
||||
"chain_fields_are_real_parent_child": true,
|
||||
"consumer_attempt": 1,
|
||||
"consumer_outcome": "succeeded",
|
||||
"db_children": 22,
|
||||
"request_id": "62e9c6b6-2851-4050-8b6f-6a60cbe05d0a",
|
||||
"s3_children": 7,
|
||||
"trace_id": "d5db164edb1e67da24c41a5432df159a"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"backend_only": false,
|
||||
"finished": "2026-11-09T23:00:03Z",
|
||||
"passed": true
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
{
|
||||
"api_replicas_with_http_metrics": [
|
||||
"907ec33d6af9",
|
||||
"b61243743d89",
|
||||
"cfba8008511e",
|
||||
"ea4900bd5808"
|
||||
],
|
||||
"captured": "2026-11-09T23:00:57Z",
|
||||
"collector_series": {
|
||||
"otelcol_exporter_enqueue_failed_log_records": 0,
|
||||
"otelcol_exporter_enqueue_failed_metric_points": 0,
|
||||
"otelcol_exporter_enqueue_failed_spans": 0,
|
||||
"otelcol_exporter_queue_capacity": 3,
|
||||
"otelcol_exporter_queue_size": 3,
|
||||
"otelcol_exporter_send_failed_log_records": 1,
|
||||
"otelcol_exporter_send_failed_metric_points": 1,
|
||||
"otelcol_exporter_send_failed_spans": 1,
|
||||
"otelcol_exporter_sent_log_records": 1,
|
||||
"otelcol_exporter_sent_metric_points": 1,
|
||||
"otelcol_exporter_sent_spans": 1,
|
||||
"otelcol_receiver_accepted_log_records": 1,
|
||||
"otelcol_receiver_accepted_metric_points": 1,
|
||||
"otelcol_receiver_accepted_spans": 1
|
||||
},
|
||||
"job_outcomes_observed": [
|
||||
"failed",
|
||||
"retry",
|
||||
"succeeded"
|
||||
],
|
||||
"passed": true,
|
||||
"problems": [],
|
||||
"queue_depth_ready_series": 4,
|
||||
"route_templates": [
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
"/api/v1/attachments/{attachment_id}/content",
|
||||
"/api/v1/auth/sessions",
|
||||
"/api/v1/auth/sessions/current",
|
||||
"/api/v1/chats",
|
||||
"/api/v1/chats/{chat_id}",
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
"/api/v1/users",
|
||||
"/api/v1/users/me",
|
||||
"/health/live",
|
||||
"/health/ready"
|
||||
],
|
||||
"signals": [
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"http_request_method",
|
||||
"http_response_status_code",
|
||||
"http_route",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "http_server_request_duration_seconds_count",
|
||||
"series": 48,
|
||||
"signal": "HTTP duration/count",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"http_request_method",
|
||||
"http_response_status_code",
|
||||
"http_route",
|
||||
"instance",
|
||||
"job",
|
||||
"le",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "http_server_request_duration_seconds_bucket",
|
||||
"series": 768,
|
||||
"signal": "HTTP duration buckets",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"http_request_method",
|
||||
"http_response_status_code",
|
||||
"http_route",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "http_server_request_duration_seconds_sum",
|
||||
"series": 48,
|
||||
"signal": "HTTP duration sum",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"job_type",
|
||||
"outcome",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_jobs_processed_total",
|
||||
"series": 4,
|
||||
"signal": "Jobs processed",
|
||||
"unit": "{job}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"job_type",
|
||||
"outcome",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_job_duration_seconds_count",
|
||||
"series": 4,
|
||||
"signal": "Job duration",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"job_type",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_attachment_time_to_ready_seconds_count",
|
||||
"series": 2,
|
||||
"signal": "Time to ready",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"messaging_destination_name",
|
||||
"service_version",
|
||||
"state"
|
||||
],
|
||||
"metric": "messenger_jobs_queue_depth",
|
||||
"series": 12,
|
||||
"signal": "Queue depth",
|
||||
"unit": "{job}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"messaging_destination_name",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_jobs_queue_oldest_age_seconds",
|
||||
"series": 4,
|
||||
"signal": "Queue oldest age",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"messaging_destination_name",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_outbox_pending",
|
||||
"series": 4,
|
||||
"signal": "Outbox pending",
|
||||
"unit": "{message}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"messaging_destination_name",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_outbox_oldest_age_seconds",
|
||||
"series": 4,
|
||||
"signal": "Outbox oldest age",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"db_client_connection_pool_name",
|
||||
"db_client_connection_state",
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "db_client_connection_count",
|
||||
"series": 12,
|
||||
"signal": "DB pool connections",
|
||||
"unit": "{connection}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"db_client_connection_pool_name",
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "db_client_connection_max",
|
||||
"series": 6,
|
||||
"signal": "DB pool max",
|
||||
"unit": "{connection}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"db_client_connection_pool_name",
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "db_client_connection_pending_requests",
|
||||
"series": 6,
|
||||
"signal": "DB pool pending",
|
||||
"unit": "{request}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"db_client_connection_pool_name",
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "db_client_connection_wait_time_seconds_count",
|
||||
"series": 6,
|
||||
"signal": "DB pool wait",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"db_operation_name",
|
||||
"db_query_summary",
|
||||
"db_system_name",
|
||||
"deployment_environment_name",
|
||||
"error_type",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "db_client_operation_duration_seconds_count",
|
||||
"series": 70,
|
||||
"signal": "DB operation duration",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_worker_jobs_active",
|
||||
"series": 2,
|
||||
"signal": "Worker active",
|
||||
"unit": "{job}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_worker_jobs_capacity",
|
||||
"series": 2,
|
||||
"signal": "Worker capacity",
|
||||
"unit": "{job}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"messaging_destination_name",
|
||||
"outcome",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "messenger_broker_published_total",
|
||||
"series": 4,
|
||||
"signal": "Broker published",
|
||||
"unit": "{message}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"outcome",
|
||||
"service_version",
|
||||
"signal"
|
||||
],
|
||||
"metric": "messenger_telemetry_export_items_total",
|
||||
"series": 12,
|
||||
"signal": "Telemetry export",
|
||||
"unit": "{item}"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"cpu_mode",
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "process_cpu_time_seconds_total",
|
||||
"series": 12,
|
||||
"signal": "Process CPU",
|
||||
"unit": "s"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "process_memory_usage_bytes",
|
||||
"series": 6,
|
||||
"signal": "Process RAM",
|
||||
"unit": "By"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "v8js_memory_heap_used_bytes",
|
||||
"series": 6,
|
||||
"signal": "V8 heap",
|
||||
"unit": "By"
|
||||
},
|
||||
{
|
||||
"labels": [
|
||||
"deployment_environment_name",
|
||||
"instance",
|
||||
"job",
|
||||
"service_version"
|
||||
],
|
||||
"metric": "nodejs_eventloop_delay_p99_seconds",
|
||||
"series": 6,
|
||||
"signal": "Event loop delay",
|
||||
"unit": "s"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user