lab6: EAI_AGAIN as unavailable, report, evidence, cold start

This commit is contained in:
lab6
2026-11-10 02:01:35 +03:00
parent d558148d0d
commit 1dd7b4a2e3
9 changed files with 558 additions and 1 deletions
+6 -1
View File
@@ -79,7 +79,8 @@ Evidence — под `evidence/lab6/`; навигация — `evidence/lab6/summ
| 4: два инцидента + диагностика | `make lab6-incident-worker lab6-incident-db` | PASS | `incident-*.json` |
| 5: трек 1 | `make lab6-track1` | PASS, p25 выбран и провалидирован | `track1/` |
| История/persistence backend | `make lab6-history` | старые sample/log/trace читаются после restart | `history.json` |
| Регрессия lab1–5 | `make check validate test test-e2e test-dependencies` | см. `regression/` | `regression/` |
| Регрессия lab1–5 | `make check validate test test-e2e test-dependencies test-persistence test-failover lab5-demo`; `npm test` | PASS: e2e 44 passed/9 skipped; dependencies 7/7; persistence (hook рестартует и postgres redis s3 broker api worker haproxy, и otel-collector prometheus loki tempo grafana); failover (0 failed attempts, recovery 3.3 s); lab5-demo; npm 89/89 | `regression/` |
| Холодный старт | отдельный clone коммита `d558148`, проект `lab6-cold`, порты 18080/18404/15432/16379/19000/19091/13000 (19090 занят чужим процессом) | `make stand-up`, `test`, `lab6-ready`, `lab6-correlate`, `lab6-signals` PASS; dashboards/datasources появились автоматически; проект удалён `down -v` только для `lab6-cold` | `cold-start/` |
## Эксперименты и отказы
@@ -115,6 +116,10 @@ Evidence — под `evidence/lab6/`; навигация — `evidence/lab6/summ
**Выбор.** p25: критерии приёмки пройдены (`selected.json`: все 6 проверок True: valid-clean, ≥1 error и slow в каждом run, 0 missing, spans ≤50% full, p95 ≤110% full). Независимая проверка на свежем run (`track1/validation/p25`, `selected-validation.json`): PASS против **зафиксированного** full-reference (1.39 spans/op ≤ 50% от 4.9; p95 13.27 мс ≤ 110% от 13.38). Это компромисс, не гарантия: любую конкретную редкую ошибку p25 сохраняет с вероятностью ≈25%.
### Дефект, найденный регрессией
`make test-dependencies` (остановленный postgres/S3) сначала падал: из-за `dns_opt timeout:1 attempts:1` остановленный сервис давал `EAI_AGAIN`, который приложение отдавало 500 вместо 503. Исправлено добавлением `EAI_AGAIN` в `UNAVAILABLE_ERROR_CODES` (`api-exception.filter.ts`); после этого 7/7. Замеры трека 1 и инциденты сняты на образе до этой правки; она влияет только на путь недоступной зависимости, не на healthy-путь.
## Ограничения и следующие шаги
- Один хост (8 ядер, 15 GiB, занят другими контейнерами), фиксированный синтетический workload 20 rps, короткие окна 180 с, три повтора; p95 ~13 мс близок к шуму, поэтому различия latency/CPU между режимами не интерпретируются как эффект телеметрии.
+9
View File
@@ -0,0 +1,9 @@
stand is ready (http://localhost:18080)
python3 tests/smoke.py --base-url "http://localhost:18080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
python3 scripts/lab6/verify.py ready --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000"
PASS: ready
python3 scripts/lab6/verify.py correlation --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000"
PASS: correlation (http, upload, retry, failed)
python3 scripts/lab6/verify.py signals --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000"
PASS: signals (23 logical signals)
+6
View File
@@ -0,0 +1,6 @@
ports: app18080 stats18404 pg15432 redis16379 s3 19000 prometheus19091(19090 busy: mihomo) grafana13000
Container lab6-cold-api-1 Healthy
Container lab6-cold-s3-init-1 Exited
Container lab6-cold-haproxy-1 Healthy
Container lab6-cold-migrate-1 Exited
Container lab6-cold-worker-1 Healthy
@@ -0,0 +1,28 @@
{
"attachment_id": "5217f331-2025-4b40-8cb9-0f903815a963",
"captured": "2026-11-09T23:00:57Z",
"consumer_spans": [
{
"attempt": 1,
"instance": "54ab1af5724b",
"kind": "CONSUMER",
"name": "process attachments.process",
"outcome": "failed",
"parent_span_id": "d7c00750aa8307df",
"service": "messenger-worker",
"span_id": "496ff12ff42c7060",
"status": "ERROR"
}
],
"final_attachment_status": "failed",
"final_job": {
"attempt": 1,
"last_error_code": "injected_fault",
"status": "failed"
},
"job_id": "161ffedb-0a73-4834-b9cd-9e3720062dad",
"kind": "permanent",
"pipeline_version": 1,
"request_id": "fd8ea75b-a7fd-46d7-bff0-75093abefe92",
"trace_id": "0eeec1516eba748b30d404a147cc0033"
}
@@ -0,0 +1,35 @@
{
"captured": "2026-11-09T23:00:57Z",
"db_children": [
{
"instance": "b61243743d89",
"kind": "CLIENT",
"name": "SELECT users",
"parent_span_id": "050698d2997e9641",
"service": "messenger-api",
"span_id": "c598408f5d0834f5",
"status": "UNSET"
},
{
"instance": "b61243743d89",
"kind": "CLIENT",
"name": "WITH idempotency_keys",
"parent_span_id": "050698d2997e9641",
"service": "messenger-api",
"span_id": "e44ee4f9da9e18c7",
"status": "UNSET"
}
],
"log_span_id": "050698d2997e9641",
"request_id": "1df5f3c3-d3e7-4c8f-87c0-7332393a5182",
"server_span": {
"instance": "b61243743d89",
"kind": "SERVER",
"name": "POST /api/v1/chats/{chat_id}/messages",
"parent_span_id": "",
"service": "messenger-api",
"span_id": "050698d2997e9641",
"status": "UNSET"
},
"trace_id": "611265fa918e94d3606f7521e20a9627"
}
@@ -0,0 +1,61 @@
{
"attachment_id": "ddd0a403-af0b-447a-b41a-4ea2d791db6c",
"captured": "2026-11-09T23:00:57Z",
"consumer_spans": [
{
"attempt": 1,
"instance": "54ab1af5724b",
"kind": "CONSUMER",
"name": "process attachments.process",
"outcome": "retry",
"parent_span_id": "be90f2e1ee549aa4",
"service": "messenger-worker",
"span_id": "33a7b57255493c51",
"status": "ERROR"
},
{
"attempt": 2,
"instance": "54ab1af5724b",
"kind": "CONSUMER",
"name": "process attachments.process",
"outcome": "retry",
"parent_span_id": "5b73c06a02d4f844",
"service": "messenger-worker",
"span_id": "486c12ef2231fc33",
"status": "ERROR"
},
{
"attempt": 3,
"instance": "54ab1af5724b",
"kind": "CONSUMER",
"name": "process attachments.process",
"outcome": "retry",
"parent_span_id": "a320a0009b317475",
"service": "messenger-worker",
"span_id": "d9ef4f0d6e0193f3",
"status": "ERROR"
},
{
"attempt": 4,
"instance": "54ab1af5724b",
"kind": "CONSUMER",
"name": "process attachments.process",
"outcome": "retry",
"parent_span_id": "ad953e5ca632b9eb",
"service": "messenger-worker",
"span_id": "9a8c4f22e3ec9bbe",
"status": "ERROR"
}
],
"final_attachment_status": "ready",
"final_job": {
"attempt": 5,
"last_error_code": "injected_fault",
"status": "succeeded"
},
"job_id": "a4c71b25-74ee-431b-a595-4cee3cb153f8",
"kind": "transient",
"pipeline_version": 1,
"request_id": "06f85dce-b7a2-4991-bb90-8a70aa261ffb",
"trace_id": "16e58ce6545b82e3d8258b35eca400c1"
}
@@ -0,0 +1,49 @@
{
"attachment_id": "7ad73537-cb62-4c33-90cb-765509f22205",
"captured": "2026-11-09T23:00:57Z",
"chain_consumer_to_root": [
{
"instance": "54ab1af5724b",
"kind": "CONSUMER",
"name": "process attachments.process",
"parent_span_id": "643f5038a4f4d54c",
"service": "messenger-worker",
"span_id": "2d619e1f288ee544",
"status": "UNSET"
},
{
"instance": "b61243743d89",
"kind": "PRODUCER",
"name": "publish attachments.jobs",
"parent_span_id": "4307e07bf374f047",
"service": "messenger-api",
"span_id": "643f5038a4f4d54c",
"status": "UNSET"
},
{
"instance": "b61243743d89",
"kind": "CLIENT",
"name": "TRANSACTION jobs.accept_upload",
"parent_span_id": "111c15c5e3d4880c",
"service": "messenger-api",
"span_id": "4307e07bf374f047",
"status": "UNSET"
},
{
"instance": "b61243743d89",
"kind": "SERVER",
"name": "POST /api/v1/chats/{chat_id}/attachments",
"parent_span_id": "",
"service": "messenger-api",
"span_id": "111c15c5e3d4880c",
"status": "UNSET"
}
],
"chain_fields_are_real_parent_child": true,
"consumer_attempt": 1,
"consumer_outcome": "succeeded",
"db_children": 22,
"request_id": "62e9c6b6-2851-4050-8b6f-6a60cbe05d0a",
"s3_children": 7,
"trace_id": "d5db164edb1e67da24c41a5432df159a"
}
+5
View File
@@ -0,0 +1,5 @@
{
"backend_only": false,
"finished": "2026-11-09T23:00:03Z",
"passed": true
}
+359
View File
@@ -0,0 +1,359 @@
{
"api_replicas_with_http_metrics": [
"907ec33d6af9",
"b61243743d89",
"cfba8008511e",
"ea4900bd5808"
],
"captured": "2026-11-09T23:00:57Z",
"collector_series": {
"otelcol_exporter_enqueue_failed_log_records": 0,
"otelcol_exporter_enqueue_failed_metric_points": 0,
"otelcol_exporter_enqueue_failed_spans": 0,
"otelcol_exporter_queue_capacity": 3,
"otelcol_exporter_queue_size": 3,
"otelcol_exporter_send_failed_log_records": 1,
"otelcol_exporter_send_failed_metric_points": 1,
"otelcol_exporter_send_failed_spans": 1,
"otelcol_exporter_sent_log_records": 1,
"otelcol_exporter_sent_metric_points": 1,
"otelcol_exporter_sent_spans": 1,
"otelcol_receiver_accepted_log_records": 1,
"otelcol_receiver_accepted_metric_points": 1,
"otelcol_receiver_accepted_spans": 1
},
"job_outcomes_observed": [
"failed",
"retry",
"succeeded"
],
"passed": true,
"problems": [],
"queue_depth_ready_series": 4,
"route_templates": [
"/api/v1/attachments/{attachment_id}",
"/api/v1/attachments/{attachment_id}/content",
"/api/v1/auth/sessions",
"/api/v1/auth/sessions/current",
"/api/v1/chats",
"/api/v1/chats/{chat_id}",
"/api/v1/chats/{chat_id}/attachments",
"/api/v1/chats/{chat_id}/messages",
"/api/v1/users",
"/api/v1/users/me",
"/health/live",
"/health/ready"
],
"signals": [
{
"labels": [
"deployment_environment_name",
"http_request_method",
"http_response_status_code",
"http_route",
"instance",
"job",
"service_version"
],
"metric": "http_server_request_duration_seconds_count",
"series": 48,
"signal": "HTTP duration/count",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"http_request_method",
"http_response_status_code",
"http_route",
"instance",
"job",
"le",
"service_version"
],
"metric": "http_server_request_duration_seconds_bucket",
"series": 768,
"signal": "HTTP duration buckets",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"http_request_method",
"http_response_status_code",
"http_route",
"instance",
"job",
"service_version"
],
"metric": "http_server_request_duration_seconds_sum",
"series": 48,
"signal": "HTTP duration sum",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"job_type",
"outcome",
"service_version"
],
"metric": "messenger_jobs_processed_total",
"series": 4,
"signal": "Jobs processed",
"unit": "{job}"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"job_type",
"outcome",
"service_version"
],
"metric": "messenger_job_duration_seconds_count",
"series": 4,
"signal": "Job duration",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"job_type",
"service_version"
],
"metric": "messenger_attachment_time_to_ready_seconds_count",
"series": 2,
"signal": "Time to ready",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"messaging_destination_name",
"service_version",
"state"
],
"metric": "messenger_jobs_queue_depth",
"series": 12,
"signal": "Queue depth",
"unit": "{job}"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"messaging_destination_name",
"service_version"
],
"metric": "messenger_jobs_queue_oldest_age_seconds",
"series": 4,
"signal": "Queue oldest age",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"messaging_destination_name",
"service_version"
],
"metric": "messenger_outbox_pending",
"series": 4,
"signal": "Outbox pending",
"unit": "{message}"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"messaging_destination_name",
"service_version"
],
"metric": "messenger_outbox_oldest_age_seconds",
"series": 4,
"signal": "Outbox oldest age",
"unit": "s"
},
{
"labels": [
"db_client_connection_pool_name",
"db_client_connection_state",
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "db_client_connection_count",
"series": 12,
"signal": "DB pool connections",
"unit": "{connection}"
},
{
"labels": [
"db_client_connection_pool_name",
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "db_client_connection_max",
"series": 6,
"signal": "DB pool max",
"unit": "{connection}"
},
{
"labels": [
"db_client_connection_pool_name",
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "db_client_connection_pending_requests",
"series": 6,
"signal": "DB pool pending",
"unit": "{request}"
},
{
"labels": [
"db_client_connection_pool_name",
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "db_client_connection_wait_time_seconds_count",
"series": 6,
"signal": "DB pool wait",
"unit": "s"
},
{
"labels": [
"db_operation_name",
"db_query_summary",
"db_system_name",
"deployment_environment_name",
"error_type",
"instance",
"job",
"service_version"
],
"metric": "db_client_operation_duration_seconds_count",
"series": 70,
"signal": "DB operation duration",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "messenger_worker_jobs_active",
"series": 2,
"signal": "Worker active",
"unit": "{job}"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "messenger_worker_jobs_capacity",
"series": 2,
"signal": "Worker capacity",
"unit": "{job}"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"messaging_destination_name",
"outcome",
"service_version"
],
"metric": "messenger_broker_published_total",
"series": 4,
"signal": "Broker published",
"unit": "{message}"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"outcome",
"service_version",
"signal"
],
"metric": "messenger_telemetry_export_items_total",
"series": 12,
"signal": "Telemetry export",
"unit": "{item}"
},
{
"labels": [
"cpu_mode",
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "process_cpu_time_seconds_total",
"series": 12,
"signal": "Process CPU",
"unit": "s"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "process_memory_usage_bytes",
"series": 6,
"signal": "Process RAM",
"unit": "By"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "v8js_memory_heap_used_bytes",
"series": 6,
"signal": "V8 heap",
"unit": "By"
},
{
"labels": [
"deployment_environment_name",
"instance",
"job",
"service_version"
],
"metric": "nodejs_eventloop_delay_p99_seconds",
"series": 6,
"signal": "Event loop delay",
"unit": "s"
}
]
}