From 1dd7b4a2e3593ba76299ba2a8f50ff8880f05c00 Mon Sep 17 00:00:00 2001 From: lab6 Date: Tue, 10 Nov 2026 02:01:35 +0300 Subject: [PATCH] lab6: EAI_AGAIN as unavailable, report, evidence, cold start --- REPORT.md | 7 +- evidence/lab6/cold-start/cold-checks.txt | 9 + evidence/lab6/cold-start/cold-up.txt | 6 + .../lab6/cold-start/correlation-failed.json | 28 ++ .../lab6/cold-start/correlation-http.json | 35 ++ .../lab6/cold-start/correlation-retry.json | 61 +++ .../lab6/cold-start/correlation-upload.json | 49 +++ evidence/lab6/cold-start/ready.json | 5 + evidence/lab6/cold-start/signals.json | 359 ++++++++++++++++++ 9 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 evidence/lab6/cold-start/cold-checks.txt create mode 100644 evidence/lab6/cold-start/cold-up.txt create mode 100644 evidence/lab6/cold-start/correlation-failed.json create mode 100644 evidence/lab6/cold-start/correlation-http.json create mode 100644 evidence/lab6/cold-start/correlation-retry.json create mode 100644 evidence/lab6/cold-start/correlation-upload.json create mode 100644 evidence/lab6/cold-start/ready.json create mode 100644 evidence/lab6/cold-start/signals.json diff --git a/REPORT.md b/REPORT.md index e476c6f..0297aa4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -79,7 +79,8 @@ Evidence — под `evidence/lab6/`; навигация — `evidence/lab6/summ | 4: два инцидента + диагностика | `make lab6-incident-worker lab6-incident-db` | PASS | `incident-*.json` | | 5: трек 1 | `make lab6-track1` | PASS, p25 выбран и провалидирован | `track1/` | | История/persistence backend | `make lab6-history` | старые sample/log/trace читаются после restart | `history.json` | -| Регрессия lab1–5 | `make check validate test test-e2e test-dependencies` | см. `regression/` | `regression/` | +| Регрессия lab1–5 | `make check validate test test-e2e test-dependencies test-persistence test-failover lab5-demo`; `npm test` | PASS: e2e 44 passed/9 skipped; dependencies 7/7; persistence (hook рестартует и postgres redis s3 broker api worker haproxy, и otel-collector prometheus loki tempo grafana); failover (0 failed attempts, recovery 3.3 s); lab5-demo; npm 89/89 | `regression/` | +| Холодный старт | отдельный clone коммита `d558148`, проект `lab6-cold`, порты 18080/18404/15432/16379/19000/19091/13000 (19090 занят чужим процессом) | `make stand-up`, `test`, `lab6-ready`, `lab6-correlate`, `lab6-signals` PASS; dashboards/datasources появились автоматически; проект удалён `down -v` только для `lab6-cold` | `cold-start/` | ## Эксперименты и отказы @@ -115,6 +116,10 @@ Evidence — под `evidence/lab6/`; навигация — `evidence/lab6/summ **Выбор.** p25: критерии приёмки пройдены (`selected.json`: все 6 проверок True: valid-clean, ≥1 error и slow в каждом run, 0 missing, spans ≤50% full, p95 ≤110% full). Независимая проверка на свежем run (`track1/validation/p25`, `selected-validation.json`): PASS против **зафиксированного** full-reference (1.39 spans/op ≤ 50% от 4.9; p95 13.27 мс ≤ 110% от 13.38). Это компромисс, не гарантия: любую конкретную редкую ошибку p25 сохраняет с вероятностью ≈25%. +### Дефект, найденный регрессией + +`make test-dependencies` (остановленный postgres/S3) сначала падал: из-за `dns_opt timeout:1 attempts:1` остановленный сервис давал `EAI_AGAIN`, который приложение отдавало 500 вместо 503. Исправлено добавлением `EAI_AGAIN` в `UNAVAILABLE_ERROR_CODES` (`api-exception.filter.ts`); после этого 7/7. Замеры трека 1 и инциденты сняты на образе до этой правки; она влияет только на путь недоступной зависимости, не на healthy-путь. + ## Ограничения и следующие шаги - Один хост (8 ядер, 15 GiB, занят другими контейнерами), фиксированный синтетический workload 20 rps, короткие окна 180 с, три повтора; p95 ~13 мс близок к шуму, поэтому различия latency/CPU между режимами не интерпретируются как эффект телеметрии. diff --git a/evidence/lab6/cold-start/cold-checks.txt b/evidence/lab6/cold-start/cold-checks.txt new file mode 100644 index 0000000..6d2dc75 --- /dev/null +++ b/evidence/lab6/cold-start/cold-checks.txt @@ -0,0 +1,9 @@ +stand is ready (http://localhost:18080) +python3 tests/smoke.py --base-url "http://localhost:18080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +python3 scripts/lab6/verify.py ready --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000" +PASS: ready +python3 scripts/lab6/verify.py correlation --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000" +PASS: correlation (http, upload, retry, failed) +python3 scripts/lab6/verify.py signals --base-url "http://localhost:18080" --prometheus-url "http://localhost:19091" --grafana-url "http://localhost:13000" +PASS: signals (23 logical signals) diff --git a/evidence/lab6/cold-start/cold-up.txt b/evidence/lab6/cold-start/cold-up.txt new file mode 100644 index 0000000..b6fe01e --- /dev/null +++ b/evidence/lab6/cold-start/cold-up.txt @@ -0,0 +1,6 @@ +ports: app18080 stats18404 pg15432 redis16379 s3 19000 prometheus19091(19090 busy: mihomo) grafana13000 + Container lab6-cold-api-1 Healthy + Container lab6-cold-s3-init-1 Exited + Container lab6-cold-haproxy-1 Healthy + Container lab6-cold-migrate-1 Exited + Container lab6-cold-worker-1 Healthy diff --git a/evidence/lab6/cold-start/correlation-failed.json b/evidence/lab6/cold-start/correlation-failed.json new file mode 100644 index 0000000..aa70995 --- /dev/null +++ b/evidence/lab6/cold-start/correlation-failed.json @@ -0,0 +1,28 @@ +{ + "attachment_id": "5217f331-2025-4b40-8cb9-0f903815a963", + "captured": "2026-11-09T23:00:57Z", + "consumer_spans": [ + { + "attempt": 1, + "instance": "54ab1af5724b", + "kind": "CONSUMER", + "name": "process attachments.process", + "outcome": "failed", + "parent_span_id": "d7c00750aa8307df", + "service": "messenger-worker", + "span_id": "496ff12ff42c7060", + "status": "ERROR" + } + ], + "final_attachment_status": "failed", + "final_job": { + "attempt": 1, + "last_error_code": "injected_fault", + "status": "failed" + }, + "job_id": "161ffedb-0a73-4834-b9cd-9e3720062dad", + "kind": "permanent", + "pipeline_version": 1, + "request_id": "fd8ea75b-a7fd-46d7-bff0-75093abefe92", + "trace_id": "0eeec1516eba748b30d404a147cc0033" +} diff --git a/evidence/lab6/cold-start/correlation-http.json b/evidence/lab6/cold-start/correlation-http.json new file mode 100644 index 0000000..3d968c0 --- /dev/null +++ b/evidence/lab6/cold-start/correlation-http.json @@ -0,0 +1,35 @@ +{ + "captured": "2026-11-09T23:00:57Z", + "db_children": [ + { + "instance": "b61243743d89", + "kind": "CLIENT", + "name": "SELECT users", + "parent_span_id": "050698d2997e9641", + "service": "messenger-api", + "span_id": "c598408f5d0834f5", + "status": "UNSET" + }, + { + "instance": "b61243743d89", + "kind": "CLIENT", + "name": "WITH idempotency_keys", + "parent_span_id": "050698d2997e9641", + "service": "messenger-api", + "span_id": "e44ee4f9da9e18c7", + "status": "UNSET" + } + ], + "log_span_id": "050698d2997e9641", + "request_id": "1df5f3c3-d3e7-4c8f-87c0-7332393a5182", + "server_span": { + "instance": "b61243743d89", + "kind": "SERVER", + "name": "POST /api/v1/chats/{chat_id}/messages", + "parent_span_id": "", + "service": "messenger-api", + "span_id": "050698d2997e9641", + "status": "UNSET" + }, + "trace_id": "611265fa918e94d3606f7521e20a9627" +} diff --git a/evidence/lab6/cold-start/correlation-retry.json b/evidence/lab6/cold-start/correlation-retry.json new file mode 100644 index 0000000..b629d7d --- /dev/null +++ b/evidence/lab6/cold-start/correlation-retry.json @@ -0,0 +1,61 @@ +{ + "attachment_id": "ddd0a403-af0b-447a-b41a-4ea2d791db6c", + "captured": "2026-11-09T23:00:57Z", + "consumer_spans": [ + { + "attempt": 1, + "instance": "54ab1af5724b", + "kind": "CONSUMER", + "name": "process attachments.process", + "outcome": "retry", + "parent_span_id": "be90f2e1ee549aa4", + "service": "messenger-worker", + "span_id": "33a7b57255493c51", + "status": "ERROR" + }, + { + "attempt": 2, + "instance": "54ab1af5724b", + "kind": "CONSUMER", + "name": "process attachments.process", + "outcome": "retry", + "parent_span_id": "5b73c06a02d4f844", + "service": "messenger-worker", + "span_id": "486c12ef2231fc33", + "status": "ERROR" + }, + { + "attempt": 3, + "instance": "54ab1af5724b", + "kind": "CONSUMER", + "name": "process attachments.process", + "outcome": "retry", + "parent_span_id": "a320a0009b317475", + "service": "messenger-worker", + "span_id": "d9ef4f0d6e0193f3", + "status": "ERROR" + }, + { + "attempt": 4, + "instance": "54ab1af5724b", + "kind": "CONSUMER", + "name": "process attachments.process", + "outcome": "retry", + "parent_span_id": "ad953e5ca632b9eb", + "service": "messenger-worker", + "span_id": "9a8c4f22e3ec9bbe", + "status": "ERROR" + } + ], + "final_attachment_status": "ready", + "final_job": { + "attempt": 5, + "last_error_code": "injected_fault", + "status": "succeeded" + }, + "job_id": "a4c71b25-74ee-431b-a595-4cee3cb153f8", + "kind": "transient", + "pipeline_version": 1, + "request_id": "06f85dce-b7a2-4991-bb90-8a70aa261ffb", + "trace_id": "16e58ce6545b82e3d8258b35eca400c1" +} diff --git a/evidence/lab6/cold-start/correlation-upload.json b/evidence/lab6/cold-start/correlation-upload.json new file mode 100644 index 0000000..a14c05f --- /dev/null +++ b/evidence/lab6/cold-start/correlation-upload.json @@ -0,0 +1,49 @@ +{ + "attachment_id": "7ad73537-cb62-4c33-90cb-765509f22205", + "captured": "2026-11-09T23:00:57Z", + "chain_consumer_to_root": [ + { + "instance": "54ab1af5724b", + "kind": "CONSUMER", + "name": "process attachments.process", + "parent_span_id": "643f5038a4f4d54c", + "service": "messenger-worker", + "span_id": "2d619e1f288ee544", + "status": "UNSET" + }, + { + "instance": "b61243743d89", + "kind": "PRODUCER", + "name": "publish attachments.jobs", + "parent_span_id": "4307e07bf374f047", + "service": "messenger-api", + "span_id": "643f5038a4f4d54c", + "status": "UNSET" + }, + { + "instance": "b61243743d89", + "kind": "CLIENT", + "name": "TRANSACTION jobs.accept_upload", + "parent_span_id": "111c15c5e3d4880c", + "service": "messenger-api", + "span_id": "4307e07bf374f047", + "status": "UNSET" + }, + { + "instance": "b61243743d89", + "kind": "SERVER", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "parent_span_id": "", + "service": "messenger-api", + "span_id": "111c15c5e3d4880c", + "status": "UNSET" + } + ], + "chain_fields_are_real_parent_child": true, + "consumer_attempt": 1, + "consumer_outcome": "succeeded", + "db_children": 22, + "request_id": "62e9c6b6-2851-4050-8b6f-6a60cbe05d0a", + "s3_children": 7, + "trace_id": "d5db164edb1e67da24c41a5432df159a" +} diff --git a/evidence/lab6/cold-start/ready.json b/evidence/lab6/cold-start/ready.json new file mode 100644 index 0000000..a3e4643 --- /dev/null +++ b/evidence/lab6/cold-start/ready.json @@ -0,0 +1,5 @@ +{ + "backend_only": false, + "finished": "2026-11-09T23:00:03Z", + "passed": true +} diff --git a/evidence/lab6/cold-start/signals.json b/evidence/lab6/cold-start/signals.json new file mode 100644 index 0000000..c4a1e34 --- /dev/null +++ b/evidence/lab6/cold-start/signals.json @@ -0,0 +1,359 @@ +{ + "api_replicas_with_http_metrics": [ + "907ec33d6af9", + "b61243743d89", + "cfba8008511e", + "ea4900bd5808" + ], + "captured": "2026-11-09T23:00:57Z", + "collector_series": { + "otelcol_exporter_enqueue_failed_log_records": 0, + "otelcol_exporter_enqueue_failed_metric_points": 0, + "otelcol_exporter_enqueue_failed_spans": 0, + "otelcol_exporter_queue_capacity": 3, + "otelcol_exporter_queue_size": 3, + "otelcol_exporter_send_failed_log_records": 1, + "otelcol_exporter_send_failed_metric_points": 1, + "otelcol_exporter_send_failed_spans": 1, + "otelcol_exporter_sent_log_records": 1, + "otelcol_exporter_sent_metric_points": 1, + "otelcol_exporter_sent_spans": 1, + "otelcol_receiver_accepted_log_records": 1, + "otelcol_receiver_accepted_metric_points": 1, + "otelcol_receiver_accepted_spans": 1 + }, + "job_outcomes_observed": [ + "failed", + "retry", + "succeeded" + ], + "passed": true, + "problems": [], + "queue_depth_ready_series": 4, + "route_templates": [ + "/api/v1/attachments/{attachment_id}", + "/api/v1/attachments/{attachment_id}/content", + "/api/v1/auth/sessions", + "/api/v1/auth/sessions/current", + "/api/v1/chats", + "/api/v1/chats/{chat_id}", + "/api/v1/chats/{chat_id}/attachments", + "/api/v1/chats/{chat_id}/messages", + "/api/v1/users", + "/api/v1/users/me", + "/health/live", + "/health/ready" + ], + "signals": [ + { + "labels": [ + "deployment_environment_name", + "http_request_method", + "http_response_status_code", + "http_route", + "instance", + "job", + "service_version" + ], + "metric": "http_server_request_duration_seconds_count", + "series": 48, + "signal": "HTTP duration/count", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "http_request_method", + "http_response_status_code", + "http_route", + "instance", + "job", + "le", + "service_version" + ], + "metric": "http_server_request_duration_seconds_bucket", + "series": 768, + "signal": "HTTP duration buckets", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "http_request_method", + "http_response_status_code", + "http_route", + "instance", + "job", + "service_version" + ], + "metric": "http_server_request_duration_seconds_sum", + "series": 48, + "signal": "HTTP duration sum", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "job_type", + "outcome", + "service_version" + ], + "metric": "messenger_jobs_processed_total", + "series": 4, + "signal": "Jobs processed", + "unit": "{job}" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "job_type", + "outcome", + "service_version" + ], + "metric": "messenger_job_duration_seconds_count", + "series": 4, + "signal": "Job duration", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "job_type", + "service_version" + ], + "metric": "messenger_attachment_time_to_ready_seconds_count", + "series": 2, + "signal": "Time to ready", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "messaging_destination_name", + "service_version", + "state" + ], + "metric": "messenger_jobs_queue_depth", + "series": 12, + "signal": "Queue depth", + "unit": "{job}" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "messaging_destination_name", + "service_version" + ], + "metric": "messenger_jobs_queue_oldest_age_seconds", + "series": 4, + "signal": "Queue oldest age", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "messaging_destination_name", + "service_version" + ], + "metric": "messenger_outbox_pending", + "series": 4, + "signal": "Outbox pending", + "unit": "{message}" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "messaging_destination_name", + "service_version" + ], + "metric": "messenger_outbox_oldest_age_seconds", + "series": 4, + "signal": "Outbox oldest age", + "unit": "s" + }, + { + "labels": [ + "db_client_connection_pool_name", + "db_client_connection_state", + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "db_client_connection_count", + "series": 12, + "signal": "DB pool connections", + "unit": "{connection}" + }, + { + "labels": [ + "db_client_connection_pool_name", + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "db_client_connection_max", + "series": 6, + "signal": "DB pool max", + "unit": "{connection}" + }, + { + "labels": [ + "db_client_connection_pool_name", + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "db_client_connection_pending_requests", + "series": 6, + "signal": "DB pool pending", + "unit": "{request}" + }, + { + "labels": [ + "db_client_connection_pool_name", + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "db_client_connection_wait_time_seconds_count", + "series": 6, + "signal": "DB pool wait", + "unit": "s" + }, + { + "labels": [ + "db_operation_name", + "db_query_summary", + "db_system_name", + "deployment_environment_name", + "error_type", + "instance", + "job", + "service_version" + ], + "metric": "db_client_operation_duration_seconds_count", + "series": 70, + "signal": "DB operation duration", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "messenger_worker_jobs_active", + "series": 2, + "signal": "Worker active", + "unit": "{job}" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "messenger_worker_jobs_capacity", + "series": 2, + "signal": "Worker capacity", + "unit": "{job}" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "messaging_destination_name", + "outcome", + "service_version" + ], + "metric": "messenger_broker_published_total", + "series": 4, + "signal": "Broker published", + "unit": "{message}" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "outcome", + "service_version", + "signal" + ], + "metric": "messenger_telemetry_export_items_total", + "series": 12, + "signal": "Telemetry export", + "unit": "{item}" + }, + { + "labels": [ + "cpu_mode", + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "process_cpu_time_seconds_total", + "series": 12, + "signal": "Process CPU", + "unit": "s" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "process_memory_usage_bytes", + "series": 6, + "signal": "Process RAM", + "unit": "By" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "v8js_memory_heap_used_bytes", + "series": 6, + "signal": "V8 heap", + "unit": "By" + }, + { + "labels": [ + "deployment_environment_name", + "instance", + "job", + "service_version" + ], + "metric": "nodejs_eventloop_delay_p99_seconds", + "series": 6, + "signal": "Event loop delay", + "unit": "s" + } + ] +}