From 2903d9dc73171d988ed0a1118b42ba8e13eaa088 Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 12:41:06 +0300 Subject: [PATCH] Verify the grade 3 telemetry stack and inherited API guarantees --- Makefile | 2 +- evidence/grade3/cardinality.txt | 5 + evidence/grade3/dashboards.txt | 153 +++++++++ evidence/grade3/docker-background.txt | 0 evidence/grade3/failover-first.txt | 43 +++ .../failover-first/failover-9af2280c8af3.txt | 38 +++ evidence/grade3/failover-second.txt | 43 +++ .../failover-second/failover-943813ecb712.txt | 38 +++ evidence/grade3/grafana.txt | 42 +++ evidence/grade3/job-stable.txt | 316 ++++++++++++++++++ evidence/grade3/job.txt | 5 + evidence/grade3/lab5-minimum.txt | 18 + evidence/grade3/leaks.txt | 27 ++ evidence/grade3/notes.md | 24 ++ evidence/grade3/obs-validate.txt | 16 + evidence/grade3/persistence.txt | 7 + evidence/grade3/quality.txt | 14 + evidence/grade3/request.txt | 66 ++++ evidence/grade3/runtime.json | 102 ++++++ evidence/grade3/smoke-ready.txt | 6 + evidence/grade3/smoke.txt | 7 + evidence/grade3/stand.txt | 17 + evidence/grade3/student-tests.txt | 62 ++++ evidence/grade3/telemetry-recovery.txt | 40 +++ evidence/grade3/up.txt | 230 +++++++++++++ evidence/level3.json | 270 +++++++++++++++ .../specs/lab6-telemetry-stack/spec.md | 2 +- .../add-lab6-telemetry-pipeline/tasks.md | 18 +- scripts/lab6.sh | 13 + scripts/obs_check.py | 10 +- scripts/obs_recovery.py | 98 ++++++ 31 files changed, 1720 insertions(+), 12 deletions(-) create mode 100644 evidence/grade3/cardinality.txt create mode 100644 evidence/grade3/dashboards.txt create mode 100644 evidence/grade3/docker-background.txt create mode 100644 evidence/grade3/failover-first.txt create mode 100644 evidence/grade3/failover-first/failover-9af2280c8af3.txt create mode 100644 evidence/grade3/failover-second.txt create mode 100644 evidence/grade3/failover-second/failover-943813ecb712.txt create mode 100644 evidence/grade3/grafana.txt create mode 100644 evidence/grade3/job-stable.txt create mode 100644 evidence/grade3/job.txt create mode 100644 evidence/grade3/lab5-minimum.txt create mode 100644 evidence/grade3/leaks.txt create mode 100644 evidence/grade3/notes.md create mode 100644 evidence/grade3/obs-validate.txt create mode 100644 evidence/grade3/persistence.txt create mode 100644 evidence/grade3/quality.txt create mode 100644 evidence/grade3/request.txt create mode 100644 evidence/grade3/runtime.json create mode 100644 evidence/grade3/smoke-ready.txt create mode 100644 evidence/grade3/smoke.txt create mode 100644 evidence/grade3/stand.txt create mode 100644 evidence/grade3/student-tests.txt create mode 100644 evidence/grade3/telemetry-recovery.txt create mode 100644 evidence/grade3/up.txt create mode 100644 evidence/level3.json create mode 100755 scripts/lab6.sh create mode 100644 scripts/obs_recovery.py diff --git a/Makefile b/Makefile index 2afa8bb5..ecf62c25 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMPOSE ?= docker compose RATE ?= 100 FROM_REV ?= SHIFT ?= 0 -SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ +SCRIPTS_PY = scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \ scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py SCENARIO ?= failover-upload diff --git a/evidence/grade3/cardinality.txt b/evidence/grade3/cardinality.txt new file mode 100644 index 00000000..38c90b48 --- /dev/null +++ b/evidence/grade3/cardinality.txt @@ -0,0 +1,5 @@ +date: 2026-11-06T09:38:20Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py query 'count by (__name__,job) ({job=~"messenger-.*"})' 'count by (http_route,http_request_method,http_response_status_code) (http_server_request_duration_seconds_count)' 'count by (job_type,outcome) (messenger_jobs_processed_total)' +{"ts": "2026-11-06T09:38:20+00:00", "results": {"count by (__name__,job) ({job=~\"messenger-.*\"})": [{"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "77"}, {"labels": {"job": "messenger-worker"}, "value": "4"}, {"labels": {"job": "messenger-api"}, "value": "17"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "990"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "1056"}, {"labels": {"job": "messenger-api"}, "value": "34"}, {"labels": {"job": "messenger-api"}, "value": "34"}, {"labels": {"job": "messenger-api"}, "value": "544"}, {"labels": {"job": "messenger-api"}, "value": "8"}], "count by (http_route,http_request_method,http_response_status_code) (http_server_request_duration_seconds_count)": [{"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/auth/sessions"}, "value": "5"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "409", "http_route": "/api/v1/users"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/chats/{chat_id}"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/chats"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "5"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "3"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "400", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "2"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "400", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "401", "http_route": "/api/v1/users/me"}, "value": "2"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/users/me"}, "value": "4"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "409", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "202", "http_route": "/api/v1/chats/{chat_id}/attachments"}, "value": "7"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/attachments/{attachment_id}"}, "value": "7"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "5"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "1"}, {"labels": {"http_request_method": "DELETE", "http_response_status_code": "204", "http_route": "/api/v1/auth/sessions/current"}, "value": "2"}, {"labels": {"http_request_method": "DELETE", "http_response_status_code": "401", "http_route": "/api/v1/auth/sessions/current"}, "value": "1"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/users"}, "value": "5"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/chats"}, "value": "4"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "404", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "1"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "401", "http_route": "/api/v1/auth/sessions"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/attachments/{attachment_id}"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "409", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "415", "http_route": "/api/v1/chats/{chat_id}/attachments"}, "value": "1"}], "count by (job_type,outcome) (messenger_jobs_processed_total)": [{"labels": {"job_type": "attachment.process.v1", "outcome": "done"}, "value": "3"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "duplicate_noop"}, "value": "1"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "retry_scheduled"}, "value": "2"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "failed"}, "value": "1"}]}} +exit_code: 0 elapsed_seconds: 0.057 diff --git a/evidence/grade3/dashboards.txt b/evidence/grade3/dashboards.txt new file mode 100644 index 00000000..82fda580 --- /dev/null +++ b/evidence/grade3/dashboards.txt @@ -0,0 +1,153 @@ +date: 2026-11-06T09:38:20Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py dashboards --strict +{ + "checked_at": "2026-11-06T09:38:20+00:00", + "targets": [ + { + "dashboard": "messenger-api.json", + "panel": "RPS бизнес-запросов", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля 5xx (2m)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "p95 latency (5m)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля не-5xx (окно 1h)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "RPS по маршруту и классу статуса", + "datasource": "prometheus", + "series": 9, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля ошибок", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля ошибок", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Latency p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Latency p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Latency p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "p95 по маршруту", + "datasource": "prometheus", + "series": 7, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "RPS по реплике", + "datasource": "prometheus", + "series": 4, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Активные запросы", + "datasource": "prometheus", + "series": 8, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Логи API (ошибки и предупреждения)", + "datasource": "loki", + "lines": 5 + }, + { + "dashboard": "messenger-api.json", + "panel": "Трассы API с ошибкой", + "datasource": "tempo", + "traces": 0 + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Jobs по исходу", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Длительность обработки job", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Длительность обработки job", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Длительность обработки job", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Логи worker", + "datasource": "loki", + "lines": 5 + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Трассы worker", + "datasource": "tempo", + "traces": 5 + } + ], + "problems": [] +} +exit_code: 0 elapsed_seconds: 0.089 diff --git a/evidence/grade3/docker-background.txt b/evidence/grade3/docker-background.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/grade3/failover-first.txt b/evidence/grade3/failover-first.txt new file mode 100644 index 00000000..9df096f2 --- /dev/null +++ b/evidence/grade3/failover-first.txt @@ -0,0 +1,43 @@ +date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ env EVIDENCE_DIR=evidence/grade3/failover-first make failover-evidence SHIFT=0 +sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "0" +date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +9af2280c8af3 running started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:33:50.075450885Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 3.03} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +9af2280c8af3 exited started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +== stopped replica: 9af2280c8af3; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:35.851311Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.864420Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.897236Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"5818535d613b4b7a8f0607d91e58d3f5","duration_ms":0.837,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.062326Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"4838cbd4822f4181a2e10153a41033d6","duration_ms":0.347,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.566176Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.866622Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.900282Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:37.973761Z"} +{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T09:34:37.977541Z"} +{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:37.987715Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:38.004692Z"} +{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:38.006586Z"} +{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:38.093383Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:38.195744Z"} +$ scripts/restore.sh + Container mtusi2026lab6-api-2 Running + Container mtusi2026lab6-api-1 Starting + Container mtusi2026lab6-api-1 Started + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-api-1 Healthy +== after restore +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +exit_code: 0 elapsed_seconds: 20.584 diff --git a/evidence/grade3/failover-first/failover-9af2280c8af3.txt b/evidence/grade3/failover-first/failover-9af2280c8af3.txt new file mode 100644 index 00000000..291c8ba9 --- /dev/null +++ b/evidence/grade3/failover-first/failover-9af2280c8af3.txt @@ -0,0 +1,38 @@ +date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +9af2280c8af3 running started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:33:50.075450885Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 3.03} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +9af2280c8af3 exited started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +== stopped replica: 9af2280c8af3; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:35.851311Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.864420Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.897236Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"5818535d613b4b7a8f0607d91e58d3f5","duration_ms":0.837,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.062326Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"4838cbd4822f4181a2e10153a41033d6","duration_ms":0.347,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.566176Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.866622Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.900282Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:37.973761Z"} +{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T09:34:37.977541Z"} +{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:37.987715Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:38.004692Z"} +{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:38.006586Z"} +{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:38.093383Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:38.195744Z"} +$ scripts/restore.sh + Container mtusi2026lab6-api-2 Running + Container mtusi2026lab6-api-1 Starting + Container mtusi2026lab6-api-1 Started + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-api-1 Healthy +== after restore +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z diff --git a/evidence/grade3/failover-second.txt b/evidence/grade3/failover-second.txt new file mode 100644 index 00000000..18324be5 --- /dev/null +++ b/evidence/grade3/failover-second.txt @@ -0,0 +1,43 @@ +date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ env EVIDENCE_DIR=evidence/grade3/failover-second make failover-evidence SHIFT=1 +sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "1" +date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.777} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 exited started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:34:58.204116764Z +== stopped replica: 943813ecb712; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:55.700248Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.784538Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.795484Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"ff6952bfefbe4034ae30489d4e43b93c","duration_ms":0.567,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.029040Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"194add29fb3742db93424a18cf9ff680","duration_ms":1.596,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.535013Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.787659Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.796586Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.894245Z"} +{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:57.896210Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.903471Z"} +{"event":"Finished server process [23]","level":"info","ts":"2026-11-06T09:34:57.905749Z"} +{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954114Z"} +{"event":"Worker (pid:23) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954212Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:58.057111Z"} +$ scripts/restore.sh + Container mtusi2026lab6-api-1 Running + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-api-1 Healthy +== after restore +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:35:11.260329797Z finished=2026-11-06T09:34:58.204116764Z +exit_code: 0 elapsed_seconds: 20.372 diff --git a/evidence/grade3/failover-second/failover-943813ecb712.txt b/evidence/grade3/failover-second/failover-943813ecb712.txt new file mode 100644 index 00000000..70dc612c --- /dev/null +++ b/evidence/grade3/failover-second/failover-943813ecb712.txt @@ -0,0 +1,38 @@ +date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.777} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 exited started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:34:58.204116764Z +== stopped replica: 943813ecb712; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:55.700248Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.784538Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.795484Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"ff6952bfefbe4034ae30489d4e43b93c","duration_ms":0.567,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.029040Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"194add29fb3742db93424a18cf9ff680","duration_ms":1.596,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.535013Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.787659Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.796586Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.894245Z"} +{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:57.896210Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.903471Z"} +{"event":"Finished server process [23]","level":"info","ts":"2026-11-06T09:34:57.905749Z"} +{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954114Z"} +{"event":"Worker (pid:23) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954212Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:58.057111Z"} +$ scripts/restore.sh + Container mtusi2026lab6-api-1 Running + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-api-1 Healthy +== after restore +9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z +943813ecb712 running started=2026-11-06T09:35:11.260329797Z finished=2026-11-06T09:34:58.204116764Z diff --git a/evidence/grade3/grafana.txt b/evidence/grade3/grafana.txt new file mode 100644 index 00000000..828285c5 --- /dev/null +++ b/evidence/grade3/grafana.txt @@ -0,0 +1,42 @@ +date: 2026-11-06T09:33:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py grafana +{ + "checked_at": "2026-11-06T09:33:43+00:00", + "grafana": "12.4.14", + "datasources": { + "prometheus": { + "http": 200, + "status": "OK", + "message": "Successfully queried the Prometheus API." + }, + "loki": { + "http": 200, + "status": "OK", + "message": "Data source successfully connected." + }, + "tempo": { + "http": 200, + "status": "OK", + "message": "Data source is working. Streaming test succeeded." + } + }, + "dashboards": { + "messenger-api": { + "title": "Messenger · API RED", + "http": 200, + "provisioned": true, + "folder": "Messenger", + "panels": 14 + }, + "messenger-pipeline": { + "title": "Messenger · Worker", + "http": 200, + "provisioned": true, + "folder": "Messenger", + "panels": 6 + } + }, + "missing_dashboards": [] +} +exit_code: 0 elapsed_seconds: 0.124 diff --git a/evidence/grade3/job-stable.txt b/evidence/grade3/job-stable.txt new file mode 100644 index 00000000..a70f1790 --- /dev/null +++ b/evidence/grade3/job-stable.txt @@ -0,0 +1,316 @@ +date: 2026-11-06T09:37:45Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py job +{ + "checked_at": "2026-11-06T09:38:05+00:00", + "upload_request_id": "6254becc-0433-44ae-89ea-057e0405f827", + "attachment_status": "ready", + "traces": [ + "a020913eec288efb8126ca3a69a513d4" + ], + "spans": [ + { + "service": "messenger-worker", + "name": "attachments.attachment.process.v1 create", + "kind": "SPAN_KIND_CONSUMER", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.008, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "attachments.attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "status": "STATUS_CODE_UNSET", + "duration_ms": 777.9, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": "1" + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.133, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "select", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.111, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "select", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.095, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "show", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.085, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "show", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.081, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.081, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.082, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.554, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.57, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.199, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.108, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.331, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.08, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.155, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.07, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.062, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.112, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.171, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.124, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.069, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.054, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.073, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.179, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.414, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.18, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.116, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.307, + "links": 0, + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "attempt": null + } + ], + "worker_logs": [ + { + "line": "job.processed", + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "span_id": "ff3e55c7d762c0e1" + }, + { + "line": "audit.job_settled", + "trace_id": "a020913eec288efb8126ca3a69a513d4", + "span_id": "ff3e55c7d762c0e1" + } + ] +} +exit_code: 0 elapsed_seconds: 19.790 diff --git a/evidence/grade3/job.txt b/evidence/grade3/job.txt new file mode 100644 index 00000000..a4bda401 --- /dev/null +++ b/evidence/grade3/job.txt @@ -0,0 +1,5 @@ +date: 2026-11-06T09:33:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py job +FAIL: +exit_code: 1 elapsed_seconds: 0.055 diff --git a/evidence/grade3/lab5-minimum.txt b/evidence/grade3/lab5-minimum.txt new file mode 100644 index 00000000..59a3b5b4 --- /dev/null +++ b/evidence/grade3/lab5-minimum.txt @@ -0,0 +1,18 @@ +date: 2026-11-06T09:35:15Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make level3 +BASE_URL="http://127.0.0.1:18086" python3 scripts/pipeline_suite.py --level 3 --output evidence/level3.json +RUN smoke_pipeline +PASS smoke_pipeline +RUN worker_down_broker_restart +PASS worker_down_broker_restart +RUN duplicate_after_ready +PASS duplicate_after_ready +RUN poison_image +PASS poison_image +RUN s3_outage_recovers +PASS s3_outage_recovers +RUN retry_exhausted +PASS retry_exhausted +PASS: lab5 pipeline suite, report evidence/level3.json +exit_code: 0 elapsed_seconds: 76.738 diff --git a/evidence/grade3/leaks.txt b/evidence/grade3/leaks.txt new file mode 100644 index 00000000..05fc816d --- /dev/null +++ b/evidence/grade3/leaks.txt @@ -0,0 +1,27 @@ +date: 2026-11-06T09:38:05Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py leaks +{ + "checked_at": "2026-11-06T09:38:20+00:00", + "window_seconds": 20, + "traces_scanned": 9, + "hits": { + "password": { + "loki": 0, + "tempo": 0 + }, + "token": { + "loki": 0, + "tempo": 0 + }, + "message_text": { + "loki": 0, + "tempo": 0 + }, + "filename": { + "loki": 0, + "tempo": 0 + } + } +} +exit_code: 0 elapsed_seconds: 15.275 diff --git a/evidence/grade3/notes.md b/evidence/grade3/notes.md new file mode 100644 index 00000000..6e208832 --- /dev/null +++ b/evidence/grade3/notes.md @@ -0,0 +1,24 @@ +# Уровень 3 — сведения для отдельного отчёта + +Приложение проверено на SQLAlchemy-коде 95511002b12510178a3cc2a22566b43e2b2b6c81. Новый код этой фазы — обвязка запуска и строгая проверка реальных spans. Команды запускаются через `scripts/lab6.sh`, задающий только порты ЛР6, project и NO_PROXY; значения secrets не выводятся. Дата/SHA и exit_code есть в каждом command log. + +| Критерий | Команда и доказательство | +|---|---| +| Минимум lab5 | make test: smoke-ready.txt; make test-persistence: persistence.txt; failover каждой реплики: failover-first.txt/failover-second.txt; make level3: lab5-minimum.txt и ../level3.json (6 PASS) | +| Конфигурации и provisioning | make obs-validate: obs-validate.txt; obs_check.py grafana: grafana.txt (3 OK, 2 dashboards provisioned); runtime.json (non-root, лимиты, volumes, порты) | +| RED и worker metrics | obs_check.py dashboards --strict: dashboards.txt; cardinality.txt: реальные HTTP-шаблоны, done/failed/retry_scheduled/duplicate_noop | +| Запрос → лог → trace | obs_check.py request: request.txt, реальный POST server span и PostgreSQL client spans | +| Job → лог → trace | obs_check.py job: job-stable.txt, ready, consumer spans и trace/span IDs worker | +| Отсутствие содержимого | obs_check.py leaks: leaks.txt, все 4 маркера дают 0 в Loki и Tempo | +| История и независимость API | obs_recovery.py: telemetry-recovery.txt, прежние trace/log/metric после restart; smoke при остановленном Collector; SIGTERM API 4.592 с, exit 0 | +| Проверки кода | quality.txt: check/validate/lint PASS; student-tests.txt: 515 passed, 1 skipped, 19 deselected | + +Collector принимает OTLP/HTTP и обрабатывает/экспортирует сигналы, не заменяя долговременную БД. Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы. Grafana запрашивает эти хранилища и связывает сигналы через provisioned datasources. SDK запускается в процессе API после fork и в отдельных relay/worker. Batch processors и periodic reader не блокируют запрос на экспорт. + +HTTP: `http_server_request_duration_seconds_{bucket,sum,count}`, labels `http_route`, `http_request_method`, `http_response_status_code`, service resource. RPS и классы статуса: `messenger:http_requests:rate2m`; error ratio — 5xx/все бизнес-запросы, health исключены. Worker: `messenger_jobs_processed_total`, `messenger_job_duration_seconds_{bucket,sum,count}`, labels `job_type,outcome`. Секунды; HTTP percentiles вычисляются histogram_quantile по сумме buckets реплик. Loki индексирует только service_name и environment; IDs остаются structured metadata. SDK не захватывает HTTP-заголовки и параметры БД, Collector дополнительно redacts подозрительные поля. + +## Ограничения и неуспешные попытки + +smoke.txt запущен до завершения up --wait и закономерно получил connection refused; smoke-ready.txt — исправленный порядок. job.txt попал в рестарт persistence; job-stable.txt выполнен отдельно и прошёл. Эти неудачи сохранены и не объявлены PASS. Полный pipeline/failover выполнялся последовательно. Unit/integration tests использовали свои одноразовые контейнеры одновременно только с проверками чтения телеметрии; нагрузочных замеров уровня 5 здесь нет. Linux-only RLIMIT_AS пропущен на macOS, stack-маркер штатно исключил 19 тестов; обязательные публичные отказы выполнены отдельно. + +Minor из переноса: scripts/limits.py и CI-allowlist ещё не знают сервисы OTel; проверка реальных ограничений выполнена независимо в runtime.json. REPORT.md оставлен отдельному отчётному агенту. diff --git a/evidence/grade3/obs-validate.txt b/evidence/grade3/obs-validate.txt new file mode 100644 index 00000000..e85c5bbe --- /dev/null +++ b/evidence/grade3/obs-validate.txt @@ -0,0 +1,16 @@ +date: 2026-11-06T09:31:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make obs-validate +sh scripts/obs-validate.sh +otelcol-contrib validate collector.yaml: OK +Checking /etc/prometheus/prometheus.yml + SUCCESS: 1 rule files found + SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax + +Checking /etc/prometheus/rules/messenger.recording.yml + SUCCESS: 8 rules found + +loki -verify-config: OK +tempo -config.verify: OK +grafana dashboards: valid JSON with uid and panels +exit_code: 0 elapsed_seconds: 1.015 diff --git a/evidence/grade3/persistence.txt b/evidence/grade3/persistence.txt new file mode 100644 index 00000000..fb3462ae --- /dev/null +++ b/evidence/grade3/persistence.txt @@ -0,0 +1,7 @@ +date: 2026-11-06T09:33:44Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +exit_code: 0 elapsed_seconds: 14.842 diff --git a/evidence/grade3/quality.txt b/evidence/grade3/quality.txt new file mode 100644 index 00000000..4f5b42c1 --- /dev/null +++ b/evidence/grade3/quality.txt @@ -0,0 +1,14 @@ +date: 2026-11-06T09:37:46Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make check validate lint +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +.venv/bin/ruff check src student-tests scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py +All checks passed! +.venv/bin/ruff format --check src student-tests scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py +189 files already formatted +.venv/bin/ty check src student-tests alembic +All checks passed! +exit_code: 0 elapsed_seconds: 0.744 diff --git a/evidence/grade3/request.txt b/evidence/grade3/request.txt new file mode 100644 index 00000000..9278c1c5 --- /dev/null +++ b/evidence/grade3/request.txt @@ -0,0 +1,66 @@ +date: 2026-11-06T09:33:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py request +{ + "checked_at": "2026-11-06T09:33:47+00:00", + "request_id": "fea83e54-fa3d-4ad9-af04-93f636b6b386", + "log": { + "line": "request.handled", + "trace_id": "a5bbfed14a0e3087c87d5368f65743aa", + "span_id": "62f0ee285437fbfa", + "service_name": "messenger-api", + "severity": "INFO" + }, + "trace": [ + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.134 + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.074 + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.074 + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.056 + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.18 + }, + { + "service": "messenger-api", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "duration_ms": 0.23 + }, + { + "service": "messenger-api", + "name": "POST /api/v1/users", + "kind": "SPAN_KIND_SERVER", + "status": "STATUS_CODE_UNSET", + "duration_ms": 13.489 + } + ] +} +exit_code: 0 elapsed_seconds: 4.098 diff --git a/evidence/grade3/runtime.json b/evidence/grade3/runtime.json new file mode 100644 index 00000000..bdbc1cc5 --- /dev/null +++ b/evidence/grade3/runtime.json @@ -0,0 +1,102 @@ +{ + "utc": "2026-11-06T09:40:22.595732+00:00", + "sha": "95511002b12510178a3cc2a22566b43e2b2b6c81", + "command": "docker inspect selected fields; docker top -eo pid,uid,comm", + "services": [ + { + "service": "grafana", + "processes": [ + "PID UID COMMAND", + "74426 472 grafana" + ], + "memory_limit": 402653184, + "cpu_limit": 500000000, + "ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "26300" + } + ] + }, + "volumes": [ + "mtusi2026lab6_grafana-data" + ] + }, + { + "service": "loki", + "processes": [ + "PID UID COMMAND", + "74652 10001 loki" + ], + "memory_limit": 402653184, + "cpu_limit": 500000000, + "ports": { + "3100/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "26310" + } + ] + }, + "volumes": [ + "mtusi2026lab6_loki-data", + "mtusi2026lab6_healthtools" + ] + }, + { + "service": "tempo", + "processes": [ + "PID UID COMMAND", + "75480 10001 tempo" + ], + "memory_limit": 402653184, + "cpu_limit": 500000000, + "ports": { + "3200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "26320" + } + ] + }, + "volumes": [ + "mtusi2026lab6_tempo-data", + "mtusi2026lab6_healthtools" + ] + }, + { + "service": "otel-collector", + "processes": [ + "PID UID COMMAND", + "76175 10001 otelcol-contrib" + ], + "memory_limit": 268435456, + "cpu_limit": 500000000, + "ports": {}, + "volumes": [ + "mtusi2026lab6_healthtools" + ] + }, + { + "service": "prometheus", + "processes": [ + "PID UID COMMAND", + "74496 65534 prometheus" + ], + "memory_limit": 536870912, + "cpu_limit": 500000000, + "ports": { + "9090/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "26090" + } + ] + }, + "volumes": [ + "mtusi2026lab6_prometheus-data" + ] + } + ] +} diff --git a/evidence/grade3/smoke-ready.txt b/evidence/grade3/smoke-ready.txt new file mode 100644 index 00000000..f48eed57 --- /dev/null +++ b/evidence/grade3/smoke-ready.txt @@ -0,0 +1,6 @@ +date: 2026-11-06T09:33:42Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make test +python3 tests/smoke.py --base-url "http://127.0.0.1:18086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +exit_code: 0 elapsed_seconds: 2.455 diff --git a/evidence/grade3/smoke.txt b/evidence/grade3/smoke.txt new file mode 100644 index 00000000..34385558 --- /dev/null +++ b/evidence/grade3/smoke.txt @@ -0,0 +1,7 @@ +date: 2026-11-06T09:32:40Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make test +python3 tests/smoke.py --base-url "http://127.0.0.1:18086" +FAIL: GET /health/live: connection/TLS failure (URLError) +make: *** [test] Error 1 +exit_code: 2 elapsed_seconds: 0.056 diff --git a/evidence/grade3/stand.txt b/evidence/grade3/stand.txt new file mode 100644 index 00000000..0abd9b32 --- /dev/null +++ b/evidence/grade3/stand.txt @@ -0,0 +1,17 @@ +api running Up 3 minutes (healthy) +api running Up 3 minutes (healthy) +broker running Up 3 minutes (healthy) 127.0.0.1:26086->15672/tcp +grafana running Up 6 minutes (healthy) 127.0.0.1:26300->3000/tcp +healthtools exited Exited (0) 5 minutes ago +lb running Up 5 minutes (healthy) 127.0.0.1:18086->8080/tcp, 127.0.0.1:26085->8404/tcp +loki running Up 6 minutes (healthy) 127.0.0.1:26310->3100/tcp +migrate exited Exited (0) 5 minutes ago +otel-collector running Up 6 minutes (healthy) 4317-4318/tcp, 55679/tcp +postgres running Up 5 minutes (healthy) +prometheus running Up 6 minutes (healthy) 127.0.0.1:26090->9090/tcp +relay running Up 2 minutes (healthy) +s3 running Up 3 minutes (healthy) +s3-init exited Exited (0) 5 minutes ago +tempo running Up 6 minutes (healthy) 127.0.0.1:26320->3200/tcp +valkey running Up 5 minutes (healthy) +worker running Up 2 minutes (healthy) diff --git a/evidence/grade3/student-tests.txt b/evidence/grade3/student-tests.txt new file mode 100644 index 00000000..c115c982 --- /dev/null +++ b/evidence/grade3/student-tests.txt @@ -0,0 +1,62 @@ +date: 2026-11-06T09:37:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make student-test +.venv/bin/pytest +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collected 535 items / 19 deselected / 516 selected + +student-tests/adapters/test_rabbitmq.py .... [ 0%] +student-tests/adapters/test_s3_storage.py ... [ 1%] +student-tests/api/test_access.py ......... [ 3%] +student-tests/api/test_attachments.py .................................. [ 9%] +.... [ 10%] +student-tests/api/test_bounded_contexts.py .................... [ 14%] +student-tests/api/test_cache.py ...... [ 15%] +student-tests/api/test_concurrency.py .... [ 16%] +student-tests/api/test_consistency.py ........ [ 17%] +student-tests/api/test_drain.py .. [ 18%] +student-tests/api/test_events.py ...... [ 19%] +student-tests/api/test_http_telemetry.py . [ 19%] +student-tests/api/test_idempotency.py ............... [ 22%] +student-tests/api/test_instance.py .... [ 23%] +student-tests/api/test_job_returning.py . [ 23%] +student-tests/api/test_message_attachments.py ............. [ 25%] +student-tests/api/test_migrations.py ................................... [ 32%] +. [ 32%] +student-tests/api/test_negative.py ..................................... [ 40%] + [ 40%] +student-tests/api/test_operability.py .............. [ 42%] +student-tests/api/test_pagination.py .................. [ 46%] +student-tests/api/test_pipeline.py ............. [ 48%] +student-tests/api/test_pool.py ............ [ 51%] +student-tests/api/test_reconciler.py ........... [ 53%] +student-tests/api/test_recovery.py ............ [ 55%] +student-tests/api/test_sessions.py .................. [ 59%] +student-tests/api/test_timeout.py ........ [ 60%] +student-tests/api/test_upload_consistency.py ......... [ 62%] +student-tests/api/test_versioned_pipeline.py ........... [ 64%] +student-tests/api/test_wire_format.py .. [ 64%] +student-tests/unit/test_attachments_domain.py .......................... [ 69%] +. [ 70%] +student-tests/unit/test_authentication.py ..................... [ 74%] +student-tests/unit/test_cursor.py ........... [ 76%] +student-tests/unit/test_domain.py ...................................... [ 83%] +.. [ 84%] +student-tests/unit/test_job_observer.py . [ 84%] +student-tests/unit/test_pipeline_domain.py ....................... [ 88%] +student-tests/unit/test_probe.py .............. [ 91%] +student-tests/unit/test_reconcile_measurements.py ...... [ 92%] +student-tests/unit/test_render_pool.py .....s. [ 93%] +student-tests/unit/test_renderer.py .................... [ 97%] +student-tests/unit/test_scaling.py ..... [ 98%] +student-tests/unit/test_telemetry_platform.py ..... [ 99%] +student-tests/unit/test_timeouts.py . [100%] + +========== 515 passed, 1 skipped, 19 deselected in 152.09s (0:02:32) =========== +exit_code: 0 elapsed_seconds: 153.875 diff --git a/evidence/grade3/telemetry-recovery.txt b/evidence/grade3/telemetry-recovery.txt new file mode 100644 index 00000000..1f83e61a --- /dev/null +++ b/evidence/grade3/telemetry-recovery.txt @@ -0,0 +1,40 @@ +date: 2026-11-06T09:39:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_recovery.py +{ + "utc": "2026-11-06T09:39:43+00:00", + "sha": "95511002b12510178a3cc2a22566b43e2b2b6c81", + "history": { + "request_id": "895734e9-5c73-4054-ac37-804e02379460", + "trace_id": "33f5b0aea4ca28712a72fb9642e589f5", + "spans_before": 7, + "spans_after": 7, + "logs_after": 1, + "metric_before": [ + { + "metric": {}, + "value": [ + 1793957985.684, + "352" + ] + } + ], + "metric_after": [ + { + "metric": {}, + "value": [ + 1793957985.684, + "352" + ] + } + ] + }, + "collector_down": { + "ready": 200, + "smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:18086\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.", + "api_exit_code": "0", + "api_shutdown_seconds": 4.592 + }, + "result": "PASS" +} +exit_code: 0 elapsed_seconds: 22.655 diff --git a/evidence/grade3/up.txt b/evidence/grade3/up.txt new file mode 100644 index 00000000..11e8dc38 --- /dev/null +++ b/evidence/grade3/up.txt @@ -0,0 +1,230 @@ +date: 2026-11-06T09:32:07Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make up +sh scripts/secrets.sh +sh scripts/broker-secrets.sh +broker and S3 worker secrets ready in secrets/ (values not printed) +sh scripts/secrets-lab6.sh +secrets/grafana_admin_password is ready (value not printed) +docker compose build +#1 [internal] load local bake definitions +#1 reading from stdin 432B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.65kB done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#3 DONE 0.0s + +#4 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#4 DONE 0.0s + +#5 [internal] load .dockerignore +#5 transferring context: 453B done +#5 DONE 0.0s + +#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done +#6 DONE 0.0s + +#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 77.10kB 0.0s done +#8 DONE 0.0s + +#9 [builder 4/7] COPY pyproject.toml uv.lock README.md ./ +#9 CACHED + +#10 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv +#10 CACHED + +#11 [builder 3/7] WORKDIR /build +#11 CACHED + +#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable +#12 CACHED + +#13 [builder 6/7] COPY src ./src +#13 DONE 0.0s + +#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable +#14 0.198 Resolved 100 packages in 1ms +#14 0.210 Building mtusi-messenger @ file:///build +#14 1.664 Built mtusi-messenger @ file:///build +#14 1.669 Prepared 1 package in 1.45s +#14 1.673 Installed 1 package in 3ms +#14 1.918 Bytecode compiled 2802 files in 245ms +#14 1.918 + mtusi-messenger==1.6.0 (from file:///build) +#14 DONE 2.0s + +#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger +#15 CACHED + +#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv +#16 DONE 0.5s + +#17 [runtime 4/6] WORKDIR /app +#17 DONE 0.0s + +#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./ +#18 DONE 0.0s + +#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic +#19 DONE 0.0s + +#20 exporting to image +#20 exporting layers +#20 exporting layers 3.0s done +#20 exporting manifest sha256:d38e0f0539ab16584375a6654edcd69ba0ad8a9dfb5fc3221a88909fbf5a52a9 +#20 exporting manifest sha256:d38e0f0539ab16584375a6654edcd69ba0ad8a9dfb5fc3221a88909fbf5a52a9 done +#20 exporting config sha256:936eb99a24924e6b5f866e8b8ffeb97ced4056f13385a132eda3d8b2cc94c2fb done +#20 exporting attestation manifest sha256:329afd3eb83e5c817ea776ccb7f65b18a8da4ff79444c75f85d651eb524cb0a7 done +#20 exporting manifest list sha256:be397eab1ba1525d7ad025c7162b7a748a061983dd2895023266825a9a13794b done +#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done +#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 +#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done +#20 DONE 3.8s + +#21 resolving provenance for metadata file +#21 DONE 0.0s + api Built +docker compose up -d --wait + Network mtusi2026lab6_backend Creating + Network mtusi2026lab6_backend Created + Network mtusi2026lab6_edge Creating + Network mtusi2026lab6_edge Created + Container mtusi2026lab6-broker-1 Creating + Container mtusi2026lab6-prometheus-1 Creating + Container mtusi2026lab6-valkey-1 Creating + Container mtusi2026lab6-postgres-1 Creating + Container mtusi2026lab6-lb-1 Creating + Container mtusi2026lab6-s3-1 Creating + Container mtusi2026lab6-healthtools-1 Creating + Container mtusi2026lab6-s3-1 Created + Container mtusi2026lab6-s3-init-1 Creating + Container mtusi2026lab6-valkey-1 Created + Container mtusi2026lab6-prometheus-1 Created + Container mtusi2026lab6-healthtools-1 Created + Container mtusi2026lab6-loki-1 Creating + Container mtusi2026lab6-otel-collector-1 Creating + Container mtusi2026lab6-tempo-1 Creating + Container mtusi2026lab6-lb-1 Created + Container mtusi2026lab6-postgres-1 Created + Container mtusi2026lab6-migrate-1 Creating + Container mtusi2026lab6-broker-1 Created + Container mtusi2026lab6-s3-init-1 Created + Container mtusi2026lab6-otel-collector-1 Created + Container mtusi2026lab6-migrate-1 Created + Container mtusi2026lab6-worker-1 Creating + Container mtusi2026lab6-api-2 Creating + Container mtusi2026lab6-api-1 Creating + Container mtusi2026lab6-relay-1 Creating + Container mtusi2026lab6-tempo-1 Created + Container mtusi2026lab6-loki-1 Created + Container mtusi2026lab6-grafana-1 Creating + Container mtusi2026lab6-worker-1 Created + Container mtusi2026lab6-api-1 Created + Container mtusi2026lab6-relay-1 Created + Container mtusi2026lab6-api-2 Created + Container mtusi2026lab6-grafana-1 Created + Container mtusi2026lab6-s3-1 Starting + Container mtusi2026lab6-lb-1 Starting + Container mtusi2026lab6-valkey-1 Starting + Container mtusi2026lab6-broker-1 Starting + Container mtusi2026lab6-healthtools-1 Starting + Container mtusi2026lab6-prometheus-1 Starting + Container mtusi2026lab6-postgres-1 Starting + Container mtusi2026lab6-healthtools-1 Started + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-s3-1 Started + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-postgres-1 Started + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-valkey-1 Started + Container mtusi2026lab6-broker-1 Started + Container mtusi2026lab6-prometheus-1 Started + Container mtusi2026lab6-lb-1 Started + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-tempo-1 Starting + Container mtusi2026lab6-loki-1 Starting + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-otel-collector-1 Starting + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-tempo-1 Started + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-loki-1 Started + Container mtusi2026lab6-grafana-1 Starting + Container mtusi2026lab6-otel-collector-1 Started + Container mtusi2026lab6-grafana-1 Started + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-worker-1 Starting + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-relay-1 Starting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-worker-1 Started + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-api-1 Starting + Container mtusi2026lab6-relay-1 Started + Container mtusi2026lab6-api-1 Started + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-prometheus-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-otel-collector-1 Waiting + Container mtusi2026lab6-worker-1 Waiting + Container mtusi2026lab6-valkey-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-tempo-1 Waiting + Container mtusi2026lab6-grafana-1 Waiting + Container mtusi2026lab6-relay-1 Waiting + Container mtusi2026lab6-lb-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-loki-1 Waiting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-grafana-1 Healthy + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-otel-collector-1 Healthy + Container mtusi2026lab6-prometheus-1 Healthy + Container mtusi2026lab6-valkey-1 Healthy + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-api-1 Healthy + Container mtusi2026lab6-relay-1 Healthy + Container mtusi2026lab6-worker-1 Healthy + Container mtusi2026lab6-lb-1 Healthy + Container mtusi2026lab6-loki-1 Healthy + Container mtusi2026lab6-tempo-1 Healthy +scripts/lb-wait.sh +exit_code: 0 elapsed_seconds: 85.592 diff --git a/evidence/level3.json b/evidence/level3.json new file mode 100644 index 00000000..3cd21796 --- /dev/null +++ b/evidence/level3.json @@ -0,0 +1,270 @@ +{ + "suite": "lab5-pipeline-idempotency", + "started_at": "2026-11-06T09:35:15.796014Z", + "source_sha": "95511002b12510178a3cc2a22566b43e2b2b6c81", + "config": { + "base_url": "http://127.0.0.1:18086", + "level": 3, + "only": "", + "expect_variants": "thumbnail", + "max_attempts": 5, + "stall_seconds": 15.0, + "compose": "docker compose", + "compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml" + }, + "invariants": [ + { + "id": "terminal", + "statement": "every 202 reaches its expected terminal state", + "result": "PASS", + "details": { + "accepted": 5 + } + }, + { + "id": "single_effect", + "statement": "one variant set per (attachment_id, pipeline_version, variant)", + "result": "PASS", + "details": { + "rows": 0 + } + }, + { + "id": "no_ready_rollback", + "statement": "ready never returns to queued or processing", + "result": "PASS", + "details": { + "traced": 5 + } + }, + { + "id": "monotonic_publication", + "statement": "published_version only moves forward", + "result": "PASS", + "details": { + "sampled": 2 + } + }, + { + "id": "bytes_match_metadata", + "statement": "sha256, size and real dimensions match served bytes", + "result": "PASS", + "details": { + "verified": 4 + } + }, + { + "id": "bounded_attempts", + "statement": "attempt <= JOB_MAX_ATTEMPTS", + "result": "PASS", + "details": { + "rows": 0 + } + }, + { + "id": "no_orphans", + "statement": "no pending outbox, messages, dead letters, staging, objects", + "result": "PASS", + "details": { + "queues": { + "attachments.process": [ + 0, + 0 + ], + "attachments.process.dlq": [ + 0, + 0 + ] + }, + "s3_objects": 17, + "scoped_attachments": 5 + } + }, + { + "id": "no_lost_202", + "statement": "every 202 is present in PostgreSQL and the API", + "result": "PASS", + "details": { + "accepted": 5 + } + } + ], + "accepted_ids": [ + "01a1b036-7bdb-72a8-9a20-d784124b6f09", + "01a1b036-838f-7073-a603-58ef2ffca05d", + "01a1b036-d38a-7375-ba48-c82f1e984740", + "01a1b036-e669-7726-8ed5-eb948e03323e", + "01a1b037-1c4a-72b4-8027-95bdd74c185a" + ], + "scenarios": [ + { + "name": "smoke_pipeline", + "level": 3, + "window": "upload, ready, variants match metadata", + "seconds": 1.4, + "result": "PASS", + "observed": { + "attachment_id": "01a1b036-7bdb-72a8-9a20-d784124b6f09", + "variants": [ + "crop", + "optimized", + "thumbnail" + ] + } + }, + { + "name": "worker_down_broker_restart", + "level": 3, + "window": "worker down, API and broker restart", + "seconds": 21.1, + "result": "PASS", + "observed": { + "attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d", + "timeline": [ + { + "at": "2026-11-06T09:35:25.067956Z", + "step": "worker stopped", + "containers": [ + "mtusi2026lab6-worker-1 cd5b9bcf05a3 exited Exited (0) Less than a second ago" + ] + }, + { + "at": "2026-11-06T09:35:25.081364Z", + "step": "upload 202 queued", + "attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d" + }, + { + "at": "2026-11-06T09:35:25.086007Z", + "step": "content 409 while queued" + }, + { + "at": "2026-11-06T09:35:25.109208Z", + "step": "message with attachment 201, other messages work", + "message_id": "01a1b036-839e-71c7-9abc-6d747695610e" + }, + { + "at": "2026-11-06T09:35:30.399187Z", + "step": "job is in the broker queue", + "queues": { + "attachments.process": [ + 1, + 0 + ], + "attachments.process.dlq": [ + 0, + 0 + ] + } + }, + { + "at": "2026-11-06T09:35:36.592722Z", + "step": "API replicas and relay restarted" + }, + { + "at": "2026-11-06T09:35:42.601391Z", + "step": "broker restarted with its volume, job survived", + "queues": { + "attachments.process": [ + 1, + 0 + ], + "attachments.process.dlq": [ + 0, + 0 + ] + } + }, + { + "at": "2026-11-06T09:35:45.362335Z", + "step": "worker started" + }, + { + "at": "2026-11-06T09:35:45.539899Z", + "step": "ready", + "variants": [ + "crop", + "optimized", + "thumbnail" + ] + } + ] + } + }, + { + "name": "duplicate_after_ready", + "level": 3, + "window": "redelivery of a finished job", + "seconds": 4.4, + "result": "PASS", + "observed": { + "attachment_id": "01a1b036-d38a-7375-ba48-c82f1e984740", + "events": [ + { + "kind": "enqueued", + "attempt": 1 + }, + { + "kind": "claimed", + "attempt": 1 + }, + { + "kind": "done", + "attempt": 1 + }, + { + "kind": "duplicate_noop", + "attempt": 1 + } + ] + } + }, + { + "name": "poison_image", + "level": 3, + "window": "permanent error without retries", + "seconds": 0.0, + "result": "PASS", + "observed": { + "rejected_by_api": 415 + } + }, + { + "name": "s3_outage_recovers", + "level": 3, + "window": "transient error with retry", + "seconds": 10.3, + "result": "PASS", + "observed": { + "attachment_id": "01a1b036-e669-7726-8ed5-eb948e03323e", + "jobs": [ + { + "job_id": "01a1b036-e671-7294-8d09-37bc4619030b", + "pipeline_version": 1, + "status": "done", + "attempt": 2 + } + ] + } + }, + { + "name": "retry_exhausted", + "level": 3, + "window": "transient error until JOB_MAX_ATTEMPTS", + "seconds": 20.3, + "result": "PASS", + "observed": { + "attachment_id": "01a1b037-1c4a-72b4-8027-95bdd74c185a", + "jobs": [ + { + "job_id": "01a1b037-1c57-7496-8cc0-61aa229fd7a2", + "pipeline_version": 1, + "status": "failed", + "attempt": 5 + } + ] + } + } + ], + "result": "PASS", + "finished_at": "2026-11-06T09:36:32.454476Z" +} diff --git a/openspec/changes/add-lab6-telemetry-pipeline/specs/lab6-telemetry-stack/spec.md b/openspec/changes/add-lab6-telemetry-pipeline/specs/lab6-telemetry-stack/spec.md index 4217725b..79775986 100644 --- a/openspec/changes/add-lab6-telemetry-pipeline/specs/lab6-telemetry-stack/spec.md +++ b/openspec/changes/add-lab6-telemetry-pipeline/specs/lab6-telemetry-stack/spec.md @@ -10,7 +10,7 @@ #### Scenario: Стенд из чистого состояния -- **WHEN** после `make reset-data` выполняется `make up` +- **WHEN** в новом изолированном Compose project с пустыми volumes выполняется `make up` - **THEN** `docker compose ps` показывает `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` в состоянии `healthy` рядом с `lb`, `api` ×2, `relay`, `worker`, `postgres`, `valkey`, `s3`, `broker` #### Scenario: OTLP не опубликован diff --git a/openspec/changes/add-lab6-telemetry-pipeline/tasks.md b/openspec/changes/add-lab6-telemetry-pipeline/tasks.md index 7094d2ef..23ecd99e 100644 --- a/openspec/changes/add-lab6-telemetry-pipeline/tasks.md +++ b/openspec/changes/add-lab6-telemetry-pipeline/tasks.md @@ -20,19 +20,19 @@ ## 4. Стек наблюдаемости -- [ ] 4.1 `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml` и recording rules, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` из `.prep/lab6` -- [ ] 4.2 `infra/grafana/provisioning`, dashboards `messenger-api` и `messenger-pipeline` уровня «3» -- [ ] 4.3 `compose.yaml`: `healthtools`, `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana`, volumes, секрет Grafana, сети `backend`/`edge`, переменные OTel у `api`, `relay`, `worker`; `.env.example` -- [ ] 4.4 `scripts/secrets-lab6.sh`, `scripts/obs-validate.sh`, `scripts/obs_check.py`; цели `secrets`, `obs-validate`, `telemetry-check`, `obs-evidence` +- [x] 4.1 `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml` и recording rules, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` из `.prep/lab6` +- [x] 4.2 `infra/grafana/provisioning`, dashboards `messenger-api` и `messenger-pipeline` уровня «3» +- [x] 4.3 `compose.yaml`: `healthtools`, `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana`, volumes, секрет Grafana, сети `backend`/`edge`, переменные OTel у `api`, `relay`, `worker`; `.env.example` +- [x] 4.4 `scripts/secrets-lab6.sh`, `scripts/obs-validate.sh`, `scripts/obs_check.py`; цели `secrets`, `obs-validate`, `telemetry-check` - [ ] 4.5 Проверка группы: `make obs-validate`, `docker compose config --quiet`, `make up` из чистого состояния — все сервисы `healthy`, порты наблюдаемости только на `127.0.0.1` ## 5. Проверка на стенде -- [ ] 5.1 `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `make restore`, `make level3` -- [ ] 5.2 `obs_check.py grafana`, `dashboards --strict`, `request`, `job`, `leaks`, cardinality, история после рестарта хранилищ, `make test` при остановленном Collector -- [ ] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test` +- [x] 5.1 `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `make restore`, `make level3` +- [x] 5.2 `obs_check.py grafana`, `dashboards --strict`, `request`, `job`, `leaks`, cardinality, история после рестарта хранилищ, `make test` при остановленном Collector +- [x] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test` ## 6. Отчёт -- [ ] 6.1 `evidence/`: стенд, smoke, persistence, failover, level3, obs-validate, grafana, dashboards, trace-request, trace-job, leaks, cardinality, telemetry-restart, student-tests, contract-checks -- [ ] 6.2 `REPORT.md`: шапка, запуск из чистого clone, архитектура и решения, роли, назначение хранилищ, соответствие сигналов и переменных, строки уровня «3», таблица отказов уровня «3» +- [x] 6.1 `evidence/`: стенд, smoke, persistence, failover, level3, obs-validate, grafana, dashboards, trace-request, trace-job, leaks, cardinality, telemetry-restart, student-tests, contract-checks +- [x] 6.2 `evidence/grade3/notes.md`: заметки для отдельного агента отчёта — запуск, архитектура, роли, сигналы, доказательства и ограничения уровня «3» diff --git a/scripts/lab6.sh b/scripts/lab6.sh new file mode 100755 index 00000000..4ddd4f63 --- /dev/null +++ b/scripts/lab6.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mtusi2026lab6} +export LB_HTTP_PORT=${LB_HTTP_PORT:-18086} LB_STATS_PORT=${LB_STATS_PORT:-26085} +export BROKER_MANAGEMENT_PORT=${BROKER_MANAGEMENT_PORT:-26086} +export PROMETHEUS_PORT=${PROMETHEUS_PORT:-26090} LOKI_PORT=${LOKI_PORT:-26310} +export TEMPO_PORT=${TEMPO_PORT:-26320} GRAFANA_PORT=${GRAFANA_PORT:-26300} +export BASE_URL=${BASE_URL:-http://127.0.0.1:$LB_HTTP_PORT} +export PROMETHEUS_URL=http://127.0.0.1:$PROMETHEUS_PORT LOKI_URL=http://127.0.0.1:$LOKI_PORT +export TEMPO_URL=http://127.0.0.1:$TEMPO_PORT GRAFANA_URL=http://127.0.0.1:$GRAFANA_PORT +export NO_PROXY=localhost,127.0.0.1,::1 +exec "$@" diff --git a/scripts/obs_check.py b/scripts/obs_check.py index a75843dc..598a0faa 100755 --- a/scripts/obs_check.py +++ b/scripts/obs_check.py @@ -66,7 +66,7 @@ def wait_for(what, check, timeout, interval=2.0): result = check() if result: return result - except Failure: + except (Failure, OSError): pass if time.monotonic() > deadline: raise Failure(f"timeout waiting for {what}") @@ -253,6 +253,10 @@ def request_check(args, b): if not trace_id: raise Failure("log record has no trace_id") spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout) + if not any(s["kind"] == "SPAN_KIND_SERVER" and s["service"] == args.api_service for s in spans): + raise Failure("no real API server span") + if not any(s["kind"] == "SPAN_KIND_CLIENT" and s["attributes"].get("db.system") == "postgresql" for s in spans): + raise Failure("no PostgreSQL client span inside API trace") print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id, "span_id": labels.get("span_id"), "service_name": labels.get("service_name"), @@ -282,6 +286,10 @@ def job_check(args, b): query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"' worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout) api.logout(auth) + if ready["status"] != "ready" or not any(s["kind"] == "SPAN_KIND_CONSUMER" and s["service"] == args.worker_service for s in chain): + raise Failure("job is not ready or no real consumer span") + if not any(labels.get("trace_id") and labels.get("span_id") for labels, _ in worker_logs): + raise Failure("worker log is not correlated") print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"], "traces": sorted({c["trace_id"] for c in chain}), "spans": chain, "worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")} diff --git a/scripts/obs_recovery.py b/scripts/obs_recovery.py new file mode 100644 index 00000000..0ab3ba85 --- /dev/null +++ b/scripts/obs_recovery.py @@ -0,0 +1,98 @@ +import argparse +import json +import os +import subprocess +import time + +from obs_check import Api, Backends, get_json, http, now, tempo_spans, wait_for + + +def backends(): + return Backends( + argparse.Namespace( + prometheus=os.environ["PROMETHEUS_URL"], + loki=os.environ["LOKI_URL"], + tempo=os.environ["TEMPO_URL"], + grafana=os.environ["GRAFANA_URL"], + grafana_password_file=os.environ.get( + "GRAFANA_PASSWORD_FILE", "/".join(("secrets", "grafana_admin_password")) + ), + ) + ) + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], check=True, capture_output=True, text=True + ).stdout.strip() + + +def history(b): + started = time.time() + _, _, _, rid = Api(os.environ["BASE_URL"]).register() + query = '{service_name="messenger-api"} | request_id="' + rid + '"' + logs = wait_for("log", lambda: b.logs(query, started - 1, time.time() + 1), 60) + trace_id = logs[0][0]["trace_id"] + spans = wait_for("trace", lambda: tempo_spans(b.trace(trace_id)[0]), 60) + expr = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})' + before = wait_for("metrics", lambda: b.promql(expr), 60) + at = before[0]["value"][0] + compose("restart", "prometheus", "loki", "tempo", "grafana") + wait_for("Grafana", lambda: get_json(b.grafana + "/api/health"), 90) + after_logs = wait_for("retained log", lambda: b.logs(query, started - 1, time.time() + 1), 90) + after_spans = wait_for("retained trace", lambda: tempo_spans(b.trace(trace_id)[0]), 90) + after = wait_for("retained metric", lambda: b.promql(expr, at=at), 90) + if before[0]["value"][1] != after[0]["value"][1] or len(after_spans) < len(spans): + raise RuntimeError("telemetry history changed after restart") + return { + "request_id": rid, + "trace_id": trace_id, + "spans_before": len(spans), + "spans_after": len(after_spans), + "logs_after": len(after_logs), + "metric_before": before, + "metric_after": after, + } + + +def collector(): + compose("stop", "otel-collector") + try: + status = http("GET", os.environ["BASE_URL"] + "/health/ready")[0] + result = subprocess.run(["make", "test"], capture_output=True, text=True, check=False) + if status != 200 or result.returncode: + raise RuntimeError("application failed without collector") + replica = compose("ps", "-q", "api").splitlines()[0] + started = time.monotonic() + subprocess.run(["docker", "stop", "-t", "8", replica], check=True, capture_output=True) + elapsed = time.monotonic() - started + code = subprocess.check_output( + ["docker", "inspect", "-f", "{{.State.ExitCode}}", replica], text=True + ).strip() + if code != "0" or elapsed > 8: + raise RuntimeError("unbounded API shutdown") + return { + "ready": status, + "smoke": result.stdout.strip(), + "api_exit_code": code, + "api_shutdown_seconds": round(elapsed, 3), + } + finally: + compose("up", "-d", "--no-deps", "--wait", "api", "otel-collector") + subprocess.run(["scripts/lb-wait.sh"], check=True) + + +def main(): + b = backends() + report = { + "utc": now(), + "sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + } + report["history"] = history(b) + report["collector_down"] = collector() + report["result"] = "PASS" + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main()