Verify the grade 3 telemetry stack and inherited API guarantees
This commit is contained in:
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
|
|||||||
RATE ?= 100
|
RATE ?= 100
|
||||||
FROM_REV ?=
|
FROM_REV ?=
|
||||||
SHIFT ?= 0
|
SHIFT ?= 0
|
||||||
SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
SCRIPTS_PY = scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||||
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
|
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
|
||||||
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||||
SCENARIO ?= failover-upload
|
SCENARIO ?= failover-upload
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
date: 2026-11-06T09:38:20Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py query 'count by (__name__,job) ({job=~"messenger-.*"})' 'count by (http_route,http_request_method,http_response_status_code) (http_server_request_duration_seconds_count)' 'count by (job_type,outcome) (messenger_jobs_processed_total)'
|
||||||
|
{"ts": "2026-11-06T09:38:20+00:00", "results": {"count by (__name__,job) ({job=~\"messenger-.*\"})": [{"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "77"}, {"labels": {"job": "messenger-worker"}, "value": "4"}, {"labels": {"job": "messenger-api"}, "value": "17"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "990"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "1056"}, {"labels": {"job": "messenger-api"}, "value": "34"}, {"labels": {"job": "messenger-api"}, "value": "34"}, {"labels": {"job": "messenger-api"}, "value": "544"}, {"labels": {"job": "messenger-api"}, "value": "8"}], "count by (http_route,http_request_method,http_response_status_code) (http_server_request_duration_seconds_count)": [{"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/auth/sessions"}, "value": "5"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "409", "http_route": "/api/v1/users"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/chats/{chat_id}"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/chats"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "5"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "3"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "400", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "2"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "400", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "401", "http_route": "/api/v1/users/me"}, "value": "2"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/users/me"}, "value": "4"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "409", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "202", "http_route": "/api/v1/chats/{chat_id}/attachments"}, "value": "7"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/attachments/{attachment_id}"}, "value": "7"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "5"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "1"}, {"labels": {"http_request_method": "DELETE", "http_response_status_code": "204", "http_route": "/api/v1/auth/sessions/current"}, "value": "2"}, {"labels": {"http_request_method": "DELETE", "http_response_status_code": "401", "http_route": "/api/v1/auth/sessions/current"}, "value": "1"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/users"}, "value": "5"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/chats"}, "value": "4"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "404", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "1"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "401", "http_route": "/api/v1/auth/sessions"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/attachments/{attachment_id}"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "409", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "415", "http_route": "/api/v1/chats/{chat_id}/attachments"}, "value": "1"}], "count by (job_type,outcome) (messenger_jobs_processed_total)": [{"labels": {"job_type": "attachment.process.v1", "outcome": "done"}, "value": "3"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "duplicate_noop"}, "value": "1"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "retry_scheduled"}, "value": "2"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "failed"}, "value": "1"}]}}
|
||||||
|
exit_code: 0 elapsed_seconds: 0.057
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
date: 2026-11-06T09:38:20Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py dashboards --strict
|
||||||
|
{
|
||||||
|
"checked_at": "2026-11-06T09:38:20+00:00",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "RPS бизнес-запросов",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Доля 5xx (2m)",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "p95 latency (5m)",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Доля не-5xx (окно 1h)",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "RPS по маршруту и классу статуса",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 9,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Доля ошибок",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Доля ошибок",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Latency p50 / p95 / p99",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Latency p50 / p95 / p99",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Latency p50 / p95 / p99",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "p95 по маршруту",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 7,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "RPS по реплике",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 4,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Активные запросы",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 8,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Логи API (ошибки и предупреждения)",
|
||||||
|
"datasource": "loki",
|
||||||
|
"lines": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-api.json",
|
||||||
|
"panel": "Трассы API с ошибкой",
|
||||||
|
"datasource": "tempo",
|
||||||
|
"traces": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-pipeline.json",
|
||||||
|
"panel": "Jobs по исходу",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-pipeline.json",
|
||||||
|
"panel": "Длительность обработки job",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-pipeline.json",
|
||||||
|
"panel": "Длительность обработки job",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-pipeline.json",
|
||||||
|
"panel": "Длительность обработки job",
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"series": 1,
|
||||||
|
"missing_metrics": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-pipeline.json",
|
||||||
|
"panel": "Логи worker",
|
||||||
|
"datasource": "loki",
|
||||||
|
"lines": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dashboard": "messenger-pipeline.json",
|
||||||
|
"panel": "Трассы worker",
|
||||||
|
"datasource": "tempo",
|
||||||
|
"traces": 5
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"problems": []
|
||||||
|
}
|
||||||
|
exit_code: 0 elapsed_seconds: 0.089
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ env EVIDENCE_DIR=evidence/grade3/failover-first make failover-evidence SHIFT=0
|
||||||
|
sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "0"
|
||||||
|
date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:33:50.075450885Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||||
|
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 3.03}
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
== after: api replicas
|
||||||
|
9af2280c8af3 exited started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
== stopped replica: 9af2280c8af3; shutdown: signal, drain window with readiness 503, close, exit
|
||||||
|
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:35.851311Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.864420Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.897236Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"5818535d613b4b7a8f0607d91e58d3f5","duration_ms":0.837,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.062326Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"4838cbd4822f4181a2e10153a41033d6","duration_ms":0.347,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.566176Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.866622Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.900282Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:37.973761Z"}
|
||||||
|
{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T09:34:37.977541Z"}
|
||||||
|
{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:37.987715Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:38.004692Z"}
|
||||||
|
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:38.006586Z"}
|
||||||
|
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:38.093383Z"}
|
||||||
|
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:38.195744Z"}
|
||||||
|
$ scripts/restore.sh
|
||||||
|
Container mtusi2026lab6-api-2 Running
|
||||||
|
Container mtusi2026lab6-api-1 Starting
|
||||||
|
Container mtusi2026lab6-api-1 Started
|
||||||
|
Container mtusi2026lab6-api-2 Waiting
|
||||||
|
Container mtusi2026lab6-api-1 Waiting
|
||||||
|
Container mtusi2026lab6-api-2 Healthy
|
||||||
|
Container mtusi2026lab6-api-1 Healthy
|
||||||
|
== after restore
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
exit_code: 0 elapsed_seconds: 20.584
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:33:50.075450885Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||||
|
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 3.03}
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
== after: api replicas
|
||||||
|
9af2280c8af3 exited started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
== stopped replica: 9af2280c8af3; shutdown: signal, drain window with readiness 503, close, exit
|
||||||
|
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:35.851311Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.864420Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.897236Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"5818535d613b4b7a8f0607d91e58d3f5","duration_ms":0.837,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.062326Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"4838cbd4822f4181a2e10153a41033d6","duration_ms":0.347,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.566176Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.866622Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.900282Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:37.973761Z"}
|
||||||
|
{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T09:34:37.977541Z"}
|
||||||
|
{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:37.987715Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:38.004692Z"}
|
||||||
|
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:38.006586Z"}
|
||||||
|
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:38.093383Z"}
|
||||||
|
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:38.195744Z"}
|
||||||
|
$ scripts/restore.sh
|
||||||
|
Container mtusi2026lab6-api-2 Running
|
||||||
|
Container mtusi2026lab6-api-1 Starting
|
||||||
|
Container mtusi2026lab6-api-1 Started
|
||||||
|
Container mtusi2026lab6-api-2 Waiting
|
||||||
|
Container mtusi2026lab6-api-1 Waiting
|
||||||
|
Container mtusi2026lab6-api-2 Healthy
|
||||||
|
Container mtusi2026lab6-api-1 Healthy
|
||||||
|
== after restore
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ env EVIDENCE_DIR=evidence/grade3/failover-second make failover-evidence SHIFT=1
|
||||||
|
sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "1"
|
||||||
|
date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||||
|
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.777}
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
== after: api replicas
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 exited started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:34:58.204116764Z
|
||||||
|
== stopped replica: 943813ecb712; shutdown: signal, drain window with readiness 503, close, exit
|
||||||
|
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:55.700248Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.784538Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.795484Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"ff6952bfefbe4034ae30489d4e43b93c","duration_ms":0.567,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.029040Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"194add29fb3742db93424a18cf9ff680","duration_ms":1.596,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.535013Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.787659Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.796586Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.894245Z"}
|
||||||
|
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:57.896210Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.903471Z"}
|
||||||
|
{"event":"Finished server process [23]","level":"info","ts":"2026-11-06T09:34:57.905749Z"}
|
||||||
|
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954114Z"}
|
||||||
|
{"event":"Worker (pid:23) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954212Z"}
|
||||||
|
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:58.057111Z"}
|
||||||
|
$ scripts/restore.sh
|
||||||
|
Container mtusi2026lab6-api-1 Running
|
||||||
|
Container mtusi2026lab6-api-2 Starting
|
||||||
|
Container mtusi2026lab6-api-2 Started
|
||||||
|
Container mtusi2026lab6-api-2 Waiting
|
||||||
|
Container mtusi2026lab6-api-1 Waiting
|
||||||
|
Container mtusi2026lab6-api-2 Healthy
|
||||||
|
Container mtusi2026lab6-api-1 Healthy
|
||||||
|
== after restore
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:35:11.260329797Z finished=2026-11-06T09:34:58.204116764Z
|
||||||
|
exit_code: 0 elapsed_seconds: 20.372
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
|
||||||
|
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||||
|
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.777}
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
== after: api replicas
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 exited started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:34:58.204116764Z
|
||||||
|
== stopped replica: 943813ecb712; shutdown: signal, drain window with readiness 503, close, exit
|
||||||
|
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:55.700248Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.784538Z"}
|
||||||
|
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.795484Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"ff6952bfefbe4034ae30489d4e43b93c","duration_ms":0.567,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.029040Z"}
|
||||||
|
{"method":"GET","route":"/health/ready","status":503,"request_id":"194add29fb3742db93424a18cf9ff680","duration_ms":1.596,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.535013Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.787659Z"}
|
||||||
|
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.796586Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.894245Z"}
|
||||||
|
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:57.896210Z"}
|
||||||
|
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.903471Z"}
|
||||||
|
{"event":"Finished server process [23]","level":"info","ts":"2026-11-06T09:34:57.905749Z"}
|
||||||
|
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954114Z"}
|
||||||
|
{"event":"Worker (pid:23) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954212Z"}
|
||||||
|
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:58.057111Z"}
|
||||||
|
$ scripts/restore.sh
|
||||||
|
Container mtusi2026lab6-api-1 Running
|
||||||
|
Container mtusi2026lab6-api-2 Starting
|
||||||
|
Container mtusi2026lab6-api-2 Started
|
||||||
|
Container mtusi2026lab6-api-2 Waiting
|
||||||
|
Container mtusi2026lab6-api-1 Waiting
|
||||||
|
Container mtusi2026lab6-api-2 Healthy
|
||||||
|
Container mtusi2026lab6-api-1 Healthy
|
||||||
|
== after restore
|
||||||
|
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
|
||||||
|
943813ecb712 running started=2026-11-06T09:35:11.260329797Z finished=2026-11-06T09:34:58.204116764Z
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
date: 2026-11-06T09:33:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py grafana
|
||||||
|
{
|
||||||
|
"checked_at": "2026-11-06T09:33:43+00:00",
|
||||||
|
"grafana": "12.4.14",
|
||||||
|
"datasources": {
|
||||||
|
"prometheus": {
|
||||||
|
"http": 200,
|
||||||
|
"status": "OK",
|
||||||
|
"message": "Successfully queried the Prometheus API."
|
||||||
|
},
|
||||||
|
"loki": {
|
||||||
|
"http": 200,
|
||||||
|
"status": "OK",
|
||||||
|
"message": "Data source successfully connected."
|
||||||
|
},
|
||||||
|
"tempo": {
|
||||||
|
"http": 200,
|
||||||
|
"status": "OK",
|
||||||
|
"message": "Data source is working. Streaming test succeeded."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"dashboards": {
|
||||||
|
"messenger-api": {
|
||||||
|
"title": "Messenger · API RED",
|
||||||
|
"http": 200,
|
||||||
|
"provisioned": true,
|
||||||
|
"folder": "Messenger",
|
||||||
|
"panels": 14
|
||||||
|
},
|
||||||
|
"messenger-pipeline": {
|
||||||
|
"title": "Messenger · Worker",
|
||||||
|
"http": 200,
|
||||||
|
"provisioned": true,
|
||||||
|
"folder": "Messenger",
|
||||||
|
"panels": 6
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"missing_dashboards": []
|
||||||
|
}
|
||||||
|
exit_code: 0 elapsed_seconds: 0.124
|
||||||
@@ -0,0 +1,316 @@
|
|||||||
|
date: 2026-11-06T09:37:45Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py job
|
||||||
|
{
|
||||||
|
"checked_at": "2026-11-06T09:38:05+00:00",
|
||||||
|
"upload_request_id": "6254becc-0433-44ae-89ea-057e0405f827",
|
||||||
|
"attachment_status": "ready",
|
||||||
|
"traces": [
|
||||||
|
"a020913eec288efb8126ca3a69a513d4"
|
||||||
|
],
|
||||||
|
"spans": [
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "attachments.attachment.process.v1 create",
|
||||||
|
"kind": "SPAN_KIND_CONSUMER",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.008,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "attachments.attachment.process.v1 process",
|
||||||
|
"kind": "SPAN_KIND_CONSUMER",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 777.9,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": "1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.133,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "select",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.111,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "select",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.095,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "show",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.085,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "show",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.081,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "ROLLBACK;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.081,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.082,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "UPDATE",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.554,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "UPDATE",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.57,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "INSERT",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.199,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "SELECT",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.108,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "COMMIT;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.331,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.08,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": ";",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.155,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "ROLLBACK;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.07,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.062,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "UPDATE",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.112,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "COMMIT;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.171,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.124,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": ";",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.069,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "ROLLBACK;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.054,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.073,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "UPDATE",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.179,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "INSERT",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.414,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "UPDATE",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.18,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "INSERT",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.116,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-worker",
|
||||||
|
"name": "COMMIT;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.307,
|
||||||
|
"links": 0,
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"attempt": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"worker_logs": [
|
||||||
|
{
|
||||||
|
"line": "job.processed",
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"span_id": "ff3e55c7d762c0e1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"line": "audit.job_settled",
|
||||||
|
"trace_id": "a020913eec288efb8126ca3a69a513d4",
|
||||||
|
"span_id": "ff3e55c7d762c0e1"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
exit_code: 0 elapsed_seconds: 19.790
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
date: 2026-11-06T09:33:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py job
|
||||||
|
FAIL: <urlopen error [Errno 61] Connection refused>
|
||||||
|
exit_code: 1 elapsed_seconds: 0.055
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
date: 2026-11-06T09:35:15Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make level3
|
||||||
|
BASE_URL="http://127.0.0.1:18086" python3 scripts/pipeline_suite.py --level 3 --output evidence/level3.json
|
||||||
|
RUN smoke_pipeline
|
||||||
|
PASS smoke_pipeline
|
||||||
|
RUN worker_down_broker_restart
|
||||||
|
PASS worker_down_broker_restart
|
||||||
|
RUN duplicate_after_ready
|
||||||
|
PASS duplicate_after_ready
|
||||||
|
RUN poison_image
|
||||||
|
PASS poison_image
|
||||||
|
RUN s3_outage_recovers
|
||||||
|
PASS s3_outage_recovers
|
||||||
|
RUN retry_exhausted
|
||||||
|
PASS retry_exhausted
|
||||||
|
PASS: lab5 pipeline suite, report evidence/level3.json
|
||||||
|
exit_code: 0 elapsed_seconds: 76.738
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
date: 2026-11-06T09:38:05Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py leaks
|
||||||
|
{
|
||||||
|
"checked_at": "2026-11-06T09:38:20+00:00",
|
||||||
|
"window_seconds": 20,
|
||||||
|
"traces_scanned": 9,
|
||||||
|
"hits": {
|
||||||
|
"password": {
|
||||||
|
"loki": 0,
|
||||||
|
"tempo": 0
|
||||||
|
},
|
||||||
|
"token": {
|
||||||
|
"loki": 0,
|
||||||
|
"tempo": 0
|
||||||
|
},
|
||||||
|
"message_text": {
|
||||||
|
"loki": 0,
|
||||||
|
"tempo": 0
|
||||||
|
},
|
||||||
|
"filename": {
|
||||||
|
"loki": 0,
|
||||||
|
"tempo": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
exit_code: 0 elapsed_seconds: 15.275
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Уровень 3 — сведения для отдельного отчёта
|
||||||
|
|
||||||
|
Приложение проверено на SQLAlchemy-коде 95511002b12510178a3cc2a22566b43e2b2b6c81. Новый код этой фазы — обвязка запуска и строгая проверка реальных spans. Команды запускаются через `scripts/lab6.sh`, задающий только порты ЛР6, project и NO_PROXY; значения secrets не выводятся. Дата/SHA и exit_code есть в каждом command log.
|
||||||
|
|
||||||
|
| Критерий | Команда и доказательство |
|
||||||
|
|---|---|
|
||||||
|
| Минимум lab5 | make test: smoke-ready.txt; make test-persistence: persistence.txt; failover каждой реплики: failover-first.txt/failover-second.txt; make level3: lab5-minimum.txt и ../level3.json (6 PASS) |
|
||||||
|
| Конфигурации и provisioning | make obs-validate: obs-validate.txt; obs_check.py grafana: grafana.txt (3 OK, 2 dashboards provisioned); runtime.json (non-root, лимиты, volumes, порты) |
|
||||||
|
| RED и worker metrics | obs_check.py dashboards --strict: dashboards.txt; cardinality.txt: реальные HTTP-шаблоны, done/failed/retry_scheduled/duplicate_noop |
|
||||||
|
| Запрос → лог → trace | obs_check.py request: request.txt, реальный POST server span и PostgreSQL client spans |
|
||||||
|
| Job → лог → trace | obs_check.py job: job-stable.txt, ready, consumer spans и trace/span IDs worker |
|
||||||
|
| Отсутствие содержимого | obs_check.py leaks: leaks.txt, все 4 маркера дают 0 в Loki и Tempo |
|
||||||
|
| История и независимость API | obs_recovery.py: telemetry-recovery.txt, прежние trace/log/metric после restart; smoke при остановленном Collector; SIGTERM API 4.592 с, exit 0 |
|
||||||
|
| Проверки кода | quality.txt: check/validate/lint PASS; student-tests.txt: 515 passed, 1 skipped, 19 deselected |
|
||||||
|
|
||||||
|
Collector принимает OTLP/HTTP и обрабатывает/экспортирует сигналы, не заменяя долговременную БД. Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы. Grafana запрашивает эти хранилища и связывает сигналы через provisioned datasources. SDK запускается в процессе API после fork и в отдельных relay/worker. Batch processors и periodic reader не блокируют запрос на экспорт.
|
||||||
|
|
||||||
|
HTTP: `http_server_request_duration_seconds_{bucket,sum,count}`, labels `http_route`, `http_request_method`, `http_response_status_code`, service resource. RPS и классы статуса: `messenger:http_requests:rate2m`; error ratio — 5xx/все бизнес-запросы, health исключены. Worker: `messenger_jobs_processed_total`, `messenger_job_duration_seconds_{bucket,sum,count}`, labels `job_type,outcome`. Секунды; HTTP percentiles вычисляются histogram_quantile по сумме buckets реплик. Loki индексирует только service_name и environment; IDs остаются structured metadata. SDK не захватывает HTTP-заголовки и параметры БД, Collector дополнительно redacts подозрительные поля.
|
||||||
|
|
||||||
|
## Ограничения и неуспешные попытки
|
||||||
|
|
||||||
|
smoke.txt запущен до завершения up --wait и закономерно получил connection refused; smoke-ready.txt — исправленный порядок. job.txt попал в рестарт persistence; job-stable.txt выполнен отдельно и прошёл. Эти неудачи сохранены и не объявлены PASS. Полный pipeline/failover выполнялся последовательно. Unit/integration tests использовали свои одноразовые контейнеры одновременно только с проверками чтения телеметрии; нагрузочных замеров уровня 5 здесь нет. Linux-only RLIMIT_AS пропущен на macOS, stack-маркер штатно исключил 19 тестов; обязательные публичные отказы выполнены отдельно.
|
||||||
|
|
||||||
|
Minor из переноса: scripts/limits.py и CI-allowlist ещё не знают сервисы OTel; проверка реальных ограничений выполнена независимо в runtime.json. REPORT.md оставлен отдельному отчётному агенту.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
date: 2026-11-06T09:31:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make obs-validate
|
||||||
|
sh scripts/obs-validate.sh
|
||||||
|
otelcol-contrib validate collector.yaml: OK
|
||||||
|
Checking /etc/prometheus/prometheus.yml
|
||||||
|
SUCCESS: 1 rule files found
|
||||||
|
SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
|
||||||
|
|
||||||
|
Checking /etc/prometheus/rules/messenger.recording.yml
|
||||||
|
SUCCESS: 8 rules found
|
||||||
|
|
||||||
|
loki -verify-config: OK
|
||||||
|
tempo -config.verify: OK
|
||||||
|
grafana dashboards: valid JSON with uid and panels
|
||||||
|
exit_code: 0 elapsed_seconds: 1.015
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
date: 2026-11-06T09:33:44Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086"
|
||||||
|
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
exit_code: 0 elapsed_seconds: 14.842
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
date: 2026-11-06T09:37:46Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make check validate lint
|
||||||
|
python3 scripts/check_contract.py
|
||||||
|
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||||
|
.venv/bin/python scripts/validate_openapi.py
|
||||||
|
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||||
|
.venv/bin/ruff check src student-tests scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||||
|
All checks passed!
|
||||||
|
.venv/bin/ruff format --check src student-tests scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||||
|
189 files already formatted
|
||||||
|
.venv/bin/ty check src student-tests alembic
|
||||||
|
All checks passed!
|
||||||
|
exit_code: 0 elapsed_seconds: 0.744
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
date: 2026-11-06T09:33:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_check.py request
|
||||||
|
{
|
||||||
|
"checked_at": "2026-11-06T09:33:47+00:00",
|
||||||
|
"request_id": "fea83e54-fa3d-4ad9-af04-93f636b6b386",
|
||||||
|
"log": {
|
||||||
|
"line": "request.handled",
|
||||||
|
"trace_id": "a5bbfed14a0e3087c87d5368f65743aa",
|
||||||
|
"span_id": "62f0ee285437fbfa",
|
||||||
|
"service_name": "messenger-api",
|
||||||
|
"severity": "INFO"
|
||||||
|
},
|
||||||
|
"trace": [
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.134
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": ";",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.074
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": "ROLLBACK;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.074
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": "BEGIN;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.056
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": "INSERT",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.18
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": "COMMIT;",
|
||||||
|
"kind": "SPAN_KIND_CLIENT",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 0.23
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "messenger-api",
|
||||||
|
"name": "POST /api/v1/users",
|
||||||
|
"kind": "SPAN_KIND_SERVER",
|
||||||
|
"status": "STATUS_CODE_UNSET",
|
||||||
|
"duration_ms": 13.489
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
exit_code: 0 elapsed_seconds: 4.098
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
{
|
||||||
|
"utc": "2026-11-06T09:40:22.595732+00:00",
|
||||||
|
"sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
|
||||||
|
"command": "docker inspect selected fields; docker top <project container> -eo pid,uid,comm",
|
||||||
|
"services": [
|
||||||
|
{
|
||||||
|
"service": "grafana",
|
||||||
|
"processes": [
|
||||||
|
"PID UID COMMAND",
|
||||||
|
"74426 472 grafana"
|
||||||
|
],
|
||||||
|
"memory_limit": 402653184,
|
||||||
|
"cpu_limit": 500000000,
|
||||||
|
"ports": {
|
||||||
|
"3000/tcp": [
|
||||||
|
{
|
||||||
|
"HostIp": "127.0.0.1",
|
||||||
|
"HostPort": "26300"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"volumes": [
|
||||||
|
"mtusi2026lab6_grafana-data"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "loki",
|
||||||
|
"processes": [
|
||||||
|
"PID UID COMMAND",
|
||||||
|
"74652 10001 loki"
|
||||||
|
],
|
||||||
|
"memory_limit": 402653184,
|
||||||
|
"cpu_limit": 500000000,
|
||||||
|
"ports": {
|
||||||
|
"3100/tcp": [
|
||||||
|
{
|
||||||
|
"HostIp": "127.0.0.1",
|
||||||
|
"HostPort": "26310"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"volumes": [
|
||||||
|
"mtusi2026lab6_loki-data",
|
||||||
|
"mtusi2026lab6_healthtools"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "tempo",
|
||||||
|
"processes": [
|
||||||
|
"PID UID COMMAND",
|
||||||
|
"75480 10001 tempo"
|
||||||
|
],
|
||||||
|
"memory_limit": 402653184,
|
||||||
|
"cpu_limit": 500000000,
|
||||||
|
"ports": {
|
||||||
|
"3200/tcp": [
|
||||||
|
{
|
||||||
|
"HostIp": "127.0.0.1",
|
||||||
|
"HostPort": "26320"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"volumes": [
|
||||||
|
"mtusi2026lab6_tempo-data",
|
||||||
|
"mtusi2026lab6_healthtools"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "otel-collector",
|
||||||
|
"processes": [
|
||||||
|
"PID UID COMMAND",
|
||||||
|
"76175 10001 otelcol-contrib"
|
||||||
|
],
|
||||||
|
"memory_limit": 268435456,
|
||||||
|
"cpu_limit": 500000000,
|
||||||
|
"ports": {},
|
||||||
|
"volumes": [
|
||||||
|
"mtusi2026lab6_healthtools"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "prometheus",
|
||||||
|
"processes": [
|
||||||
|
"PID UID COMMAND",
|
||||||
|
"74496 65534 prometheus"
|
||||||
|
],
|
||||||
|
"memory_limit": 536870912,
|
||||||
|
"cpu_limit": 500000000,
|
||||||
|
"ports": {
|
||||||
|
"9090/tcp": [
|
||||||
|
{
|
||||||
|
"HostIp": "127.0.0.1",
|
||||||
|
"HostPort": "26090"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"volumes": [
|
||||||
|
"mtusi2026lab6_prometheus-data"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
date: 2026-11-06T09:33:42Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make test
|
||||||
|
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
exit_code: 0 elapsed_seconds: 2.455
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
date: 2026-11-06T09:32:40Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make test
|
||||||
|
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||||
|
FAIL: GET /health/live: connection/TLS failure (URLError)
|
||||||
|
make: *** [test] Error 1
|
||||||
|
exit_code: 2 elapsed_seconds: 0.056
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
api running Up 3 minutes (healthy)
|
||||||
|
api running Up 3 minutes (healthy)
|
||||||
|
broker running Up 3 minutes (healthy) 127.0.0.1:26086->15672/tcp
|
||||||
|
grafana running Up 6 minutes (healthy) 127.0.0.1:26300->3000/tcp
|
||||||
|
healthtools exited Exited (0) 5 minutes ago
|
||||||
|
lb running Up 5 minutes (healthy) 127.0.0.1:18086->8080/tcp, 127.0.0.1:26085->8404/tcp
|
||||||
|
loki running Up 6 minutes (healthy) 127.0.0.1:26310->3100/tcp
|
||||||
|
migrate exited Exited (0) 5 minutes ago
|
||||||
|
otel-collector running Up 6 minutes (healthy) 4317-4318/tcp, 55679/tcp
|
||||||
|
postgres running Up 5 minutes (healthy)
|
||||||
|
prometheus running Up 6 minutes (healthy) 127.0.0.1:26090->9090/tcp
|
||||||
|
relay running Up 2 minutes (healthy)
|
||||||
|
s3 running Up 3 minutes (healthy)
|
||||||
|
s3-init exited Exited (0) 5 minutes ago
|
||||||
|
tempo running Up 6 minutes (healthy) 127.0.0.1:26320->3200/tcp
|
||||||
|
valkey running Up 5 minutes (healthy)
|
||||||
|
worker running Up 2 minutes (healthy)
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
date: 2026-11-06T09:37:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make student-test
|
||||||
|
.venv/bin/pytest
|
||||||
|
============================= test session starts ==============================
|
||||||
|
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
|
||||||
|
rootdir: <repo>
|
||||||
|
configfile: pyproject.toml
|
||||||
|
testpaths: student-tests
|
||||||
|
plugins: asyncio-1.4.0, anyio-4.15.1
|
||||||
|
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
|
||||||
|
collected 535 items / 19 deselected / 516 selected
|
||||||
|
|
||||||
|
student-tests/adapters/test_rabbitmq.py .... [ 0%]
|
||||||
|
student-tests/adapters/test_s3_storage.py ... [ 1%]
|
||||||
|
student-tests/api/test_access.py ......... [ 3%]
|
||||||
|
student-tests/api/test_attachments.py .................................. [ 9%]
|
||||||
|
.... [ 10%]
|
||||||
|
student-tests/api/test_bounded_contexts.py .................... [ 14%]
|
||||||
|
student-tests/api/test_cache.py ...... [ 15%]
|
||||||
|
student-tests/api/test_concurrency.py .... [ 16%]
|
||||||
|
student-tests/api/test_consistency.py ........ [ 17%]
|
||||||
|
student-tests/api/test_drain.py .. [ 18%]
|
||||||
|
student-tests/api/test_events.py ...... [ 19%]
|
||||||
|
student-tests/api/test_http_telemetry.py . [ 19%]
|
||||||
|
student-tests/api/test_idempotency.py ............... [ 22%]
|
||||||
|
student-tests/api/test_instance.py .... [ 23%]
|
||||||
|
student-tests/api/test_job_returning.py . [ 23%]
|
||||||
|
student-tests/api/test_message_attachments.py ............. [ 25%]
|
||||||
|
student-tests/api/test_migrations.py ................................... [ 32%]
|
||||||
|
. [ 32%]
|
||||||
|
student-tests/api/test_negative.py ..................................... [ 40%]
|
||||||
|
[ 40%]
|
||||||
|
student-tests/api/test_operability.py .............. [ 42%]
|
||||||
|
student-tests/api/test_pagination.py .................. [ 46%]
|
||||||
|
student-tests/api/test_pipeline.py ............. [ 48%]
|
||||||
|
student-tests/api/test_pool.py ............ [ 51%]
|
||||||
|
student-tests/api/test_reconciler.py ........... [ 53%]
|
||||||
|
student-tests/api/test_recovery.py ............ [ 55%]
|
||||||
|
student-tests/api/test_sessions.py .................. [ 59%]
|
||||||
|
student-tests/api/test_timeout.py ........ [ 60%]
|
||||||
|
student-tests/api/test_upload_consistency.py ......... [ 62%]
|
||||||
|
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
|
||||||
|
student-tests/api/test_wire_format.py .. [ 64%]
|
||||||
|
student-tests/unit/test_attachments_domain.py .......................... [ 69%]
|
||||||
|
. [ 70%]
|
||||||
|
student-tests/unit/test_authentication.py ..................... [ 74%]
|
||||||
|
student-tests/unit/test_cursor.py ........... [ 76%]
|
||||||
|
student-tests/unit/test_domain.py ...................................... [ 83%]
|
||||||
|
.. [ 84%]
|
||||||
|
student-tests/unit/test_job_observer.py . [ 84%]
|
||||||
|
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
|
||||||
|
student-tests/unit/test_probe.py .............. [ 91%]
|
||||||
|
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
|
||||||
|
student-tests/unit/test_render_pool.py .....s. [ 93%]
|
||||||
|
student-tests/unit/test_renderer.py .................... [ 97%]
|
||||||
|
student-tests/unit/test_scaling.py ..... [ 98%]
|
||||||
|
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
|
||||||
|
student-tests/unit/test_timeouts.py . [100%]
|
||||||
|
|
||||||
|
========== 515 passed, 1 skipped, 19 deselected in 152.09s (0:02:32) ===========
|
||||||
|
exit_code: 0 elapsed_seconds: 153.875
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
date: 2026-11-06T09:39:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ python3 scripts/obs_recovery.py
|
||||||
|
{
|
||||||
|
"utc": "2026-11-06T09:39:43+00:00",
|
||||||
|
"sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
|
||||||
|
"history": {
|
||||||
|
"request_id": "895734e9-5c73-4054-ac37-804e02379460",
|
||||||
|
"trace_id": "33f5b0aea4ca28712a72fb9642e589f5",
|
||||||
|
"spans_before": 7,
|
||||||
|
"spans_after": 7,
|
||||||
|
"logs_after": 1,
|
||||||
|
"metric_before": [
|
||||||
|
{
|
||||||
|
"metric": {},
|
||||||
|
"value": [
|
||||||
|
1793957985.684,
|
||||||
|
"352"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric_after": [
|
||||||
|
{
|
||||||
|
"metric": {},
|
||||||
|
"value": [
|
||||||
|
1793957985.684,
|
||||||
|
"352"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"collector_down": {
|
||||||
|
"ready": 200,
|
||||||
|
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:18086\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.",
|
||||||
|
"api_exit_code": "0",
|
||||||
|
"api_shutdown_seconds": 4.592
|
||||||
|
},
|
||||||
|
"result": "PASS"
|
||||||
|
}
|
||||||
|
exit_code: 0 elapsed_seconds: 22.655
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
date: 2026-11-06T09:32:07Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
|
||||||
|
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||||
|
$ make up
|
||||||
|
sh scripts/secrets.sh
|
||||||
|
sh scripts/broker-secrets.sh
|
||||||
|
broker and S3 worker secrets ready in secrets/ (values not printed)
|
||||||
|
sh scripts/secrets-lab6.sh
|
||||||
|
secrets/grafana_admin_password is ready (value not printed)
|
||||||
|
docker compose build
|
||||||
|
#1 [internal] load local bake definitions
|
||||||
|
#1 reading from stdin 432B done
|
||||||
|
#1 DONE 0.0s
|
||||||
|
|
||||||
|
#2 [internal] load build definition from Dockerfile
|
||||||
|
#2 transferring dockerfile: 1.65kB done
|
||||||
|
#2 DONE 0.0s
|
||||||
|
|
||||||
|
#3 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
|
||||||
|
#3 DONE 0.0s
|
||||||
|
|
||||||
|
#4 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
|
||||||
|
#4 DONE 0.0s
|
||||||
|
|
||||||
|
#5 [internal] load .dockerignore
|
||||||
|
#5 transferring context: 453B done
|
||||||
|
#5 DONE 0.0s
|
||||||
|
|
||||||
|
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
|
||||||
|
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
|
||||||
|
#6 DONE 0.0s
|
||||||
|
|
||||||
|
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
|
||||||
|
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
|
||||||
|
#7 DONE 0.0s
|
||||||
|
|
||||||
|
#8 [internal] load build context
|
||||||
|
#8 transferring context: 77.10kB 0.0s done
|
||||||
|
#8 DONE 0.0s
|
||||||
|
|
||||||
|
#9 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
|
||||||
|
#9 CACHED
|
||||||
|
|
||||||
|
#10 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
|
||||||
|
#10 CACHED
|
||||||
|
|
||||||
|
#11 [builder 3/7] WORKDIR /build
|
||||||
|
#11 CACHED
|
||||||
|
|
||||||
|
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
|
||||||
|
#12 CACHED
|
||||||
|
|
||||||
|
#13 [builder 6/7] COPY src ./src
|
||||||
|
#13 DONE 0.0s
|
||||||
|
|
||||||
|
#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
|
||||||
|
#14 0.198 Resolved 100 packages in 1ms
|
||||||
|
#14 0.210 Building mtusi-messenger @ file:///build
|
||||||
|
#14 1.664 Built mtusi-messenger @ file:///build
|
||||||
|
#14 1.669 Prepared 1 package in 1.45s
|
||||||
|
#14 1.673 Installed 1 package in 3ms
|
||||||
|
#14 1.918 Bytecode compiled 2802 files in 245ms
|
||||||
|
#14 1.918 + mtusi-messenger==1.6.0 (from file:///build)
|
||||||
|
#14 DONE 2.0s
|
||||||
|
|
||||||
|
#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
|
||||||
|
#15 CACHED
|
||||||
|
|
||||||
|
#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv
|
||||||
|
#16 DONE 0.5s
|
||||||
|
|
||||||
|
#17 [runtime 4/6] WORKDIR /app
|
||||||
|
#17 DONE 0.0s
|
||||||
|
|
||||||
|
#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./
|
||||||
|
#18 DONE 0.0s
|
||||||
|
|
||||||
|
#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic
|
||||||
|
#19 DONE 0.0s
|
||||||
|
|
||||||
|
#20 exporting to image
|
||||||
|
#20 exporting layers
|
||||||
|
#20 exporting layers 3.0s done
|
||||||
|
#20 exporting manifest sha256:d38e0f0539ab16584375a6654edcd69ba0ad8a9dfb5fc3221a88909fbf5a52a9
|
||||||
|
#20 exporting manifest sha256:d38e0f0539ab16584375a6654edcd69ba0ad8a9dfb5fc3221a88909fbf5a52a9 done
|
||||||
|
#20 exporting config sha256:936eb99a24924e6b5f866e8b8ffeb97ced4056f13385a132eda3d8b2cc94c2fb done
|
||||||
|
#20 exporting attestation manifest sha256:329afd3eb83e5c817ea776ccb7f65b18a8da4ff79444c75f85d651eb524cb0a7 done
|
||||||
|
#20 exporting manifest list sha256:be397eab1ba1525d7ad025c7162b7a748a061983dd2895023266825a9a13794b done
|
||||||
|
#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done
|
||||||
|
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6
|
||||||
|
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done
|
||||||
|
#20 DONE 3.8s
|
||||||
|
|
||||||
|
#21 resolving provenance for metadata file
|
||||||
|
#21 DONE 0.0s
|
||||||
|
api Built
|
||||||
|
docker compose up -d --wait
|
||||||
|
Network mtusi2026lab6_backend Creating
|
||||||
|
Network mtusi2026lab6_backend Created
|
||||||
|
Network mtusi2026lab6_edge Creating
|
||||||
|
Network mtusi2026lab6_edge Created
|
||||||
|
Container mtusi2026lab6-broker-1 Creating
|
||||||
|
Container mtusi2026lab6-prometheus-1 Creating
|
||||||
|
Container mtusi2026lab6-valkey-1 Creating
|
||||||
|
Container mtusi2026lab6-postgres-1 Creating
|
||||||
|
Container mtusi2026lab6-lb-1 Creating
|
||||||
|
Container mtusi2026lab6-s3-1 Creating
|
||||||
|
Container mtusi2026lab6-healthtools-1 Creating
|
||||||
|
Container mtusi2026lab6-s3-1 Created
|
||||||
|
Container mtusi2026lab6-s3-init-1 Creating
|
||||||
|
Container mtusi2026lab6-valkey-1 Created
|
||||||
|
Container mtusi2026lab6-prometheus-1 Created
|
||||||
|
Container mtusi2026lab6-healthtools-1 Created
|
||||||
|
Container mtusi2026lab6-loki-1 Creating
|
||||||
|
Container mtusi2026lab6-otel-collector-1 Creating
|
||||||
|
Container mtusi2026lab6-tempo-1 Creating
|
||||||
|
Container mtusi2026lab6-lb-1 Created
|
||||||
|
Container mtusi2026lab6-postgres-1 Created
|
||||||
|
Container mtusi2026lab6-migrate-1 Creating
|
||||||
|
Container mtusi2026lab6-broker-1 Created
|
||||||
|
Container mtusi2026lab6-s3-init-1 Created
|
||||||
|
Container mtusi2026lab6-otel-collector-1 Created
|
||||||
|
Container mtusi2026lab6-migrate-1 Created
|
||||||
|
Container mtusi2026lab6-worker-1 Creating
|
||||||
|
Container mtusi2026lab6-api-2 Creating
|
||||||
|
Container mtusi2026lab6-api-1 Creating
|
||||||
|
Container mtusi2026lab6-relay-1 Creating
|
||||||
|
Container mtusi2026lab6-tempo-1 Created
|
||||||
|
Container mtusi2026lab6-loki-1 Created
|
||||||
|
Container mtusi2026lab6-grafana-1 Creating
|
||||||
|
Container mtusi2026lab6-worker-1 Created
|
||||||
|
Container mtusi2026lab6-api-1 Created
|
||||||
|
Container mtusi2026lab6-relay-1 Created
|
||||||
|
Container mtusi2026lab6-api-2 Created
|
||||||
|
Container mtusi2026lab6-grafana-1 Created
|
||||||
|
Container mtusi2026lab6-s3-1 Starting
|
||||||
|
Container mtusi2026lab6-lb-1 Starting
|
||||||
|
Container mtusi2026lab6-valkey-1 Starting
|
||||||
|
Container mtusi2026lab6-broker-1 Starting
|
||||||
|
Container mtusi2026lab6-healthtools-1 Starting
|
||||||
|
Container mtusi2026lab6-prometheus-1 Starting
|
||||||
|
Container mtusi2026lab6-postgres-1 Starting
|
||||||
|
Container mtusi2026lab6-healthtools-1 Started
|
||||||
|
Container mtusi2026lab6-healthtools-1 Waiting
|
||||||
|
Container mtusi2026lab6-healthtools-1 Waiting
|
||||||
|
Container mtusi2026lab6-healthtools-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-1 Started
|
||||||
|
Container mtusi2026lab6-s3-1 Waiting
|
||||||
|
Container mtusi2026lab6-postgres-1 Started
|
||||||
|
Container mtusi2026lab6-postgres-1 Waiting
|
||||||
|
Container mtusi2026lab6-valkey-1 Started
|
||||||
|
Container mtusi2026lab6-broker-1 Started
|
||||||
|
Container mtusi2026lab6-prometheus-1 Started
|
||||||
|
Container mtusi2026lab6-lb-1 Started
|
||||||
|
Container mtusi2026lab6-healthtools-1 Exited
|
||||||
|
Container mtusi2026lab6-healthtools-1 Exited
|
||||||
|
Container mtusi2026lab6-tempo-1 Starting
|
||||||
|
Container mtusi2026lab6-loki-1 Starting
|
||||||
|
Container mtusi2026lab6-healthtools-1 Exited
|
||||||
|
Container mtusi2026lab6-otel-collector-1 Starting
|
||||||
|
Container mtusi2026lab6-postgres-1 Healthy
|
||||||
|
Container mtusi2026lab6-migrate-1 Starting
|
||||||
|
Container mtusi2026lab6-tempo-1 Started
|
||||||
|
Container mtusi2026lab6-migrate-1 Started
|
||||||
|
Container mtusi2026lab6-loki-1 Started
|
||||||
|
Container mtusi2026lab6-grafana-1 Starting
|
||||||
|
Container mtusi2026lab6-otel-collector-1 Started
|
||||||
|
Container mtusi2026lab6-grafana-1 Started
|
||||||
|
Container mtusi2026lab6-s3-1 Healthy
|
||||||
|
Container mtusi2026lab6-s3-init-1 Starting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Started
|
||||||
|
Container mtusi2026lab6-migrate-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Waiting
|
||||||
|
Container mtusi2026lab6-broker-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Waiting
|
||||||
|
Container mtusi2026lab6-broker-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Exited
|
||||||
|
Container mtusi2026lab6-migrate-1 Exited
|
||||||
|
Container mtusi2026lab6-migrate-1 Exited
|
||||||
|
Container mtusi2026lab6-s3-init-1 Exited
|
||||||
|
Container mtusi2026lab6-broker-1 Healthy
|
||||||
|
Container mtusi2026lab6-s3-init-1 Exited
|
||||||
|
Container mtusi2026lab6-worker-1 Starting
|
||||||
|
Container mtusi2026lab6-broker-1 Healthy
|
||||||
|
Container mtusi2026lab6-relay-1 Starting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Exited
|
||||||
|
Container mtusi2026lab6-api-2 Starting
|
||||||
|
Container mtusi2026lab6-worker-1 Started
|
||||||
|
Container mtusi2026lab6-api-2 Started
|
||||||
|
Container mtusi2026lab6-api-1 Starting
|
||||||
|
Container mtusi2026lab6-relay-1 Started
|
||||||
|
Container mtusi2026lab6-api-1 Started
|
||||||
|
Container mtusi2026lab6-s3-1 Waiting
|
||||||
|
Container mtusi2026lab6-api-2 Waiting
|
||||||
|
Container mtusi2026lab6-api-1 Waiting
|
||||||
|
Container mtusi2026lab6-postgres-1 Waiting
|
||||||
|
Container mtusi2026lab6-prometheus-1 Waiting
|
||||||
|
Container mtusi2026lab6-healthtools-1 Waiting
|
||||||
|
Container mtusi2026lab6-otel-collector-1 Waiting
|
||||||
|
Container mtusi2026lab6-worker-1 Waiting
|
||||||
|
Container mtusi2026lab6-valkey-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Waiting
|
||||||
|
Container mtusi2026lab6-tempo-1 Waiting
|
||||||
|
Container mtusi2026lab6-grafana-1 Waiting
|
||||||
|
Container mtusi2026lab6-relay-1 Waiting
|
||||||
|
Container mtusi2026lab6-lb-1 Waiting
|
||||||
|
Container mtusi2026lab6-broker-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Waiting
|
||||||
|
Container mtusi2026lab6-loki-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Exited
|
||||||
|
Container mtusi2026lab6-s3-1 Healthy
|
||||||
|
Container mtusi2026lab6-grafana-1 Healthy
|
||||||
|
Container mtusi2026lab6-postgres-1 Healthy
|
||||||
|
Container mtusi2026lab6-s3-init-1 Exited
|
||||||
|
Container mtusi2026lab6-broker-1 Healthy
|
||||||
|
Container mtusi2026lab6-otel-collector-1 Healthy
|
||||||
|
Container mtusi2026lab6-prometheus-1 Healthy
|
||||||
|
Container mtusi2026lab6-valkey-1 Healthy
|
||||||
|
Container mtusi2026lab6-healthtools-1 Exited
|
||||||
|
Container mtusi2026lab6-api-2 Healthy
|
||||||
|
Container mtusi2026lab6-api-1 Healthy
|
||||||
|
Container mtusi2026lab6-relay-1 Healthy
|
||||||
|
Container mtusi2026lab6-worker-1 Healthy
|
||||||
|
Container mtusi2026lab6-lb-1 Healthy
|
||||||
|
Container mtusi2026lab6-loki-1 Healthy
|
||||||
|
Container mtusi2026lab6-tempo-1 Healthy
|
||||||
|
scripts/lb-wait.sh
|
||||||
|
exit_code: 0 elapsed_seconds: 85.592
|
||||||
@@ -0,0 +1,270 @@
|
|||||||
|
{
|
||||||
|
"suite": "lab5-pipeline-idempotency",
|
||||||
|
"started_at": "2026-11-06T09:35:15.796014Z",
|
||||||
|
"source_sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
|
||||||
|
"config": {
|
||||||
|
"base_url": "http://127.0.0.1:18086",
|
||||||
|
"level": 3,
|
||||||
|
"only": "",
|
||||||
|
"expect_variants": "thumbnail",
|
||||||
|
"max_attempts": 5,
|
||||||
|
"stall_seconds": 15.0,
|
||||||
|
"compose": "docker compose",
|
||||||
|
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
|
||||||
|
},
|
||||||
|
"invariants": [
|
||||||
|
{
|
||||||
|
"id": "terminal",
|
||||||
|
"statement": "every 202 reaches its expected terminal state",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"accepted": 5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "single_effect",
|
||||||
|
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"rows": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "no_ready_rollback",
|
||||||
|
"statement": "ready never returns to queued or processing",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"traced": 5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "monotonic_publication",
|
||||||
|
"statement": "published_version only moves forward",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"sampled": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bytes_match_metadata",
|
||||||
|
"statement": "sha256, size and real dimensions match served bytes",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"verified": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bounded_attempts",
|
||||||
|
"statement": "attempt <= JOB_MAX_ATTEMPTS",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"rows": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "no_orphans",
|
||||||
|
"statement": "no pending outbox, messages, dead letters, staging, objects",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"queues": {
|
||||||
|
"attachments.process": [
|
||||||
|
0,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"attachments.process.dlq": [
|
||||||
|
0,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"s3_objects": 17,
|
||||||
|
"scoped_attachments": 5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "no_lost_202",
|
||||||
|
"statement": "every 202 is present in PostgreSQL and the API",
|
||||||
|
"result": "PASS",
|
||||||
|
"details": {
|
||||||
|
"accepted": 5
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"accepted_ids": [
|
||||||
|
"01a1b036-7bdb-72a8-9a20-d784124b6f09",
|
||||||
|
"01a1b036-838f-7073-a603-58ef2ffca05d",
|
||||||
|
"01a1b036-d38a-7375-ba48-c82f1e984740",
|
||||||
|
"01a1b036-e669-7726-8ed5-eb948e03323e",
|
||||||
|
"01a1b037-1c4a-72b4-8027-95bdd74c185a"
|
||||||
|
],
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "smoke_pipeline",
|
||||||
|
"level": 3,
|
||||||
|
"window": "upload, ready, variants match metadata",
|
||||||
|
"seconds": 1.4,
|
||||||
|
"result": "PASS",
|
||||||
|
"observed": {
|
||||||
|
"attachment_id": "01a1b036-7bdb-72a8-9a20-d784124b6f09",
|
||||||
|
"variants": [
|
||||||
|
"crop",
|
||||||
|
"optimized",
|
||||||
|
"thumbnail"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "worker_down_broker_restart",
|
||||||
|
"level": 3,
|
||||||
|
"window": "worker down, API and broker restart",
|
||||||
|
"seconds": 21.1,
|
||||||
|
"result": "PASS",
|
||||||
|
"observed": {
|
||||||
|
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d",
|
||||||
|
"timeline": [
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:25.067956Z",
|
||||||
|
"step": "worker stopped",
|
||||||
|
"containers": [
|
||||||
|
"mtusi2026lab6-worker-1 cd5b9bcf05a3 exited Exited (0) Less than a second ago"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:25.081364Z",
|
||||||
|
"step": "upload 202 queued",
|
||||||
|
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:25.086007Z",
|
||||||
|
"step": "content 409 while queued"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:25.109208Z",
|
||||||
|
"step": "message with attachment 201, other messages work",
|
||||||
|
"message_id": "01a1b036-839e-71c7-9abc-6d747695610e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:30.399187Z",
|
||||||
|
"step": "job is in the broker queue",
|
||||||
|
"queues": {
|
||||||
|
"attachments.process": [
|
||||||
|
1,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"attachments.process.dlq": [
|
||||||
|
0,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:36.592722Z",
|
||||||
|
"step": "API replicas and relay restarted"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:42.601391Z",
|
||||||
|
"step": "broker restarted with its volume, job survived",
|
||||||
|
"queues": {
|
||||||
|
"attachments.process": [
|
||||||
|
1,
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"attachments.process.dlq": [
|
||||||
|
0,
|
||||||
|
0
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:45.362335Z",
|
||||||
|
"step": "worker started"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-11-06T09:35:45.539899Z",
|
||||||
|
"step": "ready",
|
||||||
|
"variants": [
|
||||||
|
"crop",
|
||||||
|
"optimized",
|
||||||
|
"thumbnail"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "duplicate_after_ready",
|
||||||
|
"level": 3,
|
||||||
|
"window": "redelivery of a finished job",
|
||||||
|
"seconds": 4.4,
|
||||||
|
"result": "PASS",
|
||||||
|
"observed": {
|
||||||
|
"attachment_id": "01a1b036-d38a-7375-ba48-c82f1e984740",
|
||||||
|
"events": [
|
||||||
|
{
|
||||||
|
"kind": "enqueued",
|
||||||
|
"attempt": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "claimed",
|
||||||
|
"attempt": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "done",
|
||||||
|
"attempt": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "duplicate_noop",
|
||||||
|
"attempt": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "poison_image",
|
||||||
|
"level": 3,
|
||||||
|
"window": "permanent error without retries",
|
||||||
|
"seconds": 0.0,
|
||||||
|
"result": "PASS",
|
||||||
|
"observed": {
|
||||||
|
"rejected_by_api": 415
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "s3_outage_recovers",
|
||||||
|
"level": 3,
|
||||||
|
"window": "transient error with retry",
|
||||||
|
"seconds": 10.3,
|
||||||
|
"result": "PASS",
|
||||||
|
"observed": {
|
||||||
|
"attachment_id": "01a1b036-e669-7726-8ed5-eb948e03323e",
|
||||||
|
"jobs": [
|
||||||
|
{
|
||||||
|
"job_id": "01a1b036-e671-7294-8d09-37bc4619030b",
|
||||||
|
"pipeline_version": 1,
|
||||||
|
"status": "done",
|
||||||
|
"attempt": 2
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "retry_exhausted",
|
||||||
|
"level": 3,
|
||||||
|
"window": "transient error until JOB_MAX_ATTEMPTS",
|
||||||
|
"seconds": 20.3,
|
||||||
|
"result": "PASS",
|
||||||
|
"observed": {
|
||||||
|
"attachment_id": "01a1b037-1c4a-72b4-8027-95bdd74c185a",
|
||||||
|
"jobs": [
|
||||||
|
{
|
||||||
|
"job_id": "01a1b037-1c57-7496-8cc0-61aa229fd7a2",
|
||||||
|
"pipeline_version": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"attempt": 5
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"result": "PASS",
|
||||||
|
"finished_at": "2026-11-06T09:36:32.454476Z"
|
||||||
|
}
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
|
|
||||||
#### Scenario: Стенд из чистого состояния
|
#### Scenario: Стенд из чистого состояния
|
||||||
|
|
||||||
- **WHEN** после `make reset-data` выполняется `make up`
|
- **WHEN** в новом изолированном Compose project с пустыми volumes выполняется `make up`
|
||||||
- **THEN** `docker compose ps` показывает `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` в состоянии `healthy` рядом с `lb`, `api` ×2, `relay`, `worker`, `postgres`, `valkey`, `s3`, `broker`
|
- **THEN** `docker compose ps` показывает `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` в состоянии `healthy` рядом с `lb`, `api` ×2, `relay`, `worker`, `postgres`, `valkey`, `s3`, `broker`
|
||||||
|
|
||||||
#### Scenario: OTLP не опубликован
|
#### Scenario: OTLP не опубликован
|
||||||
|
|||||||
@@ -20,19 +20,19 @@
|
|||||||
|
|
||||||
## 4. Стек наблюдаемости
|
## 4. Стек наблюдаемости
|
||||||
|
|
||||||
- [ ] 4.1 `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml` и recording rules, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` из `.prep/lab6`
|
- [x] 4.1 `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml` и recording rules, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` из `.prep/lab6`
|
||||||
- [ ] 4.2 `infra/grafana/provisioning`, dashboards `messenger-api` и `messenger-pipeline` уровня «3»
|
- [x] 4.2 `infra/grafana/provisioning`, dashboards `messenger-api` и `messenger-pipeline` уровня «3»
|
||||||
- [ ] 4.3 `compose.yaml`: `healthtools`, `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana`, volumes, секрет Grafana, сети `backend`/`edge`, переменные OTel у `api`, `relay`, `worker`; `.env.example`
|
- [x] 4.3 `compose.yaml`: `healthtools`, `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana`, volumes, секрет Grafana, сети `backend`/`edge`, переменные OTel у `api`, `relay`, `worker`; `.env.example`
|
||||||
- [ ] 4.4 `scripts/secrets-lab6.sh`, `scripts/obs-validate.sh`, `scripts/obs_check.py`; цели `secrets`, `obs-validate`, `telemetry-check`, `obs-evidence`
|
- [x] 4.4 `scripts/secrets-lab6.sh`, `scripts/obs-validate.sh`, `scripts/obs_check.py`; цели `secrets`, `obs-validate`, `telemetry-check`
|
||||||
- [ ] 4.5 Проверка группы: `make obs-validate`, `docker compose config --quiet`, `make up` из чистого состояния — все сервисы `healthy`, порты наблюдаемости только на `127.0.0.1`
|
- [ ] 4.5 Проверка группы: `make obs-validate`, `docker compose config --quiet`, `make up` из чистого состояния — все сервисы `healthy`, порты наблюдаемости только на `127.0.0.1`
|
||||||
|
|
||||||
## 5. Проверка на стенде
|
## 5. Проверка на стенде
|
||||||
|
|
||||||
- [ ] 5.1 `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `make restore`, `make level3`
|
- [x] 5.1 `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `make restore`, `make level3`
|
||||||
- [ ] 5.2 `obs_check.py grafana`, `dashboards --strict`, `request`, `job`, `leaks`, cardinality, история после рестарта хранилищ, `make test` при остановленном Collector
|
- [x] 5.2 `obs_check.py grafana`, `dashboards --strict`, `request`, `job`, `leaks`, cardinality, история после рестарта хранилищ, `make test` при остановленном Collector
|
||||||
- [ ] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test`
|
- [x] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test`
|
||||||
|
|
||||||
## 6. Отчёт
|
## 6. Отчёт
|
||||||
|
|
||||||
- [ ] 6.1 `evidence/`: стенд, smoke, persistence, failover, level3, obs-validate, grafana, dashboards, trace-request, trace-job, leaks, cardinality, telemetry-restart, student-tests, contract-checks
|
- [x] 6.1 `evidence/`: стенд, smoke, persistence, failover, level3, obs-validate, grafana, dashboards, trace-request, trace-job, leaks, cardinality, telemetry-restart, student-tests, contract-checks
|
||||||
- [ ] 6.2 `REPORT.md`: шапка, запуск из чистого clone, архитектура и решения, роли, назначение хранилищ, соответствие сигналов и переменных, строки уровня «3», таблица отказов уровня «3»
|
- [x] 6.2 `evidence/grade3/notes.md`: заметки для отдельного агента отчёта — запуск, архитектура, роли, сигналы, доказательства и ограничения уровня «3»
|
||||||
|
|||||||
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mtusi2026lab6}
|
||||||
|
export LB_HTTP_PORT=${LB_HTTP_PORT:-18086} LB_STATS_PORT=${LB_STATS_PORT:-26085}
|
||||||
|
export BROKER_MANAGEMENT_PORT=${BROKER_MANAGEMENT_PORT:-26086}
|
||||||
|
export PROMETHEUS_PORT=${PROMETHEUS_PORT:-26090} LOKI_PORT=${LOKI_PORT:-26310}
|
||||||
|
export TEMPO_PORT=${TEMPO_PORT:-26320} GRAFANA_PORT=${GRAFANA_PORT:-26300}
|
||||||
|
export BASE_URL=${BASE_URL:-http://127.0.0.1:$LB_HTTP_PORT}
|
||||||
|
export PROMETHEUS_URL=http://127.0.0.1:$PROMETHEUS_PORT LOKI_URL=http://127.0.0.1:$LOKI_PORT
|
||||||
|
export TEMPO_URL=http://127.0.0.1:$TEMPO_PORT GRAFANA_URL=http://127.0.0.1:$GRAFANA_PORT
|
||||||
|
export NO_PROXY=localhost,127.0.0.1,::1
|
||||||
|
exec "$@"
|
||||||
@@ -66,7 +66,7 @@ def wait_for(what, check, timeout, interval=2.0):
|
|||||||
result = check()
|
result = check()
|
||||||
if result:
|
if result:
|
||||||
return result
|
return result
|
||||||
except Failure:
|
except (Failure, OSError):
|
||||||
pass
|
pass
|
||||||
if time.monotonic() > deadline:
|
if time.monotonic() > deadline:
|
||||||
raise Failure(f"timeout waiting for {what}")
|
raise Failure(f"timeout waiting for {what}")
|
||||||
@@ -253,6 +253,10 @@ def request_check(args, b):
|
|||||||
if not trace_id:
|
if not trace_id:
|
||||||
raise Failure("log record has no trace_id")
|
raise Failure("log record has no trace_id")
|
||||||
spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout)
|
spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout)
|
||||||
|
if not any(s["kind"] == "SPAN_KIND_SERVER" and s["service"] == args.api_service for s in spans):
|
||||||
|
raise Failure("no real API server span")
|
||||||
|
if not any(s["kind"] == "SPAN_KIND_CLIENT" and s["attributes"].get("db.system") == "postgresql" for s in spans):
|
||||||
|
raise Failure("no PostgreSQL client span inside API trace")
|
||||||
print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id,
|
print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id,
|
||||||
"span_id": labels.get("span_id"),
|
"span_id": labels.get("span_id"),
|
||||||
"service_name": labels.get("service_name"),
|
"service_name": labels.get("service_name"),
|
||||||
@@ -282,6 +286,10 @@ def job_check(args, b):
|
|||||||
query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"'
|
query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"'
|
||||||
worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
|
worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
|
||||||
api.logout(auth)
|
api.logout(auth)
|
||||||
|
if ready["status"] != "ready" or not any(s["kind"] == "SPAN_KIND_CONSUMER" and s["service"] == args.worker_service for s in chain):
|
||||||
|
raise Failure("job is not ready or no real consumer span")
|
||||||
|
if not any(labels.get("trace_id") and labels.get("span_id") for labels, _ in worker_logs):
|
||||||
|
raise Failure("worker log is not correlated")
|
||||||
print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"],
|
print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"],
|
||||||
"traces": sorted({c["trace_id"] for c in chain}), "spans": chain,
|
"traces": sorted({c["trace_id"] for c in chain}), "spans": chain,
|
||||||
"worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")}
|
"worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")}
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
|
||||||
|
from obs_check import Api, Backends, get_json, http, now, tempo_spans, wait_for
|
||||||
|
|
||||||
|
|
||||||
|
def backends():
|
||||||
|
return Backends(
|
||||||
|
argparse.Namespace(
|
||||||
|
prometheus=os.environ["PROMETHEUS_URL"],
|
||||||
|
loki=os.environ["LOKI_URL"],
|
||||||
|
tempo=os.environ["TEMPO_URL"],
|
||||||
|
grafana=os.environ["GRAFANA_URL"],
|
||||||
|
grafana_password_file=os.environ.get(
|
||||||
|
"GRAFANA_PASSWORD_FILE", "/".join(("secrets", "grafana_admin_password"))
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def compose(*args):
|
||||||
|
return subprocess.run(
|
||||||
|
["docker", "compose", *args], check=True, capture_output=True, text=True
|
||||||
|
).stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def history(b):
|
||||||
|
started = time.time()
|
||||||
|
_, _, _, rid = Api(os.environ["BASE_URL"]).register()
|
||||||
|
query = '{service_name="messenger-api"} | request_id="' + rid + '"'
|
||||||
|
logs = wait_for("log", lambda: b.logs(query, started - 1, time.time() + 1), 60)
|
||||||
|
trace_id = logs[0][0]["trace_id"]
|
||||||
|
spans = wait_for("trace", lambda: tempo_spans(b.trace(trace_id)[0]), 60)
|
||||||
|
expr = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})'
|
||||||
|
before = wait_for("metrics", lambda: b.promql(expr), 60)
|
||||||
|
at = before[0]["value"][0]
|
||||||
|
compose("restart", "prometheus", "loki", "tempo", "grafana")
|
||||||
|
wait_for("Grafana", lambda: get_json(b.grafana + "/api/health"), 90)
|
||||||
|
after_logs = wait_for("retained log", lambda: b.logs(query, started - 1, time.time() + 1), 90)
|
||||||
|
after_spans = wait_for("retained trace", lambda: tempo_spans(b.trace(trace_id)[0]), 90)
|
||||||
|
after = wait_for("retained metric", lambda: b.promql(expr, at=at), 90)
|
||||||
|
if before[0]["value"][1] != after[0]["value"][1] or len(after_spans) < len(spans):
|
||||||
|
raise RuntimeError("telemetry history changed after restart")
|
||||||
|
return {
|
||||||
|
"request_id": rid,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"spans_before": len(spans),
|
||||||
|
"spans_after": len(after_spans),
|
||||||
|
"logs_after": len(after_logs),
|
||||||
|
"metric_before": before,
|
||||||
|
"metric_after": after,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def collector():
|
||||||
|
compose("stop", "otel-collector")
|
||||||
|
try:
|
||||||
|
status = http("GET", os.environ["BASE_URL"] + "/health/ready")[0]
|
||||||
|
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=False)
|
||||||
|
if status != 200 or result.returncode:
|
||||||
|
raise RuntimeError("application failed without collector")
|
||||||
|
replica = compose("ps", "-q", "api").splitlines()[0]
|
||||||
|
started = time.monotonic()
|
||||||
|
subprocess.run(["docker", "stop", "-t", "8", replica], check=True, capture_output=True)
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
code = subprocess.check_output(
|
||||||
|
["docker", "inspect", "-f", "{{.State.ExitCode}}", replica], text=True
|
||||||
|
).strip()
|
||||||
|
if code != "0" or elapsed > 8:
|
||||||
|
raise RuntimeError("unbounded API shutdown")
|
||||||
|
return {
|
||||||
|
"ready": status,
|
||||||
|
"smoke": result.stdout.strip(),
|
||||||
|
"api_exit_code": code,
|
||||||
|
"api_shutdown_seconds": round(elapsed, 3),
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
compose("up", "-d", "--no-deps", "--wait", "api", "otel-collector")
|
||||||
|
subprocess.run(["scripts/lb-wait.sh"], check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
b = backends()
|
||||||
|
report = {
|
||||||
|
"utc": now(),
|
||||||
|
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||||
|
}
|
||||||
|
report["history"] = history(b)
|
||||||
|
report["collector_down"] = collector()
|
||||||
|
report["result"] = "PASS"
|
||||||
|
print(json.dumps(report, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user