Verify the grade 3 telemetry stack and inherited API guarantees

This commit is contained in:
Artyom
2026-11-06 12:41:06 +03:00
parent 95511002b1
commit 2903d9dc73
31 changed files with 1720 additions and 12 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
RATE ?= 100 RATE ?= 100
FROM_REV ?= FROM_REV ?=
SHIFT ?= 0 SHIFT ?= 0
SCRIPTS_PY = scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ SCRIPTS_PY = scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \ scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
SCENARIO ?= failover-upload SCENARIO ?= failover-upload
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T09:38:20Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py query 'count by (__name__,job) ({job=~"messenger-.*"})' 'count by (http_route,http_request_method,http_response_status_code) (http_server_request_duration_seconds_count)' 'count by (job_type,outcome) (messenger_jobs_processed_total)'
{"ts": "2026-11-06T09:38:20+00:00", "results": {"count by (__name__,job) ({job=~\"messenger-.*\"})": [{"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "7"}, {"labels": {"job": "messenger-worker"}, "value": "77"}, {"labels": {"job": "messenger-worker"}, "value": "4"}, {"labels": {"job": "messenger-api"}, "value": "17"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "990"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "66"}, {"labels": {"job": "messenger-api"}, "value": "1056"}, {"labels": {"job": "messenger-api"}, "value": "34"}, {"labels": {"job": "messenger-api"}, "value": "34"}, {"labels": {"job": "messenger-api"}, "value": "544"}, {"labels": {"job": "messenger-api"}, "value": "8"}], "count by (http_route,http_request_method,http_response_status_code) (http_server_request_duration_seconds_count)": [{"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/auth/sessions"}, "value": "5"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "409", "http_route": "/api/v1/users"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/chats/{chat_id}"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/chats"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "5"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "3"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "400", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "2"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "400", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "401", "http_route": "/api/v1/users/me"}, "value": "2"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/users/me"}, "value": "4"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "409", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "202", "http_route": "/api/v1/chats/{chat_id}/attachments"}, "value": "7"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/attachments/{attachment_id}"}, "value": "7"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "200", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "5"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "1"}, {"labels": {"http_request_method": "DELETE", "http_response_status_code": "204", "http_route": "/api/v1/auth/sessions/current"}, "value": "2"}, {"labels": {"http_request_method": "DELETE", "http_response_status_code": "401", "http_route": "/api/v1/auth/sessions/current"}, "value": "1"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/users"}, "value": "5"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "201", "http_route": "/api/v1/chats"}, "value": "4"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "404", "http_route": "/api/v1/chats/{chat_id}/messages"}, "value": "1"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "401", "http_route": "/api/v1/auth/sessions"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "404", "http_route": "/api/v1/attachments/{attachment_id}"}, "value": "1"}, {"labels": {"http_request_method": "GET", "http_response_status_code": "409", "http_route": "/api/v1/attachments/{attachment_id}/content"}, "value": "2"}, {"labels": {"http_request_method": "POST", "http_response_status_code": "415", "http_route": "/api/v1/chats/{chat_id}/attachments"}, "value": "1"}], "count by (job_type,outcome) (messenger_jobs_processed_total)": [{"labels": {"job_type": "attachment.process.v1", "outcome": "done"}, "value": "3"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "duplicate_noop"}, "value": "1"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "retry_scheduled"}, "value": "2"}, {"labels": {"job_type": "attachment.process.v1", "outcome": "failed"}, "value": "1"}]}}
exit_code: 0 elapsed_seconds: 0.057
+153
View File
@@ -0,0 +1,153 @@
date: 2026-11-06T09:38:20Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py dashboards --strict
{
"checked_at": "2026-11-06T09:38:20+00:00",
"targets": [
{
"dashboard": "messenger-api.json",
"panel": "RPS бизнес-запросов",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Доля 5xx (2m)",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "p95 latency (5m)",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Доля не-5xx (окно 1h)",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "RPS по маршруту и классу статуса",
"datasource": "prometheus",
"series": 9,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Доля ошибок",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Доля ошибок",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Latency p50 / p95 / p99",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Latency p50 / p95 / p99",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Latency p50 / p95 / p99",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "p95 по маршруту",
"datasource": "prometheus",
"series": 7,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "RPS по реплике",
"datasource": "prometheus",
"series": 4,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Активные запросы",
"datasource": "prometheus",
"series": 8,
"missing_metrics": []
},
{
"dashboard": "messenger-api.json",
"panel": "Логи API (ошибки и предупреждения)",
"datasource": "loki",
"lines": 5
},
{
"dashboard": "messenger-api.json",
"panel": "Трассы API с ошибкой",
"datasource": "tempo",
"traces": 0
},
{
"dashboard": "messenger-pipeline.json",
"panel": "Jobs по исходу",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-pipeline.json",
"panel": "Длительность обработки job",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-pipeline.json",
"panel": "Длительность обработки job",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-pipeline.json",
"panel": "Длительность обработки job",
"datasource": "prometheus",
"series": 1,
"missing_metrics": []
},
{
"dashboard": "messenger-pipeline.json",
"panel": "Логи worker",
"datasource": "loki",
"lines": 5
},
{
"dashboard": "messenger-pipeline.json",
"panel": "Трассы worker",
"datasource": "tempo",
"traces": 5
}
],
"problems": []
}
exit_code: 0 elapsed_seconds: 0.089
+43
View File
@@ -0,0 +1,43 @@
date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ env EVIDENCE_DIR=evidence/grade3/failover-first make failover-evidence SHIFT=0
sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "0"
date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
9af2280c8af3 running started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:33:50.075450885Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 3.03}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
== after: api replicas
9af2280c8af3 exited started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
== stopped replica: 9af2280c8af3; shutdown: signal, drain window with readiness 503, close, exit
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:35.851311Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.864420Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.897236Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"5818535d613b4b7a8f0607d91e58d3f5","duration_ms":0.837,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.062326Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"4838cbd4822f4181a2e10153a41033d6","duration_ms":0.347,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.566176Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.866622Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.900282Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:37.973761Z"}
{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T09:34:37.977541Z"}
{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:37.987715Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:38.004692Z"}
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:38.006586Z"}
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:38.093383Z"}
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:38.195744Z"}
$ scripts/restore.sh
Container mtusi2026lab6-api-2 Running
Container mtusi2026lab6-api-1 Starting
Container mtusi2026lab6-api-1 Started
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-api-1 Healthy
== after restore
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
exit_code: 0 elapsed_seconds: 20.584
@@ -0,0 +1,38 @@
date: 2026-11-06T09:34:33Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
9af2280c8af3 running started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:33:50.075450885Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 3.03}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
== after: api replicas
9af2280c8af3 exited started=2026-11-06T09:33:56.602420846Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
== stopped replica: 9af2280c8af3; shutdown: signal, drain window with readiness 503, close, exit
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:35.851311Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.864420Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:35.897236Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"5818535d613b4b7a8f0607d91e58d3f5","duration_ms":0.837,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.062326Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"4838cbd4822f4181a2e10153a41033d6","duration_ms":0.347,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:36.566176Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.866622Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:37.900282Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:37.973761Z"}
{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T09:34:37.977541Z"}
{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:37.987715Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:38.004692Z"}
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:38.006586Z"}
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:38.093383Z"}
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:38.195744Z"}
$ scripts/restore.sh
Container mtusi2026lab6-api-2 Running
Container mtusi2026lab6-api-1 Starting
Container mtusi2026lab6-api-1 Started
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-api-1 Healthy
== after restore
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
+43
View File
@@ -0,0 +1,43 @@
date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ env EVIDENCE_DIR=evidence/grade3/failover-second make failover-evidence SHIFT=1
sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "1"
date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.777}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
== after: api replicas
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 exited started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:34:58.204116764Z
== stopped replica: 943813ecb712; shutdown: signal, drain window with readiness 503, close, exit
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:55.700248Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.784538Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.795484Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"ff6952bfefbe4034ae30489d4e43b93c","duration_ms":0.567,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.029040Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"194add29fb3742db93424a18cf9ff680","duration_ms":1.596,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.535013Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.787659Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.796586Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.894245Z"}
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:57.896210Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.903471Z"}
{"event":"Finished server process [23]","level":"info","ts":"2026-11-06T09:34:57.905749Z"}
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954114Z"}
{"event":"Worker (pid:23) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954212Z"}
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:58.057111Z"}
$ scripts/restore.sh
Container mtusi2026lab6-api-1 Running
Container mtusi2026lab6-api-2 Starting
Container mtusi2026lab6-api-2 Started
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-api-1 Healthy
== after restore
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:35:11.260329797Z finished=2026-11-06T09:34:58.204116764Z
exit_code: 0 elapsed_seconds: 20.372
@@ -0,0 +1,38 @@
date: 2026-11-06T09:34:53Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:33:50.07508976Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.777}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
== after: api replicas
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 exited started=2026-11-06T09:33:56.539288471Z finished=2026-11-06T09:34:58.204116764Z
== stopped replica: 943813ecb712; shutdown: signal, drain window with readiness 503, close, exit
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T09:34:55.700248Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.784538Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T09:34:55.795484Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"ff6952bfefbe4034ae30489d4e43b93c","duration_ms":0.567,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.029040Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"194add29fb3742db93424a18cf9ff680","duration_ms":1.596,"event":"request.handled","level":"info","ts":"2026-11-06T09:34:56.535013Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.787659Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-06T09:34:57.796586Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.894245Z"}
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T09:34:57.896210Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-06T09:34:57.903471Z"}
{"event":"Finished server process [23]","level":"info","ts":"2026-11-06T09:34:57.905749Z"}
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954114Z"}
{"event":"Worker (pid:23) was sent SIGTERM!","level":"info","ts":"2026-11-06T09:34:57.954212Z"}
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T09:34:58.057111Z"}
$ scripts/restore.sh
Container mtusi2026lab6-api-1 Running
Container mtusi2026lab6-api-2 Starting
Container mtusi2026lab6-api-2 Started
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-api-1 Healthy
== after restore
9af2280c8af3 running started=2026-11-06T09:34:51.34604076Z finished=2026-11-06T09:34:38.360642671Z
943813ecb712 running started=2026-11-06T09:35:11.260329797Z finished=2026-11-06T09:34:58.204116764Z
+42
View File
@@ -0,0 +1,42 @@
date: 2026-11-06T09:33:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py grafana
{
"checked_at": "2026-11-06T09:33:43+00:00",
"grafana": "12.4.14",
"datasources": {
"prometheus": {
"http": 200,
"status": "OK",
"message": "Successfully queried the Prometheus API."
},
"loki": {
"http": 200,
"status": "OK",
"message": "Data source successfully connected."
},
"tempo": {
"http": 200,
"status": "OK",
"message": "Data source is working. Streaming test succeeded."
}
},
"dashboards": {
"messenger-api": {
"title": "Messenger · API RED",
"http": 200,
"provisioned": true,
"folder": "Messenger",
"panels": 14
},
"messenger-pipeline": {
"title": "Messenger · Worker",
"http": 200,
"provisioned": true,
"folder": "Messenger",
"panels": 6
}
},
"missing_dashboards": []
}
exit_code: 0 elapsed_seconds: 0.124
+316
View File
@@ -0,0 +1,316 @@
date: 2026-11-06T09:37:45Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py job
{
"checked_at": "2026-11-06T09:38:05+00:00",
"upload_request_id": "6254becc-0433-44ae-89ea-057e0405f827",
"attachment_status": "ready",
"traces": [
"a020913eec288efb8126ca3a69a513d4"
],
"spans": [
{
"service": "messenger-worker",
"name": "attachments.attachment.process.v1 create",
"kind": "SPAN_KIND_CONSUMER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.008,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "attachments.attachment.process.v1 process",
"kind": "SPAN_KIND_CONSUMER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 777.9,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": "1"
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.133,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "select",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.111,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "select",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.095,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "show",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.085,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "show",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.081,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.081,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.082,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.554,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.57,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.199,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.108,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.331,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.08,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.155,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.07,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.062,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.112,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.171,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.124,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.069,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.054,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.073,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.179,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.414,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.18,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.116,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
},
{
"service": "messenger-worker",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.307,
"links": 0,
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"attempt": null
}
],
"worker_logs": [
{
"line": "job.processed",
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"span_id": "ff3e55c7d762c0e1"
},
{
"line": "audit.job_settled",
"trace_id": "a020913eec288efb8126ca3a69a513d4",
"span_id": "ff3e55c7d762c0e1"
}
]
}
exit_code: 0 elapsed_seconds: 19.790
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T09:33:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py job
FAIL: <urlopen error [Errno 61] Connection refused>
exit_code: 1 elapsed_seconds: 0.055
+18
View File
@@ -0,0 +1,18 @@
date: 2026-11-06T09:35:15Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make level3
BASE_URL="http://127.0.0.1:18086" python3 scripts/pipeline_suite.py --level 3 --output evidence/level3.json
RUN smoke_pipeline
PASS smoke_pipeline
RUN worker_down_broker_restart
PASS worker_down_broker_restart
RUN duplicate_after_ready
PASS duplicate_after_ready
RUN poison_image
PASS poison_image
RUN s3_outage_recovers
PASS s3_outage_recovers
RUN retry_exhausted
PASS retry_exhausted
PASS: lab5 pipeline suite, report evidence/level3.json
exit_code: 0 elapsed_seconds: 76.738
+27
View File
@@ -0,0 +1,27 @@
date: 2026-11-06T09:38:05Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py leaks
{
"checked_at": "2026-11-06T09:38:20+00:00",
"window_seconds": 20,
"traces_scanned": 9,
"hits": {
"password": {
"loki": 0,
"tempo": 0
},
"token": {
"loki": 0,
"tempo": 0
},
"message_text": {
"loki": 0,
"tempo": 0
},
"filename": {
"loki": 0,
"tempo": 0
}
}
}
exit_code: 0 elapsed_seconds: 15.275
+24
View File
@@ -0,0 +1,24 @@
# Уровень 3 — сведения для отдельного отчёта
Приложение проверено на SQLAlchemy-коде 95511002b12510178a3cc2a22566b43e2b2b6c81. Новый код этой фазы — обвязка запуска и строгая проверка реальных spans. Команды запускаются через `scripts/lab6.sh`, задающий только порты ЛР6, project и NO_PROXY; значения secrets не выводятся. Дата/SHA и exit_code есть в каждом command log.
| Критерий | Команда и доказательство |
|---|---|
| Минимум lab5 | make test: smoke-ready.txt; make test-persistence: persistence.txt; failover каждой реплики: failover-first.txt/failover-second.txt; make level3: lab5-minimum.txt и ../level3.json (6 PASS) |
| Конфигурации и provisioning | make obs-validate: obs-validate.txt; obs_check.py grafana: grafana.txt (3 OK, 2 dashboards provisioned); runtime.json (non-root, лимиты, volumes, порты) |
| RED и worker metrics | obs_check.py dashboards --strict: dashboards.txt; cardinality.txt: реальные HTTP-шаблоны, done/failed/retry_scheduled/duplicate_noop |
| Запрос → лог → trace | obs_check.py request: request.txt, реальный POST server span и PostgreSQL client spans |
| Job → лог → trace | obs_check.py job: job-stable.txt, ready, consumer spans и trace/span IDs worker |
| Отсутствие содержимого | obs_check.py leaks: leaks.txt, все 4 маркера дают 0 в Loki и Tempo |
| История и независимость API | obs_recovery.py: telemetry-recovery.txt, прежние trace/log/metric после restart; smoke при остановленном Collector; SIGTERM API 4.592 с, exit 0 |
| Проверки кода | quality.txt: check/validate/lint PASS; student-tests.txt: 515 passed, 1 skipped, 19 deselected |
Collector принимает OTLP/HTTP и обрабатывает/экспортирует сигналы, не заменяя долговременную БД. Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы. Grafana запрашивает эти хранилища и связывает сигналы через provisioned datasources. SDK запускается в процессе API после fork и в отдельных relay/worker. Batch processors и periodic reader не блокируют запрос на экспорт.
HTTP: `http_server_request_duration_seconds_{bucket,sum,count}`, labels `http_route`, `http_request_method`, `http_response_status_code`, service resource. RPS и классы статуса: `messenger:http_requests:rate2m`; error ratio — 5xx/все бизнес-запросы, health исключены. Worker: `messenger_jobs_processed_total`, `messenger_job_duration_seconds_{bucket,sum,count}`, labels `job_type,outcome`. Секунды; HTTP percentiles вычисляются histogram_quantile по сумме buckets реплик. Loki индексирует только service_name и environment; IDs остаются structured metadata. SDK не захватывает HTTP-заголовки и параметры БД, Collector дополнительно redacts подозрительные поля.
## Ограничения и неуспешные попытки
smoke.txt запущен до завершения up --wait и закономерно получил connection refused; smoke-ready.txt — исправленный порядок. job.txt попал в рестарт persistence; job-stable.txt выполнен отдельно и прошёл. Эти неудачи сохранены и не объявлены PASS. Полный pipeline/failover выполнялся последовательно. Unit/integration tests использовали свои одноразовые контейнеры одновременно только с проверками чтения телеметрии; нагрузочных замеров уровня 5 здесь нет. Linux-only RLIMIT_AS пропущен на macOS, stack-маркер штатно исключил 19 тестов; обязательные публичные отказы выполнены отдельно.
Minor из переноса: scripts/limits.py и CI-allowlist ещё не знают сервисы OTel; проверка реальных ограничений выполнена независимо в runtime.json. REPORT.md оставлен отдельному отчётному агенту.
+16
View File
@@ -0,0 +1,16 @@
date: 2026-11-06T09:31:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make obs-validate
sh scripts/obs-validate.sh
otelcol-contrib validate collector.yaml: OK
Checking /etc/prometheus/prometheus.yml
SUCCESS: 1 rule files found
SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
Checking /etc/prometheus/rules/messenger.recording.yml
SUCCESS: 8 rules found
loki -verify-config: OK
tempo -config.verify: OK
grafana dashboards: valid JSON with uid and panels
exit_code: 0 elapsed_seconds: 1.015
+7
View File
@@ -0,0 +1,7 @@
date: 2026-11-06T09:33:44Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit_code: 0 elapsed_seconds: 14.842
+14
View File
@@ -0,0 +1,14 @@
date: 2026-11-06T09:37:46Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make check validate lint
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
189 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
exit_code: 0 elapsed_seconds: 0.744
+66
View File
@@ -0,0 +1,66 @@
date: 2026-11-06T09:33:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py request
{
"checked_at": "2026-11-06T09:33:47+00:00",
"request_id": "fea83e54-fa3d-4ad9-af04-93f636b6b386",
"log": {
"line": "request.handled",
"trace_id": "a5bbfed14a0e3087c87d5368f65743aa",
"span_id": "62f0ee285437fbfa",
"service_name": "messenger-api",
"severity": "INFO"
},
"trace": [
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.134
},
{
"service": "messenger-api",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.074
},
{
"service": "messenger-api",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.074
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.056
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.18
},
{
"service": "messenger-api",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.23
},
{
"service": "messenger-api",
"name": "POST /api/v1/users",
"kind": "SPAN_KIND_SERVER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 13.489
}
]
}
exit_code: 0 elapsed_seconds: 4.098
+102
View File
@@ -0,0 +1,102 @@
{
"utc": "2026-11-06T09:40:22.595732+00:00",
"sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
"command": "docker inspect selected fields; docker top <project container> -eo pid,uid,comm",
"services": [
{
"service": "grafana",
"processes": [
"PID UID COMMAND",
"74426 472 grafana"
],
"memory_limit": 402653184,
"cpu_limit": 500000000,
"ports": {
"3000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "26300"
}
]
},
"volumes": [
"mtusi2026lab6_grafana-data"
]
},
{
"service": "loki",
"processes": [
"PID UID COMMAND",
"74652 10001 loki"
],
"memory_limit": 402653184,
"cpu_limit": 500000000,
"ports": {
"3100/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "26310"
}
]
},
"volumes": [
"mtusi2026lab6_loki-data",
"mtusi2026lab6_healthtools"
]
},
{
"service": "tempo",
"processes": [
"PID UID COMMAND",
"75480 10001 tempo"
],
"memory_limit": 402653184,
"cpu_limit": 500000000,
"ports": {
"3200/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "26320"
}
]
},
"volumes": [
"mtusi2026lab6_tempo-data",
"mtusi2026lab6_healthtools"
]
},
{
"service": "otel-collector",
"processes": [
"PID UID COMMAND",
"76175 10001 otelcol-contrib"
],
"memory_limit": 268435456,
"cpu_limit": 500000000,
"ports": {},
"volumes": [
"mtusi2026lab6_healthtools"
]
},
{
"service": "prometheus",
"processes": [
"PID UID COMMAND",
"74496 65534 prometheus"
],
"memory_limit": 536870912,
"cpu_limit": 500000000,
"ports": {
"9090/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "26090"
}
]
},
"volumes": [
"mtusi2026lab6_prometheus-data"
]
}
]
}
+6
View File
@@ -0,0 +1,6 @@
date: 2026-11-06T09:33:42Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit_code: 0 elapsed_seconds: 2.455
+7
View File
@@ -0,0 +1,7 @@
date: 2026-11-06T09:32:40Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
FAIL: GET /health/live: connection/TLS failure (URLError)
make: *** [test] Error 1
exit_code: 2 elapsed_seconds: 0.056
+17
View File
@@ -0,0 +1,17 @@
api running Up 3 minutes (healthy)
api running Up 3 minutes (healthy)
broker running Up 3 minutes (healthy) 127.0.0.1:26086->15672/tcp
grafana running Up 6 minutes (healthy) 127.0.0.1:26300->3000/tcp
healthtools exited Exited (0) 5 minutes ago
lb running Up 5 minutes (healthy) 127.0.0.1:18086->8080/tcp, 127.0.0.1:26085->8404/tcp
loki running Up 6 minutes (healthy) 127.0.0.1:26310->3100/tcp
migrate exited Exited (0) 5 minutes ago
otel-collector running Up 6 minutes (healthy) 4317-4318/tcp, 55679/tcp
postgres running Up 5 minutes (healthy)
prometheus running Up 6 minutes (healthy) 127.0.0.1:26090->9090/tcp
relay running Up 2 minutes (healthy)
s3 running Up 3 minutes (healthy)
s3-init exited Exited (0) 5 minutes ago
tempo running Up 6 minutes (healthy) 127.0.0.1:26320->3200/tcp
valkey running Up 5 minutes (healthy)
worker running Up 2 minutes (healthy)
+62
View File
@@ -0,0 +1,62 @@
date: 2026-11-06T09:37:47Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make student-test
.venv/bin/pytest
============================= test session starts ==============================
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
rootdir: <repo>
configfile: pyproject.toml
testpaths: student-tests
plugins: asyncio-1.4.0, anyio-4.15.1
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
collected 535 items / 19 deselected / 516 selected
student-tests/adapters/test_rabbitmq.py .... [ 0%]
student-tests/adapters/test_s3_storage.py ... [ 1%]
student-tests/api/test_access.py ......... [ 3%]
student-tests/api/test_attachments.py .................................. [ 9%]
.... [ 10%]
student-tests/api/test_bounded_contexts.py .................... [ 14%]
student-tests/api/test_cache.py ...... [ 15%]
student-tests/api/test_concurrency.py .... [ 16%]
student-tests/api/test_consistency.py ........ [ 17%]
student-tests/api/test_drain.py .. [ 18%]
student-tests/api/test_events.py ...... [ 19%]
student-tests/api/test_http_telemetry.py . [ 19%]
student-tests/api/test_idempotency.py ............... [ 22%]
student-tests/api/test_instance.py .... [ 23%]
student-tests/api/test_job_returning.py . [ 23%]
student-tests/api/test_message_attachments.py ............. [ 25%]
student-tests/api/test_migrations.py ................................... [ 32%]
. [ 32%]
student-tests/api/test_negative.py ..................................... [ 40%]
[ 40%]
student-tests/api/test_operability.py .............. [ 42%]
student-tests/api/test_pagination.py .................. [ 46%]
student-tests/api/test_pipeline.py ............. [ 48%]
student-tests/api/test_pool.py ............ [ 51%]
student-tests/api/test_reconciler.py ........... [ 53%]
student-tests/api/test_recovery.py ............ [ 55%]
student-tests/api/test_sessions.py .................. [ 59%]
student-tests/api/test_timeout.py ........ [ 60%]
student-tests/api/test_upload_consistency.py ......... [ 62%]
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
student-tests/api/test_wire_format.py .. [ 64%]
student-tests/unit/test_attachments_domain.py .......................... [ 69%]
. [ 70%]
student-tests/unit/test_authentication.py ..................... [ 74%]
student-tests/unit/test_cursor.py ........... [ 76%]
student-tests/unit/test_domain.py ...................................... [ 83%]
.. [ 84%]
student-tests/unit/test_job_observer.py . [ 84%]
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
student-tests/unit/test_probe.py .............. [ 91%]
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
student-tests/unit/test_render_pool.py .....s. [ 93%]
student-tests/unit/test_renderer.py .................... [ 97%]
student-tests/unit/test_scaling.py ..... [ 98%]
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
student-tests/unit/test_timeouts.py . [100%]
========== 515 passed, 1 skipped, 19 deselected in 152.09s (0:02:32) ===========
exit_code: 0 elapsed_seconds: 153.875
+40
View File
@@ -0,0 +1,40 @@
date: 2026-11-06T09:39:43Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_recovery.py
{
"utc": "2026-11-06T09:39:43+00:00",
"sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
"history": {
"request_id": "895734e9-5c73-4054-ac37-804e02379460",
"trace_id": "33f5b0aea4ca28712a72fb9642e589f5",
"spans_before": 7,
"spans_after": 7,
"logs_after": 1,
"metric_before": [
{
"metric": {},
"value": [
1793957985.684,
"352"
]
}
],
"metric_after": [
{
"metric": {},
"value": [
1793957985.684,
"352"
]
}
]
},
"collector_down": {
"ready": 200,
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:18086\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.",
"api_exit_code": "0",
"api_shutdown_seconds": 4.592
},
"result": "PASS"
}
exit_code: 0 elapsed_seconds: 22.655
+230
View File
@@ -0,0 +1,230 @@
date: 2026-11-06T09:32:07Z commit: 95511002b12510178a3cc2a22566b43e2b2b6c81
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make up
sh scripts/secrets.sh
sh scripts/broker-secrets.sh
broker and S3 worker secrets ready in secrets/ (values not printed)
sh scripts/secrets-lab6.sh
secrets/grafana_admin_password is ready (value not printed)
docker compose build
#1 [internal] load local bake definitions
#1 reading from stdin 432B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.65kB done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#3 DONE 0.0s
#4 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#4 DONE 0.0s
#5 [internal] load .dockerignore
#5 transferring context: 453B done
#5 DONE 0.0s
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
#6 DONE 0.0s
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
#7 DONE 0.0s
#8 [internal] load build context
#8 transferring context: 77.10kB 0.0s done
#8 DONE 0.0s
#9 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
#9 CACHED
#10 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
#10 CACHED
#11 [builder 3/7] WORKDIR /build
#11 CACHED
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
#12 CACHED
#13 [builder 6/7] COPY src ./src
#13 DONE 0.0s
#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
#14 0.198 Resolved 100 packages in 1ms
#14 0.210 Building mtusi-messenger @ file:///build
#14 1.664 Built mtusi-messenger @ file:///build
#14 1.669 Prepared 1 package in 1.45s
#14 1.673 Installed 1 package in 3ms
#14 1.918 Bytecode compiled 2802 files in 245ms
#14 1.918 + mtusi-messenger==1.6.0 (from file:///build)
#14 DONE 2.0s
#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
#15 CACHED
#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv
#16 DONE 0.5s
#17 [runtime 4/6] WORKDIR /app
#17 DONE 0.0s
#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./
#18 DONE 0.0s
#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic
#19 DONE 0.0s
#20 exporting to image
#20 exporting layers
#20 exporting layers 3.0s done
#20 exporting manifest sha256:d38e0f0539ab16584375a6654edcd69ba0ad8a9dfb5fc3221a88909fbf5a52a9
#20 exporting manifest sha256:d38e0f0539ab16584375a6654edcd69ba0ad8a9dfb5fc3221a88909fbf5a52a9 done
#20 exporting config sha256:936eb99a24924e6b5f866e8b8ffeb97ced4056f13385a132eda3d8b2cc94c2fb done
#20 exporting attestation manifest sha256:329afd3eb83e5c817ea776ccb7f65b18a8da4ff79444c75f85d651eb524cb0a7 done
#20 exporting manifest list sha256:be397eab1ba1525d7ad025c7162b7a748a061983dd2895023266825a9a13794b done
#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done
#20 DONE 3.8s
#21 resolving provenance for metadata file
#21 DONE 0.0s
api Built
docker compose up -d --wait
Network mtusi2026lab6_backend Creating
Network mtusi2026lab6_backend Created
Network mtusi2026lab6_edge Creating
Network mtusi2026lab6_edge Created
Container mtusi2026lab6-broker-1 Creating
Container mtusi2026lab6-prometheus-1 Creating
Container mtusi2026lab6-valkey-1 Creating
Container mtusi2026lab6-postgres-1 Creating
Container mtusi2026lab6-lb-1 Creating
Container mtusi2026lab6-s3-1 Creating
Container mtusi2026lab6-healthtools-1 Creating
Container mtusi2026lab6-s3-1 Created
Container mtusi2026lab6-s3-init-1 Creating
Container mtusi2026lab6-valkey-1 Created
Container mtusi2026lab6-prometheus-1 Created
Container mtusi2026lab6-healthtools-1 Created
Container mtusi2026lab6-loki-1 Creating
Container mtusi2026lab6-otel-collector-1 Creating
Container mtusi2026lab6-tempo-1 Creating
Container mtusi2026lab6-lb-1 Created
Container mtusi2026lab6-postgres-1 Created
Container mtusi2026lab6-migrate-1 Creating
Container mtusi2026lab6-broker-1 Created
Container mtusi2026lab6-s3-init-1 Created
Container mtusi2026lab6-otel-collector-1 Created
Container mtusi2026lab6-migrate-1 Created
Container mtusi2026lab6-worker-1 Creating
Container mtusi2026lab6-api-2 Creating
Container mtusi2026lab6-api-1 Creating
Container mtusi2026lab6-relay-1 Creating
Container mtusi2026lab6-tempo-1 Created
Container mtusi2026lab6-loki-1 Created
Container mtusi2026lab6-grafana-1 Creating
Container mtusi2026lab6-worker-1 Created
Container mtusi2026lab6-api-1 Created
Container mtusi2026lab6-relay-1 Created
Container mtusi2026lab6-api-2 Created
Container mtusi2026lab6-grafana-1 Created
Container mtusi2026lab6-s3-1 Starting
Container mtusi2026lab6-lb-1 Starting
Container mtusi2026lab6-valkey-1 Starting
Container mtusi2026lab6-broker-1 Starting
Container mtusi2026lab6-healthtools-1 Starting
Container mtusi2026lab6-prometheus-1 Starting
Container mtusi2026lab6-postgres-1 Starting
Container mtusi2026lab6-healthtools-1 Started
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-s3-1 Started
Container mtusi2026lab6-s3-1 Waiting
Container mtusi2026lab6-postgres-1 Started
Container mtusi2026lab6-postgres-1 Waiting
Container mtusi2026lab6-valkey-1 Started
Container mtusi2026lab6-broker-1 Started
Container mtusi2026lab6-prometheus-1 Started
Container mtusi2026lab6-lb-1 Started
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-tempo-1 Starting
Container mtusi2026lab6-loki-1 Starting
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-otel-collector-1 Starting
Container mtusi2026lab6-postgres-1 Healthy
Container mtusi2026lab6-migrate-1 Starting
Container mtusi2026lab6-tempo-1 Started
Container mtusi2026lab6-migrate-1 Started
Container mtusi2026lab6-loki-1 Started
Container mtusi2026lab6-grafana-1 Starting
Container mtusi2026lab6-otel-collector-1 Started
Container mtusi2026lab6-grafana-1 Started
Container mtusi2026lab6-s3-1 Healthy
Container mtusi2026lab6-s3-init-1 Starting
Container mtusi2026lab6-s3-init-1 Started
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-broker-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-broker-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-broker-1 Healthy
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-worker-1 Starting
Container mtusi2026lab6-broker-1 Healthy
Container mtusi2026lab6-relay-1 Starting
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-api-2 Starting
Container mtusi2026lab6-worker-1 Started
Container mtusi2026lab6-api-2 Started
Container mtusi2026lab6-api-1 Starting
Container mtusi2026lab6-relay-1 Started
Container mtusi2026lab6-api-1 Started
Container mtusi2026lab6-s3-1 Waiting
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-postgres-1 Waiting
Container mtusi2026lab6-prometheus-1 Waiting
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-otel-collector-1 Waiting
Container mtusi2026lab6-worker-1 Waiting
Container mtusi2026lab6-valkey-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-tempo-1 Waiting
Container mtusi2026lab6-grafana-1 Waiting
Container mtusi2026lab6-relay-1 Waiting
Container mtusi2026lab6-lb-1 Waiting
Container mtusi2026lab6-broker-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-loki-1 Waiting
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-s3-1 Healthy
Container mtusi2026lab6-grafana-1 Healthy
Container mtusi2026lab6-postgres-1 Healthy
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-broker-1 Healthy
Container mtusi2026lab6-otel-collector-1 Healthy
Container mtusi2026lab6-prometheus-1 Healthy
Container mtusi2026lab6-valkey-1 Healthy
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-api-1 Healthy
Container mtusi2026lab6-relay-1 Healthy
Container mtusi2026lab6-worker-1 Healthy
Container mtusi2026lab6-lb-1 Healthy
Container mtusi2026lab6-loki-1 Healthy
Container mtusi2026lab6-tempo-1 Healthy
scripts/lb-wait.sh
exit_code: 0 elapsed_seconds: 85.592
+270
View File
@@ -0,0 +1,270 @@
{
"suite": "lab5-pipeline-idempotency",
"started_at": "2026-11-06T09:35:15.796014Z",
"source_sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
"config": {
"base_url": "http://127.0.0.1:18086",
"level": 3,
"only": "",
"expect_variants": "thumbnail",
"max_attempts": 5,
"stall_seconds": 15.0,
"compose": "docker compose",
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
},
"invariants": [
{
"id": "terminal",
"statement": "every 202 reaches its expected terminal state",
"result": "PASS",
"details": {
"accepted": 5
}
},
{
"id": "single_effect",
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_ready_rollback",
"statement": "ready never returns to queued or processing",
"result": "PASS",
"details": {
"traced": 5
}
},
{
"id": "monotonic_publication",
"statement": "published_version only moves forward",
"result": "PASS",
"details": {
"sampled": 2
}
},
{
"id": "bytes_match_metadata",
"statement": "sha256, size and real dimensions match served bytes",
"result": "PASS",
"details": {
"verified": 4
}
},
{
"id": "bounded_attempts",
"statement": "attempt <= JOB_MAX_ATTEMPTS",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_orphans",
"statement": "no pending outbox, messages, dead letters, staging, objects",
"result": "PASS",
"details": {
"queues": {
"attachments.process": [
0,
0
],
"attachments.process.dlq": [
0,
0
]
},
"s3_objects": 17,
"scoped_attachments": 5
}
},
{
"id": "no_lost_202",
"statement": "every 202 is present in PostgreSQL and the API",
"result": "PASS",
"details": {
"accepted": 5
}
}
],
"accepted_ids": [
"01a1b036-7bdb-72a8-9a20-d784124b6f09",
"01a1b036-838f-7073-a603-58ef2ffca05d",
"01a1b036-d38a-7375-ba48-c82f1e984740",
"01a1b036-e669-7726-8ed5-eb948e03323e",
"01a1b037-1c4a-72b4-8027-95bdd74c185a"
],
"scenarios": [
{
"name": "smoke_pipeline",
"level": 3,
"window": "upload, ready, variants match metadata",
"seconds": 1.4,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-7bdb-72a8-9a20-d784124b6f09",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
},
{
"name": "worker_down_broker_restart",
"level": 3,
"window": "worker down, API and broker restart",
"seconds": 21.1,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d",
"timeline": [
{
"at": "2026-11-06T09:35:25.067956Z",
"step": "worker stopped",
"containers": [
"mtusi2026lab6-worker-1 cd5b9bcf05a3 exited Exited (0) Less than a second ago"
]
},
{
"at": "2026-11-06T09:35:25.081364Z",
"step": "upload 202 queued",
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d"
},
{
"at": "2026-11-06T09:35:25.086007Z",
"step": "content 409 while queued"
},
{
"at": "2026-11-06T09:35:25.109208Z",
"step": "message with attachment 201, other messages work",
"message_id": "01a1b036-839e-71c7-9abc-6d747695610e"
},
{
"at": "2026-11-06T09:35:30.399187Z",
"step": "job is in the broker queue",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
0,
0
]
}
},
{
"at": "2026-11-06T09:35:36.592722Z",
"step": "API replicas and relay restarted"
},
{
"at": "2026-11-06T09:35:42.601391Z",
"step": "broker restarted with its volume, job survived",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
0,
0
]
}
},
{
"at": "2026-11-06T09:35:45.362335Z",
"step": "worker started"
},
{
"at": "2026-11-06T09:35:45.539899Z",
"step": "ready",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
]
}
},
{
"name": "duplicate_after_ready",
"level": 3,
"window": "redelivery of a finished job",
"seconds": 4.4,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-d38a-7375-ba48-c82f1e984740",
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "done",
"attempt": 1
},
{
"kind": "duplicate_noop",
"attempt": 1
}
]
}
},
{
"name": "poison_image",
"level": 3,
"window": "permanent error without retries",
"seconds": 0.0,
"result": "PASS",
"observed": {
"rejected_by_api": 415
}
},
{
"name": "s3_outage_recovers",
"level": 3,
"window": "transient error with retry",
"seconds": 10.3,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-e669-7726-8ed5-eb948e03323e",
"jobs": [
{
"job_id": "01a1b036-e671-7294-8d09-37bc4619030b",
"pipeline_version": 1,
"status": "done",
"attempt": 2
}
]
}
},
{
"name": "retry_exhausted",
"level": 3,
"window": "transient error until JOB_MAX_ATTEMPTS",
"seconds": 20.3,
"result": "PASS",
"observed": {
"attachment_id": "01a1b037-1c4a-72b4-8027-95bdd74c185a",
"jobs": [
{
"job_id": "01a1b037-1c57-7496-8cc0-61aa229fd7a2",
"pipeline_version": 1,
"status": "failed",
"attempt": 5
}
]
}
}
],
"result": "PASS",
"finished_at": "2026-11-06T09:36:32.454476Z"
}
@@ -10,7 +10,7 @@
#### Scenario: Стенд из чистого состояния #### Scenario: Стенд из чистого состояния
- **WHEN** после `make reset-data` выполняется `make up` - **WHEN** в новом изолированном Compose project с пустыми volumes выполняется `make up`
- **THEN** `docker compose ps` показывает `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` в состоянии `healthy` рядом с `lb`, `api` ×2, `relay`, `worker`, `postgres`, `valkey`, `s3`, `broker` - **THEN** `docker compose ps` показывает `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` в состоянии `healthy` рядом с `lb`, `api` ×2, `relay`, `worker`, `postgres`, `valkey`, `s3`, `broker`
#### Scenario: OTLP не опубликован #### Scenario: OTLP не опубликован
@@ -20,19 +20,19 @@
## 4. Стек наблюдаемости ## 4. Стек наблюдаемости
- [ ] 4.1 `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml` и recording rules, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` из `.prep/lab6` - [x] 4.1 `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml` и recording rules, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` из `.prep/lab6`
- [ ] 4.2 `infra/grafana/provisioning`, dashboards `messenger-api` и `messenger-pipeline` уровня «3» - [x] 4.2 `infra/grafana/provisioning`, dashboards `messenger-api` и `messenger-pipeline` уровня «3»
- [ ] 4.3 `compose.yaml`: `healthtools`, `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana`, volumes, секрет Grafana, сети `backend`/`edge`, переменные OTel у `api`, `relay`, `worker`; `.env.example` - [x] 4.3 `compose.yaml`: `healthtools`, `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana`, volumes, секрет Grafana, сети `backend`/`edge`, переменные OTel у `api`, `relay`, `worker`; `.env.example`
- [ ] 4.4 `scripts/secrets-lab6.sh`, `scripts/obs-validate.sh`, `scripts/obs_check.py`; цели `secrets`, `obs-validate`, `telemetry-check`, `obs-evidence` - [x] 4.4 `scripts/secrets-lab6.sh`, `scripts/obs-validate.sh`, `scripts/obs_check.py`; цели `secrets`, `obs-validate`, `telemetry-check`
- [ ] 4.5 Проверка группы: `make obs-validate`, `docker compose config --quiet`, `make up` из чистого состояния — все сервисы `healthy`, порты наблюдаемости только на `127.0.0.1` - [ ] 4.5 Проверка группы: `make obs-validate`, `docker compose config --quiet`, `make up` из чистого состояния — все сервисы `healthy`, порты наблюдаемости только на `127.0.0.1`
## 5. Проверка на стенде ## 5. Проверка на стенде
- [ ] 5.1 `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `make restore`, `make level3` - [x] 5.1 `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `make restore`, `make level3`
- [ ] 5.2 `obs_check.py grafana`, `dashboards --strict`, `request`, `job`, `leaks`, cardinality, история после рестарта хранилищ, `make test` при остановленном Collector - [x] 5.2 `obs_check.py grafana`, `dashboards --strict`, `request`, `job`, `leaks`, cardinality, история после рестарта хранилищ, `make test` при остановленном Collector
- [ ] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test` - [x] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test`
## 6. Отчёт ## 6. Отчёт
- [ ] 6.1 `evidence/`: стенд, smoke, persistence, failover, level3, obs-validate, grafana, dashboards, trace-request, trace-job, leaks, cardinality, telemetry-restart, student-tests, contract-checks - [x] 6.1 `evidence/`: стенд, smoke, persistence, failover, level3, obs-validate, grafana, dashboards, trace-request, trace-job, leaks, cardinality, telemetry-restart, student-tests, contract-checks
- [ ] 6.2 `REPORT.md`: шапка, запуск из чистого clone, архитектура и решения, роли, назначение хранилищ, соответствие сигналов и переменных, строки уровня «3», таблица отказов уровня «3» - [x] 6.2 `evidence/grade3/notes.md`: заметки для отдельного агента отчёта — запуск, архитектура, роли, сигналы, доказательства и ограничения уровня «3»
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mtusi2026lab6}
export LB_HTTP_PORT=${LB_HTTP_PORT:-18086} LB_STATS_PORT=${LB_STATS_PORT:-26085}
export BROKER_MANAGEMENT_PORT=${BROKER_MANAGEMENT_PORT:-26086}
export PROMETHEUS_PORT=${PROMETHEUS_PORT:-26090} LOKI_PORT=${LOKI_PORT:-26310}
export TEMPO_PORT=${TEMPO_PORT:-26320} GRAFANA_PORT=${GRAFANA_PORT:-26300}
export BASE_URL=${BASE_URL:-http://127.0.0.1:$LB_HTTP_PORT}
export PROMETHEUS_URL=http://127.0.0.1:$PROMETHEUS_PORT LOKI_URL=http://127.0.0.1:$LOKI_PORT
export TEMPO_URL=http://127.0.0.1:$TEMPO_PORT GRAFANA_URL=http://127.0.0.1:$GRAFANA_PORT
export NO_PROXY=localhost,127.0.0.1,::1
exec "$@"
+9 -1
View File
@@ -66,7 +66,7 @@ def wait_for(what, check, timeout, interval=2.0):
result = check() result = check()
if result: if result:
return result return result
except Failure: except (Failure, OSError):
pass pass
if time.monotonic() > deadline: if time.monotonic() > deadline:
raise Failure(f"timeout waiting for {what}") raise Failure(f"timeout waiting for {what}")
@@ -253,6 +253,10 @@ def request_check(args, b):
if not trace_id: if not trace_id:
raise Failure("log record has no trace_id") raise Failure("log record has no trace_id")
spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout) spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout)
if not any(s["kind"] == "SPAN_KIND_SERVER" and s["service"] == args.api_service for s in spans):
raise Failure("no real API server span")
if not any(s["kind"] == "SPAN_KIND_CLIENT" and s["attributes"].get("db.system") == "postgresql" for s in spans):
raise Failure("no PostgreSQL client span inside API trace")
print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id, print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id,
"span_id": labels.get("span_id"), "span_id": labels.get("span_id"),
"service_name": labels.get("service_name"), "service_name": labels.get("service_name"),
@@ -282,6 +286,10 @@ def job_check(args, b):
query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"' query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"'
worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout) worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
api.logout(auth) api.logout(auth)
if ready["status"] != "ready" or not any(s["kind"] == "SPAN_KIND_CONSUMER" and s["service"] == args.worker_service for s in chain):
raise Failure("job is not ready or no real consumer span")
if not any(labels.get("trace_id") and labels.get("span_id") for labels, _ in worker_logs):
raise Failure("worker log is not correlated")
print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"], print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"],
"traces": sorted({c["trace_id"] for c in chain}), "spans": chain, "traces": sorted({c["trace_id"] for c in chain}), "spans": chain,
"worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")} "worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")}
+98
View File
@@ -0,0 +1,98 @@
import argparse
import json
import os
import subprocess
import time
from obs_check import Api, Backends, get_json, http, now, tempo_spans, wait_for
def backends():
return Backends(
argparse.Namespace(
prometheus=os.environ["PROMETHEUS_URL"],
loki=os.environ["LOKI_URL"],
tempo=os.environ["TEMPO_URL"],
grafana=os.environ["GRAFANA_URL"],
grafana_password_file=os.environ.get(
"GRAFANA_PASSWORD_FILE", "/".join(("secrets", "grafana_admin_password"))
),
)
)
def compose(*args):
return subprocess.run(
["docker", "compose", *args], check=True, capture_output=True, text=True
).stdout.strip()
def history(b):
started = time.time()
_, _, _, rid = Api(os.environ["BASE_URL"]).register()
query = '{service_name="messenger-api"} | request_id="' + rid + '"'
logs = wait_for("log", lambda: b.logs(query, started - 1, time.time() + 1), 60)
trace_id = logs[0][0]["trace_id"]
spans = wait_for("trace", lambda: tempo_spans(b.trace(trace_id)[0]), 60)
expr = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})'
before = wait_for("metrics", lambda: b.promql(expr), 60)
at = before[0]["value"][0]
compose("restart", "prometheus", "loki", "tempo", "grafana")
wait_for("Grafana", lambda: get_json(b.grafana + "/api/health"), 90)
after_logs = wait_for("retained log", lambda: b.logs(query, started - 1, time.time() + 1), 90)
after_spans = wait_for("retained trace", lambda: tempo_spans(b.trace(trace_id)[0]), 90)
after = wait_for("retained metric", lambda: b.promql(expr, at=at), 90)
if before[0]["value"][1] != after[0]["value"][1] or len(after_spans) < len(spans):
raise RuntimeError("telemetry history changed after restart")
return {
"request_id": rid,
"trace_id": trace_id,
"spans_before": len(spans),
"spans_after": len(after_spans),
"logs_after": len(after_logs),
"metric_before": before,
"metric_after": after,
}
def collector():
compose("stop", "otel-collector")
try:
status = http("GET", os.environ["BASE_URL"] + "/health/ready")[0]
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=False)
if status != 200 or result.returncode:
raise RuntimeError("application failed without collector")
replica = compose("ps", "-q", "api").splitlines()[0]
started = time.monotonic()
subprocess.run(["docker", "stop", "-t", "8", replica], check=True, capture_output=True)
elapsed = time.monotonic() - started
code = subprocess.check_output(
["docker", "inspect", "-f", "{{.State.ExitCode}}", replica], text=True
).strip()
if code != "0" or elapsed > 8:
raise RuntimeError("unbounded API shutdown")
return {
"ready": status,
"smoke": result.stdout.strip(),
"api_exit_code": code,
"api_shutdown_seconds": round(elapsed, 3),
}
finally:
compose("up", "-d", "--no-deps", "--wait", "api", "otel-collector")
subprocess.run(["scripts/lb-wait.sh"], check=True)
def main():
b = backends()
report = {
"utc": now(),
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
}
report["history"] = history(b)
report["collector_down"] = collector()
report["result"] = "PASS"
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()