Complete observability cost study with fifteen measured series

This commit is contained in:
Artyom
2026-11-06 22:02:13 +03:00
parent bc0b162082
commit ebafbc3099
65 changed files with 126533 additions and 6 deletions
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:35:22Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode balanced --repeat 1 --output evidence/track5/balanced-1.json
{"mode": "balanced", "repeat": 1, "result": "PASS", "retained": {"normal": 40, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 125.650
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:45:48Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode balanced --repeat 2 --output evidence/track5/balanced-2.json
{"mode": "balanced", "repeat": 2, "result": "PASS", "retained": {"normal": 39, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 126.348
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:56:15Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode balanced --repeat 3 --output evidence/track5/balanced-3.json
{"mode": "balanced", "repeat": 3, "result": "PASS", "retained": {"normal": 27, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 126.026
+19
View File
@@ -0,0 +1,19 @@
{
"utc": "2026-11-06T19:01:45.790303+00:00",
"sha": "bc0b1620825b53fd3a3019623326ebf68cbbbeec",
"running_own_containers": [],
"preserved_volumes": [
"mtusi2026lab6_broker-data",
"mtusi2026lab6_grafana-data",
"mtusi2026lab6_healthtools",
"mtusi2026lab6_loki-data",
"mtusi2026lab6_pgdata",
"mtusi2026lab6_prometheus-data",
"mtusi2026lab6_s3data",
"mtusi2026lab6_tempo-data",
"mtusi2026lab6_valkeydata"
],
"protected_files": 22,
"protected_files_changed": [],
"result": "PASS"
}
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 131 KiB

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 100 KiB

+4
View File
@@ -0,0 +1,4 @@
date: 2026-11-06T18:23:50Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol/collector-balanced.yaml
exit_code: 0 elapsed_seconds: 0.418
+4
View File
@@ -0,0 +1,4 @@
date: 2026-11-06T18:23:50Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol/collector-tail.yaml
exit_code: 0 elapsed_seconds: 0.435
@@ -0,0 +1,5 @@
date: 2026-11-06T18:59:38Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ docker compose exec -T api python -c 'import json,os; print(json.dumps({k:os.getenv(k) for k in ("OTEL_SDK_DISABLED","OTEL_TRACE_RATIO","OTEL_METRIC_EXPORT_INTERVAL","COST_EXPERIMENT")}))'
{"OTEL_SDK_DISABLED": "false", "OTEL_TRACE_RATIO": "1", "OTEL_METRIC_EXPORT_INTERVAL": "10000", "COST_EXPERIMENT": "false"}
exit_code: 0 elapsed_seconds: 0.162
+8
View File
@@ -0,0 +1,8 @@
date: 2026-11-06T18:59:05Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.772}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit_code: 0 elapsed_seconds: 17.495
+576
View File
@@ -0,0 +1,576 @@
date: 2026-11-06T18:59:44Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py job --base-url http://127.0.0.1:18086
{
"checked_at": "2026-11-06T19:00:05+00:00",
"upload_request_id": "6288e51c-14e4-4d6b-b306-195f0ae08205",
"attachment_status": "ready",
"traces": [
"72ec9617da83d915a1a8c68bd4dd32ca"
],
"spans": [
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.088,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.052,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.044,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.058,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.295,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.067,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.083,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.052,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.04,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.059,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.288,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.231,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.081,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 19.546,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.103,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.087,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.328,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.287,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.299,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.177,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.264,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 36.666,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-api",
"name": "S3 put_object",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 4.862,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-relay",
"name": "attachments.attachment.process.v1 create",
"kind": "SPAN_KIND_PRODUCER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.014,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-relay",
"name": "attachments.attachment.process.v1 publish",
"kind": "SPAN_KIND_PRODUCER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 1.728,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.168,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.118,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.098,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.096,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.257,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.236,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.135,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.112,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.243,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.079,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.061,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.053,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.056,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.148,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.193,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.125,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.061,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.05,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.05,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.232,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.324,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.174,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.084,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.401,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "S3 get_object",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 2.003,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "S3 copy_object",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 5.013,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "S3 put_object",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 2.91,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "S3 put_object",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 2.125,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "S3 put_object",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 2.046,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": null
},
{
"service": "messenger-worker",
"name": "attachments.attachment.process.v1 process",
"kind": "SPAN_KIND_CONSUMER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 733.073,
"links": 0,
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"attempt": "1"
}
],
"worker_logs": [
{
"line": "job.processed",
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"span_id": "ecd668f30f4ccf88"
},
{
"line": "audit.job_settled",
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
"span_id": "ecd668f30f4ccf88"
}
]
}
exit_code: 0 elapsed_seconds: 21.416
+27
View File
@@ -0,0 +1,27 @@
date: 2026-11-06T19:00:05Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py leaks --base-url http://127.0.0.1:18086
{
"checked_at": "2026-11-06T19:00:21+00:00",
"window_seconds": 20,
"traces_scanned": 8,
"hits": {
"password": {
"loki": 0,
"tempo": 0
},
"token": {
"loki": 0,
"tempo": 0
},
"message_text": {
"loki": 0,
"tempo": 0
},
"filename": {
"loki": 0,
"tempo": 0
}
}
}
exit_code: 0 elapsed_seconds: 15.256
+19
View File
@@ -0,0 +1,19 @@
date: 2026-11-06T18:59:37Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make obs-validate
sh scripts/obs-validate.sh
otelcol-contrib validate collector.yaml: OK
Checking /etc/prometheus/prometheus.yml
SUCCESS: 2 rule files found
SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
Checking /etc/prometheus/rules/messenger.alerts.yml
SUCCESS: 5 rules found
Checking /etc/prometheus/rules/messenger.recording.yml
SUCCESS: 8 rules found
loki -verify-config: OK
tempo -config.verify: OK
grafana dashboards: valid JSON with uid and panels
exit_code: 0 elapsed_seconds: 0.966
+7
View File
@@ -0,0 +1,7 @@
date: 2026-11-06T18:58:50Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit_code: 0 elapsed_seconds: 14.251
+14
View File
@@ -0,0 +1,14 @@
date: 2026-11-06T18:58:38Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make check validate lint
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
200 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
exit_code: 0 elapsed_seconds: 0.880
+66
View File
@@ -0,0 +1,66 @@
date: 2026-11-06T18:59:38Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_check.py request --base-url http://127.0.0.1:18086
{
"checked_at": "2026-11-06T18:59:44+00:00",
"request_id": "c6cc7b12-b135-4795-8187-0618495a383f",
"log": {
"line": "request.handled",
"trace_id": "8af5c9f15c06bf7ae5dd82ff11639d9e",
"span_id": "7e8a5b9286de54c2",
"service_name": "messenger-api",
"severity": "INFO"
},
"trace": [
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.168
},
{
"service": "messenger-api",
"name": ";",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.13
},
{
"service": "messenger-api",
"name": "ROLLBACK;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.132
},
{
"service": "messenger-api",
"name": "BEGIN;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.119
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.328
},
{
"service": "messenger-api",
"name": "COMMIT;",
"kind": "SPAN_KIND_CLIENT",
"status": "STATUS_CODE_UNSET",
"duration_ms": 0.602
},
{
"service": "messenger-api",
"name": "POST /api/v1/users",
"kind": "SPAN_KIND_SERVER",
"status": "STATUS_CODE_UNSET",
"duration_ms": 26.748
}
]
}
exit_code: 0 elapsed_seconds: 6.162
+12
View File
@@ -0,0 +1,12 @@
date: 2026-11-06T18:59:22Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make restore
scripts/restore.sh
Container mtusi2026lab6-api-1 Running
Container mtusi2026lab6-api-2 Starting
Container mtusi2026lab6-api-2 Started
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Healthy
Container mtusi2026lab6-api-2 Healthy
exit_code: 0 elapsed_seconds: 3.620
+6
View File
@@ -0,0 +1,6 @@
date: 2026-11-06T18:58:48Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit_code: 0 elapsed_seconds: 2.160
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:31:11Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode head --repeat 1 --output evidence/track5/head-1.json
{"mode": "head", "repeat": 1, "result": "PASS", "retained": {"normal": 73, "error": 1, "slow": 0}}
exit_code: 0 elapsed_seconds: 124.365
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:41:37Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode head --repeat 2 --output evidence/track5/head-2.json
{"mode": "head", "repeat": 2, "result": "PASS", "retained": {"normal": 97, "error": 2, "slow": 2}}
exit_code: 0 elapsed_seconds: 124.824
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:52:04Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode head --repeat 3 --output evidence/track5/head-3.json
{"mode": "head", "repeat": 3, "result": "PASS", "retained": {"normal": 78, "error": 1, "slow": 1}}
exit_code: 0 elapsed_seconds: 124.173
+67
View File
@@ -0,0 +1,67 @@
# Трек 1 — сведения для отдельного отчёта
Код приложения/образ: 202b985deacda79b6d8df65dcefe8a1af43f1a8e; исправление измерителя: bc0b1620825b53fd3a3019623326ebf68cbbbeec. Raw JSON содержит SHA, UTC, все ответы клиента, samples Docker, counters до/после и IDs найденных traces. Команды и exit_code — в одноимённых txt. REPORT.md не изменяется.
## Гипотеза и метод
Head 10% должен снижать trace ingest и часть расходов SDK, теряя редкие события независимо от их важности. Tail должен сохранять ошибки и slow, уменьшая запись в Tempo, сохраняя инструментирование приложения. Balanced = tail 5% обычных traces, все ERROR и >=300 ms, экспорт метрик раз в 30 s вместо 10 s.
`scripts/lab6.sh python3 scripts/obs_cost.py --mode MODE --repeat N --output evidence/track5/MODE-N.json`. Каждый запуск пересоздаёт только свои API/relay/worker/Collector; хранилища и volumes сохраняются. 20 прогревочных GET и 35 s для экспорта. Затем 40 s open-loop: 800 arrivals, 64 клиентских потока, фиксированные 16 ошибок 503 и 16 задержек 350 ms, 768 обычных GET users/me. После — 35 s на batch/tail/periodic export. `finally` возвращает SDK on, ratio1, обычный Collector и выключает fault hook. Нет параллельных нагрузок. Порядок первого круга on/off/head/tail/balanced (on проверяет измеритель), следующих — off/on/head/tail/balanced.
Latency — wall time каждого HTTP у клиента; schedule_lag хранится отдельно. p95 normal исключает контролируемые ошибки/задержки; общий p99 ожидаемо около350 ms. CPU — разность cgroup total_usage / время внешних samples, включая дочерние процессы; application = API+relay+worker, отдельные observability и business stores/LB. RAM — working set usage−inactive_file, выборки примерно раз в 2 s. Источник — Docker Unix socket, SDK для этих данных не используется.
Ingest: bytes_received Tempo и Loki; appended samples Prometheus отдельно (несовместимые единицы не суммируются). Окно ingest включает доставку после нагрузки и постоянные self-scrapes. SDK ended считает только sampled spans; сравнение с Collector accepted проверяет доставку, sent — сохранённые экспортером spans. Для tail разность accepted−sent включает намеренную фильтрацию; send_failed отдельно. Для head отброшенные spans не создаются: точно измеряется потеря server roots по категории, неизвестное число несозданных DB spans не выдумывается. Off означает отсутствие SDK, не «ошибку экспорта».
## Воспроизводимость и ограничения
Один Docker Desktop host, клиент на том же компьютере; host не эксклюзивный. В JSON есть inventory всех запущенных контейнеров, ресурсы только своего project. Других Docker нагрузок в сериях не запускалось. Подготовка кода/графиков на host могла влиять на малые задержки; schedule lag позволяет видеть опоздания. Это HTTP/DB workload, worker находится в обычном ожидании: выводы не описывают стоимость массовой обработки изображений. Pipeline и retry отдельно доказаны уровнями3/4.
Три повтора дают наблюдаемый диапазон, не доверительный интервал и не доказательство production capacity. CPU измеряется в окне нагрузки; ingest дополнительно учитывает доставку после него. Tail decision_wait10s: поздние spans/retries после решения могут зависеть от decision cache, эксперимент не доказывает сохранность всех редких долговременных цепочек. Сохранённые evidence годятся для последующего аудита без автоматического повторного прогона.
Неуспешные попытки: smoke.txt стартовал до окончания up, smoke-ready.txt прошёл после healthy. on-1.txt остановился до нагрузки, поскольку Stats API не возвращал name; имя перенесено в отдельный inspect, on-1-retry.txt содержит полноценную серию. Ни одна из этих неудач не объявляется PASS.
`python3 scripts/obs_cost_summary.py evidence/track5` строит comparison.json из raw. Графики: `uv run --no-project --with matplotlib==3.11.2 python tools/plot_observability.py`; bars=median, dots=реальные повторы, whiskers=min/max.
## Итоговые результаты — 15 завершённых серий
12000 запросов, по 3 повтора каждой настройки. Все HTTP-ответы ожидаемые. Во всех сериях SDK ended = Collector accepted; источник/приёмник не потеряли sampled spans. Фактические export failures: 0. Максимальное опоздание генератора: 26.473 ms.
| Режим | normal p95 ms: median [min;max] | общий p99 ms | CPU app / obs cores | RAM app / obs MiB (median peak) |
|---|---:|---:|---:|---:|
| off | 11.622 [4.328; 12.139] | 357.23 | 0.1211 / 0.0192 | 653.0 / 447.8 |
| on | 12.846 [5.365; 13.582] | 359.16 | 0.1658 / 0.0269 | 701.7 / 548.2 |
| head | 12.441 [5.759; 13.091] | 357.70 | 0.1559 / 0.0249 | 703.9 / 525.9 |
| tail | 12.894 [6.887; 13.047] | 358.86 | 0.1624 / 0.0268 | 720.7 / 526.6 |
| balanced | 12.712 [7.046; 12.778] | 358.07 | 0.1657 / 0.0244 | 702.0 / 523.7 |
| Режим | Tempo KiB | Loki KiB | Prom samples | sampled / sent spans (median) | normal traces /2304 | error /48 | slow /48 |
|---|---:|---:|---:|---:|---:|---:|---:|
| off | 0.0 | 0.0 | 28360 | 0 / 0 | 0 | 0 | 0 |
| on | 1662.7 | 288.3 | 31517 | 5489 / 5489 | 2304 | 48 | 48 |
| head | 176.5 | 287.6 | 32413 | 551 / 551 | 248 | 4 | 3 |
| tail | 255.0 | 289.3 | 31837 | 5492 / 709 | 247 | 48 | 48 |
| balanced | 147.3 | 288.5 | 29693 | 5489 / 401 | 106 | 48 | 48 |
Рекомендация в границах этого workload: balanced сохраняет 48/48 errors и 48/48 slow traces, снижая медианный Tempo ingest на 91.1% относительно on. Он не устраняет цену SDK: все spans создаются до tail-решения. Head пропустил 44 ошибок и 45 медленных запросов; их важность неизвестна в момент начала trace.
Верифицированный запуск balanced: `OTEL_COLLECTOR_CONFIG=collector-balanced.yaml OTEL_METRIC_EXPORT_INTERVAL=30000 scripts/lab6.sh docker compose up -d --no-deps --wait otel-collector api relay worker`. Fault hook для обычного запуска остаётся выключен. Default после эксперимента возвращён к on/100%; рекомендация не включается скрыто.
Гипотеза о сокращении хранения подтверждается непосредственно counters. Малые различия latency/CPU следует читать вместе с min/max и ограничениями host; статистическая значимость и предельный RPS не заявляются. У CPU-окна не учитывается работа последних tail-решений во время drain, поэтому это CPU под заданной нагрузкой, не полная CPU-стоимость обработки всей серии.
Графики comparison.png/comparison.svg построены из comparison.json, исходные series остаются в Git.
## Завершение реализации
Все 15 raw-series относятся к bc0b1620825b53fd3a3019623326ebf68cbbbeec; образ приложения — 202b985deacda79b6d8df65dcefe8a1af43f1a8e. В каждой серии 21 внешний resource sample, фактическая скорость 20.014–20.023 requests/s. Inventory содержит только 14 контейнеров этого project; других Docker-нагрузок в зафиксированных снимках нет. Образ и нагрузочный код после измерений не менялись; финальный коммит добавляет сводку, графики и evidence.
`make student-test`: 519 passed, 1 Linux-only skipped, 19 stack deselected (student-tests.txt); stack-поведение отдельно проверено на реальном стенде. `make check validate lint`: PASS (final-quality.txt). `make obs-validate`: PASS (final-obs-config.txt). `make test`, persistence и failover: PASS (final-smoke/persistence/failover.txt), после failover выполнен restore. Проверены свежие server/DB spans и consumer spans, связанные логи worker (final-request/job.txt), 0 утечек по контрольным значениям (final-leaks.txt).
В контейнере проверены SDK disabled=false, ratio=1, interval=10000, COST_EXPERIMENT=false (final-default-settings.txt); Collector запущен с collector.yaml. График comparison.png визуально проверен: подписи читаются, отображаются все три точки каждого режима. Защищённые 22 файла, включая REPORT/AGENTS/README/contracts/tests/evidence README, сверены с исходными SHA-256 — без изменений.
Оставшиеся minor findings уровня4 перечислены в ../grade4/findings.json и notes.md: неполные allowlists старого аудита лимитов, отсутствие LB-generated 503 в API SLI и малый запас RAM Collector при outage. Следующий агент должен перенести их в ограничения REPORT; здесь REPORT намеренно сохранён без изменений.
Свой стенд остановлен через make down без -v (shutdown.txt); 0 запущенных собственных контейнеров, все 9 volumes сохранены (closure.json). Нагрузочные процессы и проверки завершены. Публикации не выполнялись.
OpenSpec: change measure-lab6-observability-cost прошёл strict validation; 2 requirements синхронизированы в openspec/specs/lab6-observability-cost/spec.md, все задачи завершены, архив 2026-11-06-measure-lab6-observability-cost.
Пустые файлы предварительных попыток balanced-validate.txt, tail-validate.txt не используются как доказательство; реальные проверки конфигураций — config-tail.txt и config-balanced.txt.
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:29:08Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode off --repeat 1 --output evidence/track5/off-1.json
{"mode": "off", "repeat": 1, "result": "PASS", "retained": {"normal": 0, "error": 0, "slow": 0}}
exit_code: 0 elapsed_seconds: 123.134
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:37:28Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode off --repeat 2 --output evidence/track5/off-2.json
{"mode": "off", "repeat": 2, "result": "PASS", "retained": {"normal": 0, "error": 0, "slow": 0}}
exit_code: 0 elapsed_seconds: 124.716
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:47:54Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode off --repeat 3 --output evidence/track5/off-3.json
{"mode": "off", "repeat": 3, "result": "PASS", "retained": {"normal": 0, "error": 0, "slow": 0}}
exit_code: 0 elapsed_seconds: 125.342
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:26:45Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode on --repeat 1 --output evidence/track5/on-1.json
{"mode": "on", "repeat": 1, "result": "PASS", "retained": {"normal": 768, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 124.926
File diff suppressed because it is too large Load Diff
+21
View File
@@ -0,0 +1,21 @@
date: 2026-11-06T18:24:41Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode on --repeat 1 --output evidence/track5/on-1.json
Traceback (most recent call last):
File "<repo>/scripts/obs_cost.py", line 264, in <module>
main()
~~~~^^
File "<repo>/scripts/obs_cost.py", line 251, in main
run(args.mode, args.repeat, args.duration, args.rate, args.output)
~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<repo>/scripts/obs_cost.py", line 162, in run
resources.begin()
~~~~~~~~~~~~~~~^^
File "<repo>/scripts/obs_cost.py", line 107, in begin
self.samples.append(self.sample())
~~~~~~~~~~~^^
File "<repo>/scripts/obs_cost.py", line 93, in sample
"name": data["name"],
~~~~^^^^^^^^
KeyError: 'name'
exit_code: 1 elapsed_seconds: 49.067
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:39:33Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode on --repeat 2 --output evidence/track5/on-2.json
{"mode": "on", "repeat": 2, "result": "PASS", "retained": {"normal": 768, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 124.590
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:50:00Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode on --repeat 3 --output evidence/track5/on-3.json
{"mode": "on", "repeat": 3, "result": "PASS", "retained": {"normal": 768, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 124.209
+10
View File
@@ -0,0 +1,10 @@
date: 2026-11-06T19:02:06Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make lint
.venv/bin/ruff check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
200 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
exit_code: 0 elapsed_seconds: 0.131
+77
View File
@@ -0,0 +1,77 @@
date: 2026-11-06T19:01:10Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make down
docker compose --profile bench --profile tools --profile reconcile down
Container mtusi2026lab6-worker-1 Stopping
Container mtusi2026lab6-lb-1 Stopping
Container mtusi2026lab6-otel-collector-1 Stopping
Container mtusi2026lab6-api-1 Stopping
Container mtusi2026lab6-api-2 Stopping
Container mtusi2026lab6-grafana-1 Stopping
Container mtusi2026lab6-relay-1 Stopping
Container mtusi2026lab6-grafana-1 Stopped
Container mtusi2026lab6-grafana-1 Removing
Container mtusi2026lab6-grafana-1 Removed
Container mtusi2026lab6-prometheus-1 Stopping
Container mtusi2026lab6-loki-1 Stopping
Container mtusi2026lab6-tempo-1 Stopping
Container mtusi2026lab6-lb-1 Stopped
Container mtusi2026lab6-lb-1 Removing
Container mtusi2026lab6-lb-1 Removed
Container mtusi2026lab6-otel-collector-1 Stopped
Container mtusi2026lab6-otel-collector-1 Removing
Container mtusi2026lab6-prometheus-1 Stopped
Container mtusi2026lab6-prometheus-1 Removing
Container mtusi2026lab6-otel-collector-1 Removed
Container mtusi2026lab6-prometheus-1 Removed
Container mtusi2026lab6-tempo-1 Stopped
Container mtusi2026lab6-tempo-1 Removing
Container mtusi2026lab6-tempo-1 Removed
Container mtusi2026lab6-loki-1 Stopped
Container mtusi2026lab6-loki-1 Removing
Container mtusi2026lab6-loki-1 Removed
Container mtusi2026lab6-healthtools-1 Stopping
Container mtusi2026lab6-healthtools-1 Stopped
Container mtusi2026lab6-healthtools-1 Removing
Container mtusi2026lab6-healthtools-1 Removed
Container mtusi2026lab6-relay-1 Stopped
Container mtusi2026lab6-relay-1 Removing
Container mtusi2026lab6-relay-1 Removed
Container mtusi2026lab6-worker-1 Stopped
Container mtusi2026lab6-worker-1 Removing
Container mtusi2026lab6-worker-1 Removed
Container mtusi2026lab6-broker-1 Stopping
Container mtusi2026lab6-broker-1 Stopped
Container mtusi2026lab6-broker-1 Removing
Container mtusi2026lab6-broker-1 Removed
Container mtusi2026lab6-api-1 Stopped
Container mtusi2026lab6-api-1 Removing
Container mtusi2026lab6-api-1 Removed
Container mtusi2026lab6-api-2 Stopped
Container mtusi2026lab6-api-2 Removing
Container mtusi2026lab6-api-2 Removed
Container mtusi2026lab6-migrate-1 Stopping
Container mtusi2026lab6-valkey-1 Stopping
Container mtusi2026lab6-s3-init-1 Stopping
Container mtusi2026lab6-migrate-1 Stopped
Container mtusi2026lab6-migrate-1 Removing
Container mtusi2026lab6-s3-init-1 Stopped
Container mtusi2026lab6-s3-init-1 Removing
Container mtusi2026lab6-migrate-1 Removed
Container mtusi2026lab6-postgres-1 Stopping
Container mtusi2026lab6-s3-init-1 Removed
Container mtusi2026lab6-s3-1 Stopping
Container mtusi2026lab6-valkey-1 Stopped
Container mtusi2026lab6-valkey-1 Removing
Container mtusi2026lab6-valkey-1 Removed
Container mtusi2026lab6-s3-1 Stopped
Container mtusi2026lab6-s3-1 Removing
Container mtusi2026lab6-postgres-1 Stopped
Container mtusi2026lab6-postgres-1 Removing
Container mtusi2026lab6-s3-1 Removed
Container mtusi2026lab6-postgres-1 Removed
Network mtusi2026lab6_edge Removing
Network mtusi2026lab6_backend Removing
Network mtusi2026lab6_edge Removed
Network mtusi2026lab6_backend Removed
exit_code: 0 elapsed_seconds: 5.291
+6
View File
@@ -0,0 +1,6 @@
date: 2026-11-06T18:24:39Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit_code: 0 elapsed_seconds: 1.833
+7
View File
@@ -0,0 +1,7 @@
date: 2026-11-06T18:24:01Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
FAIL: GET /health/live: connection/TLS failure (URLError)
make: *** [test] Error 1
exit_code: 2 elapsed_seconds: 0.054
+64
View File
@@ -0,0 +1,64 @@
date: 2026-11-06T18:20:02Z commit: 20e7759f8771a14e9acba5fe0165c0fed2294cc6
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make student-test
.venv/bin/pytest
============================= test session starts ==============================
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
rootdir: <repo>
configfile: pyproject.toml
testpaths: student-tests
plugins: asyncio-1.4.0, anyio-4.15.1
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
collected 539 items / 19 deselected / 520 selected
student-tests/adapters/test_rabbitmq.py .... [ 0%]
student-tests/adapters/test_s3_storage.py ... [ 1%]
student-tests/api/test_access.py ......... [ 3%]
student-tests/api/test_attachments.py .................................. [ 9%]
.... [ 10%]
student-tests/api/test_bounded_contexts.py .................... [ 14%]
student-tests/api/test_cache.py ...... [ 15%]
student-tests/api/test_concurrency.py .... [ 16%]
student-tests/api/test_consistency.py ........ [ 17%]
student-tests/api/test_cost_experiment.py ... [ 18%]
student-tests/api/test_drain.py .. [ 18%]
student-tests/api/test_events.py ...... [ 19%]
student-tests/api/test_http_telemetry.py . [ 20%]
student-tests/api/test_idempotency.py ............... [ 22%]
student-tests/api/test_instance.py .... [ 23%]
student-tests/api/test_job_returning.py . [ 23%]
student-tests/api/test_message_attachments.py ............. [ 26%]
student-tests/api/test_migrations.py ................................... [ 33%]
. [ 33%]
student-tests/api/test_negative.py ..................................... [ 40%]
[ 40%]
student-tests/api/test_operability.py .............. [ 43%]
student-tests/api/test_pagination.py .................. [ 46%]
student-tests/api/test_pipeline.py ............. [ 49%]
student-tests/api/test_pool.py ............ [ 51%]
student-tests/api/test_reconciler.py ........... [ 53%]
student-tests/api/test_recovery.py ............ [ 55%]
student-tests/api/test_sessions.py .................. [ 59%]
student-tests/api/test_timeout.py ........ [ 60%]
student-tests/api/test_trace_context.py . [ 60%]
student-tests/api/test_upload_consistency.py ......... [ 62%]
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
student-tests/api/test_wire_format.py .. [ 65%]
student-tests/unit/test_attachments_domain.py .......................... [ 70%]
. [ 70%]
student-tests/unit/test_authentication.py ..................... [ 74%]
student-tests/unit/test_cursor.py ........... [ 76%]
student-tests/unit/test_domain.py ...................................... [ 83%]
.. [ 84%]
student-tests/unit/test_job_observer.py . [ 84%]
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
student-tests/unit/test_probe.py .............. [ 91%]
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
student-tests/unit/test_render_pool.py .....s. [ 94%]
student-tests/unit/test_renderer.py .................... [ 97%]
student-tests/unit/test_scaling.py ..... [ 98%]
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
student-tests/unit/test_timeouts.py . [100%]
========== 519 passed, 1 skipped, 19 deselected in 139.31s (0:02:19) ===========
exit_code: 0 elapsed_seconds: 142.075
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:33:16Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode tail --repeat 1 --output evidence/track5/tail-1.json
{"mode": "tail", "repeat": 1, "result": "PASS", "retained": {"normal": 93, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 126.284
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:43:42Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode tail --repeat 2 --output evidence/track5/tail-2.json
{"mode": "tail", "repeat": 2, "result": "PASS", "retained": {"normal": 71, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 125.572
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
date: 2026-11-06T18:54:08Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_cost.py --mode tail --repeat 3 --output evidence/track5/tail-3.json
{"mode": "tail", "repeat": 3, "result": "PASS", "retained": {"normal": 83, "error": 16, "slow": 16}}
exit_code: 0 elapsed_seconds: 127.117
View File
+202
View File
@@ -0,0 +1,202 @@
date: 2026-11-06T18:22:38Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ make up
sh scripts/secrets.sh
sh scripts/broker-secrets.sh
broker and S3 worker secrets ready in secrets/ (values not printed)
sh scripts/secrets-lab6.sh
secrets/grafana_admin_password is ready (value not printed)
docker compose build
#1 [internal] load local bake definitions
#1 reading from stdin 432B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.65kB done
#2 DONE 0.0s
#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#3 DONE 0.0s
#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#4 DONE 1.5s
#5 [internal] load .dockerignore
#5 transferring context: 453B done
#5 DONE 0.1s
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
#6 DONE 0.0s
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
#7 DONE 0.0s
#8 [internal] load build context
#8 transferring context: 100.64kB 0.0s done
#8 DONE 0.0s
#9 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
#9 CACHED
#10 [builder 3/7] WORKDIR /build
#10 CACHED
#11 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
#11 CACHED
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
#12 CACHED
#13 [builder 6/7] COPY src ./src
#13 DONE 0.0s
#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
#14 0.205 Resolved 100 packages in 9ms
#14 0.218 Building mtusi-messenger @ file:///build
#14 1.445 Built mtusi-messenger @ file:///build
#14 1.449 Prepared 1 package in 1.23s
#14 1.453 Installed 1 package in 3ms
#14 1.688 Bytecode compiled 2805 files in 234ms
#14 1.688 + mtusi-messenger==1.6.0 (from file:///build)
#14 DONE 1.7s
#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
#15 CACHED
#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv
#16 DONE 0.4s
#17 [runtime 4/6] WORKDIR /app
#17 DONE 0.0s
#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./
#18 DONE 0.0s
#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic
#19 DONE 0.0s
#20 exporting to image
#20 exporting layers
#20 exporting layers 3.0s done
#20 exporting manifest sha256:e1a0b2d6065a4b2ea4117533457320c803b6877e28a2e40f164d9d0fd5fdbe7f
#20 exporting manifest sha256:e1a0b2d6065a4b2ea4117533457320c803b6877e28a2e40f164d9d0fd5fdbe7f done
#20 exporting config sha256:314703588174a1170dfe2808331582e9f1e1b615a4523af1d5b518fdbfe08e2a done
#20 exporting attestation manifest sha256:fb6c618fb08132e1a6b6d22bae609a713eeb3551c74c60d57996236a0eb25df7 done
#20 exporting manifest list sha256:7b053283544fab4fec9e36930ed4cf5dccf88ad5026452f29640f4d63121eb07 done
#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done
#20 DONE 3.7s
#21 resolving provenance for metadata file
#21 DONE 0.0s
api Built
docker compose up -d --wait
Container mtusi2026lab6-migrate-1 Recreate
Container mtusi2026lab6-migrate-1 Recreated
Container mtusi2026lab6-relay-1 Recreate
Container mtusi2026lab6-api-2 Recreate
Container mtusi2026lab6-api-1 Recreate
Container mtusi2026lab6-worker-1 Recreate
Container mtusi2026lab6-worker-1 Recreated
Container mtusi2026lab6-relay-1 Recreated
Container mtusi2026lab6-api-1 Recreated
Container mtusi2026lab6-api-2 Recreated
Container mtusi2026lab6-prometheus-1 Starting
Container mtusi2026lab6-healthtools-1 Starting
Container mtusi2026lab6-broker-1 Starting
Container mtusi2026lab6-postgres-1 Starting
Container mtusi2026lab6-s3-1 Starting
Container mtusi2026lab6-lb-1 Starting
Container mtusi2026lab6-valkey-1 Starting
Container mtusi2026lab6-healthtools-1 Started
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-s3-1 Started
Container mtusi2026lab6-s3-1 Waiting
Container mtusi2026lab6-valkey-1 Started
Container mtusi2026lab6-prometheus-1 Started
Container mtusi2026lab6-postgres-1 Started
Container mtusi2026lab6-postgres-1 Waiting
Container mtusi2026lab6-lb-1 Started
Container mtusi2026lab6-broker-1 Started
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-otel-collector-1 Starting
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-loki-1 Starting
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-tempo-1 Starting
Container mtusi2026lab6-postgres-1 Healthy
Container mtusi2026lab6-migrate-1 Starting
Container mtusi2026lab6-migrate-1 Started
Container mtusi2026lab6-tempo-1 Started
Container mtusi2026lab6-otel-collector-1 Started
Container mtusi2026lab6-loki-1 Started
Container mtusi2026lab6-grafana-1 Starting
Container mtusi2026lab6-grafana-1 Started
Container mtusi2026lab6-s3-1 Healthy
Container mtusi2026lab6-s3-init-1 Starting
Container mtusi2026lab6-s3-init-1 Started
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-broker-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-broker-1 Waiting
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-broker-1 Healthy
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-worker-1 Starting
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-api-1 Starting
Container mtusi2026lab6-broker-1 Healthy
Container mtusi2026lab6-relay-1 Starting
Container mtusi2026lab6-relay-1 Started
Container mtusi2026lab6-api-1 Started
Container mtusi2026lab6-api-2 Starting
Container mtusi2026lab6-worker-1 Started
Container mtusi2026lab6-api-2 Started
Container mtusi2026lab6-relay-1 Waiting
Container mtusi2026lab6-otel-collector-1 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-lb-1 Waiting
Container mtusi2026lab6-prometheus-1 Waiting
Container mtusi2026lab6-healthtools-1 Waiting
Container mtusi2026lab6-valkey-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-grafana-1 Waiting
Container mtusi2026lab6-s3-1 Waiting
Container mtusi2026lab6-worker-1 Waiting
Container mtusi2026lab6-loki-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-postgres-1 Waiting
Container mtusi2026lab6-broker-1 Waiting
Container mtusi2026lab6-tempo-1 Waiting
Container mtusi2026lab6-valkey-1 Healthy
Container mtusi2026lab6-postgres-1 Healthy
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-healthtools-1 Exited
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-s3-1 Healthy
Container mtusi2026lab6-grafana-1 Healthy
Container mtusi2026lab6-broker-1 Healthy
Container mtusi2026lab6-otel-collector-1 Healthy
Container mtusi2026lab6-prometheus-1 Healthy
Container mtusi2026lab6-api-1 Healthy
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-relay-1 Healthy
Container mtusi2026lab6-worker-1 Healthy
Container mtusi2026lab6-lb-1 Healthy
Container mtusi2026lab6-tempo-1 Healthy
Container mtusi2026lab6-loki-1 Healthy
scripts/lb-wait.sh
exit_code: 0 elapsed_seconds: 115.841
@@ -0,0 +1,6 @@
- [x] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests
- [x] 2. Tail/balanced Collector configs and sequential external measurement driver
- [x] 3. make check/lint/student-test, config validation and smoke
- [x] 4. Three real rounds of off/on/head/tail/balanced; raw evidence
- [x] 5. Compare observations, recommend verified balance, record limitations
- [x] 6. Restore default configuration, final checks, sync/archive and stop own stand
@@ -1,6 +0,0 @@
- [ ] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests
- [ ] 2. Tail/balanced Collector configs and sequential external measurement driver
- [ ] 3. make check/lint/student-test, config validation and smoke
- [ ] 4. Three real rounds of off/on/head/tail/balanced; raw evidence
- [ ] 5. Compare observations, recommend verified balance, record limitations
- [ ] 6. Restore default configuration, final checks, sync/archive and stop own stand
@@ -0,0 +1,17 @@
# lab6-observability-cost Specification
## Purpose
TBD - created by archiving change measure-lab6-observability-cost. Update Purpose after archive.
## Requirements
### Requirement: Reproducible cost comparison
The system SHALL support SDK off/on, head and tail sampling and a verified balanced configuration, measured under the same open-loop workload with raw latency, external CPU/RAM, ingest and span-loss evidence.
#### Scenario: Three repeat rounds
- **WHEN** the cost driver runs all five modes in three sequential rounds
- **THEN** each run records actual requests, resources, ingest counters and retained normal/error/slow traces without fabricating missing observations
### Requirement: Disabled experimental faults
Controlled error and slow hooks SHALL be opt-in and SHALL NOT affect default requests.
#### Scenario: Ordinary deployment
- **WHEN** the experiment setting is false
- **THEN** experiment headers cannot inject failure or delay
+14
View File
@@ -21,6 +21,7 @@ def summarize(path):
first = {r["id"]: r for r in resources[0]["containers"]}
cpu = {}
memory = {}
membership = {}
for row in resources[-1]["containers"]:
name = row["name"]
group = (
@@ -33,6 +34,7 @@ def summarize(path):
)
else "business_stores_lb"
)
membership[row["id"]] = group
cpu[group] = (
cpu.get(group, 0) + (row["cpu_ns"] - first[row["id"]]["cpu_ns"]) / 1e9 / elapsed
)
@@ -71,6 +73,18 @@ def summarize(path):
"max_schedule_lag_ms": max(r["schedule_lag_ms"] for r in run["requests"]),
"cpu_cores": cpu,
"memory_peak_bytes": memory,
"memory_peak_group_bytes": {
group: max(
sum(
r["working_set_bytes"]
for r in sample["containers"]
if membership[r["id"]] == group
)
for sample in resources
)
for group in set(membership.values())
},
"achieved_rps": len(latency) / (run["finished"] - run["started"]),
"resource_samples": len(resources),
"ingest": ingest,
"sdk_ended_spans": after["sdk_ended_spans"] - before["sdk_ended_spans"],
+63
View File
@@ -0,0 +1,63 @@
import json
import statistics
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
DIRECTORY = Path("evidence/track5")
MODES = ("off", "on", "head", "tail", "balanced")
COLORS = ("#8a99a6", "#ce6943", "#429da8", "#4864a0", "#459465")
def values(rows, mode, key):
runs = [row for row in rows if row["mode"] == mode]
if key == "latency":
return [row["normal_p95_ms"] for row in runs]
if key == "cpu":
return [row["cpu_cores"]["application"] for row in runs]
if key == "ingest":
return [sum(row["ingest"]["tempo"].values()) / 1024 for row in runs]
return [
100 * row["traces"]["error"]["retained"] / row["traces"]["error"]["expected"]
for row in runs
]
def main():
rows = json.loads((DIRECTORY / "comparison.json").read_text())["runs"]
figure, axes = plt.subplots(2, 2, figsize=(11, 7), constrained_layout=True)
panels = (
("latency", "Normal request p95", "milliseconds"),
("cpu", "Application CPU (Docker counters)", "cores"),
("ingest", "Tempo received bytes per series", "KiB, load + export drain"),
("errors", "Injected error traces retained", "percent"),
)
for axis, (key, title, unit) in zip(axes.flat, panels, strict=True):
data = [values(rows, mode, key) for mode in MODES]
medians = [statistics.median(group) for group in data]
errors = [
[median - min(group) for median, group in zip(medians, data, strict=True)],
[max(group) - median for median, group in zip(medians, data, strict=True)],
]
axis.bar(MODES, medians, color=COLORS, yerr=errors, capsize=4)
for index, group in enumerate(data):
axis.scatter([index] * len(group), group, color="#253447", s=15, zorder=3)
axis.set_title(title)
axis.set_ylabel(unit)
axis.spines[["top", "right"]].set_visible(False)
axis.grid(axis="y", alpha=0.2)
axis.set_axisbelow(True)
figure.suptitle(
"Observability cost: 20 arrivals/s, 40 s, three repeats\n"
"Dots: actual runs; bars: median; whiskers: observed range",
fontsize=13,
)
figure.savefig(DIRECTORY / "comparison.png", dpi=180)
figure.savefig(DIRECTORY / "comparison.svg")
if __name__ == "__main__":
main()