Complete observability cost study with fifteen measured series
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:35:22Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode balanced --repeat 1 --output evidence/track5/balanced-1.json
|
||||
{"mode": "balanced", "repeat": 1, "result": "PASS", "retained": {"normal": 40, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 125.650
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:45:48Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode balanced --repeat 2 --output evidence/track5/balanced-2.json
|
||||
{"mode": "balanced", "repeat": 2, "result": "PASS", "retained": {"normal": 39, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 126.348
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:56:15Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode balanced --repeat 3 --output evidence/track5/balanced-3.json
|
||||
{"mode": "balanced", "repeat": 3, "result": "PASS", "retained": {"normal": 27, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 126.026
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"utc": "2026-11-06T19:01:45.790303+00:00",
|
||||
"sha": "bc0b1620825b53fd3a3019623326ebf68cbbbeec",
|
||||
"running_own_containers": [],
|
||||
"preserved_volumes": [
|
||||
"mtusi2026lab6_broker-data",
|
||||
"mtusi2026lab6_grafana-data",
|
||||
"mtusi2026lab6_healthtools",
|
||||
"mtusi2026lab6_loki-data",
|
||||
"mtusi2026lab6_pgdata",
|
||||
"mtusi2026lab6_prometheus-data",
|
||||
"mtusi2026lab6_s3data",
|
||||
"mtusi2026lab6_tempo-data",
|
||||
"mtusi2026lab6_valkeydata"
|
||||
],
|
||||
"protected_files": 22,
|
||||
"protected_files_changed": [],
|
||||
"result": "PASS"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 131 KiB |
File diff suppressed because it is too large
Load Diff
|
After Width: | Height: | Size: 100 KiB |
@@ -0,0 +1,4 @@
|
||||
date: 2026-11-06T18:23:50Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol/collector-balanced.yaml
|
||||
exit_code: 0 elapsed_seconds: 0.418
|
||||
@@ -0,0 +1,4 @@
|
||||
date: 2026-11-06T18:23:50Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol/collector-tail.yaml
|
||||
exit_code: 0 elapsed_seconds: 0.435
|
||||
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:59:38Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ docker compose exec -T api python -c 'import json,os; print(json.dumps({k:os.getenv(k) for k in ("OTEL_SDK_DISABLED","OTEL_TRACE_RATIO","OTEL_METRIC_EXPORT_INTERVAL","COST_EXPERIMENT")}))'
|
||||
{"OTEL_SDK_DISABLED": "false", "OTEL_TRACE_RATIO": "1", "OTEL_METRIC_EXPORT_INTERVAL": "10000", "COST_EXPERIMENT": "false"}
|
||||
exit_code: 0 elapsed_seconds: 0.162
|
||||
@@ -0,0 +1,8 @@
|
||||
date: 2026-11-06T18:59:05Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.772}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit_code: 0 elapsed_seconds: 17.495
|
||||
@@ -0,0 +1,576 @@
|
||||
date: 2026-11-06T18:59:44Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_check.py job --base-url http://127.0.0.1:18086
|
||||
{
|
||||
"checked_at": "2026-11-06T19:00:05+00:00",
|
||||
"upload_request_id": "6288e51c-14e4-4d6b-b306-195f0ae08205",
|
||||
"attachment_status": "ready",
|
||||
"traces": [
|
||||
"72ec9617da83d915a1a8c68bd4dd32ca"
|
||||
],
|
||||
"spans": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.088,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.052,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.044,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.058,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.295,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.067,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.083,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.052,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.04,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.059,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.288,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.231,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.081,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 19.546,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.103,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.087,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.328,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.287,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.299,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.177,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.264,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 36.666,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 4.862,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 create",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.014,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 1.728,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.168,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.118,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.098,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.096,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.257,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.236,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.135,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.112,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.243,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.079,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.061,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.053,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.056,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.148,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.193,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.125,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.061,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.05,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.05,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.232,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.324,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.174,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.084,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.401,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 2.003,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 copy_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 5.013,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 2.91,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 2.125,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 2.046,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachments.attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 733.073,
|
||||
"links": 0,
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"attempt": "1"
|
||||
}
|
||||
],
|
||||
"worker_logs": [
|
||||
{
|
||||
"line": "job.processed",
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"span_id": "ecd668f30f4ccf88"
|
||||
},
|
||||
{
|
||||
"line": "audit.job_settled",
|
||||
"trace_id": "72ec9617da83d915a1a8c68bd4dd32ca",
|
||||
"span_id": "ecd668f30f4ccf88"
|
||||
}
|
||||
]
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 21.416
|
||||
@@ -0,0 +1,27 @@
|
||||
date: 2026-11-06T19:00:05Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_check.py leaks --base-url http://127.0.0.1:18086
|
||||
{
|
||||
"checked_at": "2026-11-06T19:00:21+00:00",
|
||||
"window_seconds": 20,
|
||||
"traces_scanned": 8,
|
||||
"hits": {
|
||||
"password": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
},
|
||||
"token": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
},
|
||||
"message_text": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
},
|
||||
"filename": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 15.256
|
||||
@@ -0,0 +1,19 @@
|
||||
date: 2026-11-06T18:59:37Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make obs-validate
|
||||
sh scripts/obs-validate.sh
|
||||
otelcol-contrib validate collector.yaml: OK
|
||||
Checking /etc/prometheus/prometheus.yml
|
||||
SUCCESS: 2 rule files found
|
||||
SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
|
||||
|
||||
Checking /etc/prometheus/rules/messenger.alerts.yml
|
||||
SUCCESS: 5 rules found
|
||||
|
||||
Checking /etc/prometheus/rules/messenger.recording.yml
|
||||
SUCCESS: 8 rules found
|
||||
|
||||
loki -verify-config: OK
|
||||
tempo -config.verify: OK
|
||||
grafana dashboards: valid JSON with uid and panels
|
||||
exit_code: 0 elapsed_seconds: 0.966
|
||||
@@ -0,0 +1,7 @@
|
||||
date: 2026-11-06T18:58:50Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit_code: 0 elapsed_seconds: 14.251
|
||||
@@ -0,0 +1,14 @@
|
||||
date: 2026-11-06T18:58:38Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make check validate lint
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
.venv/bin/ruff check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
All checks passed!
|
||||
.venv/bin/ruff format --check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
200 files already formatted
|
||||
.venv/bin/ty check src student-tests alembic
|
||||
All checks passed!
|
||||
exit_code: 0 elapsed_seconds: 0.880
|
||||
@@ -0,0 +1,66 @@
|
||||
date: 2026-11-06T18:59:38Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_check.py request --base-url http://127.0.0.1:18086
|
||||
{
|
||||
"checked_at": "2026-11-06T18:59:44+00:00",
|
||||
"request_id": "c6cc7b12-b135-4795-8187-0618495a383f",
|
||||
"log": {
|
||||
"line": "request.handled",
|
||||
"trace_id": "8af5c9f15c06bf7ae5dd82ff11639d9e",
|
||||
"span_id": "7e8a5b9286de54c2",
|
||||
"service_name": "messenger-api",
|
||||
"severity": "INFO"
|
||||
},
|
||||
"trace": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.168
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.13
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.132
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.119
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.328
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 0.602
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/users",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"duration_ms": 26.748
|
||||
}
|
||||
]
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 6.162
|
||||
@@ -0,0 +1,12 @@
|
||||
date: 2026-11-06T18:59:22Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make restore
|
||||
scripts/restore.sh
|
||||
Container mtusi2026lab6-api-1 Running
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
exit_code: 0 elapsed_seconds: 3.620
|
||||
@@ -0,0 +1,6 @@
|
||||
date: 2026-11-06T18:58:48Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit_code: 0 elapsed_seconds: 2.160
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:31:11Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode head --repeat 1 --output evidence/track5/head-1.json
|
||||
{"mode": "head", "repeat": 1, "result": "PASS", "retained": {"normal": 73, "error": 1, "slow": 0}}
|
||||
exit_code: 0 elapsed_seconds: 124.365
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:41:37Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode head --repeat 2 --output evidence/track5/head-2.json
|
||||
{"mode": "head", "repeat": 2, "result": "PASS", "retained": {"normal": 97, "error": 2, "slow": 2}}
|
||||
exit_code: 0 elapsed_seconds: 124.824
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:52:04Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode head --repeat 3 --output evidence/track5/head-3.json
|
||||
{"mode": "head", "repeat": 3, "result": "PASS", "retained": {"normal": 78, "error": 1, "slow": 1}}
|
||||
exit_code: 0 elapsed_seconds: 124.173
|
||||
@@ -0,0 +1,67 @@
|
||||
# Трек 1 — сведения для отдельного отчёта
|
||||
|
||||
Код приложения/образ: 202b985deacda79b6d8df65dcefe8a1af43f1a8e; исправление измерителя: bc0b1620825b53fd3a3019623326ebf68cbbbeec. Raw JSON содержит SHA, UTC, все ответы клиента, samples Docker, counters до/после и IDs найденных traces. Команды и exit_code — в одноимённых txt. REPORT.md не изменяется.
|
||||
|
||||
## Гипотеза и метод
|
||||
|
||||
Head 10% должен снижать trace ingest и часть расходов SDK, теряя редкие события независимо от их важности. Tail должен сохранять ошибки и slow, уменьшая запись в Tempo, сохраняя инструментирование приложения. Balanced = tail 5% обычных traces, все ERROR и >=300 ms, экспорт метрик раз в 30 s вместо 10 s.
|
||||
|
||||
`scripts/lab6.sh python3 scripts/obs_cost.py --mode MODE --repeat N --output evidence/track5/MODE-N.json`. Каждый запуск пересоздаёт только свои API/relay/worker/Collector; хранилища и volumes сохраняются. 20 прогревочных GET и 35 s для экспорта. Затем 40 s open-loop: 800 arrivals, 64 клиентских потока, фиксированные 16 ошибок 503 и 16 задержек 350 ms, 768 обычных GET users/me. После — 35 s на batch/tail/periodic export. `finally` возвращает SDK on, ratio1, обычный Collector и выключает fault hook. Нет параллельных нагрузок. Порядок первого круга on/off/head/tail/balanced (on проверяет измеритель), следующих — off/on/head/tail/balanced.
|
||||
|
||||
Latency — wall time каждого HTTP у клиента; schedule_lag хранится отдельно. p95 normal исключает контролируемые ошибки/задержки; общий p99 ожидаемо около350 ms. CPU — разность cgroup total_usage / время внешних samples, включая дочерние процессы; application = API+relay+worker, отдельные observability и business stores/LB. RAM — working set usage−inactive_file, выборки примерно раз в 2 s. Источник — Docker Unix socket, SDK для этих данных не используется.
|
||||
|
||||
Ingest: bytes_received Tempo и Loki; appended samples Prometheus отдельно (несовместимые единицы не суммируются). Окно ingest включает доставку после нагрузки и постоянные self-scrapes. SDK ended считает только sampled spans; сравнение с Collector accepted проверяет доставку, sent — сохранённые экспортером spans. Для tail разность accepted−sent включает намеренную фильтрацию; send_failed отдельно. Для head отброшенные spans не создаются: точно измеряется потеря server roots по категории, неизвестное число несозданных DB spans не выдумывается. Off означает отсутствие SDK, не «ошибку экспорта».
|
||||
|
||||
## Воспроизводимость и ограничения
|
||||
|
||||
Один Docker Desktop host, клиент на том же компьютере; host не эксклюзивный. В JSON есть inventory всех запущенных контейнеров, ресурсы только своего project. Других Docker нагрузок в сериях не запускалось. Подготовка кода/графиков на host могла влиять на малые задержки; schedule lag позволяет видеть опоздания. Это HTTP/DB workload, worker находится в обычном ожидании: выводы не описывают стоимость массовой обработки изображений. Pipeline и retry отдельно доказаны уровнями3/4.
|
||||
|
||||
Три повтора дают наблюдаемый диапазон, не доверительный интервал и не доказательство production capacity. CPU измеряется в окне нагрузки; ingest дополнительно учитывает доставку после него. Tail decision_wait10s: поздние spans/retries после решения могут зависеть от decision cache, эксперимент не доказывает сохранность всех редких долговременных цепочек. Сохранённые evidence годятся для последующего аудита без автоматического повторного прогона.
|
||||
|
||||
Неуспешные попытки: smoke.txt стартовал до окончания up, smoke-ready.txt прошёл после healthy. on-1.txt остановился до нагрузки, поскольку Stats API не возвращал name; имя перенесено в отдельный inspect, on-1-retry.txt содержит полноценную серию. Ни одна из этих неудач не объявляется PASS.
|
||||
|
||||
`python3 scripts/obs_cost_summary.py evidence/track5` строит comparison.json из raw. Графики: `uv run --no-project --with matplotlib==3.11.2 python tools/plot_observability.py`; bars=median, dots=реальные повторы, whiskers=min/max.
|
||||
|
||||
## Итоговые результаты — 15 завершённых серий
|
||||
|
||||
12000 запросов, по 3 повтора каждой настройки. Все HTTP-ответы ожидаемые. Во всех сериях SDK ended = Collector accepted; источник/приёмник не потеряли sampled spans. Фактические export failures: 0. Максимальное опоздание генератора: 26.473 ms.
|
||||
|
||||
| Режим | normal p95 ms: median [min;max] | общий p99 ms | CPU app / obs cores | RAM app / obs MiB (median peak) |
|
||||
|---|---:|---:|---:|---:|
|
||||
| off | 11.622 [4.328; 12.139] | 357.23 | 0.1211 / 0.0192 | 653.0 / 447.8 |
|
||||
| on | 12.846 [5.365; 13.582] | 359.16 | 0.1658 / 0.0269 | 701.7 / 548.2 |
|
||||
| head | 12.441 [5.759; 13.091] | 357.70 | 0.1559 / 0.0249 | 703.9 / 525.9 |
|
||||
| tail | 12.894 [6.887; 13.047] | 358.86 | 0.1624 / 0.0268 | 720.7 / 526.6 |
|
||||
| balanced | 12.712 [7.046; 12.778] | 358.07 | 0.1657 / 0.0244 | 702.0 / 523.7 |
|
||||
|
||||
| Режим | Tempo KiB | Loki KiB | Prom samples | sampled / sent spans (median) | normal traces /2304 | error /48 | slow /48 |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|
|
||||
| off | 0.0 | 0.0 | 28360 | 0 / 0 | 0 | 0 | 0 |
|
||||
| on | 1662.7 | 288.3 | 31517 | 5489 / 5489 | 2304 | 48 | 48 |
|
||||
| head | 176.5 | 287.6 | 32413 | 551 / 551 | 248 | 4 | 3 |
|
||||
| tail | 255.0 | 289.3 | 31837 | 5492 / 709 | 247 | 48 | 48 |
|
||||
| balanced | 147.3 | 288.5 | 29693 | 5489 / 401 | 106 | 48 | 48 |
|
||||
|
||||
Рекомендация в границах этого workload: balanced сохраняет 48/48 errors и 48/48 slow traces, снижая медианный Tempo ingest на 91.1% относительно on. Он не устраняет цену SDK: все spans создаются до tail-решения. Head пропустил 44 ошибок и 45 медленных запросов; их важность неизвестна в момент начала trace.
|
||||
|
||||
Верифицированный запуск balanced: `OTEL_COLLECTOR_CONFIG=collector-balanced.yaml OTEL_METRIC_EXPORT_INTERVAL=30000 scripts/lab6.sh docker compose up -d --no-deps --wait otel-collector api relay worker`. Fault hook для обычного запуска остаётся выключен. Default после эксперимента возвращён к on/100%; рекомендация не включается скрыто.
|
||||
|
||||
Гипотеза о сокращении хранения подтверждается непосредственно counters. Малые различия latency/CPU следует читать вместе с min/max и ограничениями host; статистическая значимость и предельный RPS не заявляются. У CPU-окна не учитывается работа последних tail-решений во время drain, поэтому это CPU под заданной нагрузкой, не полная CPU-стоимость обработки всей серии.
|
||||
|
||||
Графики comparison.png/comparison.svg построены из comparison.json, исходные series остаются в Git.
|
||||
|
||||
## Завершение реализации
|
||||
|
||||
Все 15 raw-series относятся к bc0b1620825b53fd3a3019623326ebf68cbbbeec; образ приложения — 202b985deacda79b6d8df65dcefe8a1af43f1a8e. В каждой серии 21 внешний resource sample, фактическая скорость 20.014–20.023 requests/s. Inventory содержит только 14 контейнеров этого project; других Docker-нагрузок в зафиксированных снимках нет. Образ и нагрузочный код после измерений не менялись; финальный коммит добавляет сводку, графики и evidence.
|
||||
|
||||
`make student-test`: 519 passed, 1 Linux-only skipped, 19 stack deselected (student-tests.txt); stack-поведение отдельно проверено на реальном стенде. `make check validate lint`: PASS (final-quality.txt). `make obs-validate`: PASS (final-obs-config.txt). `make test`, persistence и failover: PASS (final-smoke/persistence/failover.txt), после failover выполнен restore. Проверены свежие server/DB spans и consumer spans, связанные логи worker (final-request/job.txt), 0 утечек по контрольным значениям (final-leaks.txt).
|
||||
|
||||
В контейнере проверены SDK disabled=false, ratio=1, interval=10000, COST_EXPERIMENT=false (final-default-settings.txt); Collector запущен с collector.yaml. График comparison.png визуально проверен: подписи читаются, отображаются все три точки каждого режима. Защищённые 22 файла, включая REPORT/AGENTS/README/contracts/tests/evidence README, сверены с исходными SHA-256 — без изменений.
|
||||
|
||||
Оставшиеся minor findings уровня4 перечислены в ../grade4/findings.json и notes.md: неполные allowlists старого аудита лимитов, отсутствие LB-generated 503 в API SLI и малый запас RAM Collector при outage. Следующий агент должен перенести их в ограничения REPORT; здесь REPORT намеренно сохранён без изменений.
|
||||
|
||||
Свой стенд остановлен через make down без -v (shutdown.txt); 0 запущенных собственных контейнеров, все 9 volumes сохранены (closure.json). Нагрузочные процессы и проверки завершены. Публикации не выполнялись.
|
||||
|
||||
OpenSpec: change measure-lab6-observability-cost прошёл strict validation; 2 requirements синхронизированы в openspec/specs/lab6-observability-cost/spec.md, все задачи завершены, архив 2026-11-06-measure-lab6-observability-cost.
|
||||
|
||||
Пустые файлы предварительных попыток balanced-validate.txt, tail-validate.txt не используются как доказательство; реальные проверки конфигураций — config-tail.txt и config-balanced.txt.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:29:08Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode off --repeat 1 --output evidence/track5/off-1.json
|
||||
{"mode": "off", "repeat": 1, "result": "PASS", "retained": {"normal": 0, "error": 0, "slow": 0}}
|
||||
exit_code: 0 elapsed_seconds: 123.134
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:37:28Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode off --repeat 2 --output evidence/track5/off-2.json
|
||||
{"mode": "off", "repeat": 2, "result": "PASS", "retained": {"normal": 0, "error": 0, "slow": 0}}
|
||||
exit_code: 0 elapsed_seconds: 124.716
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:47:54Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode off --repeat 3 --output evidence/track5/off-3.json
|
||||
{"mode": "off", "repeat": 3, "result": "PASS", "retained": {"normal": 0, "error": 0, "slow": 0}}
|
||||
exit_code: 0 elapsed_seconds: 125.342
|
||||
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:26:45Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode on --repeat 1 --output evidence/track5/on-1.json
|
||||
{"mode": "on", "repeat": 1, "result": "PASS", "retained": {"normal": 768, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 124.926
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
date: 2026-11-06T18:24:41Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode on --repeat 1 --output evidence/track5/on-1.json
|
||||
Traceback (most recent call last):
|
||||
File "<repo>/scripts/obs_cost.py", line 264, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "<repo>/scripts/obs_cost.py", line 251, in main
|
||||
run(args.mode, args.repeat, args.duration, args.rate, args.output)
|
||||
~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "<repo>/scripts/obs_cost.py", line 162, in run
|
||||
resources.begin()
|
||||
~~~~~~~~~~~~~~~^^
|
||||
File "<repo>/scripts/obs_cost.py", line 107, in begin
|
||||
self.samples.append(self.sample())
|
||||
~~~~~~~~~~~^^
|
||||
File "<repo>/scripts/obs_cost.py", line 93, in sample
|
||||
"name": data["name"],
|
||||
~~~~^^^^^^^^
|
||||
KeyError: 'name'
|
||||
exit_code: 1 elapsed_seconds: 49.067
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:39:33Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode on --repeat 2 --output evidence/track5/on-2.json
|
||||
{"mode": "on", "repeat": 2, "result": "PASS", "retained": {"normal": 768, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 124.590
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:50:00Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode on --repeat 3 --output evidence/track5/on-3.json
|
||||
{"mode": "on", "repeat": 3, "result": "PASS", "retained": {"normal": 768, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 124.209
|
||||
@@ -0,0 +1,10 @@
|
||||
date: 2026-11-06T19:02:06Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make lint
|
||||
.venv/bin/ruff check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
All checks passed!
|
||||
.venv/bin/ruff format --check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
200 files already formatted
|
||||
.venv/bin/ty check src student-tests alembic
|
||||
All checks passed!
|
||||
exit_code: 0 elapsed_seconds: 0.131
|
||||
@@ -0,0 +1,77 @@
|
||||
date: 2026-11-06T19:01:10Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make down
|
||||
docker compose --profile bench --profile tools --profile reconcile down
|
||||
Container mtusi2026lab6-worker-1 Stopping
|
||||
Container mtusi2026lab6-lb-1 Stopping
|
||||
Container mtusi2026lab6-otel-collector-1 Stopping
|
||||
Container mtusi2026lab6-api-1 Stopping
|
||||
Container mtusi2026lab6-api-2 Stopping
|
||||
Container mtusi2026lab6-grafana-1 Stopping
|
||||
Container mtusi2026lab6-relay-1 Stopping
|
||||
Container mtusi2026lab6-grafana-1 Stopped
|
||||
Container mtusi2026lab6-grafana-1 Removing
|
||||
Container mtusi2026lab6-grafana-1 Removed
|
||||
Container mtusi2026lab6-prometheus-1 Stopping
|
||||
Container mtusi2026lab6-loki-1 Stopping
|
||||
Container mtusi2026lab6-tempo-1 Stopping
|
||||
Container mtusi2026lab6-lb-1 Stopped
|
||||
Container mtusi2026lab6-lb-1 Removing
|
||||
Container mtusi2026lab6-lb-1 Removed
|
||||
Container mtusi2026lab6-otel-collector-1 Stopped
|
||||
Container mtusi2026lab6-otel-collector-1 Removing
|
||||
Container mtusi2026lab6-prometheus-1 Stopped
|
||||
Container mtusi2026lab6-prometheus-1 Removing
|
||||
Container mtusi2026lab6-otel-collector-1 Removed
|
||||
Container mtusi2026lab6-prometheus-1 Removed
|
||||
Container mtusi2026lab6-tempo-1 Stopped
|
||||
Container mtusi2026lab6-tempo-1 Removing
|
||||
Container mtusi2026lab6-tempo-1 Removed
|
||||
Container mtusi2026lab6-loki-1 Stopped
|
||||
Container mtusi2026lab6-loki-1 Removing
|
||||
Container mtusi2026lab6-loki-1 Removed
|
||||
Container mtusi2026lab6-healthtools-1 Stopping
|
||||
Container mtusi2026lab6-healthtools-1 Stopped
|
||||
Container mtusi2026lab6-healthtools-1 Removing
|
||||
Container mtusi2026lab6-healthtools-1 Removed
|
||||
Container mtusi2026lab6-relay-1 Stopped
|
||||
Container mtusi2026lab6-relay-1 Removing
|
||||
Container mtusi2026lab6-relay-1 Removed
|
||||
Container mtusi2026lab6-worker-1 Stopped
|
||||
Container mtusi2026lab6-worker-1 Removing
|
||||
Container mtusi2026lab6-worker-1 Removed
|
||||
Container mtusi2026lab6-broker-1 Stopping
|
||||
Container mtusi2026lab6-broker-1 Stopped
|
||||
Container mtusi2026lab6-broker-1 Removing
|
||||
Container mtusi2026lab6-broker-1 Removed
|
||||
Container mtusi2026lab6-api-1 Stopped
|
||||
Container mtusi2026lab6-api-1 Removing
|
||||
Container mtusi2026lab6-api-1 Removed
|
||||
Container mtusi2026lab6-api-2 Stopped
|
||||
Container mtusi2026lab6-api-2 Removing
|
||||
Container mtusi2026lab6-api-2 Removed
|
||||
Container mtusi2026lab6-migrate-1 Stopping
|
||||
Container mtusi2026lab6-valkey-1 Stopping
|
||||
Container mtusi2026lab6-s3-init-1 Stopping
|
||||
Container mtusi2026lab6-migrate-1 Stopped
|
||||
Container mtusi2026lab6-migrate-1 Removing
|
||||
Container mtusi2026lab6-s3-init-1 Stopped
|
||||
Container mtusi2026lab6-s3-init-1 Removing
|
||||
Container mtusi2026lab6-migrate-1 Removed
|
||||
Container mtusi2026lab6-postgres-1 Stopping
|
||||
Container mtusi2026lab6-s3-init-1 Removed
|
||||
Container mtusi2026lab6-s3-1 Stopping
|
||||
Container mtusi2026lab6-valkey-1 Stopped
|
||||
Container mtusi2026lab6-valkey-1 Removing
|
||||
Container mtusi2026lab6-valkey-1 Removed
|
||||
Container mtusi2026lab6-s3-1 Stopped
|
||||
Container mtusi2026lab6-s3-1 Removing
|
||||
Container mtusi2026lab6-postgres-1 Stopped
|
||||
Container mtusi2026lab6-postgres-1 Removing
|
||||
Container mtusi2026lab6-s3-1 Removed
|
||||
Container mtusi2026lab6-postgres-1 Removed
|
||||
Network mtusi2026lab6_edge Removing
|
||||
Network mtusi2026lab6_backend Removing
|
||||
Network mtusi2026lab6_edge Removed
|
||||
Network mtusi2026lab6_backend Removed
|
||||
exit_code: 0 elapsed_seconds: 5.291
|
||||
@@ -0,0 +1,6 @@
|
||||
date: 2026-11-06T18:24:39Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit_code: 0 elapsed_seconds: 1.833
|
||||
@@ -0,0 +1,7 @@
|
||||
date: 2026-11-06T18:24:01Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||
FAIL: GET /health/live: connection/TLS failure (URLError)
|
||||
make: *** [test] Error 1
|
||||
exit_code: 2 elapsed_seconds: 0.054
|
||||
@@ -0,0 +1,64 @@
|
||||
date: 2026-11-06T18:20:02Z commit: 20e7759f8771a14e9acba5fe0165c0fed2294cc6
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make student-test
|
||||
.venv/bin/pytest
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
|
||||
rootdir: <repo>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: asyncio-1.4.0, anyio-4.15.1
|
||||
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
|
||||
collected 539 items / 19 deselected / 520 selected
|
||||
|
||||
student-tests/adapters/test_rabbitmq.py .... [ 0%]
|
||||
student-tests/adapters/test_s3_storage.py ... [ 1%]
|
||||
student-tests/api/test_access.py ......... [ 3%]
|
||||
student-tests/api/test_attachments.py .................................. [ 9%]
|
||||
.... [ 10%]
|
||||
student-tests/api/test_bounded_contexts.py .................... [ 14%]
|
||||
student-tests/api/test_cache.py ...... [ 15%]
|
||||
student-tests/api/test_concurrency.py .... [ 16%]
|
||||
student-tests/api/test_consistency.py ........ [ 17%]
|
||||
student-tests/api/test_cost_experiment.py ... [ 18%]
|
||||
student-tests/api/test_drain.py .. [ 18%]
|
||||
student-tests/api/test_events.py ...... [ 19%]
|
||||
student-tests/api/test_http_telemetry.py . [ 20%]
|
||||
student-tests/api/test_idempotency.py ............... [ 22%]
|
||||
student-tests/api/test_instance.py .... [ 23%]
|
||||
student-tests/api/test_job_returning.py . [ 23%]
|
||||
student-tests/api/test_message_attachments.py ............. [ 26%]
|
||||
student-tests/api/test_migrations.py ................................... [ 33%]
|
||||
. [ 33%]
|
||||
student-tests/api/test_negative.py ..................................... [ 40%]
|
||||
[ 40%]
|
||||
student-tests/api/test_operability.py .............. [ 43%]
|
||||
student-tests/api/test_pagination.py .................. [ 46%]
|
||||
student-tests/api/test_pipeline.py ............. [ 49%]
|
||||
student-tests/api/test_pool.py ............ [ 51%]
|
||||
student-tests/api/test_reconciler.py ........... [ 53%]
|
||||
student-tests/api/test_recovery.py ............ [ 55%]
|
||||
student-tests/api/test_sessions.py .................. [ 59%]
|
||||
student-tests/api/test_timeout.py ........ [ 60%]
|
||||
student-tests/api/test_trace_context.py . [ 60%]
|
||||
student-tests/api/test_upload_consistency.py ......... [ 62%]
|
||||
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
|
||||
student-tests/api/test_wire_format.py .. [ 65%]
|
||||
student-tests/unit/test_attachments_domain.py .......................... [ 70%]
|
||||
. [ 70%]
|
||||
student-tests/unit/test_authentication.py ..................... [ 74%]
|
||||
student-tests/unit/test_cursor.py ........... [ 76%]
|
||||
student-tests/unit/test_domain.py ...................................... [ 83%]
|
||||
.. [ 84%]
|
||||
student-tests/unit/test_job_observer.py . [ 84%]
|
||||
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
|
||||
student-tests/unit/test_probe.py .............. [ 91%]
|
||||
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
|
||||
student-tests/unit/test_render_pool.py .....s. [ 94%]
|
||||
student-tests/unit/test_renderer.py .................... [ 97%]
|
||||
student-tests/unit/test_scaling.py ..... [ 98%]
|
||||
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
|
||||
student-tests/unit/test_timeouts.py . [100%]
|
||||
|
||||
========== 519 passed, 1 skipped, 19 deselected in 139.31s (0:02:19) ===========
|
||||
exit_code: 0 elapsed_seconds: 142.075
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:33:16Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode tail --repeat 1 --output evidence/track5/tail-1.json
|
||||
{"mode": "tail", "repeat": 1, "result": "PASS", "retained": {"normal": 93, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 126.284
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:43:42Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode tail --repeat 2 --output evidence/track5/tail-2.json
|
||||
{"mode": "tail", "repeat": 2, "result": "PASS", "retained": {"normal": 71, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 125.572
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T18:54:08Z commit: bc0b1620825b53fd3a3019623326ebf68cbbbeec
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_cost.py --mode tail --repeat 3 --output evidence/track5/tail-3.json
|
||||
{"mode": "tail", "repeat": 3, "result": "PASS", "retained": {"normal": 83, "error": 16, "slow": 16}}
|
||||
exit_code: 0 elapsed_seconds: 127.117
|
||||
@@ -0,0 +1,202 @@
|
||||
date: 2026-11-06T18:22:38Z commit: 202b985deacda79b6d8df65dcefe8a1af43f1a8e
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make up
|
||||
sh scripts/secrets.sh
|
||||
sh scripts/broker-secrets.sh
|
||||
broker and S3 worker secrets ready in secrets/ (values not printed)
|
||||
sh scripts/secrets-lab6.sh
|
||||
secrets/grafana_admin_password is ready (value not printed)
|
||||
docker compose build
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 432B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 1.65kB done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
|
||||
#4 DONE 1.5s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 453B done
|
||||
#5 DONE 0.1s
|
||||
|
||||
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
|
||||
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
|
||||
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [internal] load build context
|
||||
#8 transferring context: 100.64kB 0.0s done
|
||||
#8 DONE 0.0s
|
||||
|
||||
#9 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
|
||||
#9 CACHED
|
||||
|
||||
#10 [builder 3/7] WORKDIR /build
|
||||
#10 CACHED
|
||||
|
||||
#11 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
|
||||
#11 CACHED
|
||||
|
||||
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
|
||||
#12 CACHED
|
||||
|
||||
#13 [builder 6/7] COPY src ./src
|
||||
#13 DONE 0.0s
|
||||
|
||||
#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
|
||||
#14 0.205 Resolved 100 packages in 9ms
|
||||
#14 0.218 Building mtusi-messenger @ file:///build
|
||||
#14 1.445 Built mtusi-messenger @ file:///build
|
||||
#14 1.449 Prepared 1 package in 1.23s
|
||||
#14 1.453 Installed 1 package in 3ms
|
||||
#14 1.688 Bytecode compiled 2805 files in 234ms
|
||||
#14 1.688 + mtusi-messenger==1.6.0 (from file:///build)
|
||||
#14 DONE 1.7s
|
||||
|
||||
#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
|
||||
#15 CACHED
|
||||
|
||||
#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv
|
||||
#16 DONE 0.4s
|
||||
|
||||
#17 [runtime 4/6] WORKDIR /app
|
||||
#17 DONE 0.0s
|
||||
|
||||
#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./
|
||||
#18 DONE 0.0s
|
||||
|
||||
#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic
|
||||
#19 DONE 0.0s
|
||||
|
||||
#20 exporting to image
|
||||
#20 exporting layers
|
||||
#20 exporting layers 3.0s done
|
||||
#20 exporting manifest sha256:e1a0b2d6065a4b2ea4117533457320c803b6877e28a2e40f164d9d0fd5fdbe7f
|
||||
#20 exporting manifest sha256:e1a0b2d6065a4b2ea4117533457320c803b6877e28a2e40f164d9d0fd5fdbe7f done
|
||||
#20 exporting config sha256:314703588174a1170dfe2808331582e9f1e1b615a4523af1d5b518fdbfe08e2a done
|
||||
#20 exporting attestation manifest sha256:fb6c618fb08132e1a6b6d22bae609a713eeb3551c74c60d57996236a0eb25df7 done
|
||||
#20 exporting manifest list sha256:7b053283544fab4fec9e36930ed4cf5dccf88ad5026452f29640f4d63121eb07 done
|
||||
#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done
|
||||
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6
|
||||
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done
|
||||
#20 DONE 3.7s
|
||||
|
||||
#21 resolving provenance for metadata file
|
||||
#21 DONE 0.0s
|
||||
api Built
|
||||
docker compose up -d --wait
|
||||
Container mtusi2026lab6-migrate-1 Recreate
|
||||
Container mtusi2026lab6-migrate-1 Recreated
|
||||
Container mtusi2026lab6-relay-1 Recreate
|
||||
Container mtusi2026lab6-api-2 Recreate
|
||||
Container mtusi2026lab6-api-1 Recreate
|
||||
Container mtusi2026lab6-worker-1 Recreate
|
||||
Container mtusi2026lab6-worker-1 Recreated
|
||||
Container mtusi2026lab6-relay-1 Recreated
|
||||
Container mtusi2026lab6-api-1 Recreated
|
||||
Container mtusi2026lab6-api-2 Recreated
|
||||
Container mtusi2026lab6-prometheus-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Starting
|
||||
Container mtusi2026lab6-s3-1 Starting
|
||||
Container mtusi2026lab6-lb-1 Starting
|
||||
Container mtusi2026lab6-valkey-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Started
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Started
|
||||
Container mtusi2026lab6-prometheus-1 Started
|
||||
Container mtusi2026lab6-postgres-1 Started
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-lb-1 Started
|
||||
Container mtusi2026lab6-broker-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-otel-collector-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-loki-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-tempo-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Started
|
||||
Container mtusi2026lab6-tempo-1 Started
|
||||
Container mtusi2026lab6-otel-collector-1 Started
|
||||
Container mtusi2026lab6-loki-1 Started
|
||||
Container mtusi2026lab6-grafana-1 Starting
|
||||
Container mtusi2026lab6-grafana-1 Started
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Starting
|
||||
Container mtusi2026lab6-s3-init-1 Started
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-worker-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-api-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-relay-1 Starting
|
||||
Container mtusi2026lab6-relay-1 Started
|
||||
Container mtusi2026lab6-api-1 Started
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-worker-1 Started
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-relay-1 Waiting
|
||||
Container mtusi2026lab6-otel-collector-1 Waiting
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-lb-1 Waiting
|
||||
Container mtusi2026lab6-prometheus-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-grafana-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-worker-1 Waiting
|
||||
Container mtusi2026lab6-loki-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-tempo-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Healthy
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-grafana-1 Healthy
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-otel-collector-1 Healthy
|
||||
Container mtusi2026lab6-prometheus-1 Healthy
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
Container mtusi2026lab6-relay-1 Healthy
|
||||
Container mtusi2026lab6-worker-1 Healthy
|
||||
Container mtusi2026lab6-lb-1 Healthy
|
||||
Container mtusi2026lab6-tempo-1 Healthy
|
||||
Container mtusi2026lab6-loki-1 Healthy
|
||||
scripts/lb-wait.sh
|
||||
exit_code: 0 elapsed_seconds: 115.841
|
||||
@@ -0,0 +1,6 @@
|
||||
- [x] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests
|
||||
- [x] 2. Tail/balanced Collector configs and sequential external measurement driver
|
||||
- [x] 3. make check/lint/student-test, config validation and smoke
|
||||
- [x] 4. Three real rounds of off/on/head/tail/balanced; raw evidence
|
||||
- [x] 5. Compare observations, recommend verified balance, record limitations
|
||||
- [x] 6. Restore default configuration, final checks, sync/archive and stop own stand
|
||||
@@ -1,6 +0,0 @@
|
||||
- [ ] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests
|
||||
- [ ] 2. Tail/balanced Collector configs and sequential external measurement driver
|
||||
- [ ] 3. make check/lint/student-test, config validation and smoke
|
||||
- [ ] 4. Three real rounds of off/on/head/tail/balanced; raw evidence
|
||||
- [ ] 5. Compare observations, recommend verified balance, record limitations
|
||||
- [ ] 6. Restore default configuration, final checks, sync/archive and stop own stand
|
||||
@@ -0,0 +1,17 @@
|
||||
# lab6-observability-cost Specification
|
||||
|
||||
## Purpose
|
||||
TBD - created by archiving change measure-lab6-observability-cost. Update Purpose after archive.
|
||||
## Requirements
|
||||
### Requirement: Reproducible cost comparison
|
||||
The system SHALL support SDK off/on, head and tail sampling and a verified balanced configuration, measured under the same open-loop workload with raw latency, external CPU/RAM, ingest and span-loss evidence.
|
||||
#### Scenario: Three repeat rounds
|
||||
- **WHEN** the cost driver runs all five modes in three sequential rounds
|
||||
- **THEN** each run records actual requests, resources, ingest counters and retained normal/error/slow traces without fabricating missing observations
|
||||
|
||||
### Requirement: Disabled experimental faults
|
||||
Controlled error and slow hooks SHALL be opt-in and SHALL NOT affect default requests.
|
||||
#### Scenario: Ordinary deployment
|
||||
- **WHEN** the experiment setting is false
|
||||
- **THEN** experiment headers cannot inject failure or delay
|
||||
|
||||
@@ -21,6 +21,7 @@ def summarize(path):
|
||||
first = {r["id"]: r for r in resources[0]["containers"]}
|
||||
cpu = {}
|
||||
memory = {}
|
||||
membership = {}
|
||||
for row in resources[-1]["containers"]:
|
||||
name = row["name"]
|
||||
group = (
|
||||
@@ -33,6 +34,7 @@ def summarize(path):
|
||||
)
|
||||
else "business_stores_lb"
|
||||
)
|
||||
membership[row["id"]] = group
|
||||
cpu[group] = (
|
||||
cpu.get(group, 0) + (row["cpu_ns"] - first[row["id"]]["cpu_ns"]) / 1e9 / elapsed
|
||||
)
|
||||
@@ -71,6 +73,18 @@ def summarize(path):
|
||||
"max_schedule_lag_ms": max(r["schedule_lag_ms"] for r in run["requests"]),
|
||||
"cpu_cores": cpu,
|
||||
"memory_peak_bytes": memory,
|
||||
"memory_peak_group_bytes": {
|
||||
group: max(
|
||||
sum(
|
||||
r["working_set_bytes"]
|
||||
for r in sample["containers"]
|
||||
if membership[r["id"]] == group
|
||||
)
|
||||
for sample in resources
|
||||
)
|
||||
for group in set(membership.values())
|
||||
},
|
||||
"achieved_rps": len(latency) / (run["finished"] - run["started"]),
|
||||
"resource_samples": len(resources),
|
||||
"ingest": ingest,
|
||||
"sdk_ended_spans": after["sdk_ended_spans"] - before["sdk_ended_spans"],
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
DIRECTORY = Path("evidence/track5")
|
||||
MODES = ("off", "on", "head", "tail", "balanced")
|
||||
COLORS = ("#8a99a6", "#ce6943", "#429da8", "#4864a0", "#459465")
|
||||
|
||||
|
||||
def values(rows, mode, key):
|
||||
runs = [row for row in rows if row["mode"] == mode]
|
||||
if key == "latency":
|
||||
return [row["normal_p95_ms"] for row in runs]
|
||||
if key == "cpu":
|
||||
return [row["cpu_cores"]["application"] for row in runs]
|
||||
if key == "ingest":
|
||||
return [sum(row["ingest"]["tempo"].values()) / 1024 for row in runs]
|
||||
return [
|
||||
100 * row["traces"]["error"]["retained"] / row["traces"]["error"]["expected"]
|
||||
for row in runs
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
rows = json.loads((DIRECTORY / "comparison.json").read_text())["runs"]
|
||||
figure, axes = plt.subplots(2, 2, figsize=(11, 7), constrained_layout=True)
|
||||
panels = (
|
||||
("latency", "Normal request p95", "milliseconds"),
|
||||
("cpu", "Application CPU (Docker counters)", "cores"),
|
||||
("ingest", "Tempo received bytes per series", "KiB, load + export drain"),
|
||||
("errors", "Injected error traces retained", "percent"),
|
||||
)
|
||||
for axis, (key, title, unit) in zip(axes.flat, panels, strict=True):
|
||||
data = [values(rows, mode, key) for mode in MODES]
|
||||
medians = [statistics.median(group) for group in data]
|
||||
errors = [
|
||||
[median - min(group) for median, group in zip(medians, data, strict=True)],
|
||||
[max(group) - median for median, group in zip(medians, data, strict=True)],
|
||||
]
|
||||
axis.bar(MODES, medians, color=COLORS, yerr=errors, capsize=4)
|
||||
for index, group in enumerate(data):
|
||||
axis.scatter([index] * len(group), group, color="#253447", s=15, zorder=3)
|
||||
axis.set_title(title)
|
||||
axis.set_ylabel(unit)
|
||||
axis.spines[["top", "right"]].set_visible(False)
|
||||
axis.grid(axis="y", alpha=0.2)
|
||||
axis.set_axisbelow(True)
|
||||
figure.suptitle(
|
||||
"Observability cost: 20 arrivals/s, 40 s, three repeats\n"
|
||||
"Dots: actual runs; bars: median; whiskers: observed range",
|
||||
fontsize=13,
|
||||
)
|
||||
figure.savefig(DIRECTORY / "comparison.png", dpi=180)
|
||||
figure.savefig(DIRECTORY / "comparison.svg")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user