forked from mtusi2026/mtusi2026lab6
Archive grade 4 traces, alerts and bounded outage evidence
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
date: 2026-11-06T11:37:50Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ docker ps --format '{{.Names}} {{.Label "com.docker.compose.project"}}'
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6
|
||||
mtusi2026lab6-api-1 mtusi2026lab6
|
||||
mtusi2026lab6-api-2 mtusi2026lab6
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6
|
||||
mtusi2026lab6-grafana-1 mtusi2026lab6
|
||||
mtusi2026lab6-tempo-1 mtusi2026lab6
|
||||
mtusi2026lab6-loki-1 mtusi2026lab6
|
||||
mtusi2026lab6-otel-collector-1 mtusi2026lab6
|
||||
mtusi2026lab6-postgres-1 mtusi2026lab6
|
||||
mtusi2026lab6-broker-1 mtusi2026lab6
|
||||
mtusi2026lab6-lb-1 mtusi2026lab6
|
||||
mtusi2026lab6-prometheus-1 mtusi2026lab6
|
||||
mtusi2026lab6-s3-1 mtusi2026lab6
|
||||
mtusi2026lab6-valkey-1 mtusi2026lab6
|
||||
exit_code: 0 elapsed_seconds: 0.065
|
||||
@@ -0,0 +1,10 @@
|
||||
date: 2026-11-06T18:17:30Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_check.py cardinality
|
||||
usage: obs_check.py [-h] [--prometheus PROMETHEUS] [--loki LOKI]
|
||||
[--tempo TEMPO] [--grafana GRAFANA]
|
||||
[--grafana-password-file GRAFANA_PASSWORD_FILE]
|
||||
[--timeout TIMEOUT]
|
||||
{probe,request,job,leaks,uploads,query,dashboards,grafana,alerts} ...
|
||||
obs_check.py: error: argument command: invalid choice: 'cardinality' (choose from 'probe', 'request', 'job', 'leaks', 'uploads', 'query', 'dashboards', 'grafana', 'alerts')
|
||||
exit_code: 2 elapsed_seconds: 0.062
|
||||
@@ -0,0 +1,600 @@
|
||||
date: 2026-11-06T09:59:19Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py chain
|
||||
{
|
||||
"started": "2026-11-06T09:59:19+00:00",
|
||||
"finished": "2026-11-06T09:59:26+00:00",
|
||||
"result": "PASS",
|
||||
"scenario": "chain",
|
||||
"observations": {
|
||||
"upload_request_id": "c66198d3-2ee1-4f0d-8b14-3b96158af739",
|
||||
"trace_id": "01b6718401f4aa51e87b3486f50500f0",
|
||||
"retry": false,
|
||||
"outcomes": [
|
||||
"done"
|
||||
],
|
||||
"spans": [
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "ec4R0Bkm03M=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.125,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "select",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "iP97TxdsDLc=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.111,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "select",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "kL5eYNKV2R8=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.082,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "show",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "BHoYqet3w6A=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.054,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "show",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "zORTYvXABOE=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.085,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Q0o68RbCwFo=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.088,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "cPrr2Q9ozzI=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.085,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "qmZ1v/7izUs=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.532,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "wFTBQugE7AA=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.69,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "xEM7rVOybXM=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.228,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "k+pcb/sNrDo=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.143,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "nqLQAASj7ag=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.339,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "KdARxGM8uU4=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.081,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "vzu7yeNCQ38=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.154,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "YICpBi0lB60=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.307,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "MUMKQRPy9v0=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.077,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "8a6QeUZ92tU=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.14,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "pzSnDERQ274=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.188,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "T7i1Nuj9yZI=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.115,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "5Ug86nhSihQ=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.066,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "RDYZQQi877o=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.054,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "4yhDxc4Apqw=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.061,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "ReyPb9uzq6w=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.15,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "RD8W0N6iQFA=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.989,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "QbGIVarUs/8=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.273,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "kzQeqKPNtdQ=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.165,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "STVSIs1sK5c=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 0.397,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "IAVwwJlBgC8=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 5.942,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 copy_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "loyxPmCBLpE=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 6.386,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "vr1ugVYGVbo=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 3.402,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "5dIA22MtUOs=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 2.928,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "/qpDiGf0/ng=",
|
||||
"parent_span_id": "w/WXm6BxeQE=",
|
||||
"duration_ms": 2.51,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachments.attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "w/WXm6BxeQE=",
|
||||
"parent_span_id": "HMCCZoDJ2P8=",
|
||||
"duration_ms": 869.643,
|
||||
"attempt": "1"
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "hxQI059hSkw=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.083,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "NsPVrscA8YA=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.053,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Qop6hTTQWRc=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.041,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "6E82ranJFHM=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.089,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "iU2v2s/5Xkg=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.322,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "hgzLMkcD+cw=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.084,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "nFNN95Ux6uc=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.077,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "e4zfBC6mzdE=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.051,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Hm/K7CNm58A=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.041,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "0CfqVl34eHY=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.076,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "6bLA6xEiYKk=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.692,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "YYAXSZsGAnk=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.197,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "21NlRZIh1No=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.118,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "9R1Ln0frJkk=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.079,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "q8U6KZsG2gs=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.051,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "VOUk7s3ASuE=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.073,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "h6J/oZlqmCc=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.248,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Dp/lFj9CYZI=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.633,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "rlaTBR5DaUU=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.813,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "DSHUXsaYmpo=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.857,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "FSo9VAOcgeg=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.197,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "xaST2SpTu8w=",
|
||||
"parent_span_id": null,
|
||||
"duration_ms": 39.756,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Ds7zLc8Tpmc=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 12.464,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 create",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "HMCCZoDJ2P8=",
|
||||
"parent_span_id": "xaST2SpTu8w=",
|
||||
"duration_ms": 0.013,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "2Y5nbdOQT/E=",
|
||||
"parent_span_id": "HMCCZoDJ2P8=",
|
||||
"duration_ms": 2.424,
|
||||
"attempt": null
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 6.315
|
||||
@@ -0,0 +1,230 @@
|
||||
date: 2026-11-06T11:38:29Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_check.py dashboards --strict
|
||||
{
|
||||
"checked_at": "2026-11-06T11:38:29+00:00",
|
||||
"targets": [
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "RPS бизнес-запросов",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Доля 5xx (2m)",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "p95 latency (5m)",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Доля не-5xx (окно 1h)",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "RPS по маршруту и классу статуса",
|
||||
"datasource": "prometheus",
|
||||
"series": 9,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Доля ошибок",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Доля ошибок",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Latency p50 / p95 / p99",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Latency p50 / p95 / p99",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Latency p50 / p95 / p99",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "p95 по маршруту",
|
||||
"datasource": "prometheus",
|
||||
"series": 8,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "RPS по реплике",
|
||||
"datasource": "prometheus",
|
||||
"series": 3,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Активные запросы",
|
||||
"datasource": "prometheus",
|
||||
"series": 8,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Логи API (ошибки и предупреждения)",
|
||||
"datasource": "loki",
|
||||
"lines": 5
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-api.json",
|
||||
"panel": "Трассы API с ошибкой",
|
||||
"datasource": "tempo",
|
||||
"traces": 0
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Jobs по исходу",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Длительность обработки job",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Длительность обработки job",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Длительность обработки job",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Логи worker",
|
||||
"datasource": "loki",
|
||||
"lines": 5
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Трассы worker",
|
||||
"datasource": "tempo",
|
||||
"traces": 2
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Durable queue depth (includes retry)",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Oldest durable job age",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "DB pool used / waiting / capacity",
|
||||
"datasource": "prometheus",
|
||||
"series": 9,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Worker busy / capacity",
|
||||
"datasource": "prometheus",
|
||||
"series": 2,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Process CPU cores",
|
||||
"datasource": "prometheus",
|
||||
"series": 3,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Process resident memory",
|
||||
"datasource": "prometheus",
|
||||
"series": 3,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Time to ready p50 / p95 / p99",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Time to ready p50 / p95 / p99",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Time to ready p50 / p95 / p99",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "Ready within 30s, SLO 95% / 1h",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
},
|
||||
{
|
||||
"dashboard": "messenger-pipeline.json",
|
||||
"panel": "API availability, SLO 99% / 1h",
|
||||
"datasource": "prometheus",
|
||||
"series": 1,
|
||||
"missing_metrics": []
|
||||
}
|
||||
],
|
||||
"problems": []
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 0.105
|
||||
@@ -0,0 +1,18 @@
|
||||
date: 2026-11-06T11:38:44Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py database
|
||||
Traceback (most recent call last):
|
||||
File "<repo>/scripts/obs_incidents.py", line 349, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "<repo>/scripts/obs_incidents.py", line 332, in main
|
||||
else {"worker": worker, "database": database, "storage": storage}[args.scenario](b)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^
|
||||
File "<repo>/scripts/obs_incidents.py", line 185, in database
|
||||
firing = alert(b, "MessengerHighErrorRate", True)
|
||||
File "<repo>/scripts/obs_incidents.py", line 56, in alert
|
||||
return wait_for(name, check, 150)
|
||||
File "<repo>/scripts/obs_check.py", line 72, in wait_for
|
||||
raise Failure(f"timeout waiting for {what}")
|
||||
obs_check.Failure: timeout waiting for MessengerHighErrorRate
|
||||
exit_code: 1 elapsed_seconds: 167.429
|
||||
@@ -0,0 +1,656 @@
|
||||
date: 2026-11-06T18:12:24Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py database
|
||||
{
|
||||
"started": "2026-11-06T18:12:25+00:00",
|
||||
"finished": "2026-11-06T18:14:15+00:00",
|
||||
"result": "PASS",
|
||||
"scenario": "database",
|
||||
"observations": {
|
||||
"paused": "2026-11-06T18:12:37+00:00",
|
||||
"restored": "2026-11-06T18:13:13+00:00",
|
||||
"requests": [
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-10",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-11",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-12",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-13",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-14",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-15",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-16",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-17",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-18",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-19",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-20",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-21",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-22",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-23",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-24",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-25",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-26",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-27",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-28",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-29",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-30",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-31",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-32",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-33",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-34",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-35",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-36",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-37",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-38",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-39",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-40",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-41",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-42",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-43",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-44",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-45",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-46",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-47",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-48",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-49",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-50",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-51",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-52",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-53",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-54",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-55",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-56",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-57",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-58",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-59",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-60",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-61",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-62",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-63",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-64",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-65",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-66",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-67",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-68",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-69",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-70",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-71",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-72",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-73",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-74",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-75",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-76",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-77",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-78",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-79",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-80",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-81",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-82",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-83",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-84",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-85",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-86",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-87",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-88",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-89",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-90",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-91",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-92",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-93",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-94",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-95",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-96",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-97",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-98",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-99",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-100",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-101",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-102",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-103",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-104",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-105",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-106",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-107",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-108",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-109",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-110",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-111",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-112",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-113",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-114",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-115",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-116",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-117",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-118",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-119",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-120",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-121",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-122",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-123",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-124",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-125",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-126",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-127",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-128",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-129",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-130",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-131",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-132",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-133",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-134",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-135",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-136",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-137",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-138",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-139",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-140",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-141",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-142",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-143",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-144",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-145",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-146",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-147",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-148",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-149",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-150",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-151",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-152",
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"request_id": "db-incident-8985ff30724f-153",
|
||||
"status": 503
|
||||
}
|
||||
],
|
||||
"signals": {
|
||||
"error_ratio": 1.0,
|
||||
"pool_waiting": 0.0
|
||||
},
|
||||
"firing": {
|
||||
"utc": "2026-11-06T18:13:13+00:00",
|
||||
"state": "firing",
|
||||
"alerts": [
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "MessengerHighErrorRate",
|
||||
"severity": "warning"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.",
|
||||
"runbook": "infra/prometheus/runbooks.md#messengerhigherrorrate",
|
||||
"summary": "More than 5% business HTTP 5xx over 1 minute"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-06T18:12:58.389340978Z",
|
||||
"value": "1e+00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resolved": {
|
||||
"utc": "2026-11-06T18:14:14+00:00",
|
||||
"state": "resolved",
|
||||
"alerts": []
|
||||
},
|
||||
"dependency_traces": [
|
||||
{
|
||||
"trace_id": "56a65d4cf3881f888a2550ca344f578e",
|
||||
"db_errors": [
|
||||
{
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_ERROR",
|
||||
"duration_ms": 2002.331
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"trace_id": "36ab8690d98f80687f5983bfa9a50d3c",
|
||||
"db_errors": [
|
||||
{
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_ERROR",
|
||||
"duration_ms": 2004.439
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"trace_id": "ef191d8d6bc7ab4d4d87feefaa464ffe",
|
||||
"db_errors": [
|
||||
{
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_ERROR",
|
||||
"duration_ms": 2010.771
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 110.856
|
||||
@@ -0,0 +1,44 @@
|
||||
date: 2026-11-06T11:34:43Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ env EVIDENCE_DIR=evidence/grade4/failover make failover-evidence SHIFT=0
|
||||
sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "0"
|
||||
date: 2026-11-06T11:34:43Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
|
||||
60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z
|
||||
b3d83f79f546 running started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:33.543407339Z
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.844}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
== after: api replicas
|
||||
60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z
|
||||
b3d83f79f546 exited started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:48.429366971Z
|
||||
== stopped replica: b3d83f79f546; shutdown: signal, drain window with readiness 503, close, exit
|
||||
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T11:34:45.913524Z"}
|
||||
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.938489Z"}
|
||||
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.977221Z"}
|
||||
{"method":"GET","route":"/health/ready","status":503,"request_id":"41dd383e94a3402eb46cbb33a22d66bf","duration_ms":1.036,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.188407Z"}
|
||||
{"method":"GET","route":"/health/ready","status":503,"request_id":"977969cc9efe4c68b46a1b03730fa3d7","duration_ms":0.288,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.560434Z"}
|
||||
{"method":"GET","route":"/health/ready","status":503,"request_id":"1d1cd824e63f42d6bc35f8162ac660bd","duration_ms":0.344,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.695325Z"}
|
||||
{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.938966Z"}
|
||||
{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.978946Z"}
|
||||
{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.041941Z"}
|
||||
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T11:34:48.045395Z"}
|
||||
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.057249Z"}
|
||||
{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.081771Z"}
|
||||
{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T11:34:48.086046Z"}
|
||||
{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.158260Z"}
|
||||
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T11:34:48.263693Z"}
|
||||
$ scripts/restore.sh
|
||||
Container mtusi2026lab6-api-1 Running
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
== after restore
|
||||
60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z
|
||||
b3d83f79f546 running started=2026-11-06T11:35:01.309446421Z finished=2026-11-06T11:34:48.429366971Z
|
||||
exit_code: 0 elapsed_seconds: 21.152
|
||||
@@ -0,0 +1,39 @@
|
||||
date: 2026-11-06T11:34:43Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
|
||||
60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z
|
||||
b3d83f79f546 running started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:33.543407339Z
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.844}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
== after: api replicas
|
||||
60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z
|
||||
b3d83f79f546 exited started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:48.429366971Z
|
||||
== stopped replica: b3d83f79f546; shutdown: signal, drain window with readiness 503, close, exit
|
||||
{"event":"Handling signal: term","level":"info","ts":"2026-11-06T11:34:45.913524Z"}
|
||||
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.938489Z"}
|
||||
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.977221Z"}
|
||||
{"method":"GET","route":"/health/ready","status":503,"request_id":"41dd383e94a3402eb46cbb33a22d66bf","duration_ms":1.036,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.188407Z"}
|
||||
{"method":"GET","route":"/health/ready","status":503,"request_id":"977969cc9efe4c68b46a1b03730fa3d7","duration_ms":0.288,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.560434Z"}
|
||||
{"method":"GET","route":"/health/ready","status":503,"request_id":"1d1cd824e63f42d6bc35f8162ac660bd","duration_ms":0.344,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.695325Z"}
|
||||
{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.938966Z"}
|
||||
{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.978946Z"}
|
||||
{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.041941Z"}
|
||||
{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T11:34:48.045395Z"}
|
||||
{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.057249Z"}
|
||||
{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.081771Z"}
|
||||
{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T11:34:48.086046Z"}
|
||||
{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.158260Z"}
|
||||
{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T11:34:48.263693Z"}
|
||||
$ scripts/restore.sh
|
||||
Container mtusi2026lab6-api-1 Running
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
== after restore
|
||||
60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z
|
||||
b3d83f79f546 running started=2026-11-06T11:35:01.309446421Z finished=2026-11-06T11:34:48.429366971Z
|
||||
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{
|
||||
"severity": "minor",
|
||||
"file": "scripts/limits.py:33; .github/workflows/application.yml:85",
|
||||
"problem": "Inherited service allowlists omit OTel/helper services. Runtime limits checked independently in evidence/grade3/runtime.json; helper audit remains incompatible."
|
||||
},
|
||||
{
|
||||
"severity": "minor",
|
||||
"file": "infra/prometheus/rules/messenger.alerts.yml",
|
||||
"problem": "API error-rate SLI excludes HAProxy-generated 503 and can miss the first burst of a new counter series. Full PostgreSQL pause did not fire (database-incident.txt); sustained SQL-lock incident is verified separately."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,27 @@
|
||||
date: 2026-11-06T18:17:15Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_check.py leaks
|
||||
{
|
||||
"checked_at": "2026-11-06T18:17:30+00:00",
|
||||
"window_seconds": 20,
|
||||
"traces_scanned": 0,
|
||||
"hits": {
|
||||
"password": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
},
|
||||
"token": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
},
|
||||
"message_text": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
},
|
||||
"filename": {
|
||||
"loki": 0,
|
||||
"tempo": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 15.205
|
||||
@@ -0,0 +1,45 @@
|
||||
# Уровень 4 — заметки для отдельного отчёта
|
||||
|
||||
Код приложения: 44d81fd80ee13faa42bca069245ee0943f88b2df. Обвязка retry: 447a6507f973596e612f40c1484dda922bf0e50d. Даты UTC, SHA, команды и exit_code находятся в command logs. Команды выполняются через scripts/lab6.sh.
|
||||
|
||||
## Реализованные сигналы
|
||||
|
||||
Контекст traceparent/tracestate хранится отдельно от job payload в JSONB jobs/outbox (Alembic 0007); retry использует consumer context, восстановление аренды — сохранённый origin. FastStream relay создаёт producer spans; consumer сохраняет родителя. S3 client spans содержат только операцию, rpc.service=S3, тип ошибки; ключи и exception payload не записываются. SQLAlchemy/asyncpg дают DB client spans. Ack после COMMIT сохранён.
|
||||
|
||||
| Сигнал | Источник, единица, labels |
|
||||
|---|---|
|
||||
| messenger_queue_depth / messenger_queue_oldest_age_seconds | SQLAlchemy poll каждые 5 s: durable queued/processing, включая retry; jobs / s; queue_name=attachments.process |
|
||||
| messenger_db_pool_used/waiting/capacity | SQLAlchemy ObservedPool; connections / acquisitions / connections; service resource, без IDs в labels |
|
||||
| messenger_worker_busy/capacity | текущие выполняемые deliveries / prefetch capacity, jobs |
|
||||
| messenger_job_time_to_ready_seconds_* | histogram от Job.created_at до done после commit; buckets 0.05,0.1,0.25,0.5,1,2,5,10,30,60,+Inf; job_type |
|
||||
| process_cpu_time_seconds_total | os.times user+system процесса; s; rate даёт cores; CPU дочерних renderer не включён |
|
||||
| process_memory_usage_bytes | текущий RSS /proc/self/statm внутри Linux container; bytes |
|
||||
|
||||
API p50/p95/p99 = histogram_quantile по sum(rate(bucket[5m])) всех реплик. Доля ошибок — 5xx/все бизнес-HTTP без health. Availability SLO 99% за 1 h; time-to-ready SLO 95% <=30 s за 1 h среди завершённых jobs; pending отдельно контролирует queue lag. Все запросы и единицы находятся в provisioned dashboards. Runbooks: infra/prometheus/runbooks.md. Alerts локальные, без внешней отправки.
|
||||
|
||||
## Проверки
|
||||
|
||||
- obs-validate.txt: Collector, Prometheus, Loki, Tempo, dashboard JSON — PASS.
|
||||
- quality.txt: check, validate, lint — PASS.
|
||||
- student-tests-final.txt: 516 passed, 1 Linux-only skip, 19 stack deselected; migrations fresh/legacy/idempotence и context/lease recovery проходят.
|
||||
- chain.txt: реальный upload trace API→S3/DB→producer→consumer→DB COMMIT.
|
||||
- retry-isolated.txt: тот же trace включает retry_scheduled и done после отказа S3.
|
||||
- smoke.txt, persistence-resumed.txt, failover.txt: публичные сценарии PASS; replica restored.
|
||||
|
||||
## Неуспешные и прерванные попытки
|
||||
|
||||
student-tests-first.txt выявил раннее получение DB connection: нарушались lazy session и advisory-lock autocommit. Исправлено обёрткой ObservedPool._do_get без изменения границ транзакции. student-tests.txt затем обнаружил неверно расширенный allowlist собственного архитектурного теста; исправлено, итоговый полный прогон зелёный. retry.txt не создал отказ: docker compose start worker запускал зависимости, включая S3. retry-isolated.txt использует up --no-deps и подтверждает настоящий retry. persistence.txt прерван извне без exit_code и не является PASS; persistence-resumed.txt завершён после восстановления того же образа с прежними volumes.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Waiting включает получение/создание connection, а не только время ожидания свободного слота; используется закреплённый SQLAlchemy 2.0.45. Gauge после остановки процесса может оставаться видимым в Prometheus до lookback expiry. Queue metric описывает durable backlog, не AMQP messages_ready. Учебные пороги alerts (5%/1m и age>15s, for15s) не являются production-рекомендацией. Корректность off/on cost пока не оценивается этим уровнем.
|
||||
|
||||
Minor из foundation: scripts/limits.py и CI allowlist не учитывают OTel/helper services; отдельные runtime проверки лимитов есть в grade3/runtime.json. REPORT.md не изменяется: финальный отчёт подготовит следующий агент.
|
||||
|
||||
## Дополнительные ограничения проверки
|
||||
|
||||
Полная пауза PostgreSQL (database-incident.txt) дала реальные 503, но не firing: HAProxy быстро снял API с ротации, а первоначальный скачок новой серии 503 не дал положительного rate. Ответы LB не входят в SLI приложения. Это minor ограничение охвата alert; итоговый обязательный сценарий использует ограниченную 120 s SQL-блокировку users при работающем readiness, а не объявляет первую попытку успешной. Блокировка освобождается finally и не изменяет данные.
|
||||
|
||||
SQL-инцидент: 3 error traces PostgreSQL, error ratio 1.000; firing 2026-11-06T18:13:13+00:00, resolved 2026-11-06T18:14:14+00:00. Блокировка началась 2026-11-06T18:12:37+00:00, освобождена 2026-11-06T18:13:13+00:00. Диагностика: server 503 → client PostgreSQL ERROR около 2 s; это зависимость, не просто медленный handler. См. database-lock.txt.
|
||||
|
||||
Telemetry storage: 74 успешных probes, сохранено 72 traces, потеряно 2; максимум RAM 98.7% лимита. smoke PASS во время отказа. Raw memory samples и counters accepted/sent/send_failed/queue до/во время/после — storage-incident.txt. Backend остановлен 2026-11-06T18:14:29+00:00, восстановлен 2026-11-06T18:16:16+00:00. Потеря trace после retry budget 60 s допустима; бизнес-данные сохранены.
|
||||
@@ -0,0 +1,19 @@
|
||||
date: 2026-11-06T09:52:30Z commit: 1a171d0d0f13ab71661a0f63e695031addd88dc0
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make obs-validate
|
||||
sh scripts/obs-validate.sh
|
||||
otelcol-contrib validate collector.yaml: OK
|
||||
Checking /etc/prometheus/prometheus.yml
|
||||
SUCCESS: 2 rule files found
|
||||
SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
|
||||
|
||||
Checking /etc/prometheus/rules/messenger.alerts.yml
|
||||
SUCCESS: 5 rules found
|
||||
|
||||
Checking /etc/prometheus/rules/messenger.recording.yml
|
||||
SUCCESS: 8 rules found
|
||||
|
||||
loki -verify-config: OK
|
||||
tempo -config.verify: OK
|
||||
grafana dashboards: valid JSON with uid and panels
|
||||
exit_code: 0 elapsed_seconds: 2.837
|
||||
@@ -0,0 +1,7 @@
|
||||
date: 2026-11-06T11:34:29Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit_code: 0 elapsed_seconds: 14.086
|
||||
@@ -0,0 +1,5 @@
|
||||
date: 2026-11-06T10:04:41Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
@@ -0,0 +1,14 @@
|
||||
date: 2026-11-06T10:00:29Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make check validate lint
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
.venv/bin/ruff check src student-tests scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
All checks passed!
|
||||
.venv/bin/ruff format --check src student-tests scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
196 files already formatted
|
||||
.venv/bin/ty check src student-tests alembic
|
||||
All checks passed!
|
||||
exit_code: 0 elapsed_seconds: 0.616
|
||||
@@ -0,0 +1,98 @@
|
||||
date: 2026-11-06T18:12:05Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ docker compose up -d --no-build --wait
|
||||
Container mtusi2026lab6-lb-1 Starting
|
||||
Container mtusi2026lab6-prometheus-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Starting
|
||||
Container mtusi2026lab6-valkey-1 Starting
|
||||
Container mtusi2026lab6-s3-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Started
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Started
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Started
|
||||
Container mtusi2026lab6-prometheus-1 Started
|
||||
Container mtusi2026lab6-lb-1 Started
|
||||
Container mtusi2026lab6-broker-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-loki-1 Starting
|
||||
Container mtusi2026lab6-tempo-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-otel-collector-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Started
|
||||
Container mtusi2026lab6-otel-collector-1 Started
|
||||
Container mtusi2026lab6-tempo-1 Started
|
||||
Container mtusi2026lab6-loki-1 Started
|
||||
Container mtusi2026lab6-grafana-1 Starting
|
||||
Container mtusi2026lab6-grafana-1 Started
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Starting
|
||||
Container mtusi2026lab6-s3-init-1 Started
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-api-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-worker-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-relay-1 Starting
|
||||
Container mtusi2026lab6-api-1 Started
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-relay-1 Started
|
||||
Container mtusi2026lab6-worker-1 Started
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-relay-1 Waiting
|
||||
Container mtusi2026lab6-lb-1 Waiting
|
||||
Container mtusi2026lab6-worker-1 Waiting
|
||||
Container mtusi2026lab6-otel-collector-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-loki-1 Waiting
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-tempo-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Waiting
|
||||
Container mtusi2026lab6-grafana-1 Waiting
|
||||
Container mtusi2026lab6-prometheus-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-valkey-1 Healthy
|
||||
Container mtusi2026lab6-prometheus-1 Healthy
|
||||
Container mtusi2026lab6-grafana-1 Healthy
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-otel-collector-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
Container mtusi2026lab6-relay-1 Healthy
|
||||
Container mtusi2026lab6-worker-1 Healthy
|
||||
Container mtusi2026lab6-lb-1 Healthy
|
||||
Container mtusi2026lab6-loki-1 Healthy
|
||||
Container mtusi2026lab6-tempo-1 Healthy
|
||||
exit_code: 0 elapsed_seconds: 19.561
|
||||
@@ -0,0 +1,98 @@
|
||||
date: 2026-11-06T11:34:09Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ docker compose up -d --no-build --wait
|
||||
Container mtusi2026lab6-healthtools-1 Starting
|
||||
Container mtusi2026lab6-prometheus-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Starting
|
||||
Container mtusi2026lab6-s3-1 Starting
|
||||
Container mtusi2026lab6-lb-1 Starting
|
||||
Container mtusi2026lab6-valkey-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Started
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Started
|
||||
Container mtusi2026lab6-postgres-1 Started
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-lb-1 Started
|
||||
Container mtusi2026lab6-prometheus-1 Started
|
||||
Container mtusi2026lab6-broker-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-loki-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-otel-collector-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-tempo-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Started
|
||||
Container mtusi2026lab6-loki-1 Started
|
||||
Container mtusi2026lab6-otel-collector-1 Started
|
||||
Container mtusi2026lab6-tempo-1 Started
|
||||
Container mtusi2026lab6-grafana-1 Starting
|
||||
Container mtusi2026lab6-grafana-1 Started
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Starting
|
||||
Container mtusi2026lab6-s3-init-1 Started
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-api-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-relay-1 Starting
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-worker-1 Starting
|
||||
Container mtusi2026lab6-api-1 Started
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-worker-1 Started
|
||||
Container mtusi2026lab6-relay-1 Started
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-grafana-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Waiting
|
||||
Container mtusi2026lab6-lb-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-tempo-1 Waiting
|
||||
Container mtusi2026lab6-loki-1 Waiting
|
||||
Container mtusi2026lab6-prometheus-1 Waiting
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-relay-1 Waiting
|
||||
Container mtusi2026lab6-otel-collector-1 Waiting
|
||||
Container mtusi2026lab6-worker-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-otel-collector-1 Healthy
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-prometheus-1 Healthy
|
||||
Container mtusi2026lab6-valkey-1 Healthy
|
||||
Container mtusi2026lab6-grafana-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
Container mtusi2026lab6-worker-1 Healthy
|
||||
Container mtusi2026lab6-relay-1 Healthy
|
||||
Container mtusi2026lab6-lb-1 Healthy
|
||||
Container mtusi2026lab6-loki-1 Healthy
|
||||
Container mtusi2026lab6-tempo-1 Healthy
|
||||
exit_code: 0 elapsed_seconds: 19.572
|
||||
@@ -0,0 +1,881 @@
|
||||
date: 2026-11-06T10:01:28Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py retry
|
||||
{
|
||||
"started": "2026-11-06T10:01:28+00:00",
|
||||
"finished": "2026-11-06T10:04:39+00:00",
|
||||
"result": "PASS",
|
||||
"scenario": "retry",
|
||||
"observations": {
|
||||
"upload_request_id": "a9174dd6-03bc-4eb9-a175-97976ebb5a27",
|
||||
"trace_id": "6e54b186a23385c78757721e00540fe0",
|
||||
"retry": true,
|
||||
"outcomes": [
|
||||
"retry_scheduled",
|
||||
"done"
|
||||
],
|
||||
"spans": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "y2HJN1o754Q=",
|
||||
"parent_span_id": null,
|
||||
"duration_ms": 19.13,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "79T0OS4x6j4=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.161,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "jA+hfBOYY7Y=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.086,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "adlQpOS+yb0=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.127,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "dGfhnXTkNZE=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.138,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "HoZo26PbJ8E=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.391,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "mUidBRiK1IU=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.096,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "23ntEo2MC5w=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.08,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "efimFyLMhKE=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.101,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "5uIwYLv0XX8=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.099,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "m5ue9pVQNPw=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.081,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "ymYPNIo3M08=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.581,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "V8IG+RB8wKs=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.322,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "hnGkAWoIqVE=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.099,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "jC1VnzukYG4=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.081,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "MnbqBcTDMlU=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.101,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "7UmSm35vGHw=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.062,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "C8zkBA8JQTc=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.292,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "BZ8+T2pSLXM=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.244,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "M0ptOEdC8mM=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.266,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "YmOm9bHrN6Q=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.16,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "UfB8iSRiouE=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.199,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "rlFzBwYXjfc=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 5.044,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 create",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Ih2oPBzUUgc=",
|
||||
"parent_span_id": "y2HJN1o754Q=",
|
||||
"duration_ms": 0.009,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "rFD9hkLrc0g=",
|
||||
"parent_span_id": "Ih2oPBzUUgc=",
|
||||
"duration_ms": 1.139,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachments.attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"status": "STATUS_CODE_ERROR",
|
||||
"span_id": "kEXksGW8DXE=",
|
||||
"parent_span_id": "Ih2oPBzUUgc=",
|
||||
"duration_ms": 2713.042,
|
||||
"attempt": "1"
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "vctbMOdli9o=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.189,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "select",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "5+ACzZ817bE=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.114,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "select",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "FlbW1uK5JSY=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.105,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "show",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "8AcL2qB2jrw=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.063,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "show",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "yH4TJznSUIQ=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.047,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "i0EI/0qRMgQ=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.05,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "kpnEiG2rpBA=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.061,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "AFsWPLjfQIU=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.523,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "gzsVFvV7eXM=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.637,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "FboVP7mvVC4=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.199,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "mV4WCbh/AO8=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.109,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "0lW6W3JfBGE=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.288,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "qt8r1O7DW7k=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.075,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "k6Un3ViBqdI=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.112,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "b8bvwMEOR7E=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.043,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "e+7vKBTyle0=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.062,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "HqUbe2ofwe8=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.173,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "TOle7/uQVJw=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.168,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Pq+p8ni6BfY=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.53,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "uI9ReQX6MDc=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.266,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Dp3i567FC7Y=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.312,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "lt/BMQ99wDk=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.362,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "GLCftn0qa5k=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.608,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "xGpDCrebQlI=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.398,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "iHx6VHOSrlg=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.244,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "lJ4xp8OMr60=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.643,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "YwElqVZxfeE=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 1.178,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "VUr+jQ211S4=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.123,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "yiaG28vdQ0o=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.095,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "INvMu6Dcs7U=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.074,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "5nUCWHmRWD4=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.064,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "pzI2Tbuy7Qs=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.2,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "tupwPWFePe8=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.206,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "KD4IIvqufFI=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.099,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "K+7+L6qJcuM=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.099,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "yXe79B5ddGw=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.286,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "X5vQNrLEMnE=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.068,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "swuJPCl1Z0Q=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.066,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "UCCn71xKKdI=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.054,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "pPUmGj6KCQE=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.054,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "pCLMMPCg9q0=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.116,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "16J3ACLeVDA=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.182,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_ERROR",
|
||||
"span_id": "n3qnniKhyYs=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 2643.324,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "lrWmvDODn8Y=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 2.461,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 create",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "nYWo9sEJuO4=",
|
||||
"parent_span_id": "kEXksGW8DXE=",
|
||||
"duration_ms": 0.01,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachments.attachment.process.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "91CmUd9RVok=",
|
||||
"parent_span_id": "nYWo9sEJuO4=",
|
||||
"duration_ms": 6.188,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachments.attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "5FdSinM8g/I=",
|
||||
"parent_span_id": "nYWo9sEJuO4=",
|
||||
"duration_ms": 740.915,
|
||||
"attempt": "2"
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 copy_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "qpWqWo1+vJw=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 5.771,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "xVnbGqSwIyM=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 3.068,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "xlsNqUduAJg=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 2.116,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "S3 put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "b1yVsz3MXXU=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 2.15,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "BQ9ygc2ijJ4=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.118,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": ";",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "HDUv+6emOrs=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.056,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "ROLLBACK;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "AbMWZAykxew=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.047,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "ajiQzRZGl40=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.063,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "Gutvv1G6/W0=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.18,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "LH3hKhCnOfU=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.311,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "7Qrp8PCcAbU=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.143,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "1nP4RrrfEvE=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.217,
|
||||
"attempt": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT;",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"status": "STATUS_CODE_UNSET",
|
||||
"span_id": "fz28vhFBmpM=",
|
||||
"parent_span_id": "5FdSinM8g/I=",
|
||||
"duration_ms": 0.26,
|
||||
"attempt": null
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 13.411
|
||||
@@ -0,0 +1,17 @@
|
||||
date: 2026-11-06T09:59:26Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py retry
|
||||
Traceback (most recent call last):
|
||||
File "<repo>/scripts/obs_incidents.py", line 349, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "<repo>/scripts/obs_incidents.py", line 330, in main
|
||||
chain(b, args.scenario == "retry")
|
||||
~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "<repo>/scripts/obs_incidents.py", line 74, in chain
|
||||
wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60)
|
||||
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "<repo>/scripts/obs_check.py", line 72, in wait_for
|
||||
raise Failure(f"timeout waiting for {what}")
|
||||
obs_check.Failure: timeout waiting for real S3 retry
|
||||
exit_code: 1 elapsed_seconds: 68.320
|
||||
@@ -0,0 +1,6 @@
|
||||
date: 2026-11-06T10:04:39Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit_code: 0 elapsed_seconds: 1.991
|
||||
@@ -0,0 +1,279 @@
|
||||
date: 2026-11-06T18:14:15Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py storage
|
||||
{
|
||||
"started": "2026-11-06T18:14:15+00:00",
|
||||
"finished": "2026-11-06T18:16:39+00:00",
|
||||
"result": "PASS",
|
||||
"scenario": "storage",
|
||||
"observations": {
|
||||
"stopped": "2026-11-06T18:14:29+00:00",
|
||||
"restored": "2026-11-06T18:16:16+00:00",
|
||||
"peak_memory_percent_of_limit": 98.7,
|
||||
"smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:18086\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.\n",
|
||||
"successful_probes": 74,
|
||||
"stored_probe_traces": 72,
|
||||
"lost_probe_traces": 2,
|
||||
"memory_samples": [
|
||||
{
|
||||
"utc": "2026-11-06T18:14:16+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"MemUsage": "246.6MiB / 512MiB",
|
||||
"MemPerc": "48.16%",
|
||||
"CPUPerc": "1.94%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-2",
|
||||
"MemUsage": "241.7MiB / 512MiB",
|
||||
"MemPerc": "47.20%",
|
||||
"CPUPerc": "1.12%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"MemUsage": "246.2MiB / 256MiB",
|
||||
"MemPerc": "96.18%",
|
||||
"CPUPerc": "0.63%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-relay-1",
|
||||
"MemUsage": "119.4MiB / 192MiB",
|
||||
"MemPerc": "62.18%",
|
||||
"CPUPerc": "0.38%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"MemUsage": "121.3MiB / 1GiB",
|
||||
"MemPerc": "11.84%",
|
||||
"CPUPerc": "0.10%"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"utc": "2026-11-06T18:14:45+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"MemUsage": "259.7MiB / 512MiB",
|
||||
"MemPerc": "50.72%",
|
||||
"CPUPerc": "1.59%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-2",
|
||||
"MemUsage": "249.4MiB / 512MiB",
|
||||
"MemPerc": "48.71%",
|
||||
"CPUPerc": "1.56%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"MemUsage": "248.5MiB / 256MiB",
|
||||
"MemPerc": "97.08%",
|
||||
"CPUPerc": "0.30%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-relay-1",
|
||||
"MemUsage": "120.2MiB / 192MiB",
|
||||
"MemPerc": "62.59%",
|
||||
"CPUPerc": "0.48%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"MemUsage": "253.4MiB / 1GiB",
|
||||
"MemPerc": "24.75%",
|
||||
"CPUPerc": "0.35%"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"utc": "2026-11-06T18:15:03+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"MemUsage": "259.5MiB / 512MiB",
|
||||
"MemPerc": "50.69%",
|
||||
"CPUPerc": "1.10%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-2",
|
||||
"MemUsage": "249.4MiB / 512MiB",
|
||||
"MemPerc": "48.71%",
|
||||
"CPUPerc": "1.19%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"MemUsage": "250.5MiB / 256MiB",
|
||||
"MemPerc": "97.87%",
|
||||
"CPUPerc": "0.20%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-relay-1",
|
||||
"MemUsage": "119.9MiB / 192MiB",
|
||||
"MemPerc": "62.46%",
|
||||
"CPUPerc": "0.18%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"MemUsage": "253.5MiB / 1GiB",
|
||||
"MemPerc": "24.75%",
|
||||
"CPUPerc": "2.07%"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"utc": "2026-11-06T18:15:21+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"MemUsage": "259.6MiB / 512MiB",
|
||||
"MemPerc": "50.70%",
|
||||
"CPUPerc": "2.63%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-2",
|
||||
"MemUsage": "249.5MiB / 512MiB",
|
||||
"MemPerc": "48.73%",
|
||||
"CPUPerc": "1.35%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"MemUsage": "250.6MiB / 256MiB",
|
||||
"MemPerc": "97.89%",
|
||||
"CPUPerc": "1.18%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-relay-1",
|
||||
"MemUsage": "119.9MiB / 192MiB",
|
||||
"MemPerc": "62.47%",
|
||||
"CPUPerc": "1.77%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"MemUsage": "253.4MiB / 1GiB",
|
||||
"MemPerc": "24.75%",
|
||||
"CPUPerc": "0.75%"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"utc": "2026-11-06T18:15:39+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"MemUsage": "259.6MiB / 512MiB",
|
||||
"MemPerc": "50.71%",
|
||||
"CPUPerc": "0.50%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-2",
|
||||
"MemUsage": "249.1MiB / 512MiB",
|
||||
"MemPerc": "48.66%",
|
||||
"CPUPerc": "0.47%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"MemUsage": "250.6MiB / 256MiB",
|
||||
"MemPerc": "97.88%",
|
||||
"CPUPerc": "0.07%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-relay-1",
|
||||
"MemUsage": "120MiB / 192MiB",
|
||||
"MemPerc": "62.48%",
|
||||
"CPUPerc": "0.18%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"MemUsage": "253.4MiB / 1GiB",
|
||||
"MemPerc": "24.75%",
|
||||
"CPUPerc": "0.20%"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"utc": "2026-11-06T18:16:36+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"MemUsage": "259.6MiB / 512MiB",
|
||||
"MemPerc": "50.70%",
|
||||
"CPUPerc": "1.35%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-api-2",
|
||||
"MemUsage": "249.1MiB / 512MiB",
|
||||
"MemPerc": "48.65%",
|
||||
"CPUPerc": "1.59%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"MemUsage": "252.7MiB / 256MiB",
|
||||
"MemPerc": "98.70%",
|
||||
"CPUPerc": "0.23%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-relay-1",
|
||||
"MemUsage": "120MiB / 192MiB",
|
||||
"MemPerc": "62.49%",
|
||||
"CPUPerc": "0.77%"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"MemUsage": "253.4MiB / 1GiB",
|
||||
"MemPerc": "24.75%",
|
||||
"CPUPerc": "0.29%"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"collector_before": {
|
||||
"otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0,
|
||||
"otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0,
|
||||
"otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 0.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 0.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 0.0,
|
||||
"otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 807.0,
|
||||
"otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 656.0,
|
||||
"otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 784.0,
|
||||
"otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 810.0,
|
||||
"otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 656.0,
|
||||
"otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 784.0
|
||||
},
|
||||
"collector_during": {
|
||||
"otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0,
|
||||
"otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0,
|
||||
"otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 381.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 1287.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 465.0,
|
||||
"otelcol_exporter_send_failed_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 11.0,
|
||||
"otelcol_exporter_send_failed_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 72.0,
|
||||
"otelcol_exporter_send_failed_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 440.0,
|
||||
"otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 833.0,
|
||||
"otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 656.0,
|
||||
"otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 807.0,
|
||||
"otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 1228.0,
|
||||
"otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 2015.0,
|
||||
"otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 1712.0
|
||||
},
|
||||
"collector_after": {
|
||||
"otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0,
|
||||
"otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0,
|
||||
"otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 0.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 0.0,
|
||||
"otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 0.0,
|
||||
"otelcol_exporter_send_failed_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 11.0,
|
||||
"otelcol_exporter_send_failed_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 72.0,
|
||||
"otelcol_exporter_send_failed_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 440.0,
|
||||
"otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 1320.0,
|
||||
"otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 2515.0,
|
||||
"otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 1293.0,
|
||||
"otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 1335.0,
|
||||
"otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 2587.0,
|
||||
"otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 1733.0
|
||||
}
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 143.965
|
||||
@@ -0,0 +1,63 @@
|
||||
date: 2026-11-06T09:58:18Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make student-test
|
||||
.venv/bin/pytest
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
|
||||
rootdir: <repo>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: asyncio-1.4.0, anyio-4.15.1
|
||||
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
|
||||
collected 536 items / 19 deselected / 517 selected
|
||||
|
||||
student-tests/adapters/test_rabbitmq.py .... [ 0%]
|
||||
student-tests/adapters/test_s3_storage.py ... [ 1%]
|
||||
student-tests/api/test_access.py ......... [ 3%]
|
||||
student-tests/api/test_attachments.py .................................. [ 9%]
|
||||
.... [ 10%]
|
||||
student-tests/api/test_bounded_contexts.py .................... [ 14%]
|
||||
student-tests/api/test_cache.py ...... [ 15%]
|
||||
student-tests/api/test_concurrency.py .... [ 16%]
|
||||
student-tests/api/test_consistency.py ........ [ 17%]
|
||||
student-tests/api/test_drain.py .. [ 18%]
|
||||
student-tests/api/test_events.py ...... [ 19%]
|
||||
student-tests/api/test_http_telemetry.py . [ 19%]
|
||||
student-tests/api/test_idempotency.py ............... [ 22%]
|
||||
student-tests/api/test_instance.py .... [ 23%]
|
||||
student-tests/api/test_job_returning.py . [ 23%]
|
||||
student-tests/api/test_message_attachments.py ............. [ 25%]
|
||||
student-tests/api/test_migrations.py ................................... [ 32%]
|
||||
. [ 32%]
|
||||
student-tests/api/test_negative.py ..................................... [ 40%]
|
||||
[ 40%]
|
||||
student-tests/api/test_operability.py .............. [ 42%]
|
||||
student-tests/api/test_pagination.py .................. [ 46%]
|
||||
student-tests/api/test_pipeline.py ............. [ 48%]
|
||||
student-tests/api/test_pool.py ............ [ 51%]
|
||||
student-tests/api/test_reconciler.py ........... [ 53%]
|
||||
student-tests/api/test_recovery.py ............ [ 55%]
|
||||
student-tests/api/test_sessions.py .................. [ 58%]
|
||||
student-tests/api/test_timeout.py ........ [ 60%]
|
||||
student-tests/api/test_trace_context.py . [ 60%]
|
||||
student-tests/api/test_upload_consistency.py ......... [ 62%]
|
||||
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
|
||||
student-tests/api/test_wire_format.py .. [ 64%]
|
||||
student-tests/unit/test_attachments_domain.py .......................... [ 70%]
|
||||
. [ 70%]
|
||||
student-tests/unit/test_authentication.py ..................... [ 74%]
|
||||
student-tests/unit/test_cursor.py ........... [ 76%]
|
||||
student-tests/unit/test_domain.py ...................................... [ 83%]
|
||||
.. [ 84%]
|
||||
student-tests/unit/test_job_observer.py . [ 84%]
|
||||
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
|
||||
student-tests/unit/test_probe.py .............. [ 91%]
|
||||
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
|
||||
student-tests/unit/test_render_pool.py .....s. [ 94%]
|
||||
student-tests/unit/test_renderer.py .................... [ 97%]
|
||||
student-tests/unit/test_scaling.py ..... [ 98%]
|
||||
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
|
||||
student-tests/unit/test_timeouts.py . [100%]
|
||||
|
||||
========== 516 passed, 1 skipped, 19 deselected in 152.10s (0:02:32) ===========
|
||||
exit_code: 0 elapsed_seconds: 154.509
|
||||
@@ -0,0 +1,243 @@
|
||||
date: 2026-11-06T09:52:20Z commit: 1a171d0d0f13ab71661a0f63e695031addd88dc0
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make student-test
|
||||
.venv/bin/pytest
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
|
||||
rootdir: <repo>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: asyncio-1.4.0, anyio-4.15.1
|
||||
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
|
||||
collected 536 items / 19 deselected / 517 selected
|
||||
|
||||
student-tests/adapters/test_rabbitmq.py .... [ 0%]
|
||||
student-tests/adapters/test_s3_storage.py ... [ 1%]
|
||||
student-tests/api/test_access.py ......... [ 3%]
|
||||
student-tests/api/test_attachments.py .................................. [ 9%]
|
||||
.... [ 10%]
|
||||
student-tests/api/test_bounded_contexts.py ......F............. [ 14%]
|
||||
student-tests/api/test_cache.py ...... [ 15%]
|
||||
student-tests/api/test_concurrency.py .... [ 16%]
|
||||
student-tests/api/test_consistency.py ........ [ 17%]
|
||||
student-tests/api/test_drain.py .. [ 18%]
|
||||
student-tests/api/test_events.py ...... [ 19%]
|
||||
student-tests/api/test_http_telemetry.py . [ 19%]
|
||||
student-tests/api/test_idempotency.py ............... [ 22%]
|
||||
student-tests/api/test_instance.py .... [ 23%]
|
||||
student-tests/api/test_job_returning.py . [ 23%]
|
||||
student-tests/api/test_message_attachments.py ............. [ 25%]
|
||||
student-tests/api/test_migrations.py ................................... [ 32%]
|
||||
. [ 32%]
|
||||
student-tests/api/test_negative.py ..................................... [ 40%]
|
||||
[ 40%]
|
||||
student-tests/api/test_operability.py .............. [ 42%]
|
||||
student-tests/api/test_pagination.py .................. [ 46%]
|
||||
student-tests/api/test_pipeline.py ............. [ 48%]
|
||||
student-tests/api/test_pool.py ..F......... [ 51%]
|
||||
student-tests/api/test_reconciler.py .........F. [ 53%]
|
||||
student-tests/api/test_recovery.py ............ [ 55%]
|
||||
student-tests/api/test_sessions.py .................. [ 58%]
|
||||
student-tests/api/test_timeout.py ........ [ 60%]
|
||||
student-tests/api/test_trace_context.py . [ 60%]
|
||||
student-tests/api/test_upload_consistency.py ......... [ 62%]
|
||||
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
|
||||
student-tests/api/test_wire_format.py .. [ 64%]
|
||||
student-tests/unit/test_attachments_domain.py .......................... [ 70%]
|
||||
. [ 70%]
|
||||
student-tests/unit/test_authentication.py ..................... [ 74%]
|
||||
student-tests/unit/test_cursor.py ........... [ 76%]
|
||||
student-tests/unit/test_domain.py ...................................... [ 83%]
|
||||
.. [ 84%]
|
||||
student-tests/unit/test_job_observer.py . [ 84%]
|
||||
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
|
||||
student-tests/unit/test_probe.py .............. [ 91%]
|
||||
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
|
||||
student-tests/unit/test_render_pool.py .....s. [ 94%]
|
||||
student-tests/unit/test_renderer.py .................... [ 97%]
|
||||
student-tests/unit/test_scaling.py ..... [ 98%]
|
||||
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
|
||||
student-tests/unit/test_timeouts.py . [100%]
|
||||
|
||||
=================================== FAILURES ===================================
|
||||
_____________________ test_shared_core_holds_only_platform _____________________
|
||||
|
||||
def test_shared_core_holds_only_platform() -> None:
|
||||
names = {
|
||||
path.name
|
||||
for layer in ("domain", "application", "infrastructure")
|
||||
for path in modules_under("shared", layer)
|
||||
}
|
||||
> assert names == {
|
||||
"errors.py",
|
||||
"events.py",
|
||||
"commands.py",
|
||||
"messagebus.py",
|
||||
"ports.py",
|
||||
"clock.py",
|
||||
"ids.py",
|
||||
"logging.py",
|
||||
"postgres.py",
|
||||
"telemetry.py",
|
||||
"readiness.py",
|
||||
}
|
||||
E AssertionError: assert {'clock.py', ...ging.py', ...} == {'clock.py', ...ging.py', ...}
|
||||
E
|
||||
E Extra items in the left set:
|
||||
E 'trace_context.py'
|
||||
E 'runtime_metrics.py'
|
||||
E Use -v to get more diff
|
||||
|
||||
student-tests/api/test_bounded_contexts.py:104: AssertionError
|
||||
________________ test_idle_session_does_not_acquire_connection _________________
|
||||
|
||||
self = <messenger.shared.infrastructure.postgres.Database object at 0x32428de00>
|
||||
|
||||
@asynccontextmanager
|
||||
async def session(self) -> AsyncIterator[DatabaseSession]:
|
||||
try:
|
||||
async with self._sessions() as session:
|
||||
self._waiting += 1
|
||||
try:
|
||||
> await session.connection()
|
||||
|
||||
src/messenger/shared/infrastructure/postgres.py:106:
|
||||
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/ext/asyncio/session.py:930: in connection
|
||||
sync_connection = await greenlet_spawn(
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:201: in greenlet_spawn
|
||||
result = context.throw(*sys.exc_info())
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:2092: in connection
|
||||
return self._connection_for_bind(
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:2108: in _connection_for_bind
|
||||
return trans._connection_for_bind(engine, execution_options)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/orm/state_changes.py:137: in _go
|
||||
ret_value = fn(self, *arg, **kw)
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:1187: in _connection_for_bind
|
||||
conn = bind.connect()
|
||||
^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:3285: in connect
|
||||
return self._connection_cls(self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:143: in __init__
|
||||
self._dbapi_connection = engine.raw_connection()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:3309: in raw_connection
|
||||
return self.pool.connect()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py:447: in connect
|
||||
return _ConnectionFairy._checkout(self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py:1264: in _checkout
|
||||
fairy = _ConnectionRecord.checkout(pool)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py:711: in checkout
|
||||
rec = pool._do_get()
|
||||
^^^^^^^^^^^^^^
|
||||
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
|
||||
|
||||
self = <sqlalchemy.pool.impl.AsyncAdaptedQueuePool object at 0x324289390>
|
||||
|
||||
def _do_get(self) -> ConnectionPoolEntry:
|
||||
use_overflow = self._max_overflow > -1
|
||||
|
||||
wait = use_overflow and self._overflow >= self._max_overflow
|
||||
try:
|
||||
return self._pool.get(wait, self._timeout)
|
||||
except sqla_queue.Empty:
|
||||
# don't do things inside of "except Empty", because when we say
|
||||
# we timed out or can't connect and raise, Python 3 tells
|
||||
# people the real error is queue.Empty which it isn't.
|
||||
pass
|
||||
if use_overflow and self._overflow >= self._max_overflow:
|
||||
if not wait:
|
||||
return self._do_get()
|
||||
else:
|
||||
> raise exc.TimeoutError(
|
||||
"QueuePool limit of size %d overflow %d reached, "
|
||||
"connection timed out, timeout %0.2f"
|
||||
% (self.size(), self.overflow(), self._timeout),
|
||||
code="3o7r",
|
||||
)
|
||||
E sqlalchemy.exc.TimeoutError: QueuePool limit of size 1 overflow 0 reached, connection timed out, timeout 1.00 (Background on this error at: https://sqlalche.me/e/20/3o7r)
|
||||
|
||||
.venv/lib/python3.14/site-packages/sqlalchemy/pool/impl.py:166: TimeoutError
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
async def test_idle_session_does_not_acquire_connection() -> None:
|
||||
async with running_app(db_pool_max_size=1) as http:
|
||||
database = await resolve(http, Database)
|
||||
> async with database.session(), database.session() as active:
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
student-tests/api/test_pool.py:74:
|
||||
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
|
||||
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/contextlib.py:214: in __aenter__
|
||||
return await anext(self.gen)
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
|
||||
|
||||
self = <messenger.shared.infrastructure.postgres.Database object at 0x32428de00>
|
||||
|
||||
@asynccontextmanager
|
||||
async def session(self) -> AsyncIterator[DatabaseSession]:
|
||||
try:
|
||||
async with self._sessions() as session:
|
||||
self._waiting += 1
|
||||
try:
|
||||
await session.connection()
|
||||
finally:
|
||||
self._waiting -= 1
|
||||
yield session
|
||||
except Exception as error:
|
||||
if not unavailable(error):
|
||||
raise
|
||||
> raise Unavailable("PostgreSQL недоступен") from error
|
||||
E messenger.shared.application.errors.Unavailable: PostgreSQL недоступен
|
||||
|
||||
src/messenger/shared/infrastructure/postgres.py:113: Unavailable
|
||||
_______________ test_cancelled_pass_releases_its_autocommit_lock _______________
|
||||
|
||||
dsn = 'postgresql://postgres:test@127.0.0.1:26002/reconcile_07e983d849c3'
|
||||
|
||||
async def test_cancelled_pass_releases_its_autocommit_lock(dsn: str) -> None:
|
||||
async with stand(dsn) as s:
|
||||
entered = asyncio.Event()
|
||||
|
||||
async def hold() -> None:
|
||||
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
|
||||
assert held
|
||||
entered.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
task = asyncio.create_task(hold())
|
||||
try:
|
||||
await asyncio.wait_for(entered.wait(), 2)
|
||||
states = await s.sql(
|
||||
"SELECT a.state FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid "
|
||||
"WHERE l.locktype = 'advisory' AND l.objid = $1",
|
||||
LOCK_KEY,
|
||||
)
|
||||
> assert [row["state"] for row in states] == ["idle"]
|
||||
E AssertionError: assert ['idle in transaction'] == ['idle']
|
||||
E
|
||||
E At index 0 diff: 'idle in transaction' != 'idle'
|
||||
E Use -v to get more diff
|
||||
|
||||
student-tests/api/test_reconciler.py:403: AssertionError
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/api/test_reconciler.py: 11 warnings
|
||||
<sys>:0: SAWarning: Connection is already established for the given bind; execution_options ignored
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
=========================== short test summary info ============================
|
||||
FAILED student-tests/api/test_bounded_contexts.py::test_shared_core_holds_only_platform
|
||||
FAILED student-tests/api/test_pool.py::test_idle_session_does_not_acquire_connection
|
||||
FAILED student-tests/api/test_reconciler.py::test_cancelled_pass_releases_its_autocommit_lock
|
||||
= 3 failed, 513 passed, 1 skipped, 19 deselected, 11 warnings in 156.67s (0:02:36) =
|
||||
make: *** [student-test] Error 1
|
||||
exit_code: 2 elapsed_seconds: 159.760
|
||||
@@ -0,0 +1,91 @@
|
||||
date: 2026-11-06T09:55:35Z commit: 1a171d0d0f13ab71661a0f63e695031addd88dc0
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make student-test
|
||||
.venv/bin/pytest
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
|
||||
rootdir: <repo>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: asyncio-1.4.0, anyio-4.15.1
|
||||
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
|
||||
collected 536 items / 19 deselected / 517 selected
|
||||
|
||||
student-tests/adapters/test_rabbitmq.py .... [ 0%]
|
||||
student-tests/adapters/test_s3_storage.py ... [ 1%]
|
||||
student-tests/api/test_access.py ......... [ 3%]
|
||||
student-tests/api/test_attachments.py .................................. [ 9%]
|
||||
.... [ 10%]
|
||||
student-tests/api/test_bounded_contexts.py .......F............ [ 14%]
|
||||
student-tests/api/test_cache.py ...... [ 15%]
|
||||
student-tests/api/test_concurrency.py .... [ 16%]
|
||||
student-tests/api/test_consistency.py ........ [ 17%]
|
||||
student-tests/api/test_drain.py .. [ 18%]
|
||||
student-tests/api/test_events.py ...... [ 19%]
|
||||
student-tests/api/test_http_telemetry.py . [ 19%]
|
||||
student-tests/api/test_idempotency.py ............... [ 22%]
|
||||
student-tests/api/test_instance.py .... [ 23%]
|
||||
student-tests/api/test_job_returning.py . [ 23%]
|
||||
student-tests/api/test_message_attachments.py ............. [ 25%]
|
||||
student-tests/api/test_migrations.py ................................... [ 32%]
|
||||
. [ 32%]
|
||||
student-tests/api/test_negative.py ..................................... [ 40%]
|
||||
[ 40%]
|
||||
student-tests/api/test_operability.py .............. [ 42%]
|
||||
student-tests/api/test_pagination.py .................. [ 46%]
|
||||
student-tests/api/test_pipeline.py ............. [ 48%]
|
||||
student-tests/api/test_pool.py ............ [ 51%]
|
||||
student-tests/api/test_reconciler.py ........... [ 53%]
|
||||
student-tests/api/test_recovery.py ............ [ 55%]
|
||||
student-tests/api/test_sessions.py .................. [ 58%]
|
||||
student-tests/api/test_timeout.py ........ [ 60%]
|
||||
student-tests/api/test_trace_context.py . [ 60%]
|
||||
student-tests/api/test_upload_consistency.py ......... [ 62%]
|
||||
student-tests/api/test_versioned_pipeline.py ........... [ 64%]
|
||||
student-tests/api/test_wire_format.py .. [ 64%]
|
||||
student-tests/unit/test_attachments_domain.py .......................... [ 70%]
|
||||
. [ 70%]
|
||||
student-tests/unit/test_authentication.py ..................... [ 74%]
|
||||
student-tests/unit/test_cursor.py ........... [ 76%]
|
||||
student-tests/unit/test_domain.py ...................................... [ 83%]
|
||||
.. [ 84%]
|
||||
student-tests/unit/test_job_observer.py . [ 84%]
|
||||
student-tests/unit/test_pipeline_domain.py ....................... [ 88%]
|
||||
student-tests/unit/test_probe.py .............. [ 91%]
|
||||
student-tests/unit/test_reconcile_measurements.py ...... [ 92%]
|
||||
student-tests/unit/test_render_pool.py .....s. [ 94%]
|
||||
student-tests/unit/test_renderer.py .................... [ 97%]
|
||||
student-tests/unit/test_scaling.py ..... [ 98%]
|
||||
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
|
||||
student-tests/unit/test_timeouts.py . [100%]
|
||||
|
||||
=================================== FAILURES ===================================
|
||||
_________________ test_shared_entrypoints_hold_only_transport __________________
|
||||
|
||||
def test_shared_entrypoints_hold_only_transport() -> None:
|
||||
names = {path.name for path in modules_under("shared", "entrypoints")}
|
||||
> assert names == {
|
||||
"actor.py",
|
||||
"config.py",
|
||||
"errors.py",
|
||||
"health.py",
|
||||
"ioc.py",
|
||||
"pipeline.py",
|
||||
"schemas.py",
|
||||
"telemetry.py",
|
||||
"trace_context.py",
|
||||
"runtime_metrics.py",
|
||||
}
|
||||
E AssertionError: assert {'actor.py', ...line.py', ...} == {'actor.py', ...line.py', ...}
|
||||
E
|
||||
E Extra items in the right set:
|
||||
E 'runtime_metrics.py'
|
||||
E 'trace_context.py'
|
||||
E Use -v to get more diff
|
||||
|
||||
student-tests/api/test_bounded_contexts.py:123: AssertionError
|
||||
=========================== short test summary info ============================
|
||||
FAILED student-tests/api/test_bounded_contexts.py::test_shared_entrypoints_hold_only_transport
|
||||
===== 1 failed, 515 passed, 1 skipped, 19 deselected in 155.56s (0:02:35) ======
|
||||
make: *** [student-test] Error 1
|
||||
exit_code: 2 elapsed_seconds: 157.936
|
||||
@@ -0,0 +1,230 @@
|
||||
date: 2026-11-06T09:57:10Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make up
|
||||
sh scripts/secrets.sh
|
||||
sh scripts/broker-secrets.sh
|
||||
broker and S3 worker secrets ready in secrets/ (values not printed)
|
||||
sh scripts/secrets-lab6.sh
|
||||
secrets/grafana_admin_password is ready (value not printed)
|
||||
docker compose build
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 432B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 1.65kB done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
|
||||
#4 DONE 1.4s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 453B done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
|
||||
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
|
||||
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [internal] load build context
|
||||
#8 transferring context: 305.62kB 0.0s done
|
||||
#8 DONE 0.0s
|
||||
|
||||
#9 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
|
||||
#9 CACHED
|
||||
|
||||
#10 [builder 3/7] WORKDIR /build
|
||||
#10 CACHED
|
||||
|
||||
#11 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
|
||||
#11 CACHED
|
||||
|
||||
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
|
||||
#12 CACHED
|
||||
|
||||
#13 [builder 6/7] COPY src ./src
|
||||
#13 DONE 0.1s
|
||||
|
||||
#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
|
||||
#14 0.231 Resolved 100 packages in 11ms
|
||||
#14 0.248 Building mtusi-messenger @ file:///build
|
||||
#14 1.818 Built mtusi-messenger @ file:///build
|
||||
#14 1.824 Prepared 1 package in 1.57s
|
||||
#14 1.828 Installed 1 package in 3ms
|
||||
#14 2.068 Bytecode compiled 2805 files in 240ms
|
||||
#14 2.068 + mtusi-messenger==1.6.0 (from file:///build)
|
||||
#14 DONE 2.1s
|
||||
|
||||
#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
|
||||
#15 CACHED
|
||||
|
||||
#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv
|
||||
#16 DONE 0.5s
|
||||
|
||||
#17 [runtime 4/6] WORKDIR /app
|
||||
#17 DONE 0.1s
|
||||
|
||||
#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./
|
||||
#18 DONE 0.0s
|
||||
|
||||
#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic
|
||||
#19 DONE 0.0s
|
||||
|
||||
#20 exporting to image
|
||||
#20 exporting layers
|
||||
#20 exporting layers 3.4s done
|
||||
#20 exporting manifest sha256:e611ff8072cef7f330ca373cc74464be3c219851d9c7210c932b0f9edcd4f113
|
||||
#20 exporting manifest sha256:e611ff8072cef7f330ca373cc74464be3c219851d9c7210c932b0f9edcd4f113 done
|
||||
#20 exporting config sha256:ebe6a2493a7d49a508b76b70e0895ff95bd41c91db5c1953e2fc80d95bc8835e done
|
||||
#20 exporting attestation manifest sha256:ee30177f12179366bd66d631272804ab91a4b8c87a28b1aad8a59424e1524daf done
|
||||
#20 exporting manifest list sha256:e1701e04d5f6dcaf33b07246843b703fc3e506f5d65e0cd4d2d7603105e1c4a2 done
|
||||
#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done
|
||||
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6
|
||||
#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done
|
||||
#20 DONE 4.1s
|
||||
|
||||
#21 resolving provenance for metadata file
|
||||
#21 DONE 0.0s
|
||||
api Built
|
||||
docker compose up -d --wait
|
||||
Network mtusi2026lab6_edge Creating
|
||||
Network mtusi2026lab6_edge Created
|
||||
Network mtusi2026lab6_backend Creating
|
||||
Network mtusi2026lab6_backend Created
|
||||
Container mtusi2026lab6-broker-1 Creating
|
||||
Container mtusi2026lab6-lb-1 Creating
|
||||
Container mtusi2026lab6-postgres-1 Creating
|
||||
Container mtusi2026lab6-prometheus-1 Creating
|
||||
Container mtusi2026lab6-healthtools-1 Creating
|
||||
Container mtusi2026lab6-valkey-1 Creating
|
||||
Container mtusi2026lab6-s3-1 Creating
|
||||
Container mtusi2026lab6-valkey-1 Created
|
||||
Container mtusi2026lab6-s3-1 Created
|
||||
Container mtusi2026lab6-s3-init-1 Creating
|
||||
Container mtusi2026lab6-healthtools-1 Created
|
||||
Container mtusi2026lab6-loki-1 Creating
|
||||
Container mtusi2026lab6-tempo-1 Creating
|
||||
Container mtusi2026lab6-otel-collector-1 Creating
|
||||
Container mtusi2026lab6-prometheus-1 Created
|
||||
Container mtusi2026lab6-postgres-1 Created
|
||||
Container mtusi2026lab6-migrate-1 Creating
|
||||
Container mtusi2026lab6-lb-1 Created
|
||||
Container mtusi2026lab6-broker-1 Created
|
||||
Container mtusi2026lab6-s3-init-1 Created
|
||||
Container mtusi2026lab6-otel-collector-1 Created
|
||||
Container mtusi2026lab6-loki-1 Created
|
||||
Container mtusi2026lab6-tempo-1 Created
|
||||
Container mtusi2026lab6-grafana-1 Creating
|
||||
Container mtusi2026lab6-migrate-1 Created
|
||||
Container mtusi2026lab6-worker-1 Creating
|
||||
Container mtusi2026lab6-api-1 Creating
|
||||
Container mtusi2026lab6-relay-1 Creating
|
||||
Container mtusi2026lab6-api-2 Creating
|
||||
Container mtusi2026lab6-worker-1 Created
|
||||
Container mtusi2026lab6-api-2 Created
|
||||
Container mtusi2026lab6-grafana-1 Created
|
||||
Container mtusi2026lab6-api-1 Created
|
||||
Container mtusi2026lab6-relay-1 Created
|
||||
Container mtusi2026lab6-broker-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Starting
|
||||
Container mtusi2026lab6-valkey-1 Starting
|
||||
Container mtusi2026lab6-s3-1 Starting
|
||||
Container mtusi2026lab6-prometheus-1 Starting
|
||||
Container mtusi2026lab6-lb-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Started
|
||||
Container mtusi2026lab6-s3-1 Started
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Started
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-lb-1 Started
|
||||
Container mtusi2026lab6-prometheus-1 Started
|
||||
Container mtusi2026lab6-broker-1 Started
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-otel-collector-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-loki-1 Starting
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-tempo-1 Starting
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Starting
|
||||
Container mtusi2026lab6-tempo-1 Started
|
||||
Container mtusi2026lab6-migrate-1 Started
|
||||
Container mtusi2026lab6-otel-collector-1 Started
|
||||
Container mtusi2026lab6-loki-1 Started
|
||||
Container mtusi2026lab6-grafana-1 Starting
|
||||
Container mtusi2026lab6-grafana-1 Started
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Starting
|
||||
Container mtusi2026lab6-s3-init-1 Started
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-api-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-relay-1 Starting
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-worker-1 Starting
|
||||
Container mtusi2026lab6-api-1 Started
|
||||
Container mtusi2026lab6-api-2 Starting
|
||||
Container mtusi2026lab6-worker-1 Started
|
||||
Container mtusi2026lab6-relay-1 Started
|
||||
Container mtusi2026lab6-api-2 Started
|
||||
Container mtusi2026lab6-relay-1 Waiting
|
||||
Container mtusi2026lab6-api-1 Waiting
|
||||
Container mtusi2026lab6-api-2 Waiting
|
||||
Container mtusi2026lab6-worker-1 Waiting
|
||||
Container mtusi2026lab6-postgres-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-healthtools-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-valkey-1 Waiting
|
||||
Container mtusi2026lab6-lb-1 Waiting
|
||||
Container mtusi2026lab6-s3-1 Waiting
|
||||
Container mtusi2026lab6-loki-1 Waiting
|
||||
Container mtusi2026lab6-tempo-1 Waiting
|
||||
Container mtusi2026lab6-otel-collector-1 Waiting
|
||||
Container mtusi2026lab6-broker-1 Waiting
|
||||
Container mtusi2026lab6-prometheus-1 Waiting
|
||||
Container mtusi2026lab6-grafana-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-prometheus-1 Healthy
|
||||
Container mtusi2026lab6-grafana-1 Healthy
|
||||
Container mtusi2026lab6-valkey-1 Healthy
|
||||
Container mtusi2026lab6-otel-collector-1 Healthy
|
||||
Container mtusi2026lab6-s3-1 Healthy
|
||||
Container mtusi2026lab6-healthtools-1 Exited
|
||||
Container mtusi2026lab6-postgres-1 Healthy
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-broker-1 Healthy
|
||||
Container mtusi2026lab6-worker-1 Healthy
|
||||
Container mtusi2026lab6-relay-1 Healthy
|
||||
Container mtusi2026lab6-api-1 Healthy
|
||||
Container mtusi2026lab6-api-2 Healthy
|
||||
Container mtusi2026lab6-lb-1 Healthy
|
||||
Container mtusi2026lab6-loki-1 Healthy
|
||||
Container mtusi2026lab6-tempo-1 Healthy
|
||||
scripts/lb-wait.sh
|
||||
exit_code: 0 elapsed_seconds: 85.736
|
||||
@@ -0,0 +1,45 @@
|
||||
date: 2026-11-06T11:37:50Z commit: 447a6507f973596e612f40c1484dda922bf0e50d
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ python3 scripts/obs_incidents.py worker
|
||||
{
|
||||
"started": "2026-11-06T11:37:50+00:00",
|
||||
"finished": "2026-11-06T11:38:44+00:00",
|
||||
"result": "PASS",
|
||||
"scenario": "worker",
|
||||
"observations": {
|
||||
"stopped": "2026-11-06T11:37:51+00:00",
|
||||
"restored": "2026-11-06T11:38:32+00:00",
|
||||
"jobs": 8,
|
||||
"backlog": {
|
||||
"depth": 8.0,
|
||||
"age_seconds": 30.558317
|
||||
},
|
||||
"firing": {
|
||||
"utc": "2026-11-06T11:38:29+00:00",
|
||||
"state": "firing",
|
||||
"alerts": [
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "MessengerQueueLag",
|
||||
"severity": "warning"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "Target is 95% ready within 30 seconds over 1 hour; depth includes retry and leased work.",
|
||||
"runbook": "infra/prometheus/runbooks.md#messengerqueuelag",
|
||||
"summary": "Durable attachment backlog has waited more than 15 seconds"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-06T11:38:13.389340978Z",
|
||||
"value": "3.0558317e+01"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resolved": {
|
||||
"utc": "2026-11-06T11:38:44+00:00",
|
||||
"state": "resolved",
|
||||
"alerts": []
|
||||
},
|
||||
"depth_after": 0.0
|
||||
}
|
||||
}
|
||||
exit_code: 0 elapsed_seconds: 53.624
|
||||
+4
-4
@@ -1,9 +1,9 @@
|
||||
## Implementation
|
||||
- [ ] 1. Durable context migration, repository and publisher wiring; propagation/retry tests
|
||||
- [ ] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks
|
||||
- [ ] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation
|
||||
- [x] 1. Durable context migration, repository and publisher wiring; propagation/retry tests
|
||||
- [x] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks
|
||||
- [x] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation
|
||||
## Verification
|
||||
- [ ] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes
|
||||
- [x] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes
|
||||
- [ ] 5. End-to-end and retry traces, dashboards, leaks and cardinality
|
||||
- [ ] 6. Worker and database incidents: evidence, firing/resolved and diagnosis
|
||||
- [ ] 7. Storage outage: availability, bounded memory, lost signals and recovery
|
||||
@@ -0,0 +1,30 @@
|
||||
# lab6-observability-resilience Specification
|
||||
|
||||
## Purpose
|
||||
TBD - created by archiving change complete-lab6-observability-resilience. Update Purpose after archive.
|
||||
## Requirements
|
||||
### Requirement: Durable distributed context
|
||||
The system SHALL preserve W3C context through API, transactional outbox, relay producer and worker consumer including retries, and expose safe database and S3 client spans.
|
||||
|
||||
#### Scenario: Deferred delivery and retry
|
||||
- **WHEN** an accepted upload is delivered after relay restart or retried after S3 failure
|
||||
- **THEN** its producer and consumer spans share the originating trace and causally valid parents, without adding fields to the normative job payload
|
||||
|
||||
### Requirement: Operational signals and SLO
|
||||
The system SHALL expose queue depth and oldest age, database pool usage and waiting, worker saturation, process CPU/RAM and durable time-to-ready; dashboards SHALL show histogram percentiles and error ratio and define availability and ready SLO windows.
|
||||
|
||||
#### Scenario: Stopped worker
|
||||
- **WHEN** queued uploads wait for a stopped worker
|
||||
- **THEN** depth and age increase, a queue-lag alert fires and resolves after recovery
|
||||
|
||||
#### Scenario: Unavailable database
|
||||
- **WHEN** PostgreSQL becomes unavailable
|
||||
- **THEN** API failure metrics and database client spans distinguish the dependency failure and an error-rate alert fires and resolves
|
||||
|
||||
### Requirement: Bounded observability failure
|
||||
Collector SHALL use memory limits, batching and bounded queues; unavailable telemetry storage SHALL NOT prevent business requests or cause unbounded memory growth.
|
||||
|
||||
#### Scenario: Storage outage
|
||||
- **WHEN** telemetry storage is stopped during requests
|
||||
- **THEN** business smoke passes, measured memory remains within container limits, losses are recorded, and exports resume after recovery
|
||||
|
||||
Reference in New Issue
Block a user