diff --git a/evidence/grade4/background.txt b/evidence/grade4/background.txt new file mode 100644 index 0000000..1263bc8 --- /dev/null +++ b/evidence/grade4/background.txt @@ -0,0 +1,18 @@ +date: 2026-11-06T11:37:50Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ docker ps --format '{{.Names}} {{.Label "com.docker.compose.project"}}' +mtusi2026lab6-relay-1 mtusi2026lab6 +mtusi2026lab6-api-1 mtusi2026lab6 +mtusi2026lab6-api-2 mtusi2026lab6 +mtusi2026lab6-worker-1 mtusi2026lab6 +mtusi2026lab6-grafana-1 mtusi2026lab6 +mtusi2026lab6-tempo-1 mtusi2026lab6 +mtusi2026lab6-loki-1 mtusi2026lab6 +mtusi2026lab6-otel-collector-1 mtusi2026lab6 +mtusi2026lab6-postgres-1 mtusi2026lab6 +mtusi2026lab6-broker-1 mtusi2026lab6 +mtusi2026lab6-lb-1 mtusi2026lab6 +mtusi2026lab6-prometheus-1 mtusi2026lab6 +mtusi2026lab6-s3-1 mtusi2026lab6 +mtusi2026lab6-valkey-1 mtusi2026lab6 +exit_code: 0 elapsed_seconds: 0.065 diff --git a/evidence/grade4/cardinality.txt b/evidence/grade4/cardinality.txt new file mode 100644 index 0000000..b10c323 --- /dev/null +++ b/evidence/grade4/cardinality.txt @@ -0,0 +1,10 @@ +date: 2026-11-06T18:17:30Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py cardinality +usage: obs_check.py [-h] [--prometheus PROMETHEUS] [--loki LOKI] + [--tempo TEMPO] [--grafana GRAFANA] + [--grafana-password-file GRAFANA_PASSWORD_FILE] + [--timeout TIMEOUT] + {probe,request,job,leaks,uploads,query,dashboards,grafana,alerts} ... +obs_check.py: error: argument command: invalid choice: 'cardinality' (choose from 'probe', 'request', 'job', 'leaks', 'uploads', 'query', 'dashboards', 'grafana', 'alerts') +exit_code: 2 elapsed_seconds: 0.062 diff --git a/evidence/grade4/chain.txt b/evidence/grade4/chain.txt new file mode 100644 index 0000000..700d065 --- /dev/null +++ b/evidence/grade4/chain.txt @@ -0,0 +1,600 @@ +date: 2026-11-06T09:59:19Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py chain +{ + "started": "2026-11-06T09:59:19+00:00", + "finished": "2026-11-06T09:59:26+00:00", + "result": "PASS", + "scenario": "chain", + "observations": { + "upload_request_id": "c66198d3-2ee1-4f0d-8b14-3b96158af739", + "trace_id": "01b6718401f4aa51e87b3486f50500f0", + "retry": false, + "outcomes": [ + "done" + ], + "spans": [ + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "ec4R0Bkm03M=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.125, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "select", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "iP97TxdsDLc=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.111, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "select", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "kL5eYNKV2R8=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.082, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "show", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "BHoYqet3w6A=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.054, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "show", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "zORTYvXABOE=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.085, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Q0o68RbCwFo=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.088, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "cPrr2Q9ozzI=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.085, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "qmZ1v/7izUs=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.532, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "wFTBQugE7AA=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.69, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "xEM7rVOybXM=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.228, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "k+pcb/sNrDo=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.143, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "nqLQAASj7ag=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.339, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "KdARxGM8uU4=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.081, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "vzu7yeNCQ38=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.154, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "YICpBi0lB60=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.307, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "MUMKQRPy9v0=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.077, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "8a6QeUZ92tU=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.14, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "pzSnDERQ274=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.188, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "T7i1Nuj9yZI=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.115, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "5Ug86nhSihQ=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.066, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "RDYZQQi877o=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.054, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "4yhDxc4Apqw=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.061, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "ReyPb9uzq6w=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.15, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "RD8W0N6iQFA=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.989, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "QbGIVarUs/8=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.273, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "kzQeqKPNtdQ=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.165, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "STVSIs1sK5c=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 0.397, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 get_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "IAVwwJlBgC8=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 5.942, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 copy_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "loyxPmCBLpE=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 6.386, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "vr1ugVYGVbo=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 3.402, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "5dIA22MtUOs=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 2.928, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "/qpDiGf0/ng=", + "parent_span_id": "w/WXm6BxeQE=", + "duration_ms": 2.51, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "attachments.attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "status": "STATUS_CODE_UNSET", + "span_id": "w/WXm6BxeQE=", + "parent_span_id": "HMCCZoDJ2P8=", + "duration_ms": 869.643, + "attempt": "1" + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "hxQI059hSkw=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.083, + "attempt": null + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "NsPVrscA8YA=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.053, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Qop6hTTQWRc=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.041, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "6E82ranJFHM=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.089, + "attempt": null + }, + { + "service": "messenger-api", + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "iU2v2s/5Xkg=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.322, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "hgzLMkcD+cw=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.084, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "nFNN95Ux6uc=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.077, + "attempt": null + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "e4zfBC6mzdE=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.051, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Hm/K7CNm58A=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.041, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "0CfqVl34eHY=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.076, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "6bLA6xEiYKk=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.692, + "attempt": null + }, + { + "service": "messenger-api", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "YYAXSZsGAnk=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.197, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "21NlRZIh1No=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.118, + "attempt": null + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "9R1Ln0frJkk=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.079, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "q8U6KZsG2gs=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.051, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "VOUk7s3ASuE=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.073, + "attempt": null + }, + { + "service": "messenger-api", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "h6J/oZlqmCc=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.248, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Dp/lFj9CYZI=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.633, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "rlaTBR5DaUU=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.813, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "DSHUXsaYmpo=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.857, + "attempt": null + }, + { + "service": "messenger-api", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "FSo9VAOcgeg=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.197, + "attempt": null + }, + { + "service": "messenger-api", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "status": "STATUS_CODE_UNSET", + "span_id": "xaST2SpTu8w=", + "parent_span_id": null, + "duration_ms": 39.756, + "attempt": null + }, + { + "service": "messenger-api", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Ds7zLc8Tpmc=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 12.464, + "attempt": null + }, + { + "service": "messenger-relay", + "name": "attachments.attachment.process.v1 create", + "kind": "SPAN_KIND_PRODUCER", + "status": "STATUS_CODE_UNSET", + "span_id": "HMCCZoDJ2P8=", + "parent_span_id": "xaST2SpTu8w=", + "duration_ms": 0.013, + "attempt": null + }, + { + "service": "messenger-relay", + "name": "attachments.attachment.process.v1 publish", + "kind": "SPAN_KIND_PRODUCER", + "status": "STATUS_CODE_UNSET", + "span_id": "2Y5nbdOQT/E=", + "parent_span_id": "HMCCZoDJ2P8=", + "duration_ms": 2.424, + "attempt": null + } + ] + } +} +exit_code: 0 elapsed_seconds: 6.315 diff --git a/evidence/grade4/dashboards.txt b/evidence/grade4/dashboards.txt new file mode 100644 index 0000000..aedd59c --- /dev/null +++ b/evidence/grade4/dashboards.txt @@ -0,0 +1,230 @@ +date: 2026-11-06T11:38:29Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py dashboards --strict +{ + "checked_at": "2026-11-06T11:38:29+00:00", + "targets": [ + { + "dashboard": "messenger-api.json", + "panel": "RPS бизнес-запросов", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля 5xx (2m)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "p95 latency (5m)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля не-5xx (окно 1h)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "RPS по маршруту и классу статуса", + "datasource": "prometheus", + "series": 9, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля ошибок", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Доля ошибок", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Latency p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Latency p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Latency p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "p95 по маршруту", + "datasource": "prometheus", + "series": 8, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "RPS по реплике", + "datasource": "prometheus", + "series": 3, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Активные запросы", + "datasource": "prometheus", + "series": 8, + "missing_metrics": [] + }, + { + "dashboard": "messenger-api.json", + "panel": "Логи API (ошибки и предупреждения)", + "datasource": "loki", + "lines": 5 + }, + { + "dashboard": "messenger-api.json", + "panel": "Трассы API с ошибкой", + "datasource": "tempo", + "traces": 0 + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Jobs по исходу", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Длительность обработки job", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Длительность обработки job", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Длительность обработки job", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Логи worker", + "datasource": "loki", + "lines": 5 + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Трассы worker", + "datasource": "tempo", + "traces": 2 + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Durable queue depth (includes retry)", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Oldest durable job age", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "DB pool used / waiting / capacity", + "datasource": "prometheus", + "series": 9, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Worker busy / capacity", + "datasource": "prometheus", + "series": 2, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Process CPU cores", + "datasource": "prometheus", + "series": 3, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Process resident memory", + "datasource": "prometheus", + "series": 3, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Time to ready p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Time to ready p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Time to ready p50 / p95 / p99", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "Ready within 30s, SLO 95% / 1h", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + }, + { + "dashboard": "messenger-pipeline.json", + "panel": "API availability, SLO 99% / 1h", + "datasource": "prometheus", + "series": 1, + "missing_metrics": [] + } + ], + "problems": [] +} +exit_code: 0 elapsed_seconds: 0.105 diff --git a/evidence/grade4/database-incident.txt b/evidence/grade4/database-incident.txt new file mode 100644 index 0000000..15dc22c --- /dev/null +++ b/evidence/grade4/database-incident.txt @@ -0,0 +1,18 @@ +date: 2026-11-06T11:38:44Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py database +Traceback (most recent call last): + File "/scripts/obs_incidents.py", line 349, in + main() + ~~~~^^ + File "/scripts/obs_incidents.py", line 332, in main + else {"worker": worker, "database": database, "storage": storage}[args.scenario](b) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^ + File "/scripts/obs_incidents.py", line 185, in database + firing = alert(b, "MessengerHighErrorRate", True) + File "/scripts/obs_incidents.py", line 56, in alert + return wait_for(name, check, 150) + File "/scripts/obs_check.py", line 72, in wait_for + raise Failure(f"timeout waiting for {what}") +obs_check.Failure: timeout waiting for MessengerHighErrorRate +exit_code: 1 elapsed_seconds: 167.429 diff --git a/evidence/grade4/database-lock.txt b/evidence/grade4/database-lock.txt new file mode 100644 index 0000000..d10894c --- /dev/null +++ b/evidence/grade4/database-lock.txt @@ -0,0 +1,656 @@ +date: 2026-11-06T18:12:24Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py database +{ + "started": "2026-11-06T18:12:25+00:00", + "finished": "2026-11-06T18:14:15+00:00", + "result": "PASS", + "scenario": "database", + "observations": { + "paused": "2026-11-06T18:12:37+00:00", + "restored": "2026-11-06T18:13:13+00:00", + "requests": [ + { + "request_id": "db-incident-8985ff30724f-10", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-11", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-12", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-13", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-14", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-15", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-16", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-17", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-18", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-19", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-20", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-21", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-22", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-23", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-24", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-25", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-26", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-27", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-28", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-29", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-30", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-31", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-32", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-33", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-34", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-35", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-36", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-37", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-38", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-39", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-40", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-41", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-42", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-43", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-44", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-45", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-46", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-47", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-48", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-49", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-50", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-51", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-52", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-53", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-54", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-55", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-56", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-57", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-58", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-59", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-60", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-61", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-62", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-63", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-64", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-65", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-66", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-67", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-68", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-69", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-70", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-71", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-72", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-73", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-74", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-75", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-76", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-77", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-78", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-79", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-80", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-81", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-82", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-83", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-84", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-85", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-86", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-87", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-88", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-89", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-90", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-91", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-92", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-93", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-94", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-95", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-96", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-97", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-98", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-99", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-100", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-101", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-102", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-103", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-104", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-105", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-106", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-107", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-108", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-109", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-110", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-111", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-112", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-113", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-114", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-115", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-116", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-117", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-118", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-119", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-120", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-121", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-122", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-123", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-124", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-125", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-126", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-127", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-128", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-129", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-130", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-131", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-132", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-133", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-134", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-135", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-136", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-137", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-138", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-139", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-140", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-141", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-142", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-143", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-144", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-145", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-146", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-147", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-148", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-149", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-150", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-151", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-152", + "status": 503 + }, + { + "request_id": "db-incident-8985ff30724f-153", + "status": 503 + } + ], + "signals": { + "error_ratio": 1.0, + "pool_waiting": 0.0 + }, + "firing": { + "utc": "2026-11-06T18:13:13+00:00", + "state": "firing", + "alerts": [ + { + "labels": { + "alertname": "MessengerHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.", + "runbook": "infra/prometheus/runbooks.md#messengerhigherrorrate", + "summary": "More than 5% business HTTP 5xx over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-06T18:12:58.389340978Z", + "value": "1e+00" + } + ] + }, + "resolved": { + "utc": "2026-11-06T18:14:14+00:00", + "state": "resolved", + "alerts": [] + }, + "dependency_traces": [ + { + "trace_id": "56a65d4cf3881f888a2550ca344f578e", + "db_errors": [ + { + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_ERROR", + "duration_ms": 2002.331 + } + ] + }, + { + "trace_id": "36ab8690d98f80687f5983bfa9a50d3c", + "db_errors": [ + { + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_ERROR", + "duration_ms": 2004.439 + } + ] + }, + { + "trace_id": "ef191d8d6bc7ab4d4d87feefaa464ffe", + "db_errors": [ + { + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_ERROR", + "duration_ms": 2010.771 + } + ] + } + ] + } +} +exit_code: 0 elapsed_seconds: 110.856 diff --git a/evidence/grade4/failover.txt b/evidence/grade4/failover.txt new file mode 100644 index 0000000..3f28546 --- /dev/null +++ b/evidence/grade4/failover.txt @@ -0,0 +1,44 @@ +date: 2026-11-06T11:34:43Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ env EVIDENCE_DIR=evidence/grade4/failover make failover-evidence SHIFT=0 +sh scripts/failover-evidence.sh "http://127.0.0.1:18086" "0" +date: 2026-11-06T11:34:43Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z +b3d83f79f546 running started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:33.543407339Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.844} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z +b3d83f79f546 exited started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:48.429366971Z +== stopped replica: b3d83f79f546; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-11-06T11:34:45.913524Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.938489Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.977221Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"41dd383e94a3402eb46cbb33a22d66bf","duration_ms":1.036,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.188407Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"977969cc9efe4c68b46a1b03730fa3d7","duration_ms":0.288,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.560434Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"1d1cd824e63f42d6bc35f8162ac660bd","duration_ms":0.344,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.695325Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.938966Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.978946Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.041941Z"} +{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T11:34:48.045395Z"} +{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.057249Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.081771Z"} +{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T11:34:48.086046Z"} +{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.158260Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T11:34:48.263693Z"} +$ scripts/restore.sh + Container mtusi2026lab6-api-1 Running + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Healthy + Container mtusi2026lab6-api-2 Healthy +== after restore +60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z +b3d83f79f546 running started=2026-11-06T11:35:01.309446421Z finished=2026-11-06T11:34:48.429366971Z +exit_code: 0 elapsed_seconds: 21.152 diff --git a/evidence/grade4/failover/failover-b3d83f79f546.txt b/evidence/grade4/failover/failover-b3d83f79f546.txt new file mode 100644 index 0000000..870508e --- /dev/null +++ b/evidence/grade4/failover/failover-b3d83f79f546.txt @@ -0,0 +1,39 @@ +date: 2026-11-06T11:34:43Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z +b3d83f79f546 running started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:33.543407339Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.844} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z +b3d83f79f546 exited started=2026-11-06T11:34:40.000869551Z finished=2026-11-06T11:34:48.429366971Z +== stopped replica: b3d83f79f546; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-11-06T11:34:45.913524Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.938489Z"} +{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-06T11:34:45.977221Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"41dd383e94a3402eb46cbb33a22d66bf","duration_ms":1.036,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.188407Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"977969cc9efe4c68b46a1b03730fa3d7","duration_ms":0.288,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.560434Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"1d1cd824e63f42d6bc35f8162ac660bd","duration_ms":0.344,"event":"request.handled","level":"info","ts":"2026-11-06T11:34:46.695325Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.938966Z"} +{"event":"Shutting down","level":"info","ts":"2026-11-06T11:34:47.978946Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.041941Z"} +{"event":"Finished server process [21]","level":"info","ts":"2026-11-06T11:34:48.045395Z"} +{"event":"Worker (pid:21) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.057249Z"} +{"event":"service.stopped","level":"info","ts":"2026-11-06T11:34:48.081771Z"} +{"event":"Finished server process [26]","level":"info","ts":"2026-11-06T11:34:48.086046Z"} +{"event":"Worker (pid:26) was sent SIGTERM!","level":"info","ts":"2026-11-06T11:34:48.158260Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-11-06T11:34:48.263693Z"} +$ scripts/restore.sh + Container mtusi2026lab6-api-1 Running + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Healthy + Container mtusi2026lab6-api-2 Healthy +== after restore +60787bbfae10 running started=2026-11-06T11:34:39.928086134Z finished=2026-11-06T11:34:33.549212797Z +b3d83f79f546 running started=2026-11-06T11:35:01.309446421Z finished=2026-11-06T11:34:48.429366971Z diff --git a/evidence/grade4/findings.json b/evidence/grade4/findings.json new file mode 100644 index 0000000..1aaa7e1 --- /dev/null +++ b/evidence/grade4/findings.json @@ -0,0 +1,12 @@ +[ + { + "severity": "minor", + "file": "scripts/limits.py:33; .github/workflows/application.yml:85", + "problem": "Inherited service allowlists omit OTel/helper services. Runtime limits checked independently in evidence/grade3/runtime.json; helper audit remains incompatible." + }, + { + "severity": "minor", + "file": "infra/prometheus/rules/messenger.alerts.yml", + "problem": "API error-rate SLI excludes HAProxy-generated 503 and can miss the first burst of a new counter series. Full PostgreSQL pause did not fire (database-incident.txt); sustained SQL-lock incident is verified separately." + } +] diff --git a/evidence/grade4/leaks.txt b/evidence/grade4/leaks.txt new file mode 100644 index 0000000..efb9632 --- /dev/null +++ b/evidence/grade4/leaks.txt @@ -0,0 +1,27 @@ +date: 2026-11-06T18:17:15Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_check.py leaks +{ + "checked_at": "2026-11-06T18:17:30+00:00", + "window_seconds": 20, + "traces_scanned": 0, + "hits": { + "password": { + "loki": 0, + "tempo": 0 + }, + "token": { + "loki": 0, + "tempo": 0 + }, + "message_text": { + "loki": 0, + "tempo": 0 + }, + "filename": { + "loki": 0, + "tempo": 0 + } + } +} +exit_code: 0 elapsed_seconds: 15.205 diff --git a/evidence/grade4/notes.md b/evidence/grade4/notes.md new file mode 100644 index 0000000..0b39e1f --- /dev/null +++ b/evidence/grade4/notes.md @@ -0,0 +1,45 @@ +# Уровень 4 — заметки для отдельного отчёта + +Код приложения: 44d81fd80ee13faa42bca069245ee0943f88b2df. Обвязка retry: 447a6507f973596e612f40c1484dda922bf0e50d. Даты UTC, SHA, команды и exit_code находятся в command logs. Команды выполняются через scripts/lab6.sh. + +## Реализованные сигналы + +Контекст traceparent/tracestate хранится отдельно от job payload в JSONB jobs/outbox (Alembic 0007); retry использует consumer context, восстановление аренды — сохранённый origin. FastStream relay создаёт producer spans; consumer сохраняет родителя. S3 client spans содержат только операцию, rpc.service=S3, тип ошибки; ключи и exception payload не записываются. SQLAlchemy/asyncpg дают DB client spans. Ack после COMMIT сохранён. + +| Сигнал | Источник, единица, labels | +|---|---| +| messenger_queue_depth / messenger_queue_oldest_age_seconds | SQLAlchemy poll каждые 5 s: durable queued/processing, включая retry; jobs / s; queue_name=attachments.process | +| messenger_db_pool_used/waiting/capacity | SQLAlchemy ObservedPool; connections / acquisitions / connections; service resource, без IDs в labels | +| messenger_worker_busy/capacity | текущие выполняемые deliveries / prefetch capacity, jobs | +| messenger_job_time_to_ready_seconds_* | histogram от Job.created_at до done после commit; buckets 0.05,0.1,0.25,0.5,1,2,5,10,30,60,+Inf; job_type | +| process_cpu_time_seconds_total | os.times user+system процесса; s; rate даёт cores; CPU дочерних renderer не включён | +| process_memory_usage_bytes | текущий RSS /proc/self/statm внутри Linux container; bytes | + +API p50/p95/p99 = histogram_quantile по sum(rate(bucket[5m])) всех реплик. Доля ошибок — 5xx/все бизнес-HTTP без health. Availability SLO 99% за 1 h; time-to-ready SLO 95% <=30 s за 1 h среди завершённых jobs; pending отдельно контролирует queue lag. Все запросы и единицы находятся в provisioned dashboards. Runbooks: infra/prometheus/runbooks.md. Alerts локальные, без внешней отправки. + +## Проверки + +- obs-validate.txt: Collector, Prometheus, Loki, Tempo, dashboard JSON — PASS. +- quality.txt: check, validate, lint — PASS. +- student-tests-final.txt: 516 passed, 1 Linux-only skip, 19 stack deselected; migrations fresh/legacy/idempotence и context/lease recovery проходят. +- chain.txt: реальный upload trace API→S3/DB→producer→consumer→DB COMMIT. +- retry-isolated.txt: тот же trace включает retry_scheduled и done после отказа S3. +- smoke.txt, persistence-resumed.txt, failover.txt: публичные сценарии PASS; replica restored. + +## Неуспешные и прерванные попытки + +student-tests-first.txt выявил раннее получение DB connection: нарушались lazy session и advisory-lock autocommit. Исправлено обёрткой ObservedPool._do_get без изменения границ транзакции. student-tests.txt затем обнаружил неверно расширенный allowlist собственного архитектурного теста; исправлено, итоговый полный прогон зелёный. retry.txt не создал отказ: docker compose start worker запускал зависимости, включая S3. retry-isolated.txt использует up --no-deps и подтверждает настоящий retry. persistence.txt прерван извне без exit_code и не является PASS; persistence-resumed.txt завершён после восстановления того же образа с прежними volumes. + +## Ограничения + +Waiting включает получение/создание connection, а не только время ожидания свободного слота; используется закреплённый SQLAlchemy 2.0.45. Gauge после остановки процесса может оставаться видимым в Prometheus до lookback expiry. Queue metric описывает durable backlog, не AMQP messages_ready. Учебные пороги alerts (5%/1m и age>15s, for15s) не являются production-рекомендацией. Корректность off/on cost пока не оценивается этим уровнем. + +Minor из foundation: scripts/limits.py и CI allowlist не учитывают OTel/helper services; отдельные runtime проверки лимитов есть в grade3/runtime.json. REPORT.md не изменяется: финальный отчёт подготовит следующий агент. + +## Дополнительные ограничения проверки + +Полная пауза PostgreSQL (database-incident.txt) дала реальные 503, но не firing: HAProxy быстро снял API с ротации, а первоначальный скачок новой серии 503 не дал положительного rate. Ответы LB не входят в SLI приложения. Это minor ограничение охвата alert; итоговый обязательный сценарий использует ограниченную 120 s SQL-блокировку users при работающем readiness, а не объявляет первую попытку успешной. Блокировка освобождается finally и не изменяет данные. + +SQL-инцидент: 3 error traces PostgreSQL, error ratio 1.000; firing 2026-11-06T18:13:13+00:00, resolved 2026-11-06T18:14:14+00:00. Блокировка началась 2026-11-06T18:12:37+00:00, освобождена 2026-11-06T18:13:13+00:00. Диагностика: server 503 → client PostgreSQL ERROR около 2 s; это зависимость, не просто медленный handler. См. database-lock.txt. + +Telemetry storage: 74 успешных probes, сохранено 72 traces, потеряно 2; максимум RAM 98.7% лимита. smoke PASS во время отказа. Raw memory samples и counters accepted/sent/send_failed/queue до/во время/после — storage-incident.txt. Backend остановлен 2026-11-06T18:14:29+00:00, восстановлен 2026-11-06T18:16:16+00:00. Потеря trace после retry budget 60 s допустима; бизнес-данные сохранены. diff --git a/evidence/grade4/obs-validate.txt b/evidence/grade4/obs-validate.txt new file mode 100644 index 0000000..8e14312 --- /dev/null +++ b/evidence/grade4/obs-validate.txt @@ -0,0 +1,19 @@ +date: 2026-11-06T09:52:30Z commit: 1a171d0d0f13ab71661a0f63e695031addd88dc0 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make obs-validate +sh scripts/obs-validate.sh +otelcol-contrib validate collector.yaml: OK +Checking /etc/prometheus/prometheus.yml + SUCCESS: 2 rule files found + SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax + +Checking /etc/prometheus/rules/messenger.alerts.yml + SUCCESS: 5 rules found + +Checking /etc/prometheus/rules/messenger.recording.yml + SUCCESS: 8 rules found + +loki -verify-config: OK +tempo -config.verify: OK +grafana dashboards: valid JSON with uid and panels +exit_code: 0 elapsed_seconds: 2.837 diff --git a/evidence/grade4/persistence-resumed.txt b/evidence/grade4/persistence-resumed.txt new file mode 100644 index 0000000..d96c4dc --- /dev/null +++ b/evidence/grade4/persistence-resumed.txt @@ -0,0 +1,7 @@ +date: 2026-11-06T11:34:29Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +exit_code: 0 elapsed_seconds: 14.086 diff --git a/evidence/grade4/persistence.txt b/evidence/grade4/persistence.txt new file mode 100644 index 0000000..c2e198a --- /dev/null +++ b/evidence/grade4/persistence.txt @@ -0,0 +1,5 @@ +date: 2026-11-06T10:04:41Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18086" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. diff --git a/evidence/grade4/quality.txt b/evidence/grade4/quality.txt new file mode 100644 index 0000000..cbb6748 --- /dev/null +++ b/evidence/grade4/quality.txt @@ -0,0 +1,14 @@ +date: 2026-11-06T10:00:29Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make check validate lint +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +.venv/bin/ruff check src student-tests scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py +All checks passed! +.venv/bin/ruff format --check src student-tests scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py +196 files already formatted +.venv/bin/ty check src student-tests alembic +All checks passed! +exit_code: 0 elapsed_seconds: 0.616 diff --git a/evidence/grade4/resume-evening.txt b/evidence/grade4/resume-evening.txt new file mode 100644 index 0000000..0946a50 --- /dev/null +++ b/evidence/grade4/resume-evening.txt @@ -0,0 +1,98 @@ +date: 2026-11-06T18:12:05Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ docker compose up -d --no-build --wait + Container mtusi2026lab6-lb-1 Starting + Container mtusi2026lab6-prometheus-1 Starting + Container mtusi2026lab6-broker-1 Starting + Container mtusi2026lab6-valkey-1 Starting + Container mtusi2026lab6-s3-1 Starting + Container mtusi2026lab6-postgres-1 Starting + Container mtusi2026lab6-healthtools-1 Starting + Container mtusi2026lab6-healthtools-1 Started + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-s3-1 Started + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-postgres-1 Started + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-valkey-1 Started + Container mtusi2026lab6-prometheus-1 Started + Container mtusi2026lab6-lb-1 Started + Container mtusi2026lab6-broker-1 Started + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-loki-1 Starting + Container mtusi2026lab6-tempo-1 Starting + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-otel-collector-1 Starting + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-otel-collector-1 Started + Container mtusi2026lab6-tempo-1 Started + Container mtusi2026lab6-loki-1 Started + Container mtusi2026lab6-grafana-1 Starting + Container mtusi2026lab6-grafana-1 Started + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-api-1 Starting + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-worker-1 Starting + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-relay-1 Starting + Container mtusi2026lab6-api-1 Started + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-relay-1 Started + Container mtusi2026lab6-worker-1 Started + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-relay-1 Waiting + Container mtusi2026lab6-lb-1 Waiting + Container mtusi2026lab6-worker-1 Waiting + Container mtusi2026lab6-otel-collector-1 Waiting + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-loki-1 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-tempo-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-valkey-1 Waiting + Container mtusi2026lab6-grafana-1 Waiting + Container mtusi2026lab6-prometheus-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-valkey-1 Healthy + Container mtusi2026lab6-prometheus-1 Healthy + Container mtusi2026lab6-grafana-1 Healthy + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-otel-collector-1 Healthy + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-api-1 Healthy + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-relay-1 Healthy + Container mtusi2026lab6-worker-1 Healthy + Container mtusi2026lab6-lb-1 Healthy + Container mtusi2026lab6-loki-1 Healthy + Container mtusi2026lab6-tempo-1 Healthy +exit_code: 0 elapsed_seconds: 19.561 diff --git a/evidence/grade4/resume-up.txt b/evidence/grade4/resume-up.txt new file mode 100644 index 0000000..df353fd --- /dev/null +++ b/evidence/grade4/resume-up.txt @@ -0,0 +1,98 @@ +date: 2026-11-06T11:34:09Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ docker compose up -d --no-build --wait + Container mtusi2026lab6-healthtools-1 Starting + Container mtusi2026lab6-prometheus-1 Starting + Container mtusi2026lab6-broker-1 Starting + Container mtusi2026lab6-s3-1 Starting + Container mtusi2026lab6-lb-1 Starting + Container mtusi2026lab6-valkey-1 Starting + Container mtusi2026lab6-postgres-1 Starting + Container mtusi2026lab6-healthtools-1 Started + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-s3-1 Started + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-valkey-1 Started + Container mtusi2026lab6-postgres-1 Started + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-lb-1 Started + Container mtusi2026lab6-prometheus-1 Started + Container mtusi2026lab6-broker-1 Started + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-loki-1 Starting + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-otel-collector-1 Starting + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-tempo-1 Starting + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-loki-1 Started + Container mtusi2026lab6-otel-collector-1 Started + Container mtusi2026lab6-tempo-1 Started + Container mtusi2026lab6-grafana-1 Starting + Container mtusi2026lab6-grafana-1 Started + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-api-1 Starting + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-relay-1 Starting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-worker-1 Starting + Container mtusi2026lab6-api-1 Started + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-worker-1 Started + Container mtusi2026lab6-relay-1 Started + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-grafana-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-valkey-1 Waiting + Container mtusi2026lab6-lb-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-tempo-1 Waiting + Container mtusi2026lab6-loki-1 Waiting + Container mtusi2026lab6-prometheus-1 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-relay-1 Waiting + Container mtusi2026lab6-otel-collector-1 Waiting + Container mtusi2026lab6-worker-1 Waiting + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-otel-collector-1 Healthy + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-prometheus-1 Healthy + Container mtusi2026lab6-valkey-1 Healthy + Container mtusi2026lab6-grafana-1 Healthy + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-api-1 Healthy + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-worker-1 Healthy + Container mtusi2026lab6-relay-1 Healthy + Container mtusi2026lab6-lb-1 Healthy + Container mtusi2026lab6-loki-1 Healthy + Container mtusi2026lab6-tempo-1 Healthy +exit_code: 0 elapsed_seconds: 19.572 diff --git a/evidence/grade4/retry-isolated.txt b/evidence/grade4/retry-isolated.txt new file mode 100644 index 0000000..e2dac46 --- /dev/null +++ b/evidence/grade4/retry-isolated.txt @@ -0,0 +1,881 @@ +date: 2026-11-06T10:01:28Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py retry +{ + "started": "2026-11-06T10:01:28+00:00", + "finished": "2026-11-06T10:04:39+00:00", + "result": "PASS", + "scenario": "retry", + "observations": { + "upload_request_id": "a9174dd6-03bc-4eb9-a175-97976ebb5a27", + "trace_id": "6e54b186a23385c78757721e00540fe0", + "retry": true, + "outcomes": [ + "retry_scheduled", + "done" + ], + "spans": [ + { + "service": "messenger-api", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "status": "STATUS_CODE_UNSET", + "span_id": "y2HJN1o754Q=", + "parent_span_id": null, + "duration_ms": 19.13, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "79T0OS4x6j4=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.161, + "attempt": null + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "jA+hfBOYY7Y=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.086, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "adlQpOS+yb0=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.127, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "dGfhnXTkNZE=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.138, + "attempt": null + }, + { + "service": "messenger-api", + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "HoZo26PbJ8E=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.391, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "mUidBRiK1IU=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.096, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "23ntEo2MC5w=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.08, + "attempt": null + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "efimFyLMhKE=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.101, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "5uIwYLv0XX8=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.099, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "m5ue9pVQNPw=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.081, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "ymYPNIo3M08=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.581, + "attempt": null + }, + { + "service": "messenger-api", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "V8IG+RB8wKs=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.322, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "hnGkAWoIqVE=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.099, + "attempt": null + }, + { + "service": "messenger-api", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "jC1VnzukYG4=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.081, + "attempt": null + }, + { + "service": "messenger-api", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "MnbqBcTDMlU=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.101, + "attempt": null + }, + { + "service": "messenger-api", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "7UmSm35vGHw=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.062, + "attempt": null + }, + { + "service": "messenger-api", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "C8zkBA8JQTc=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.292, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "BZ8+T2pSLXM=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.244, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "M0ptOEdC8mM=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.266, + "attempt": null + }, + { + "service": "messenger-api", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "YmOm9bHrN6Q=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.16, + "attempt": null + }, + { + "service": "messenger-api", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "UfB8iSRiouE=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.199, + "attempt": null + }, + { + "service": "messenger-api", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "rlFzBwYXjfc=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 5.044, + "attempt": null + }, + { + "service": "messenger-relay", + "name": "attachments.attachment.process.v1 create", + "kind": "SPAN_KIND_PRODUCER", + "status": "STATUS_CODE_UNSET", + "span_id": "Ih2oPBzUUgc=", + "parent_span_id": "y2HJN1o754Q=", + "duration_ms": 0.009, + "attempt": null + }, + { + "service": "messenger-relay", + "name": "attachments.attachment.process.v1 publish", + "kind": "SPAN_KIND_PRODUCER", + "status": "STATUS_CODE_UNSET", + "span_id": "rFD9hkLrc0g=", + "parent_span_id": "Ih2oPBzUUgc=", + "duration_ms": 1.139, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "attachments.attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "status": "STATUS_CODE_ERROR", + "span_id": "kEXksGW8DXE=", + "parent_span_id": "Ih2oPBzUUgc=", + "duration_ms": 2713.042, + "attempt": "1" + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "vctbMOdli9o=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.189, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "select", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "5+ACzZ817bE=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.114, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "select", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "FlbW1uK5JSY=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.105, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "show", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "8AcL2qB2jrw=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.063, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "show", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "yH4TJznSUIQ=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.047, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "i0EI/0qRMgQ=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.05, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "kpnEiG2rpBA=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.061, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "AFsWPLjfQIU=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.523, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "gzsVFvV7eXM=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.637, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "FboVP7mvVC4=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.199, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "mV4WCbh/AO8=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.109, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "0lW6W3JfBGE=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.288, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "qt8r1O7DW7k=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.075, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "k6Un3ViBqdI=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.112, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "b8bvwMEOR7E=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.043, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "e+7vKBTyle0=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.062, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "HqUbe2ofwe8=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.173, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "TOle7/uQVJw=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.168, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Pq+p8ni6BfY=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.53, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "uI9ReQX6MDc=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.266, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Dp3i567FC7Y=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.312, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "lt/BMQ99wDk=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.362, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "GLCftn0qa5k=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.608, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "xGpDCrebQlI=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.398, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "iHx6VHOSrlg=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.244, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "lJ4xp8OMr60=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.643, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "YwElqVZxfeE=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 1.178, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "VUr+jQ211S4=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.123, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "yiaG28vdQ0o=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.095, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "INvMu6Dcs7U=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.074, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "5nUCWHmRWD4=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.064, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "pzI2Tbuy7Qs=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.2, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "tupwPWFePe8=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.206, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "KD4IIvqufFI=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.099, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "SELECT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "K+7+L6qJcuM=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.099, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "yXe79B5ddGw=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.286, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "X5vQNrLEMnE=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.068, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "swuJPCl1Z0Q=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.066, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "UCCn71xKKdI=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.054, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "pPUmGj6KCQE=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.054, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "pCLMMPCg9q0=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.116, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "16J3ACLeVDA=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.182, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 get_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_ERROR", + "span_id": "n3qnniKhyYs=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 2643.324, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 get_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "lrWmvDODn8Y=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 2.461, + "attempt": null + }, + { + "service": "messenger-relay", + "name": "attachments.attachment.process.v1 create", + "kind": "SPAN_KIND_PRODUCER", + "status": "STATUS_CODE_UNSET", + "span_id": "nYWo9sEJuO4=", + "parent_span_id": "kEXksGW8DXE=", + "duration_ms": 0.01, + "attempt": null + }, + { + "service": "messenger-relay", + "name": "attachments.attachment.process.v1 publish", + "kind": "SPAN_KIND_PRODUCER", + "status": "STATUS_CODE_UNSET", + "span_id": "91CmUd9RVok=", + "parent_span_id": "nYWo9sEJuO4=", + "duration_ms": 6.188, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "attachments.attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "status": "STATUS_CODE_UNSET", + "span_id": "5FdSinM8g/I=", + "parent_span_id": "nYWo9sEJuO4=", + "duration_ms": 740.915, + "attempt": "2" + }, + { + "service": "messenger-worker", + "name": "S3 copy_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "qpWqWo1+vJw=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 5.771, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "xVnbGqSwIyM=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 3.068, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "xlsNqUduAJg=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 2.116, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "S3 put_object", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "b1yVsz3MXXU=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 2.15, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "BQ9ygc2ijJ4=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.118, + "attempt": null + }, + { + "service": "messenger-worker", + "name": ";", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "HDUv+6emOrs=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.056, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "ROLLBACK;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "AbMWZAykxew=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.047, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "BEGIN;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "ajiQzRZGl40=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.063, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "Gutvv1G6/W0=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.18, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "LH3hKhCnOfU=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.311, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "UPDATE", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "7Qrp8PCcAbU=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.143, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "INSERT", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "1nP4RrrfEvE=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.217, + "attempt": null + }, + { + "service": "messenger-worker", + "name": "COMMIT;", + "kind": "SPAN_KIND_CLIENT", + "status": "STATUS_CODE_UNSET", + "span_id": "fz28vhFBmpM=", + "parent_span_id": "5FdSinM8g/I=", + "duration_ms": 0.26, + "attempt": null + } + ] + } +} +exit_code: 0 elapsed_seconds: 13.411 diff --git a/evidence/grade4/retry.txt b/evidence/grade4/retry.txt new file mode 100644 index 0000000..4775781 --- /dev/null +++ b/evidence/grade4/retry.txt @@ -0,0 +1,17 @@ +date: 2026-11-06T09:59:26Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py retry +Traceback (most recent call last): + File "/scripts/obs_incidents.py", line 349, in + main() + ~~~~^^ + File "/scripts/obs_incidents.py", line 330, in main + chain(b, args.scenario == "retry") + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scripts/obs_incidents.py", line 74, in chain + wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60) + ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scripts/obs_check.py", line 72, in wait_for + raise Failure(f"timeout waiting for {what}") +obs_check.Failure: timeout waiting for real S3 retry +exit_code: 1 elapsed_seconds: 68.320 diff --git a/evidence/grade4/smoke.txt b/evidence/grade4/smoke.txt new file mode 100644 index 0000000..802a92f --- /dev/null +++ b/evidence/grade4/smoke.txt @@ -0,0 +1,6 @@ +date: 2026-11-06T10:04:39Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make test +python3 tests/smoke.py --base-url "http://127.0.0.1:18086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +exit_code: 0 elapsed_seconds: 1.991 diff --git a/evidence/grade4/storage-incident.txt b/evidence/grade4/storage-incident.txt new file mode 100644 index 0000000..c72ef4c --- /dev/null +++ b/evidence/grade4/storage-incident.txt @@ -0,0 +1,279 @@ +date: 2026-11-06T18:14:15Z commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py storage +{ + "started": "2026-11-06T18:14:15+00:00", + "finished": "2026-11-06T18:16:39+00:00", + "result": "PASS", + "scenario": "storage", + "observations": { + "stopped": "2026-11-06T18:14:29+00:00", + "restored": "2026-11-06T18:16:16+00:00", + "peak_memory_percent_of_limit": 98.7, + "smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:18086\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.\n", + "successful_probes": 74, + "stored_probe_traces": 72, + "lost_probe_traces": 2, + "memory_samples": [ + { + "utc": "2026-11-06T18:14:16+00:00", + "containers": [ + { + "Name": "mtusi2026lab6-api-1", + "MemUsage": "246.6MiB / 512MiB", + "MemPerc": "48.16%", + "CPUPerc": "1.94%" + }, + { + "Name": "mtusi2026lab6-api-2", + "MemUsage": "241.7MiB / 512MiB", + "MemPerc": "47.20%", + "CPUPerc": "1.12%" + }, + { + "Name": "mtusi2026lab6-otel-collector-1", + "MemUsage": "246.2MiB / 256MiB", + "MemPerc": "96.18%", + "CPUPerc": "0.63%" + }, + { + "Name": "mtusi2026lab6-relay-1", + "MemUsage": "119.4MiB / 192MiB", + "MemPerc": "62.18%", + "CPUPerc": "0.38%" + }, + { + "Name": "mtusi2026lab6-worker-1", + "MemUsage": "121.3MiB / 1GiB", + "MemPerc": "11.84%", + "CPUPerc": "0.10%" + } + ] + }, + { + "utc": "2026-11-06T18:14:45+00:00", + "containers": [ + { + "Name": "mtusi2026lab6-api-1", + "MemUsage": "259.7MiB / 512MiB", + "MemPerc": "50.72%", + "CPUPerc": "1.59%" + }, + { + "Name": "mtusi2026lab6-api-2", + "MemUsage": "249.4MiB / 512MiB", + "MemPerc": "48.71%", + "CPUPerc": "1.56%" + }, + { + "Name": "mtusi2026lab6-otel-collector-1", + "MemUsage": "248.5MiB / 256MiB", + "MemPerc": "97.08%", + "CPUPerc": "0.30%" + }, + { + "Name": "mtusi2026lab6-relay-1", + "MemUsage": "120.2MiB / 192MiB", + "MemPerc": "62.59%", + "CPUPerc": "0.48%" + }, + { + "Name": "mtusi2026lab6-worker-1", + "MemUsage": "253.4MiB / 1GiB", + "MemPerc": "24.75%", + "CPUPerc": "0.35%" + } + ] + }, + { + "utc": "2026-11-06T18:15:03+00:00", + "containers": [ + { + "Name": "mtusi2026lab6-api-1", + "MemUsage": "259.5MiB / 512MiB", + "MemPerc": "50.69%", + "CPUPerc": "1.10%" + }, + { + "Name": "mtusi2026lab6-api-2", + "MemUsage": "249.4MiB / 512MiB", + "MemPerc": "48.71%", + "CPUPerc": "1.19%" + }, + { + "Name": "mtusi2026lab6-otel-collector-1", + "MemUsage": "250.5MiB / 256MiB", + "MemPerc": "97.87%", + "CPUPerc": "0.20%" + }, + { + "Name": "mtusi2026lab6-relay-1", + "MemUsage": "119.9MiB / 192MiB", + "MemPerc": "62.46%", + "CPUPerc": "0.18%" + }, + { + "Name": "mtusi2026lab6-worker-1", + "MemUsage": "253.5MiB / 1GiB", + "MemPerc": "24.75%", + "CPUPerc": "2.07%" + } + ] + }, + { + "utc": "2026-11-06T18:15:21+00:00", + "containers": [ + { + "Name": "mtusi2026lab6-api-1", + "MemUsage": "259.6MiB / 512MiB", + "MemPerc": "50.70%", + "CPUPerc": "2.63%" + }, + { + "Name": "mtusi2026lab6-api-2", + "MemUsage": "249.5MiB / 512MiB", + "MemPerc": "48.73%", + "CPUPerc": "1.35%" + }, + { + "Name": "mtusi2026lab6-otel-collector-1", + "MemUsage": "250.6MiB / 256MiB", + "MemPerc": "97.89%", + "CPUPerc": "1.18%" + }, + { + "Name": "mtusi2026lab6-relay-1", + "MemUsage": "119.9MiB / 192MiB", + "MemPerc": "62.47%", + "CPUPerc": "1.77%" + }, + { + "Name": "mtusi2026lab6-worker-1", + "MemUsage": "253.4MiB / 1GiB", + "MemPerc": "24.75%", + "CPUPerc": "0.75%" + } + ] + }, + { + "utc": "2026-11-06T18:15:39+00:00", + "containers": [ + { + "Name": "mtusi2026lab6-api-1", + "MemUsage": "259.6MiB / 512MiB", + "MemPerc": "50.71%", + "CPUPerc": "0.50%" + }, + { + "Name": "mtusi2026lab6-api-2", + "MemUsage": "249.1MiB / 512MiB", + "MemPerc": "48.66%", + "CPUPerc": "0.47%" + }, + { + "Name": "mtusi2026lab6-otel-collector-1", + "MemUsage": "250.6MiB / 256MiB", + "MemPerc": "97.88%", + "CPUPerc": "0.07%" + }, + { + "Name": "mtusi2026lab6-relay-1", + "MemUsage": "120MiB / 192MiB", + "MemPerc": "62.48%", + "CPUPerc": "0.18%" + }, + { + "Name": "mtusi2026lab6-worker-1", + "MemUsage": "253.4MiB / 1GiB", + "MemPerc": "24.75%", + "CPUPerc": "0.20%" + } + ] + }, + { + "utc": "2026-11-06T18:16:36+00:00", + "containers": [ + { + "Name": "mtusi2026lab6-api-1", + "MemUsage": "259.6MiB / 512MiB", + "MemPerc": "50.70%", + "CPUPerc": "1.35%" + }, + { + "Name": "mtusi2026lab6-api-2", + "MemUsage": "249.1MiB / 512MiB", + "MemPerc": "48.65%", + "CPUPerc": "1.59%" + }, + { + "Name": "mtusi2026lab6-otel-collector-1", + "MemUsage": "252.7MiB / 256MiB", + "MemPerc": "98.70%", + "CPUPerc": "0.23%" + }, + { + "Name": "mtusi2026lab6-relay-1", + "MemUsage": "120MiB / 192MiB", + "MemPerc": "62.49%", + "CPUPerc": "0.77%" + }, + { + "Name": "mtusi2026lab6-worker-1", + "MemUsage": "253.4MiB / 1GiB", + "MemPerc": "24.75%", + "CPUPerc": "0.29%" + } + ] + } + ], + "collector_before": { + "otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0, + "otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0, + "otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0, + "otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 0.0, + "otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 0.0, + "otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 0.0, + "otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 807.0, + "otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 656.0, + "otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 784.0, + "otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 810.0, + "otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 656.0, + "otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 784.0 + }, + "collector_during": { + "otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0, + "otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0, + "otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0, + "otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 381.0, + "otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 1287.0, + "otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 465.0, + "otelcol_exporter_send_failed_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 11.0, + "otelcol_exporter_send_failed_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 72.0, + "otelcol_exporter_send_failed_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 440.0, + "otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 833.0, + "otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 656.0, + "otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 807.0, + "otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 1228.0, + "otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 2015.0, + "otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 1712.0 + }, + "collector_after": { + "otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0, + "otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0, + "otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0, + "otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 0.0, + "otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 0.0, + "otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 0.0, + "otelcol_exporter_send_failed_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 11.0, + "otelcol_exporter_send_failed_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 72.0, + "otelcol_exporter_send_failed_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 440.0, + "otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 1320.0, + "otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 2515.0, + "otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 1293.0, + "otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 1335.0, + "otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 2587.0, + "otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 1733.0 + } + } +} +exit_code: 0 elapsed_seconds: 143.965 diff --git a/evidence/grade4/student-tests-final.txt b/evidence/grade4/student-tests-final.txt new file mode 100644 index 0000000..3a074f8 --- /dev/null +++ b/evidence/grade4/student-tests-final.txt @@ -0,0 +1,63 @@ +date: 2026-11-06T09:58:18Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make student-test +.venv/bin/pytest +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collected 536 items / 19 deselected / 517 selected + +student-tests/adapters/test_rabbitmq.py .... [ 0%] +student-tests/adapters/test_s3_storage.py ... [ 1%] +student-tests/api/test_access.py ......... [ 3%] +student-tests/api/test_attachments.py .................................. [ 9%] +.... [ 10%] +student-tests/api/test_bounded_contexts.py .................... [ 14%] +student-tests/api/test_cache.py ...... [ 15%] +student-tests/api/test_concurrency.py .... [ 16%] +student-tests/api/test_consistency.py ........ [ 17%] +student-tests/api/test_drain.py .. [ 18%] +student-tests/api/test_events.py ...... [ 19%] +student-tests/api/test_http_telemetry.py . [ 19%] +student-tests/api/test_idempotency.py ............... [ 22%] +student-tests/api/test_instance.py .... [ 23%] +student-tests/api/test_job_returning.py . [ 23%] +student-tests/api/test_message_attachments.py ............. [ 25%] +student-tests/api/test_migrations.py ................................... [ 32%] +. [ 32%] +student-tests/api/test_negative.py ..................................... [ 40%] + [ 40%] +student-tests/api/test_operability.py .............. [ 42%] +student-tests/api/test_pagination.py .................. [ 46%] +student-tests/api/test_pipeline.py ............. [ 48%] +student-tests/api/test_pool.py ............ [ 51%] +student-tests/api/test_reconciler.py ........... [ 53%] +student-tests/api/test_recovery.py ............ [ 55%] +student-tests/api/test_sessions.py .................. [ 58%] +student-tests/api/test_timeout.py ........ [ 60%] +student-tests/api/test_trace_context.py . [ 60%] +student-tests/api/test_upload_consistency.py ......... [ 62%] +student-tests/api/test_versioned_pipeline.py ........... [ 64%] +student-tests/api/test_wire_format.py .. [ 64%] +student-tests/unit/test_attachments_domain.py .......................... [ 70%] +. [ 70%] +student-tests/unit/test_authentication.py ..................... [ 74%] +student-tests/unit/test_cursor.py ........... [ 76%] +student-tests/unit/test_domain.py ...................................... [ 83%] +.. [ 84%] +student-tests/unit/test_job_observer.py . [ 84%] +student-tests/unit/test_pipeline_domain.py ....................... [ 88%] +student-tests/unit/test_probe.py .............. [ 91%] +student-tests/unit/test_reconcile_measurements.py ...... [ 92%] +student-tests/unit/test_render_pool.py .....s. [ 94%] +student-tests/unit/test_renderer.py .................... [ 97%] +student-tests/unit/test_scaling.py ..... [ 98%] +student-tests/unit/test_telemetry_platform.py ..... [ 99%] +student-tests/unit/test_timeouts.py . [100%] + +========== 516 passed, 1 skipped, 19 deselected in 152.10s (0:02:32) =========== +exit_code: 0 elapsed_seconds: 154.509 diff --git a/evidence/grade4/student-tests-first.txt b/evidence/grade4/student-tests-first.txt new file mode 100644 index 0000000..dd0ccbc --- /dev/null +++ b/evidence/grade4/student-tests-first.txt @@ -0,0 +1,243 @@ +date: 2026-11-06T09:52:20Z commit: 1a171d0d0f13ab71661a0f63e695031addd88dc0 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make student-test +.venv/bin/pytest +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collected 536 items / 19 deselected / 517 selected + +student-tests/adapters/test_rabbitmq.py .... [ 0%] +student-tests/adapters/test_s3_storage.py ... [ 1%] +student-tests/api/test_access.py ......... [ 3%] +student-tests/api/test_attachments.py .................................. [ 9%] +.... [ 10%] +student-tests/api/test_bounded_contexts.py ......F............. [ 14%] +student-tests/api/test_cache.py ...... [ 15%] +student-tests/api/test_concurrency.py .... [ 16%] +student-tests/api/test_consistency.py ........ [ 17%] +student-tests/api/test_drain.py .. [ 18%] +student-tests/api/test_events.py ...... [ 19%] +student-tests/api/test_http_telemetry.py . [ 19%] +student-tests/api/test_idempotency.py ............... [ 22%] +student-tests/api/test_instance.py .... [ 23%] +student-tests/api/test_job_returning.py . [ 23%] +student-tests/api/test_message_attachments.py ............. [ 25%] +student-tests/api/test_migrations.py ................................... [ 32%] +. [ 32%] +student-tests/api/test_negative.py ..................................... [ 40%] + [ 40%] +student-tests/api/test_operability.py .............. [ 42%] +student-tests/api/test_pagination.py .................. [ 46%] +student-tests/api/test_pipeline.py ............. [ 48%] +student-tests/api/test_pool.py ..F......... [ 51%] +student-tests/api/test_reconciler.py .........F. [ 53%] +student-tests/api/test_recovery.py ............ [ 55%] +student-tests/api/test_sessions.py .................. [ 58%] +student-tests/api/test_timeout.py ........ [ 60%] +student-tests/api/test_trace_context.py . [ 60%] +student-tests/api/test_upload_consistency.py ......... [ 62%] +student-tests/api/test_versioned_pipeline.py ........... [ 64%] +student-tests/api/test_wire_format.py .. [ 64%] +student-tests/unit/test_attachments_domain.py .......................... [ 70%] +. [ 70%] +student-tests/unit/test_authentication.py ..................... [ 74%] +student-tests/unit/test_cursor.py ........... [ 76%] +student-tests/unit/test_domain.py ...................................... [ 83%] +.. [ 84%] +student-tests/unit/test_job_observer.py . [ 84%] +student-tests/unit/test_pipeline_domain.py ....................... [ 88%] +student-tests/unit/test_probe.py .............. [ 91%] +student-tests/unit/test_reconcile_measurements.py ...... [ 92%] +student-tests/unit/test_render_pool.py .....s. [ 94%] +student-tests/unit/test_renderer.py .................... [ 97%] +student-tests/unit/test_scaling.py ..... [ 98%] +student-tests/unit/test_telemetry_platform.py ..... [ 99%] +student-tests/unit/test_timeouts.py . [100%] + +=================================== FAILURES =================================== +_____________________ test_shared_core_holds_only_platform _____________________ + + def test_shared_core_holds_only_platform() -> None: + names = { + path.name + for layer in ("domain", "application", "infrastructure") + for path in modules_under("shared", layer) + } +> assert names == { + "errors.py", + "events.py", + "commands.py", + "messagebus.py", + "ports.py", + "clock.py", + "ids.py", + "logging.py", + "postgres.py", + "telemetry.py", + "readiness.py", + } +E AssertionError: assert {'clock.py', ...ging.py', ...} == {'clock.py', ...ging.py', ...} +E +E Extra items in the left set: +E 'trace_context.py' +E 'runtime_metrics.py' +E Use -v to get more diff + +student-tests/api/test_bounded_contexts.py:104: AssertionError +________________ test_idle_session_does_not_acquire_connection _________________ + +self = + + @asynccontextmanager + async def session(self) -> AsyncIterator[DatabaseSession]: + try: + async with self._sessions() as session: + self._waiting += 1 + try: +> await session.connection() + +src/messenger/shared/infrastructure/postgres.py:106: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.14/site-packages/sqlalchemy/ext/asyncio/session.py:930: in connection + sync_connection = await greenlet_spawn( +.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:201: in greenlet_spawn + result = context.throw(*sys.exc_info()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:2092: in connection + return self._connection_for_bind( +.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:2108: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:1187: in _connection_for_bind + conn = bind.connect() + ^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:3285: in connect + return self._connection_cls(self) + ^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:143: in __init__ + self._dbapi_connection = engine.raw_connection() + ^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:3309: in raw_connection + return self.pool.connect() + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py:447: in connect + return _ConnectionFairy._checkout(self) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py:1264: in _checkout + fairy = _ConnectionRecord.checkout(pool) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py:711: in checkout + rec = pool._do_get() + ^^^^^^^^^^^^^^ +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = + + def _do_get(self) -> ConnectionPoolEntry: + use_overflow = self._max_overflow > -1 + + wait = use_overflow and self._overflow >= self._max_overflow + try: + return self._pool.get(wait, self._timeout) + except sqla_queue.Empty: + # don't do things inside of "except Empty", because when we say + # we timed out or can't connect and raise, Python 3 tells + # people the real error is queue.Empty which it isn't. + pass + if use_overflow and self._overflow >= self._max_overflow: + if not wait: + return self._do_get() + else: +> raise exc.TimeoutError( + "QueuePool limit of size %d overflow %d reached, " + "connection timed out, timeout %0.2f" + % (self.size(), self.overflow(), self._timeout), + code="3o7r", + ) +E sqlalchemy.exc.TimeoutError: QueuePool limit of size 1 overflow 0 reached, connection timed out, timeout 1.00 (Background on this error at: https://sqlalche.me/e/20/3o7r) + +.venv/lib/python3.14/site-packages/sqlalchemy/pool/impl.py:166: TimeoutError + +The above exception was the direct cause of the following exception: + + async def test_idle_session_does_not_acquire_connection() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) +> async with database.session(), database.session() as active: + ^^^^^^^^^^^^^^^^^^ + +student-tests/api/test_pool.py:74: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/contextlib.py:214: in __aenter__ + return await anext(self.gen) + ^^^^^^^^^^^^^^^^^^^^^ +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = + + @asynccontextmanager + async def session(self) -> AsyncIterator[DatabaseSession]: + try: + async with self._sessions() as session: + self._waiting += 1 + try: + await session.connection() + finally: + self._waiting -= 1 + yield session + except Exception as error: + if not unavailable(error): + raise +> raise Unavailable("PostgreSQL недоступен") from error +E messenger.shared.application.errors.Unavailable: PostgreSQL недоступен + +src/messenger/shared/infrastructure/postgres.py:113: Unavailable +_______________ test_cancelled_pass_releases_its_autocommit_lock _______________ + +dsn = 'postgresql://postgres:test@127.0.0.1:26002/reconcile_07e983d849c3' + + async def test_cancelled_pass_releases_its_autocommit_lock(dsn: str) -> None: + async with stand(dsn) as s: + entered = asyncio.Event() + + async def hold() -> None: + async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held: + assert held + entered.set() + await asyncio.Event().wait() + + task = asyncio.create_task(hold()) + try: + await asyncio.wait_for(entered.wait(), 2) + states = await s.sql( + "SELECT a.state FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid " + "WHERE l.locktype = 'advisory' AND l.objid = $1", + LOCK_KEY, + ) +> assert [row["state"] for row in states] == ["idle"] +E AssertionError: assert ['idle in transaction'] == ['idle'] +E +E At index 0 diff: 'idle in transaction' != 'idle' +E Use -v to get more diff + +student-tests/api/test_reconciler.py:403: AssertionError +=============================== warnings summary =============================== +student-tests/api/test_reconciler.py: 11 warnings + :0: SAWarning: Connection is already established for the given bind; execution_options ignored + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +=========================== short test summary info ============================ +FAILED student-tests/api/test_bounded_contexts.py::test_shared_core_holds_only_platform +FAILED student-tests/api/test_pool.py::test_idle_session_does_not_acquire_connection +FAILED student-tests/api/test_reconciler.py::test_cancelled_pass_releases_its_autocommit_lock += 3 failed, 513 passed, 1 skipped, 19 deselected, 11 warnings in 156.67s (0:02:36) = +make: *** [student-test] Error 1 +exit_code: 2 elapsed_seconds: 159.760 diff --git a/evidence/grade4/student-tests.txt b/evidence/grade4/student-tests.txt new file mode 100644 index 0000000..48a62c7 --- /dev/null +++ b/evidence/grade4/student-tests.txt @@ -0,0 +1,91 @@ +date: 2026-11-06T09:55:35Z commit: 1a171d0d0f13ab71661a0f63e695031addd88dc0 +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make student-test +.venv/bin/pytest +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collected 536 items / 19 deselected / 517 selected + +student-tests/adapters/test_rabbitmq.py .... [ 0%] +student-tests/adapters/test_s3_storage.py ... [ 1%] +student-tests/api/test_access.py ......... [ 3%] +student-tests/api/test_attachments.py .................................. [ 9%] +.... [ 10%] +student-tests/api/test_bounded_contexts.py .......F............ [ 14%] +student-tests/api/test_cache.py ...... [ 15%] +student-tests/api/test_concurrency.py .... [ 16%] +student-tests/api/test_consistency.py ........ [ 17%] +student-tests/api/test_drain.py .. [ 18%] +student-tests/api/test_events.py ...... [ 19%] +student-tests/api/test_http_telemetry.py . [ 19%] +student-tests/api/test_idempotency.py ............... [ 22%] +student-tests/api/test_instance.py .... [ 23%] +student-tests/api/test_job_returning.py . [ 23%] +student-tests/api/test_message_attachments.py ............. [ 25%] +student-tests/api/test_migrations.py ................................... [ 32%] +. [ 32%] +student-tests/api/test_negative.py ..................................... [ 40%] + [ 40%] +student-tests/api/test_operability.py .............. [ 42%] +student-tests/api/test_pagination.py .................. [ 46%] +student-tests/api/test_pipeline.py ............. [ 48%] +student-tests/api/test_pool.py ............ [ 51%] +student-tests/api/test_reconciler.py ........... [ 53%] +student-tests/api/test_recovery.py ............ [ 55%] +student-tests/api/test_sessions.py .................. [ 58%] +student-tests/api/test_timeout.py ........ [ 60%] +student-tests/api/test_trace_context.py . [ 60%] +student-tests/api/test_upload_consistency.py ......... [ 62%] +student-tests/api/test_versioned_pipeline.py ........... [ 64%] +student-tests/api/test_wire_format.py .. [ 64%] +student-tests/unit/test_attachments_domain.py .......................... [ 70%] +. [ 70%] +student-tests/unit/test_authentication.py ..................... [ 74%] +student-tests/unit/test_cursor.py ........... [ 76%] +student-tests/unit/test_domain.py ...................................... [ 83%] +.. [ 84%] +student-tests/unit/test_job_observer.py . [ 84%] +student-tests/unit/test_pipeline_domain.py ....................... [ 88%] +student-tests/unit/test_probe.py .............. [ 91%] +student-tests/unit/test_reconcile_measurements.py ...... [ 92%] +student-tests/unit/test_render_pool.py .....s. [ 94%] +student-tests/unit/test_renderer.py .................... [ 97%] +student-tests/unit/test_scaling.py ..... [ 98%] +student-tests/unit/test_telemetry_platform.py ..... [ 99%] +student-tests/unit/test_timeouts.py . [100%] + +=================================== FAILURES =================================== +_________________ test_shared_entrypoints_hold_only_transport __________________ + + def test_shared_entrypoints_hold_only_transport() -> None: + names = {path.name for path in modules_under("shared", "entrypoints")} +> assert names == { + "actor.py", + "config.py", + "errors.py", + "health.py", + "ioc.py", + "pipeline.py", + "schemas.py", + "telemetry.py", + "trace_context.py", + "runtime_metrics.py", + } +E AssertionError: assert {'actor.py', ...line.py', ...} == {'actor.py', ...line.py', ...} +E +E Extra items in the right set: +E 'runtime_metrics.py' +E 'trace_context.py' +E Use -v to get more diff + +student-tests/api/test_bounded_contexts.py:123: AssertionError +=========================== short test summary info ============================ +FAILED student-tests/api/test_bounded_contexts.py::test_shared_entrypoints_hold_only_transport +===== 1 failed, 515 passed, 1 skipped, 19 deselected in 155.56s (0:02:35) ====== +make: *** [student-test] Error 1 +exit_code: 2 elapsed_seconds: 157.936 diff --git a/evidence/grade4/up.txt b/evidence/grade4/up.txt new file mode 100644 index 0000000..d47e25c --- /dev/null +++ b/evidence/grade4/up.txt @@ -0,0 +1,230 @@ +date: 2026-11-06T09:57:10Z commit: 44d81fd80ee13faa42bca069245ee0943f88b2df +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ make up +sh scripts/secrets.sh +sh scripts/broker-secrets.sh +broker and S3 worker secrets ready in secrets/ (values not printed) +sh scripts/secrets-lab6.sh +secrets/grafana_admin_password is ready (value not printed) +docker compose build +#1 [internal] load local bake definitions +#1 reading from stdin 432B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.65kB done +#2 DONE 0.0s + +#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#3 DONE 0.0s + +#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#4 DONE 1.4s + +#5 [internal] load .dockerignore +#5 transferring context: 453B done +#5 DONE 0.0s + +#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done +#6 DONE 0.0s + +#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 305.62kB 0.0s done +#8 DONE 0.0s + +#9 [builder 4/7] COPY pyproject.toml uv.lock README.md ./ +#9 CACHED + +#10 [builder 3/7] WORKDIR /build +#10 CACHED + +#11 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv +#11 CACHED + +#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable +#12 CACHED + +#13 [builder 6/7] COPY src ./src +#13 DONE 0.1s + +#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable +#14 0.231 Resolved 100 packages in 11ms +#14 0.248 Building mtusi-messenger @ file:///build +#14 1.818 Built mtusi-messenger @ file:///build +#14 1.824 Prepared 1 package in 1.57s +#14 1.828 Installed 1 package in 3ms +#14 2.068 Bytecode compiled 2805 files in 240ms +#14 2.068 + mtusi-messenger==1.6.0 (from file:///build) +#14 DONE 2.1s + +#15 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger +#15 CACHED + +#16 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv +#16 DONE 0.5s + +#17 [runtime 4/6] WORKDIR /app +#17 DONE 0.1s + +#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./ +#18 DONE 0.0s + +#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic +#19 DONE 0.0s + +#20 exporting to image +#20 exporting layers +#20 exporting layers 3.4s done +#20 exporting manifest sha256:e611ff8072cef7f330ca373cc74464be3c219851d9c7210c932b0f9edcd4f113 +#20 exporting manifest sha256:e611ff8072cef7f330ca373cc74464be3c219851d9c7210c932b0f9edcd4f113 done +#20 exporting config sha256:ebe6a2493a7d49a508b76b70e0895ff95bd41c91db5c1953e2fc80d95bc8835e done +#20 exporting attestation manifest sha256:ee30177f12179366bd66d631272804ab91a4b8c87a28b1aad8a59424e1524daf done +#20 exporting manifest list sha256:e1701e04d5f6dcaf33b07246843b703fc3e506f5d65e0cd4d2d7603105e1c4a2 done +#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab6 done +#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 +#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab6 0.7s done +#20 DONE 4.1s + +#21 resolving provenance for metadata file +#21 DONE 0.0s + api Built +docker compose up -d --wait + Network mtusi2026lab6_edge Creating + Network mtusi2026lab6_edge Created + Network mtusi2026lab6_backend Creating + Network mtusi2026lab6_backend Created + Container mtusi2026lab6-broker-1 Creating + Container mtusi2026lab6-lb-1 Creating + Container mtusi2026lab6-postgres-1 Creating + Container mtusi2026lab6-prometheus-1 Creating + Container mtusi2026lab6-healthtools-1 Creating + Container mtusi2026lab6-valkey-1 Creating + Container mtusi2026lab6-s3-1 Creating + Container mtusi2026lab6-valkey-1 Created + Container mtusi2026lab6-s3-1 Created + Container mtusi2026lab6-s3-init-1 Creating + Container mtusi2026lab6-healthtools-1 Created + Container mtusi2026lab6-loki-1 Creating + Container mtusi2026lab6-tempo-1 Creating + Container mtusi2026lab6-otel-collector-1 Creating + Container mtusi2026lab6-prometheus-1 Created + Container mtusi2026lab6-postgres-1 Created + Container mtusi2026lab6-migrate-1 Creating + Container mtusi2026lab6-lb-1 Created + Container mtusi2026lab6-broker-1 Created + Container mtusi2026lab6-s3-init-1 Created + Container mtusi2026lab6-otel-collector-1 Created + Container mtusi2026lab6-loki-1 Created + Container mtusi2026lab6-tempo-1 Created + Container mtusi2026lab6-grafana-1 Creating + Container mtusi2026lab6-migrate-1 Created + Container mtusi2026lab6-worker-1 Creating + Container mtusi2026lab6-api-1 Creating + Container mtusi2026lab6-relay-1 Creating + Container mtusi2026lab6-api-2 Creating + Container mtusi2026lab6-worker-1 Created + Container mtusi2026lab6-api-2 Created + Container mtusi2026lab6-grafana-1 Created + Container mtusi2026lab6-api-1 Created + Container mtusi2026lab6-relay-1 Created + Container mtusi2026lab6-broker-1 Starting + Container mtusi2026lab6-postgres-1 Starting + Container mtusi2026lab6-valkey-1 Starting + Container mtusi2026lab6-s3-1 Starting + Container mtusi2026lab6-prometheus-1 Starting + Container mtusi2026lab6-lb-1 Starting + Container mtusi2026lab6-healthtools-1 Starting + Container mtusi2026lab6-healthtools-1 Started + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-valkey-1 Started + Container mtusi2026lab6-s3-1 Started + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-postgres-1 Started + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-lb-1 Started + Container mtusi2026lab6-prometheus-1 Started + Container mtusi2026lab6-broker-1 Started + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-otel-collector-1 Starting + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-loki-1 Starting + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-tempo-1 Starting + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-tempo-1 Started + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-otel-collector-1 Started + Container mtusi2026lab6-loki-1 Started + Container mtusi2026lab6-grafana-1 Starting + Container mtusi2026lab6-grafana-1 Started + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-api-1 Starting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-relay-1 Starting + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-worker-1 Starting + Container mtusi2026lab6-api-1 Started + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-worker-1 Started + Container mtusi2026lab6-relay-1 Started + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-relay-1 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-worker-1 Waiting + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-healthtools-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-valkey-1 Waiting + Container mtusi2026lab6-lb-1 Waiting + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-loki-1 Waiting + Container mtusi2026lab6-tempo-1 Waiting + Container mtusi2026lab6-otel-collector-1 Waiting + Container mtusi2026lab6-broker-1 Waiting + Container mtusi2026lab6-prometheus-1 Waiting + Container mtusi2026lab6-grafana-1 Waiting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-prometheus-1 Healthy + Container mtusi2026lab6-grafana-1 Healthy + Container mtusi2026lab6-valkey-1 Healthy + Container mtusi2026lab6-otel-collector-1 Healthy + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-healthtools-1 Exited + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-broker-1 Healthy + Container mtusi2026lab6-worker-1 Healthy + Container mtusi2026lab6-relay-1 Healthy + Container mtusi2026lab6-api-1 Healthy + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-lb-1 Healthy + Container mtusi2026lab6-loki-1 Healthy + Container mtusi2026lab6-tempo-1 Healthy +scripts/lb-wait.sh +exit_code: 0 elapsed_seconds: 85.736 diff --git a/evidence/grade4/worker-incident.txt b/evidence/grade4/worker-incident.txt new file mode 100644 index 0000000..cf94cc0 --- /dev/null +++ b/evidence/grade4/worker-incident.txt @@ -0,0 +1,45 @@ +date: 2026-11-06T11:37:50Z commit: 447a6507f973596e612f40c1484dda922bf0e50d +environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"} +$ python3 scripts/obs_incidents.py worker +{ + "started": "2026-11-06T11:37:50+00:00", + "finished": "2026-11-06T11:38:44+00:00", + "result": "PASS", + "scenario": "worker", + "observations": { + "stopped": "2026-11-06T11:37:51+00:00", + "restored": "2026-11-06T11:38:32+00:00", + "jobs": 8, + "backlog": { + "depth": 8.0, + "age_seconds": 30.558317 + }, + "firing": { + "utc": "2026-11-06T11:38:29+00:00", + "state": "firing", + "alerts": [ + { + "labels": { + "alertname": "MessengerQueueLag", + "severity": "warning" + }, + "annotations": { + "description": "Target is 95% ready within 30 seconds over 1 hour; depth includes retry and leased work.", + "runbook": "infra/prometheus/runbooks.md#messengerqueuelag", + "summary": "Durable attachment backlog has waited more than 15 seconds" + }, + "state": "firing", + "activeAt": "2026-11-06T11:38:13.389340978Z", + "value": "3.0558317e+01" + } + ] + }, + "resolved": { + "utc": "2026-11-06T11:38:44+00:00", + "state": "resolved", + "alerts": [] + }, + "depth_after": 0.0 + } +} +exit_code: 0 elapsed_seconds: 53.624 diff --git a/openspec/changes/complete-lab6-observability-resilience/.openspec.yaml b/openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/.openspec.yaml similarity index 100% rename from openspec/changes/complete-lab6-observability-resilience/.openspec.yaml rename to openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/.openspec.yaml diff --git a/openspec/changes/complete-lab6-observability-resilience/design.md b/openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/design.md similarity index 100% rename from openspec/changes/complete-lab6-observability-resilience/design.md rename to openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/design.md diff --git a/openspec/changes/complete-lab6-observability-resilience/proposal.md b/openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/proposal.md similarity index 100% rename from openspec/changes/complete-lab6-observability-resilience/proposal.md rename to openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/proposal.md diff --git a/openspec/changes/complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md b/openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md similarity index 100% rename from openspec/changes/complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md rename to openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md diff --git a/openspec/changes/complete-lab6-observability-resilience/tasks.md b/openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/tasks.md similarity index 63% rename from openspec/changes/complete-lab6-observability-resilience/tasks.md rename to openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/tasks.md index 391d8bf..c6e2209 100644 --- a/openspec/changes/complete-lab6-observability-resilience/tasks.md +++ b/openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/tasks.md @@ -1,9 +1,9 @@ ## Implementation -- [ ] 1. Durable context migration, repository and publisher wiring; propagation/retry tests -- [ ] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks -- [ ] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation +- [x] 1. Durable context migration, repository and publisher wiring; propagation/retry tests +- [x] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks +- [x] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation ## Verification -- [ ] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes +- [x] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes - [ ] 5. End-to-end and retry traces, dashboards, leaks and cardinality - [ ] 6. Worker and database incidents: evidence, firing/resolved and diagnosis - [ ] 7. Storage outage: availability, bounded memory, lost signals and recovery diff --git a/openspec/specs/lab6-observability-resilience/spec.md b/openspec/specs/lab6-observability-resilience/spec.md new file mode 100644 index 0000000..204a557 --- /dev/null +++ b/openspec/specs/lab6-observability-resilience/spec.md @@ -0,0 +1,30 @@ +# lab6-observability-resilience Specification + +## Purpose +TBD - created by archiving change complete-lab6-observability-resilience. Update Purpose after archive. +## Requirements +### Requirement: Durable distributed context +The system SHALL preserve W3C context through API, transactional outbox, relay producer and worker consumer including retries, and expose safe database and S3 client spans. + +#### Scenario: Deferred delivery and retry +- **WHEN** an accepted upload is delivered after relay restart or retried after S3 failure +- **THEN** its producer and consumer spans share the originating trace and causally valid parents, without adding fields to the normative job payload + +### Requirement: Operational signals and SLO +The system SHALL expose queue depth and oldest age, database pool usage and waiting, worker saturation, process CPU/RAM and durable time-to-ready; dashboards SHALL show histogram percentiles and error ratio and define availability and ready SLO windows. + +#### Scenario: Stopped worker +- **WHEN** queued uploads wait for a stopped worker +- **THEN** depth and age increase, a queue-lag alert fires and resolves after recovery + +#### Scenario: Unavailable database +- **WHEN** PostgreSQL becomes unavailable +- **THEN** API failure metrics and database client spans distinguish the dependency failure and an error-rate alert fires and resolves + +### Requirement: Bounded observability failure +Collector SHALL use memory limits, batching and bounded queues; unavailable telemetry storage SHALL NOT prevent business requests or cause unbounded memory growth. + +#### Scenario: Storage outage +- **WHEN** telemetry storage is stopped during requests +- **THEN** business smoke passes, measured memory remains within container limits, losses are recorded, and exports resume after recovery +