diff --git a/evidence/grade4/rolling-load.json b/evidence/grade4/rolling-load.json
new file mode 100644
index 00000000..2e6941a8
--- /dev/null
+++ b/evidence/grade4/rolling-load.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 50.0,
+ "elapsed_seconds": 50.101,
+ "concurrency": 4,
+ "successful_requests": 8361,
+ "failed_requests": 18,
+ "error_ratio": 0.002148,
+ "successful_rps": 166.884,
+ "successful_latency_ms": {
+ "p50": 13.481,
+ "p95": 57.817,
+ "p99": 96.441
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/grade4/summary.json b/evidence/grade4/summary.json
new file mode 100644
index 00000000..7893768b
--- /dev/null
+++ b/evidence/grade4/summary.json
@@ -0,0 +1,70 @@
+{
+ "instances_before": [
+ "03c3a0545628",
+ "b555f8327eee"
+ ],
+ "parallel_idempotency": {
+ "workers": 16,
+ "successes": 16,
+ "errors": 0,
+ "distinct_message_ids": 1,
+ "history_copies": 1
+ },
+ "failovers": [
+ {
+ "target": "03c3a0545628",
+ "hook_exit": 0,
+ "hook_seconds": 2.373,
+ "hook_stderr": "",
+ "failed_attempts": 2,
+ "successes_after_hook_start": 130,
+ "first_recovery_seconds": 0.034,
+ "tail_instance_ids": [
+ "b555f8327eee",
+ "b555f8327eee",
+ "b555f8327eee",
+ "b555f8327eee",
+ "b555f8327eee"
+ ],
+ "target_still_in_tail": false
+ },
+ {
+ "target": "b555f8327eee",
+ "hook_exit": 0,
+ "hook_seconds": 1.744,
+ "hook_stderr": "",
+ "failed_attempts": 1,
+ "successes_after_hook_start": 132,
+ "first_recovery_seconds": 0.038,
+ "tail_instance_ids": [
+ "03c3a0545628",
+ "03c3a0545628",
+ "03c3a0545628",
+ "03c3a0545628",
+ "03c3a0545628"
+ ],
+ "target_still_in_tail": false
+ }
+ ],
+ "rolling": {
+ "rolling_seconds": 5.929,
+ "rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n",
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 50.0,
+ "elapsed_seconds": 50.101,
+ "concurrency": 4,
+ "successful_requests": 8361,
+ "failed_requests": 18,
+ "error_ratio": 0.002148,
+ "successful_rps": 166.884,
+ "successful_latency_ms": {
+ "p50": 13.481,
+ "p95": 57.817,
+ "p99": 96.441
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ }
+ }
+}
diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt
new file mode 100644
index 00000000..711ae329
--- /dev/null
+++ b/evidence/public-checks.txt
@@ -0,0 +1,29 @@
+Date: 2026-11-07
+Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792
+Engine: Docker 29.5.3, Compose 5.1.4
+Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine
+
+python -X utf8 scripts/check_contract.py
+PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+
+python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
+PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
+
+python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
+15 passed, 3 deselected
+
+Official persistence and failover were executed by Linux python3 in docker:27-cli
+with the host Docker socket and BASE_URL=http://host.docker.internal:8080,
+because Windows CreateProcess cannot run scripts/restart.sh (WinError 193).
+The hooks and the API containers are the host stack.
+
+python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
+PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+
+python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh
+{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465}
+PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+
+After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0),
+hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628.
+Exit code 0 means the process left on SIGTERM before the stop timeout.
diff --git a/evidence/track1/A1.json b/evidence/track1/A1.json
new file mode 100644
index 00000000..81e232fe
--- /dev/null
+++ b/evidence/track1/A1.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.109,
+ "concurrency": 8,
+ "successful_requests": 5197,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 343.964,
+ "successful_latency_ms": {
+ "p50": 19.732,
+ "p95": 42.061,
+ "p99": 81.001
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/A1.point.json b/evidence/track1/A1.point.json
new file mode 100644
index 00000000..3a986495
--- /dev/null
+++ b/evidence/track1/A1.point.json
@@ -0,0 +1,38 @@
+{
+ "tag": "A1",
+ "replicas": 1,
+ "api_cpus_each": "1.0",
+ "api_mem_each": "512m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 8,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.109,
+ "concurrency": 8,
+ "successful_requests": 5197,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 343.964,
+ "successful_latency_ms": {
+ "p50": 19.732,
+ "p95": 42.061,
+ "p99": 81.001
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
+ "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/A2.json b/evidence/track1/A2.json
new file mode 100644
index 00000000..f228ca31
--- /dev/null
+++ b/evidence/track1/A2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.206,
+ "concurrency": 8,
+ "successful_requests": 2985,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 196.3,
+ "successful_latency_ms": {
+ "p50": 26.613,
+ "p95": 89.893,
+ "p99": 123.339
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/A2.point.json b/evidence/track1/A2.point.json
new file mode 100644
index 00000000..bb805c1c
--- /dev/null
+++ b/evidence/track1/A2.point.json
@@ -0,0 +1,39 @@
+{
+ "tag": "A2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.206,
+ "concurrency": 8,
+ "successful_requests": 2985,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 196.3,
+ "successful_latency_ms": {
+ "p50": 26.613,
+ "p95": 89.893,
+ "p99": 123.339
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
+ "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/A4.json b/evidence/track1/A4.json
new file mode 100644
index 00000000..7eefffc6
--- /dev/null
+++ b/evidence/track1/A4.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.224,
+ "concurrency": 8,
+ "successful_requests": 1895,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 124.476,
+ "successful_latency_ms": {
+ "p50": 39.136,
+ "p95": 185.654,
+ "p99": 267.62
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/A4.point.json b/evidence/track1/A4.point.json
new file mode 100644
index 00000000..87e0fcf0
--- /dev/null
+++ b/evidence/track1/A4.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "A4",
+ "replicas": 4,
+ "api_cpus_each": "0.25",
+ "api_mem_each": "128m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.224,
+ "concurrency": 8,
+ "successful_requests": 1895,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 124.476,
+ "successful_latency_ms": {
+ "p50": 39.136,
+ "p95": 185.654,
+ "p99": 267.62
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
+ "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
+ "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
+ "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
+ "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/B1.json b/evidence/track1/B1.json
new file mode 100644
index 00000000..af7b2ad4
--- /dev/null
+++ b/evidence/track1/B1.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.139,
+ "concurrency": 8,
+ "successful_requests": 1485,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 98.094,
+ "successful_latency_ms": {
+ "p50": 85.074,
+ "p95": 167.209,
+ "p99": 201.857
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/B1.point.json b/evidence/track1/B1.point.json
new file mode 100644
index 00000000..20b66ffc
--- /dev/null
+++ b/evidence/track1/B1.point.json
@@ -0,0 +1,38 @@
+{
+ "tag": "B1",
+ "replicas": 1,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 0.5,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 4,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.139,
+ "concurrency": 8,
+ "successful_requests": 1485,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 98.094,
+ "successful_latency_ms": {
+ "p50": 85.074,
+ "p95": 167.209,
+ "p99": 201.857
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
+ ],
+ "postgres_sessions": "5"
+ }
+}
diff --git a/evidence/track1/B2.json b/evidence/track1/B2.json
new file mode 100644
index 00000000..393fe4c9
--- /dev/null
+++ b/evidence/track1/B2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.136,
+ "concurrency": 8,
+ "successful_requests": 3388,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 223.842,
+ "successful_latency_ms": {
+ "p50": 24.1,
+ "p95": 81.009,
+ "p99": 142.445
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/B2.point.json b/evidence/track1/B2.point.json
new file mode 100644
index 00000000..10cd6d82
--- /dev/null
+++ b/evidence/track1/B2.point.json
@@ -0,0 +1,39 @@
+{
+ "tag": "B2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.136,
+ "concurrency": 8,
+ "successful_requests": 3388,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 223.842,
+ "successful_latency_ms": {
+ "p50": 24.1,
+ "p95": 81.009,
+ "p99": 142.445
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
+ "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/B4.json b/evidence/track1/B4.json
new file mode 100644
index 00000000..fb3cd66c
--- /dev/null
+++ b/evidence/track1/B4.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.181,
+ "concurrency": 8,
+ "successful_requests": 3592,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 236.616,
+ "successful_latency_ms": {
+ "p50": 22.559,
+ "p95": 80.196,
+ "p99": 131.013
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/B4.point.json b/evidence/track1/B4.point.json
new file mode 100644
index 00000000..7bed65df
--- /dev/null
+++ b/evidence/track1/B4.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "B4",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.181,
+ "concurrency": 8,
+ "successful_requests": 3592,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 236.616,
+ "successful_latency_ms": {
+ "p50": 22.559,
+ "p95": 80.196,
+ "p99": 131.013
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
+ "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
+ "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
+ "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+}
diff --git a/evidence/track1/C4-pg2.json b/evidence/track1/C4-pg2.json
new file mode 100644
index 00000000..53f3215b
--- /dev/null
+++ b/evidence/track1/C4-pg2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.128,
+ "concurrency": 8,
+ "successful_requests": 3800,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 251.189,
+ "successful_latency_ms": {
+ "p50": 19.749,
+ "p95": 77.161,
+ "p99": 101.17
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/C4-pg2.point.json b/evidence/track1/C4-pg2.point.json
new file mode 100644
index 00000000..7a43b388
--- /dev/null
+++ b/evidence/track1/C4-pg2.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "C4-pg2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "2.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.128,
+ "concurrency": 8,
+ "successful_requests": 3800,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 251.189,
+ "successful_latency_ms": {
+ "p50": 19.749,
+ "p95": 77.161,
+ "p99": 101.17
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
+ "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+}
diff --git a/evidence/track1/D4-pool2.json b/evidence/track1/D4-pool2.json
new file mode 100644
index 00000000..c9e683e4
--- /dev/null
+++ b/evidence/track1/D4-pool2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.156,
+ "concurrency": 8,
+ "successful_requests": 3622,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 238.98,
+ "successful_latency_ms": {
+ "p50": 22.721,
+ "p95": 74.859,
+ "p99": 98.063
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/D4-pool2.point.json b/evidence/track1/D4-pool2.point.json
new file mode 100644
index 00000000..80c5703c
--- /dev/null
+++ b/evidence/track1/D4-pool2.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "D4-pool2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.156,
+ "concurrency": 8,
+ "successful_requests": 3622,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 238.98,
+ "successful_latency_ms": {
+ "p50": 22.721,
+ "p95": 74.859,
+ "p99": 98.063
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
+ "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
+ "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
+ "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/rps.svg b/evidence/track1/rps.svg
new file mode 100644
index 00000000..d97a031a
--- /dev/null
+++ b/evidence/track1/rps.svg
@@ -0,0 +1,29 @@
+
+
diff --git a/evidence/track1/summary.json b/evidence/track1/summary.json
new file mode 100644
index 00000000..f4916581
--- /dev/null
+++ b/evidence/track1/summary.json
@@ -0,0 +1,320 @@
+[
+ {
+ "tag": "A1",
+ "replicas": 1,
+ "api_cpus_each": "1.0",
+ "api_mem_each": "512m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 8,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.109,
+ "concurrency": 8,
+ "successful_requests": 5197,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 343.964,
+ "successful_latency_ms": {
+ "p50": 19.732,
+ "p95": 42.061,
+ "p99": 81.001
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
+ "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "A2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.206,
+ "concurrency": 8,
+ "successful_requests": 2985,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 196.3,
+ "successful_latency_ms": {
+ "p50": 26.613,
+ "p95": 89.893,
+ "p99": 123.339
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
+ "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "A4",
+ "replicas": 4,
+ "api_cpus_each": "0.25",
+ "api_mem_each": "128m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.224,
+ "concurrency": 8,
+ "successful_requests": 1895,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 124.476,
+ "successful_latency_ms": {
+ "p50": 39.136,
+ "p95": 185.654,
+ "p99": 267.62
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
+ "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
+ "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
+ "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
+ "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "B1",
+ "replicas": 1,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 0.5,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 4,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.139,
+ "concurrency": 8,
+ "successful_requests": 1485,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 98.094,
+ "successful_latency_ms": {
+ "p50": 85.074,
+ "p95": 167.209,
+ "p99": 201.857
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
+ ],
+ "postgres_sessions": "5"
+ }
+ },
+ {
+ "tag": "B2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.136,
+ "concurrency": 8,
+ "successful_requests": 3388,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 223.842,
+ "successful_latency_ms": {
+ "p50": 24.1,
+ "p95": 81.009,
+ "p99": 142.445
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
+ "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "B4",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.181,
+ "concurrency": 8,
+ "successful_requests": 3592,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 236.616,
+ "successful_latency_ms": {
+ "p50": 22.559,
+ "p95": 80.196,
+ "p99": 131.013
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
+ "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
+ "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
+ "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+ },
+ {
+ "tag": "C4-pg2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "2.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.128,
+ "concurrency": 8,
+ "successful_requests": 3800,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 251.189,
+ "successful_latency_ms": {
+ "p50": 19.749,
+ "p95": 77.161,
+ "p99": 101.17
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
+ "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+ },
+ {
+ "tag": "D4-pool2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.156,
+ "concurrency": 8,
+ "successful_requests": 3622,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 238.98,
+ "successful_latency_ms": {
+ "p50": 22.721,
+ "p95": 74.859,
+ "p99": 98.063
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
+ "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
+ "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
+ "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ }
+]
diff --git a/scripts/grade4_evidence.py b/scripts/grade4_evidence.py
new file mode 100644
index 00000000..4ea1029a
--- /dev/null
+++ b/scripts/grade4_evidence.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+
+from client import Client, Failure # noqa: E402
+
+OUT = ROOT / "evidence" / "grade4"
+BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
+
+
+def compose(*args, check=True):
+ cmd = ["docker", "compose"]
+ if (ROOT / ".env").is_file():
+ cmd.extend(["--env-file", str(ROOT / ".env")])
+ cmd.extend(args)
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
+ if check and proc.returncode != 0:
+ raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
+ return proc
+
+
+def bash_argv(script: Path):
+ if os.name != "nt":
+ return [str(script)]
+ git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
+ if git_bash.is_file():
+ return [str(git_bash), str(script)]
+ return ["bash", str(script)]
+
+
+def hook_cmd():
+ return bash_argv(ROOT / "scripts" / "stop-one.sh")
+
+
+def wait_ready(seconds=60):
+ deadline = time.monotonic() + seconds
+ last = None
+ while time.monotonic() < deadline:
+ try:
+ Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
+ return
+ except Failure as exc:
+ last = exc
+ time.sleep(0.4)
+ raise SystemExit("API did not become ready: %s" % last)
+
+
+def instance_ids(actor, samples=24):
+ found = []
+ client = Client(BASE, timeout=3)
+ for _ in range(samples):
+ found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
+ return sorted(set(found))
+
+
+def parallel_idempotency():
+ client = Client(BASE, timeout=20)
+ alice, _bob, _eve, chat = client.bootstrap()
+ key = uuid.uuid4().hex
+ text = "parallel-" + key
+ ids = []
+ errors = []
+ barrier = threading.Barrier(16)
+
+ def one():
+ local = Client(BASE, timeout=20)
+ barrier.wait()
+ try:
+ ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
+ except Exception as exc:
+ errors.append(type(exc).__name__)
+
+ threads = [threading.Thread(target=one) for _ in range(16)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ unique = sorted(set(ids))
+ report = {
+ "workers": 16,
+ "successes": len(ids),
+ "errors": len(errors),
+ "distinct_message_ids": len(unique),
+ }
+ if len(unique) != 1 or errors:
+ raise SystemExit("parallel idempotency failed: %s" % report)
+ page = client.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=100" % chat["id"],
+ actor=alice,
+ schema="MessagePage",
+ )["json"]
+ copies = [item for item in page["items"] if item["text"] == text]
+ report["history_copies"] = len(copies)
+ if len(copies) != 1:
+ raise SystemExit("history has %s copies" % len(copies))
+ return report
+
+
+def failover_one(actor, chat, instance_id):
+ key = "failover-" + uuid.uuid4().hex
+ text = "during-" + key
+ successes = []
+ failures = 0
+ started_box = {}
+
+ def traffic():
+ nonlocal failures
+ local = Client(BASE, timeout=2)
+ while "t0" not in started_box:
+ time.sleep(0.01)
+ deadline = started_box["t0"] + 12
+ recovered = False
+ while time.monotonic() < deadline:
+ try:
+ if not recovered:
+ response = local.message(actor, chat, text, key=key)
+ recovered = True
+ else:
+ response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
+ successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
+ except Failure:
+ failures += 1
+ time.sleep(0.05)
+
+ worker = threading.Thread(target=traffic)
+ worker.start()
+ time.sleep(0.2)
+ started = time.monotonic()
+ started_box["t0"] = started
+ env = os.environ.copy()
+ env["TARGET_INSTANCE_ID"] = instance_id
+ proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
+ hook_s = time.monotonic() - started
+ worker.join()
+ after = [item for item in successes if item[0] >= started]
+ recovery = None
+ if after:
+ recovery = round(after[0][0] - started, 3)
+ tail = [item[1] for item in after[-5:]]
+ return {
+ "target": instance_id,
+ "hook_exit": proc.returncode,
+ "hook_seconds": round(hook_s, 3),
+ "hook_stderr": (proc.stderr or "")[-300:],
+ "failed_attempts": failures,
+ "successes_after_hook_start": len(after),
+ "first_recovery_seconds": recovery,
+ "tail_instance_ids": tail,
+ "target_still_in_tail": instance_id in tail,
+ }
+
+
+def restore_and_wait(actor):
+ compose("start", "api")
+ deadline = time.monotonic() + 40
+ while time.monotonic() < deadline:
+ try:
+ if len(instance_ids(actor, samples=16)) >= 2:
+ return
+ except Failure:
+ pass
+ time.sleep(0.5)
+ raise SystemExit("second replica did not return")
+
+
+def rolling_under_load():
+ out = OUT / "rolling-load.json"
+ load = subprocess.Popen(
+ [sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
+ "--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
+ cwd=ROOT,
+ )
+ time.sleep(3)
+ cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
+ started = time.monotonic()
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
+ elapsed = round(time.monotonic() - started, 3)
+ load.wait(timeout=90)
+ metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
+ if proc.returncode != 0:
+ raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
+ return {
+ "rolling_seconds": elapsed,
+ "rolling_stdout": (proc.stdout or "")[-400:],
+ "load": metrics,
+ }
+
+
+def main():
+ OUT.mkdir(parents=True, exist_ok=True)
+ wait_ready()
+ client = Client(BASE, timeout=10)
+ alice, _bob, _eve, chat = client.bootstrap()
+ before = instance_ids(alice)
+ if len(before) < 2:
+ raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
+ idem = parallel_idempotency()
+ failovers = []
+ for instance_id in before:
+ item = failover_one(alice, chat, instance_id)
+ failovers.append(item)
+ recovery = item["first_recovery_seconds"]
+ if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
+ raise SystemExit("failover criteria failed: %s" % item)
+ restore_and_wait(alice)
+ rolling = rolling_under_load()
+ report = {
+ "instances_before": before,
+ "parallel_idempotency": idem,
+ "failovers": failovers,
+ "rolling": rolling,
+ }
+ (OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print(json.dumps(report, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py
new file mode 100644
index 00000000..dcf96bf3
--- /dev/null
+++ b/scripts/track1_experiment.py
@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change.
+
+The load profile is the public closed-loop POST generator. Same duration and
+concurrency on every point. Raw JSON is written under evidence/track1/.
+"""
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / "evidence" / "track1"
+BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
+DURATION = float(os.environ.get("TRACK_DURATION", "15"))
+WARMUP = float(os.environ.get("TRACK_WARMUP", "5"))
+CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8"))
+
+
+def compose(*args):
+ cmd = ["docker", "compose"]
+ if (ROOT / ".env").is_file():
+ cmd.extend(["--env-file", str(ROOT / ".env")])
+ cmd.extend(args)
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy())
+ if proc.returncode != 0:
+ raise SystemExit((proc.stderr or proc.stdout)[-1000:])
+ return proc
+
+
+def wait_ready(seconds=90):
+ import urllib.request
+
+ deadline = time.monotonic() + seconds
+ while time.monotonic() < deadline:
+ try:
+ with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
+ if response.status == 200:
+ return
+ except Exception:
+ time.sleep(0.5)
+ raise SystemExit("ready timeout")
+
+
+def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool):
+ os.environ["API_CPUS"] = api_cpus
+ os.environ["API_MEM"] = api_mem
+ os.environ["POSTGRES_CPUS"] = postgres_cpus
+ os.environ["DB_POOL_MAX"] = str(pool)
+ compose("up", "-d", "--no-deps", "--force-recreate", "postgres")
+ deadline = time.monotonic() + 40
+ while time.monotonic() < deadline:
+ probe = subprocess.run(
+ ["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"],
+ cwd=ROOT,
+ capture_output=True,
+ text=True,
+ )
+ if probe.returncode == 0:
+ break
+ time.sleep(0.5)
+ else:
+ raise SystemExit("postgres did not become ready")
+ compose(
+ "up",
+ "-d",
+ "--no-deps",
+ "--force-recreate",
+ "--scale",
+ "api=%s" % replicas,
+ "api",
+ )
+ wait_ready()
+ running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()]
+ if len(running) != replicas:
+ raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running)))
+
+
+def run_load(tag, warmup=False):
+ args = [
+ sys.executable,
+ "-X",
+ "utf8",
+ str(ROOT / "tests" / "load.py"),
+ "--base-url",
+ BASE,
+ "--duration",
+ str(WARMUP if warmup else DURATION),
+ "--concurrency",
+ str(CONCURRENCY),
+ ]
+ dest = None
+ if not warmup:
+ dest = OUT / ("%s.json" % tag)
+ args.extend(["--output", str(dest)])
+ proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
+ if proc.returncode != 0:
+ raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:]))
+ if dest is None:
+ return None
+ return json.loads(dest.read_text(encoding="utf-8"))
+
+
+def sample_resources():
+ stats = subprocess.run(
+ ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
+ capture_output=True,
+ text=True,
+ )
+ activity = compose(
+ "exec",
+ "-T",
+ "postgres",
+ "psql",
+ "-U",
+ "messenger",
+ "-d",
+ "messenger",
+ "-tAc",
+ "SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'",
+ )
+ return {
+ "docker_stats": [line for line in stats.stdout.splitlines() if line.strip()],
+ "postgres_sessions": activity.stdout.strip(),
+ }
+
+
+def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool):
+ print("profile %s" % tag, flush=True)
+ apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool)
+ run_load(tag, warmup=True)
+ box = {}
+
+ def snapshot():
+ time.sleep(DURATION / 2)
+ box["resources"] = sample_resources()
+
+ thread = threading.Thread(target=snapshot)
+ thread.start()
+ metrics = run_load(tag)
+ thread.join()
+ point = {
+ "tag": tag,
+ "replicas": replicas,
+ "api_cpus_each": api_cpus,
+ "api_mem_each": api_mem,
+ "api_cpu_total": round(replicas * float(api_cpus), 3),
+ "postgres_cpus": postgres_cpus,
+ "db_pool_max_each": pool,
+ "db_pool_max_total": replicas * pool,
+ "load": metrics,
+ "resources": box.get("resources"),
+ }
+ (OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True)
+ return point
+
+
+def svg(points, path):
+ width, height = 720, 320
+ labels = [p["tag"] for p in points]
+ values = [p["load"]["successful_rps"] for p in points]
+ peak = max(values) or 1
+ left, top, plot_w, plot_h = 48, 20, width - 70, height - 70
+ bar_w = plot_w / len(points) * 0.62
+ gap = plot_w / len(points)
+ parts = [
+ '',
+ '")
+ path.write_text("\n".join(parts) + "\n", encoding="utf-8")
+
+
+def main():
+ OUT.mkdir(parents=True, exist_ok=True)
+ points = []
+ # Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8.
+ points.append(measure("A1", 1, "1.0", "512m", "1.0", 8))
+ points.append(measure("A2", 2, "0.50", "256m", "1.0", 4))
+ points.append(measure("A4", 4, "0.25", "128m", "1.0", 2))
+ # Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU.
+ points.append(measure("B1", 1, "0.50", "256m", "1.0", 4))
+ points.append(measure("B2", 2, "0.50", "256m", "1.0", 4))
+ points.append(measure("B4", 4, "0.50", "256m", "1.0", 4))
+ # Candidate improvement: give the database the extra CPU the API side already has.
+ points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4))
+ # Candidate improvement: stop the 4 pools from opening 16 server connections.
+ points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2))
+ (OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ svg(points, OUT / "rps.svg")
+ # Return the stand to the documented 2 x 0.50 CPU layout.
+ apply_profile(2, "0.50", "256m", "1.0", 4)
+ print("wrote %s" % (OUT / "summary.json"))
+
+
+if __name__ == "__main__":
+ main()