From 99ce1b1391a6bf6736e0eb7c7ae694e6f85ff83b Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 12:46:37 +0300 Subject: [PATCH] =?UTF-8?q?=D0=98=D0=B7=D0=BC=D0=B5=D1=80=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=20=D0=BA=D0=B0=D0=B6?= =?UTF-8?q?=D0=B4=D0=BE=D0=B9=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B8=20=D1=86=D0=B5=D0=BD=D1=83=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BF=D1=80=D0=BE=D1=86?= =?UTF-8?q?=D0=B5=D1=81=D1=81=D0=BE=D0=B2.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU. --- evidence/grade4/rolling-load.json | 17 ++ evidence/grade4/summary.json | 70 ++++++ evidence/public-checks.txt | 29 +++ evidence/track1/A1.json | 17 ++ evidence/track1/A1.point.json | 38 ++++ evidence/track1/A2.json | 17 ++ evidence/track1/A2.point.json | 39 ++++ evidence/track1/A4.json | 17 ++ evidence/track1/A4.point.json | 41 ++++ evidence/track1/B1.json | 17 ++ evidence/track1/B1.point.json | 38 ++++ evidence/track1/B2.json | 17 ++ evidence/track1/B2.point.json | 39 ++++ evidence/track1/B4.json | 17 ++ evidence/track1/B4.point.json | 41 ++++ evidence/track1/C4-pg2.json | 17 ++ evidence/track1/C4-pg2.point.json | 41 ++++ evidence/track1/D4-pool2.json | 17 ++ evidence/track1/D4-pool2.point.json | 41 ++++ evidence/track1/rps.svg | 29 +++ evidence/track1/summary.json | 320 ++++++++++++++++++++++++++++ scripts/grade4_evidence.py | 229 ++++++++++++++++++++ scripts/track1_experiment.py | 217 +++++++++++++++++++ 23 files changed, 1365 insertions(+) create mode 100644 evidence/grade4/rolling-load.json create mode 100644 evidence/grade4/summary.json create mode 100644 evidence/public-checks.txt create mode 100644 evidence/track1/A1.json create mode 100644 evidence/track1/A1.point.json create mode 100644 evidence/track1/A2.json create mode 100644 evidence/track1/A2.point.json create mode 100644 evidence/track1/A4.json create mode 100644 evidence/track1/A4.point.json create mode 100644 evidence/track1/B1.json create mode 100644 evidence/track1/B1.point.json create mode 100644 evidence/track1/B2.json create mode 100644 evidence/track1/B2.point.json create mode 100644 evidence/track1/B4.json create mode 100644 evidence/track1/B4.point.json create mode 100644 evidence/track1/C4-pg2.json create mode 100644 evidence/track1/C4-pg2.point.json create mode 100644 evidence/track1/D4-pool2.json create mode 100644 evidence/track1/D4-pool2.point.json create mode 100644 evidence/track1/rps.svg create mode 100644 evidence/track1/summary.json create mode 100644 scripts/grade4_evidence.py create mode 100644 scripts/track1_experiment.py diff --git a/evidence/grade4/rolling-load.json b/evidence/grade4/rolling-load.json new file mode 100644 index 00000000..2e6941a8 --- /dev/null +++ b/evidence/grade4/rolling-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 50.0, + "elapsed_seconds": 50.101, + "concurrency": 4, + "successful_requests": 8361, + "failed_requests": 18, + "error_ratio": 0.002148, + "successful_rps": 166.884, + "successful_latency_ms": { + "p50": 13.481, + "p95": 57.817, + "p99": 96.441 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/grade4/summary.json b/evidence/grade4/summary.json new file mode 100644 index 00000000..7893768b --- /dev/null +++ b/evidence/grade4/summary.json @@ -0,0 +1,70 @@ +{ + "instances_before": [ + "03c3a0545628", + "b555f8327eee" + ], + "parallel_idempotency": { + "workers": 16, + "successes": 16, + "errors": 0, + "distinct_message_ids": 1, + "history_copies": 1 + }, + "failovers": [ + { + "target": "03c3a0545628", + "hook_exit": 0, + "hook_seconds": 2.373, + "hook_stderr": "", + "failed_attempts": 2, + "successes_after_hook_start": 130, + "first_recovery_seconds": 0.034, + "tail_instance_ids": [ + "b555f8327eee", + "b555f8327eee", + "b555f8327eee", + "b555f8327eee", + "b555f8327eee" + ], + "target_still_in_tail": false + }, + { + "target": "b555f8327eee", + "hook_exit": 0, + "hook_seconds": 1.744, + "hook_stderr": "", + "failed_attempts": 1, + "successes_after_hook_start": 132, + "first_recovery_seconds": 0.038, + "tail_instance_ids": [ + "03c3a0545628", + "03c3a0545628", + "03c3a0545628", + "03c3a0545628", + "03c3a0545628" + ], + "target_still_in_tail": false + } + ], + "rolling": { + "rolling_seconds": 5.929, + "rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n", + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 50.0, + "elapsed_seconds": 50.101, + "concurrency": 4, + "successful_requests": 8361, + "failed_requests": 18, + "error_ratio": 0.002148, + "successful_rps": 166.884, + "successful_latency_ms": { + "p50": 13.481, + "p95": 57.817, + "p99": 96.441 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + } + } +} diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt new file mode 100644 index 00000000..711ae329 --- /dev/null +++ b/evidence/public-checks.txt @@ -0,0 +1,29 @@ +Date: 2026-11-07 +Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792 +Engine: Docker 29.5.3, Compose 5.1.4 +Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine + +python -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + +python -X utf8 -m pytest student_tests -q -m "not infra and not expiry" +15 passed, 3 deselected + +Official persistence and failover were executed by Linux python3 in docker:27-cli +with the host Docker socket and BASE_URL=http://host.docker.internal:8080, +because Windows CreateProcess cannot run scripts/restart.sh (WinError 193). +The hooks and the API containers are the host stack. + +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh +{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0), +hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628. +Exit code 0 means the process left on SIGTERM before the stop timeout. diff --git a/evidence/track1/A1.json b/evidence/track1/A1.json new file mode 100644 index 00000000..81e232fe --- /dev/null +++ b/evidence/track1/A1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.109, + "concurrency": 8, + "successful_requests": 5197, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 343.964, + "successful_latency_ms": { + "p50": 19.732, + "p95": 42.061, + "p99": 81.001 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/A1.point.json b/evidence/track1/A1.point.json new file mode 100644 index 00000000..3a986495 --- /dev/null +++ b/evidence/track1/A1.point.json @@ -0,0 +1,38 @@ +{ + "tag": "A1", + "replicas": 1, + "api_cpus_each": "1.0", + "api_mem_each": "512m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 8, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.109, + "concurrency": 8, + "successful_requests": 5197, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 343.964, + "successful_latency_ms": { + "p50": 19.732, + "p95": 42.061, + "p99": 81.001 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB", + "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/A2.json b/evidence/track1/A2.json new file mode 100644 index 00000000..f228ca31 --- /dev/null +++ b/evidence/track1/A2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.206, + "concurrency": 8, + "successful_requests": 2985, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 196.3, + "successful_latency_ms": { + "p50": 26.613, + "p95": 89.893, + "p99": 123.339 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/A2.point.json b/evidence/track1/A2.point.json new file mode 100644 index 00000000..bb805c1c --- /dev/null +++ b/evidence/track1/A2.point.json @@ -0,0 +1,39 @@ +{ + "tag": "A2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.206, + "concurrency": 8, + "successful_requests": 2985, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 196.3, + "successful_latency_ms": { + "p50": 26.613, + "p95": 89.893, + "p99": 123.339 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB", + "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB", + "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB", + "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/A4.json b/evidence/track1/A4.json new file mode 100644 index 00000000..7eefffc6 --- /dev/null +++ b/evidence/track1/A4.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.224, + "concurrency": 8, + "successful_requests": 1895, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 124.476, + "successful_latency_ms": { + "p50": 39.136, + "p95": 185.654, + "p99": 267.62 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/A4.point.json b/evidence/track1/A4.point.json new file mode 100644 index 00000000..87e0fcf0 --- /dev/null +++ b/evidence/track1/A4.point.json @@ -0,0 +1,41 @@ +{ + "tag": "A4", + "replicas": 4, + "api_cpus_each": "0.25", + "api_mem_each": "128m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.224, + "concurrency": 8, + "successful_requests": 1895, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 124.476, + "successful_latency_ms": { + "p50": 39.136, + "p95": 185.654, + "p99": 267.62 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB", + "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB", + "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB", + "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB", + "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB", + "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB", + "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/B1.json b/evidence/track1/B1.json new file mode 100644 index 00000000..af7b2ad4 --- /dev/null +++ b/evidence/track1/B1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.139, + "concurrency": 8, + "successful_requests": 1485, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 98.094, + "successful_latency_ms": { + "p50": 85.074, + "p95": 167.209, + "p99": 201.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/B1.point.json b/evidence/track1/B1.point.json new file mode 100644 index 00000000..20b66ffc --- /dev/null +++ b/evidence/track1/B1.point.json @@ -0,0 +1,38 @@ +{ + "tag": "B1", + "replicas": 1, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 0.5, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 4, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.139, + "concurrency": 8, + "successful_requests": 1485, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 98.094, + "successful_latency_ms": { + "p50": 85.074, + "p95": 167.209, + "p99": 201.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB", + "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB", + "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB", + "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB" + ], + "postgres_sessions": "5" + } +} diff --git a/evidence/track1/B2.json b/evidence/track1/B2.json new file mode 100644 index 00000000..393fe4c9 --- /dev/null +++ b/evidence/track1/B2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.136, + "concurrency": 8, + "successful_requests": 3388, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 223.842, + "successful_latency_ms": { + "p50": 24.1, + "p95": 81.009, + "p99": 142.445 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/B2.point.json b/evidence/track1/B2.point.json new file mode 100644 index 00000000..10cd6d82 --- /dev/null +++ b/evidence/track1/B2.point.json @@ -0,0 +1,39 @@ +{ + "tag": "B2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.136, + "concurrency": 8, + "successful_requests": 3388, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 223.842, + "successful_latency_ms": { + "p50": 24.1, + "p95": 81.009, + "p99": 142.445 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB", + "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB", + "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB", + "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB", + "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/B4.json b/evidence/track1/B4.json new file mode 100644 index 00000000..fb3cd66c --- /dev/null +++ b/evidence/track1/B4.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.181, + "concurrency": 8, + "successful_requests": 3592, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.616, + "successful_latency_ms": { + "p50": 22.559, + "p95": 80.196, + "p99": 131.013 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/B4.point.json b/evidence/track1/B4.point.json new file mode 100644 index 00000000..7bed65df --- /dev/null +++ b/evidence/track1/B4.point.json @@ -0,0 +1,41 @@ +{ + "tag": "B4", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.181, + "concurrency": 8, + "successful_requests": 3592, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.616, + "successful_latency_ms": { + "p50": 22.559, + "p95": 80.196, + "p99": 131.013 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB", + "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB", + "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB", + "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB", + "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB" + ], + "postgres_sessions": "17" + } +} diff --git a/evidence/track1/C4-pg2.json b/evidence/track1/C4-pg2.json new file mode 100644 index 00000000..53f3215b --- /dev/null +++ b/evidence/track1/C4-pg2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.128, + "concurrency": 8, + "successful_requests": 3800, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 251.189, + "successful_latency_ms": { + "p50": 19.749, + "p95": 77.161, + "p99": 101.17 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/C4-pg2.point.json b/evidence/track1/C4-pg2.point.json new file mode 100644 index 00000000..7a43b388 --- /dev/null +++ b/evidence/track1/C4-pg2.point.json @@ -0,0 +1,41 @@ +{ + "tag": "C4-pg2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "2.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.128, + "concurrency": 8, + "successful_requests": 3800, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 251.189, + "successful_latency_ms": { + "p50": 19.749, + "p95": 77.161, + "p99": 101.17 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB", + "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB", + "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "17" + } +} diff --git a/evidence/track1/D4-pool2.json b/evidence/track1/D4-pool2.json new file mode 100644 index 00000000..c9e683e4 --- /dev/null +++ b/evidence/track1/D4-pool2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.156, + "concurrency": 8, + "successful_requests": 3622, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 238.98, + "successful_latency_ms": { + "p50": 22.721, + "p95": 74.859, + "p99": 98.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/D4-pool2.point.json b/evidence/track1/D4-pool2.point.json new file mode 100644 index 00000000..80c5703c --- /dev/null +++ b/evidence/track1/D4-pool2.point.json @@ -0,0 +1,41 @@ +{ + "tag": "D4-pool2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.156, + "concurrency": 8, + "successful_requests": 3622, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 238.98, + "successful_latency_ms": { + "p50": 22.721, + "p95": 74.859, + "p99": 98.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB", + "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB", + "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB", + "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/rps.svg b/evidence/track1/rps.svg new file mode 100644 index 00000000..d97a031a --- /dev/null +++ b/evidence/track1/rps.svg @@ -0,0 +1,29 @@ + + + +Successful RPS, closed-loop POST, concurrency 8, 15s + +344.0 +A1 + +196.3 +A2 + +124.5 +A4 + +98.1 +B1 + +223.8 +B2 + +236.6 +B4 + +251.2 +C4-pg2 + +239.0 +D4-pool2 + diff --git a/evidence/track1/summary.json b/evidence/track1/summary.json new file mode 100644 index 00000000..f4916581 --- /dev/null +++ b/evidence/track1/summary.json @@ -0,0 +1,320 @@ +[ + { + "tag": "A1", + "replicas": 1, + "api_cpus_each": "1.0", + "api_mem_each": "512m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 8, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.109, + "concurrency": 8, + "successful_requests": 5197, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 343.964, + "successful_latency_ms": { + "p50": 19.732, + "p95": 42.061, + "p99": 81.001 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB", + "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "A2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.206, + "concurrency": 8, + "successful_requests": 2985, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 196.3, + "successful_latency_ms": { + "p50": 26.613, + "p95": 89.893, + "p99": 123.339 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB", + "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB", + "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB", + "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "A4", + "replicas": 4, + "api_cpus_each": "0.25", + "api_mem_each": "128m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.224, + "concurrency": 8, + "successful_requests": 1895, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 124.476, + "successful_latency_ms": { + "p50": 39.136, + "p95": 185.654, + "p99": 267.62 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB", + "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB", + "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB", + "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB", + "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB", + "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB", + "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "B1", + "replicas": 1, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 0.5, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 4, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.139, + "concurrency": 8, + "successful_requests": 1485, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 98.094, + "successful_latency_ms": { + "p50": 85.074, + "p95": 167.209, + "p99": 201.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB", + "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB", + "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB", + "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB" + ], + "postgres_sessions": "5" + } + }, + { + "tag": "B2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.136, + "concurrency": 8, + "successful_requests": 3388, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 223.842, + "successful_latency_ms": { + "p50": 24.1, + "p95": 81.009, + "p99": 142.445 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB", + "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB", + "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB", + "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB", + "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "B4", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.181, + "concurrency": 8, + "successful_requests": 3592, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.616, + "successful_latency_ms": { + "p50": 22.559, + "p95": 80.196, + "p99": 131.013 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB", + "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB", + "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB", + "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB", + "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB" + ], + "postgres_sessions": "17" + } + }, + { + "tag": "C4-pg2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "2.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.128, + "concurrency": 8, + "successful_requests": 3800, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 251.189, + "successful_latency_ms": { + "p50": 19.749, + "p95": 77.161, + "p99": 101.17 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB", + "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB", + "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "17" + } + }, + { + "tag": "D4-pool2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.156, + "concurrency": 8, + "successful_requests": 3622, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 238.98, + "successful_latency_ms": { + "p50": 22.721, + "p95": 74.859, + "p99": 98.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB", + "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB", + "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB", + "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "9" + } + } +] diff --git a/scripts/grade4_evidence.py b/scripts/grade4_evidence.py new file mode 100644 index 00000000..4ea1029a --- /dev/null +++ b/scripts/grade4_evidence.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load.""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure # noqa: E402 + +OUT = ROOT / "evidence" / "grade4" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") + + +def compose(*args, check=True): + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if check and proc.returncode != 0: + raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:])) + return proc + + +def bash_argv(script: Path): + if os.name != "nt": + return [str(script)] + git_bash = Path(r"C:\Program Files\Git\bin\bash.exe") + if git_bash.is_file(): + return [str(git_bash), str(script)] + return ["bash", str(script)] + + +def hook_cmd(): + return bash_argv(ROOT / "scripts" / "stop-one.sh") + + +def wait_ready(seconds=60): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = exc + time.sleep(0.4) + raise SystemExit("API did not become ready: %s" % last) + + +def instance_ids(actor, samples=24): + found = [] + client = Client(BASE, timeout=3) + for _ in range(samples): + found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"]) + return sorted(set(found)) + + +def parallel_idempotency(): + client = Client(BASE, timeout=20) + alice, _bob, _eve, chat = client.bootstrap() + key = uuid.uuid4().hex + text = "parallel-" + key + ids = [] + errors = [] + barrier = threading.Barrier(16) + + def one(): + local = Client(BASE, timeout=20) + barrier.wait() + try: + ids.append(local.message(alice, chat, text, key=key)["json"]["id"]) + except Exception as exc: + errors.append(type(exc).__name__) + + threads = [threading.Thread(target=one) for _ in range(16)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + unique = sorted(set(ids)) + report = { + "workers": 16, + "successes": len(ids), + "errors": len(errors), + "distinct_message_ids": len(unique), + } + if len(unique) != 1 or errors: + raise SystemExit("parallel idempotency failed: %s" % report) + page = client.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=alice, + schema="MessagePage", + )["json"] + copies = [item for item in page["items"] if item["text"] == text] + report["history_copies"] = len(copies) + if len(copies) != 1: + raise SystemExit("history has %s copies" % len(copies)) + return report + + +def failover_one(actor, chat, instance_id): + key = "failover-" + uuid.uuid4().hex + text = "during-" + key + successes = [] + failures = 0 + started_box = {} + + def traffic(): + nonlocal failures + local = Client(BASE, timeout=2) + while "t0" not in started_box: + time.sleep(0.01) + deadline = started_box["t0"] + 12 + recovered = False + while time.monotonic() < deadline: + try: + if not recovered: + response = local.message(actor, chat, text, key=key) + recovered = True + else: + response = local.message(actor, chat, "after-" + uuid.uuid4().hex) + successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"])) + except Failure: + failures += 1 + time.sleep(0.05) + + worker = threading.Thread(target=traffic) + worker.start() + time.sleep(0.2) + started = time.monotonic() + started_box["t0"] = started + env = os.environ.copy() + env["TARGET_INSTANCE_ID"] = instance_id + proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True) + hook_s = time.monotonic() - started + worker.join() + after = [item for item in successes if item[0] >= started] + recovery = None + if after: + recovery = round(after[0][0] - started, 3) + tail = [item[1] for item in after[-5:]] + return { + "target": instance_id, + "hook_exit": proc.returncode, + "hook_seconds": round(hook_s, 3), + "hook_stderr": (proc.stderr or "")[-300:], + "failed_attempts": failures, + "successes_after_hook_start": len(after), + "first_recovery_seconds": recovery, + "tail_instance_ids": tail, + "target_still_in_tail": instance_id in tail, + } + + +def restore_and_wait(actor): + compose("start", "api") + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + try: + if len(instance_ids(actor, samples=16)) >= 2: + return + except Failure: + pass + time.sleep(0.5) + raise SystemExit("second replica did not return") + + +def rolling_under_load(): + out = OUT / "rolling-load.json" + load = subprocess.Popen( + [sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"), + "--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)], + cwd=ROOT, + ) + time.sleep(3) + cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh") + started = time.monotonic() + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + elapsed = round(time.monotonic() - started, 3) + load.wait(timeout=90) + metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} + if proc.returncode != 0: + raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:]) + return { + "rolling_seconds": elapsed, + "rolling_stdout": (proc.stdout or "")[-400:], + "load": metrics, + } + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + wait_ready() + client = Client(BASE, timeout=10) + alice, _bob, _eve, chat = client.bootstrap() + before = instance_ids(alice) + if len(before) < 2: + raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before) + idem = parallel_idempotency() + failovers = [] + for instance_id in before: + item = failover_one(alice, chat, instance_id) + failovers.append(item) + recovery = item["first_recovery_seconds"] + if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5: + raise SystemExit("failover criteria failed: %s" % item) + restore_and_wait(alice) + rolling = rolling_under_load() + report = { + "instances_before": before, + "parallel_idempotency": idem, + "failovers": failovers, + "rolling": rolling, + } + (OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py new file mode 100644 index 00000000..dcf96bf3 --- /dev/null +++ b/scripts/track1_experiment.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change. + +The load profile is the public closed-loop POST generator. Same duration and +concurrency on every point. Raw JSON is written under evidence/track1/. +""" +import json +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" / "track1" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +DURATION = float(os.environ.get("TRACK_DURATION", "15")) +WARMUP = float(os.environ.get("TRACK_WARMUP", "5")) +CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8")) + + +def compose(*args): + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy()) + if proc.returncode != 0: + raise SystemExit((proc.stderr or proc.stdout)[-1000:]) + return proc + + +def wait_ready(seconds=90): + import urllib.request + + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response: + if response.status == 200: + return + except Exception: + time.sleep(0.5) + raise SystemExit("ready timeout") + + +def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool): + os.environ["API_CPUS"] = api_cpus + os.environ["API_MEM"] = api_mem + os.environ["POSTGRES_CPUS"] = postgres_cpus + os.environ["DB_POOL_MAX"] = str(pool) + compose("up", "-d", "--no-deps", "--force-recreate", "postgres") + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + probe = subprocess.run( + ["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"], + cwd=ROOT, + capture_output=True, + text=True, + ) + if probe.returncode == 0: + break + time.sleep(0.5) + else: + raise SystemExit("postgres did not become ready") + compose( + "up", + "-d", + "--no-deps", + "--force-recreate", + "--scale", + "api=%s" % replicas, + "api", + ) + wait_ready() + running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()] + if len(running) != replicas: + raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running))) + + +def run_load(tag, warmup=False): + args = [ + sys.executable, + "-X", + "utf8", + str(ROOT / "tests" / "load.py"), + "--base-url", + BASE, + "--duration", + str(WARMUP if warmup else DURATION), + "--concurrency", + str(CONCURRENCY), + ] + dest = None + if not warmup: + dest = OUT / ("%s.json" % tag) + args.extend(["--output", str(dest)]) + proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:])) + if dest is None: + return None + return json.loads(dest.read_text(encoding="utf-8")) + + +def sample_resources(): + stats = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], + capture_output=True, + text=True, + ) + activity = compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-tAc", + "SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'", + ) + return { + "docker_stats": [line for line in stats.stdout.splitlines() if line.strip()], + "postgres_sessions": activity.stdout.strip(), + } + + +def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool): + print("profile %s" % tag, flush=True) + apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool) + run_load(tag, warmup=True) + box = {} + + def snapshot(): + time.sleep(DURATION / 2) + box["resources"] = sample_resources() + + thread = threading.Thread(target=snapshot) + thread.start() + metrics = run_load(tag) + thread.join() + point = { + "tag": tag, + "replicas": replicas, + "api_cpus_each": api_cpus, + "api_mem_each": api_mem, + "api_cpu_total": round(replicas * float(api_cpus), 3), + "postgres_cpus": postgres_cpus, + "db_pool_max_each": pool, + "db_pool_max_total": replicas * pool, + "load": metrics, + "resources": box.get("resources"), + } + (OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True) + return point + + +def svg(points, path): + width, height = 720, 320 + labels = [p["tag"] for p in points] + values = [p["load"]["successful_rps"] for p in points] + peak = max(values) or 1 + left, top, plot_w, plot_h = 48, 20, width - 70, height - 70 + bar_w = plot_w / len(points) * 0.62 + gap = plot_w / len(points) + parts = [ + '', + '' % (width, height), + '', + 'Successful RPS, closed-loop POST, concurrency %d, %ss' + % (CONCURRENCY, int(DURATION)), + ] + for index, (label, value) in enumerate(zip(labels, values)): + h = plot_h * value / peak + x = left + index * gap + (gap - bar_w) / 2 + y = top + 12 + (plot_h - h) + parts.append('' % (x, y, bar_w, h)) + parts.append( + '%.1f' + % (x + bar_w / 2, y - 4, value) + ) + parts.append( + '%s' + % (x + bar_w / 2, height - 18, label) + ) + parts.append("") + path.write_text("\n".join(parts) + "\n", encoding="utf-8") + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + points = [] + # Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8. + points.append(measure("A1", 1, "1.0", "512m", "1.0", 8)) + points.append(measure("A2", 2, "0.50", "256m", "1.0", 4)) + points.append(measure("A4", 4, "0.25", "128m", "1.0", 2)) + # Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU. + points.append(measure("B1", 1, "0.50", "256m", "1.0", 4)) + points.append(measure("B2", 2, "0.50", "256m", "1.0", 4)) + points.append(measure("B4", 4, "0.50", "256m", "1.0", 4)) + # Candidate improvement: give the database the extra CPU the API side already has. + points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4)) + # Candidate improvement: stop the 4 pools from opening 16 server connections. + points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2)) + (OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + svg(points, OUT / "rps.svg") + # Return the stand to the documented 2 x 0.50 CPU layout. + apply_profile(2, "0.50", "256m", "1.0", 4) + print("wrote %s" % (OUT / "summary.json")) + + +if __name__ == "__main__": + main()