forked from mtusi2026/mtusi2026lab3
Измерить отказ каждой реплики и цену добавления процессов.
Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 50.0,
|
||||
"elapsed_seconds": 50.101,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8361,
|
||||
"failed_requests": 18,
|
||||
"error_ratio": 0.002148,
|
||||
"successful_rps": 166.884,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.481,
|
||||
"p95": 57.817,
|
||||
"p99": 96.441
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"instances_before": [
|
||||
"03c3a0545628",
|
||||
"b555f8327eee"
|
||||
],
|
||||
"parallel_idempotency": {
|
||||
"workers": 16,
|
||||
"successes": 16,
|
||||
"errors": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"history_copies": 1
|
||||
},
|
||||
"failovers": [
|
||||
{
|
||||
"target": "03c3a0545628",
|
||||
"hook_exit": 0,
|
||||
"hook_seconds": 2.373,
|
||||
"hook_stderr": "",
|
||||
"failed_attempts": 2,
|
||||
"successes_after_hook_start": 130,
|
||||
"first_recovery_seconds": 0.034,
|
||||
"tail_instance_ids": [
|
||||
"b555f8327eee",
|
||||
"b555f8327eee",
|
||||
"b555f8327eee",
|
||||
"b555f8327eee",
|
||||
"b555f8327eee"
|
||||
],
|
||||
"target_still_in_tail": false
|
||||
},
|
||||
{
|
||||
"target": "b555f8327eee",
|
||||
"hook_exit": 0,
|
||||
"hook_seconds": 1.744,
|
||||
"hook_stderr": "",
|
||||
"failed_attempts": 1,
|
||||
"successes_after_hook_start": 132,
|
||||
"first_recovery_seconds": 0.038,
|
||||
"tail_instance_ids": [
|
||||
"03c3a0545628",
|
||||
"03c3a0545628",
|
||||
"03c3a0545628",
|
||||
"03c3a0545628",
|
||||
"03c3a0545628"
|
||||
],
|
||||
"target_still_in_tail": false
|
||||
}
|
||||
],
|
||||
"rolling": {
|
||||
"rolling_seconds": 5.929,
|
||||
"rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n",
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 50.0,
|
||||
"elapsed_seconds": 50.101,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8361,
|
||||
"failed_requests": 18,
|
||||
"error_ratio": 0.002148,
|
||||
"successful_rps": 166.884,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.481,
|
||||
"p95": 57.817,
|
||||
"p99": 96.441
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
Date: 2026-11-07
|
||||
Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792
|
||||
Engine: Docker 29.5.3, Compose 5.1.4
|
||||
Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine
|
||||
|
||||
python -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
|
||||
15 passed, 3 deselected
|
||||
|
||||
Official persistence and failover were executed by Linux python3 in docker:27-cli
|
||||
with the host Docker socket and BASE_URL=http://host.docker.internal:8080,
|
||||
because Windows CreateProcess cannot run scripts/restart.sh (WinError 193).
|
||||
The hooks and the API containers are the host stack.
|
||||
|
||||
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh
|
||||
{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0),
|
||||
hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628.
|
||||
Exit code 0 means the process left on SIGTERM before the stop timeout.
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.109,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 5197,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 343.964,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.732,
|
||||
"p95": 42.061,
|
||||
"p99": 81.001
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"tag": "A1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "1.0",
|
||||
"api_mem_each": "512m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 8,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.109,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 5197,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 343.964,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.732,
|
||||
"p95": 42.061,
|
||||
"p99": 81.001
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
|
||||
"mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.206,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 2985,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 196.3,
|
||||
"successful_latency_ms": {
|
||||
"p50": 26.613,
|
||||
"p95": 89.893,
|
||||
"p99": 123.339
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"tag": "A2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.206,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 2985,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 196.3,
|
||||
"successful_latency_ms": {
|
||||
"p50": 26.613,
|
||||
"p95": 89.893,
|
||||
"p99": 123.339
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
|
||||
"mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.224,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1895,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 124.476,
|
||||
"successful_latency_ms": {
|
||||
"p50": 39.136,
|
||||
"p95": 185.654,
|
||||
"p99": 267.62
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "A4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.25",
|
||||
"api_mem_each": "128m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.224,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1895,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 124.476,
|
||||
"successful_latency_ms": {
|
||||
"p50": 39.136,
|
||||
"p95": 185.654,
|
||||
"p99": 267.62
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
|
||||
"mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
|
||||
"mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
|
||||
"mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
|
||||
"mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.139,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1485,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 98.094,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.074,
|
||||
"p95": 167.209,
|
||||
"p99": 201.857
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"tag": "B1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 0.5,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 4,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.139,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1485,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 98.094,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.074,
|
||||
"p95": 167.209,
|
||||
"p99": 201.857
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.136,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3388,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 223.842,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.1,
|
||||
"p95": 81.009,
|
||||
"p99": 142.445
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"tag": "B2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.136,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3388,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 223.842,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.1,
|
||||
"p95": 81.009,
|
||||
"p99": 142.445
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.181,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3592,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 236.616,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.559,
|
||||
"p95": 80.196,
|
||||
"p99": 131.013
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "B4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.181,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3592,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 236.616,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.559,
|
||||
"p95": 80.196,
|
||||
"p99": 131.013
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.128,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3800,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 251.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.749,
|
||||
"p95": 77.161,
|
||||
"p99": 101.17
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "C4-pg2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "2.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.128,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3800,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 251.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.749,
|
||||
"p95": 77.161,
|
||||
"p99": 101.17
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.156,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3622,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 238.98,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.721,
|
||||
"p95": 74.859,
|
||||
"p99": 98.063
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "D4-pool2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.156,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3622,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 238.98,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.721,
|
||||
"p95": 74.859,
|
||||
"p99": 98.063
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="720" height="320">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency 8, 15s</text>
|
||||
<rect x="63.4" y="32.0" width="50.4" height="250.0" fill="#2f6f4e"/>
|
||||
<text x="88.6" y="28.0" font-family="sans-serif" font-size="12" text-anchor="middle">344.0</text>
|
||||
<text x="88.6" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A1</text>
|
||||
<rect x="144.7" y="139.3" width="50.4" height="142.7" fill="#2f6f4e"/>
|
||||
<text x="169.9" y="135.3" font-family="sans-serif" font-size="12" text-anchor="middle">196.3</text>
|
||||
<text x="169.9" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A2</text>
|
||||
<rect x="225.9" y="191.5" width="50.4" height="90.5" fill="#2f6f4e"/>
|
||||
<text x="251.1" y="187.5" font-family="sans-serif" font-size="12" text-anchor="middle">124.5</text>
|
||||
<text x="251.1" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A4</text>
|
||||
<rect x="307.2" y="210.7" width="50.4" height="71.3" fill="#2f6f4e"/>
|
||||
<text x="332.4" y="206.7" font-family="sans-serif" font-size="12" text-anchor="middle">98.1</text>
|
||||
<text x="332.4" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B1</text>
|
||||
<rect x="388.4" y="119.3" width="50.4" height="162.7" fill="#2f6f4e"/>
|
||||
<text x="413.6" y="115.3" font-family="sans-serif" font-size="12" text-anchor="middle">223.8</text>
|
||||
<text x="413.6" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B2</text>
|
||||
<rect x="469.7" y="110.0" width="50.4" height="172.0" fill="#2f6f4e"/>
|
||||
<text x="494.9" y="106.0" font-family="sans-serif" font-size="12" text-anchor="middle">236.6</text>
|
||||
<text x="494.9" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B4</text>
|
||||
<rect x="550.9" y="99.4" width="50.4" height="182.6" fill="#2f6f4e"/>
|
||||
<text x="576.1" y="95.4" font-family="sans-serif" font-size="12" text-anchor="middle">251.2</text>
|
||||
<text x="576.1" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">C4-pg2</text>
|
||||
<rect x="632.2" y="108.3" width="50.4" height="173.7" fill="#2f6f4e"/>
|
||||
<text x="657.4" y="104.3" font-family="sans-serif" font-size="12" text-anchor="middle">239.0</text>
|
||||
<text x="657.4" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">D4-pool2</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.4 KiB |
@@ -0,0 +1,320 @@
|
||||
[
|
||||
{
|
||||
"tag": "A1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "1.0",
|
||||
"api_mem_each": "512m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 8,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.109,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 5197,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 343.964,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.732,
|
||||
"p95": 42.061,
|
||||
"p99": 81.001
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
|
||||
"mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "A2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.206,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 2985,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 196.3,
|
||||
"successful_latency_ms": {
|
||||
"p50": 26.613,
|
||||
"p95": 89.893,
|
||||
"p99": 123.339
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
|
||||
"mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "A4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.25",
|
||||
"api_mem_each": "128m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.224,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1895,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 124.476,
|
||||
"successful_latency_ms": {
|
||||
"p50": 39.136,
|
||||
"p95": 185.654,
|
||||
"p99": 267.62
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
|
||||
"mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
|
||||
"mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
|
||||
"mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
|
||||
"mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "B1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 0.5,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 4,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.139,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1485,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 98.094,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.074,
|
||||
"p95": 167.209,
|
||||
"p99": 201.857
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "5"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "B2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.136,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3388,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 223.842,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.1,
|
||||
"p95": 81.009,
|
||||
"p99": 142.445
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "B4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.181,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3592,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 236.616,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.559,
|
||||
"p95": 80.196,
|
||||
"p99": 131.013
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "C4-pg2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "2.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.128,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3800,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 251.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.749,
|
||||
"p95": 77.161,
|
||||
"p99": 101.17
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "D4-pool2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.156,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3622,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 238.98,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.721,
|
||||
"p95": 74.859,
|
||||
"p99": 98.063
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
OUT = ROOT / "evidence" / "grade4"
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
|
||||
|
||||
def compose(*args, check=True):
|
||||
cmd = ["docker", "compose"]
|
||||
if (ROOT / ".env").is_file():
|
||||
cmd.extend(["--env-file", str(ROOT / ".env")])
|
||||
cmd.extend(args)
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
if check and proc.returncode != 0:
|
||||
raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
|
||||
return proc
|
||||
|
||||
|
||||
def bash_argv(script: Path):
|
||||
if os.name != "nt":
|
||||
return [str(script)]
|
||||
git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
|
||||
if git_bash.is_file():
|
||||
return [str(git_bash), str(script)]
|
||||
return ["bash", str(script)]
|
||||
|
||||
|
||||
def hook_cmd():
|
||||
return bash_argv(ROOT / "scripts" / "stop-one.sh")
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure as exc:
|
||||
last = exc
|
||||
time.sleep(0.4)
|
||||
raise SystemExit("API did not become ready: %s" % last)
|
||||
|
||||
|
||||
def instance_ids(actor, samples=24):
|
||||
found = []
|
||||
client = Client(BASE, timeout=3)
|
||||
for _ in range(samples):
|
||||
found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
|
||||
return sorted(set(found))
|
||||
|
||||
|
||||
def parallel_idempotency():
|
||||
client = Client(BASE, timeout=20)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
text = "parallel-" + key
|
||||
ids = []
|
||||
errors = []
|
||||
barrier = threading.Barrier(16)
|
||||
|
||||
def one():
|
||||
local = Client(BASE, timeout=20)
|
||||
barrier.wait()
|
||||
try:
|
||||
ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
|
||||
except Exception as exc:
|
||||
errors.append(type(exc).__name__)
|
||||
|
||||
threads = [threading.Thread(target=one) for _ in range(16)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
unique = sorted(set(ids))
|
||||
report = {
|
||||
"workers": 16,
|
||||
"successes": len(ids),
|
||||
"errors": len(errors),
|
||||
"distinct_message_ids": len(unique),
|
||||
}
|
||||
if len(unique) != 1 or errors:
|
||||
raise SystemExit("parallel idempotency failed: %s" % report)
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
copies = [item for item in page["items"] if item["text"] == text]
|
||||
report["history_copies"] = len(copies)
|
||||
if len(copies) != 1:
|
||||
raise SystemExit("history has %s copies" % len(copies))
|
||||
return report
|
||||
|
||||
|
||||
def failover_one(actor, chat, instance_id):
|
||||
key = "failover-" + uuid.uuid4().hex
|
||||
text = "during-" + key
|
||||
successes = []
|
||||
failures = 0
|
||||
started_box = {}
|
||||
|
||||
def traffic():
|
||||
nonlocal failures
|
||||
local = Client(BASE, timeout=2)
|
||||
while "t0" not in started_box:
|
||||
time.sleep(0.01)
|
||||
deadline = started_box["t0"] + 12
|
||||
recovered = False
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if not recovered:
|
||||
response = local.message(actor, chat, text, key=key)
|
||||
recovered = True
|
||||
else:
|
||||
response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
|
||||
successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
|
||||
except Failure:
|
||||
failures += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
worker = threading.Thread(target=traffic)
|
||||
worker.start()
|
||||
time.sleep(0.2)
|
||||
started = time.monotonic()
|
||||
started_box["t0"] = started
|
||||
env = os.environ.copy()
|
||||
env["TARGET_INSTANCE_ID"] = instance_id
|
||||
proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
|
||||
hook_s = time.monotonic() - started
|
||||
worker.join()
|
||||
after = [item for item in successes if item[0] >= started]
|
||||
recovery = None
|
||||
if after:
|
||||
recovery = round(after[0][0] - started, 3)
|
||||
tail = [item[1] for item in after[-5:]]
|
||||
return {
|
||||
"target": instance_id,
|
||||
"hook_exit": proc.returncode,
|
||||
"hook_seconds": round(hook_s, 3),
|
||||
"hook_stderr": (proc.stderr or "")[-300:],
|
||||
"failed_attempts": failures,
|
||||
"successes_after_hook_start": len(after),
|
||||
"first_recovery_seconds": recovery,
|
||||
"tail_instance_ids": tail,
|
||||
"target_still_in_tail": instance_id in tail,
|
||||
}
|
||||
|
||||
|
||||
def restore_and_wait(actor):
|
||||
compose("start", "api")
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if len(instance_ids(actor, samples=16)) >= 2:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise SystemExit("second replica did not return")
|
||||
|
||||
|
||||
def rolling_under_load():
|
||||
out = OUT / "rolling-load.json"
|
||||
load = subprocess.Popen(
|
||||
[sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
|
||||
"--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
|
||||
cwd=ROOT,
|
||||
)
|
||||
time.sleep(3)
|
||||
cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
elapsed = round(time.monotonic() - started, 3)
|
||||
load.wait(timeout=90)
|
||||
metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
|
||||
return {
|
||||
"rolling_seconds": elapsed,
|
||||
"rolling_stdout": (proc.stdout or "")[-400:],
|
||||
"load": metrics,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
wait_ready()
|
||||
client = Client(BASE, timeout=10)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
before = instance_ids(alice)
|
||||
if len(before) < 2:
|
||||
raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
|
||||
idem = parallel_idempotency()
|
||||
failovers = []
|
||||
for instance_id in before:
|
||||
item = failover_one(alice, chat, instance_id)
|
||||
failovers.append(item)
|
||||
recovery = item["first_recovery_seconds"]
|
||||
if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
|
||||
raise SystemExit("failover criteria failed: %s" % item)
|
||||
restore_and_wait(alice)
|
||||
rolling = rolling_under_load()
|
||||
report = {
|
||||
"instances_before": before,
|
||||
"parallel_idempotency": idem,
|
||||
"failovers": failovers,
|
||||
"rolling": rolling,
|
||||
}
|
||||
(OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change.
|
||||
|
||||
The load profile is the public closed-loop POST generator. Same duration and
|
||||
concurrency on every point. Raw JSON is written under evidence/track1/.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "track1"
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
DURATION = float(os.environ.get("TRACK_DURATION", "15"))
|
||||
WARMUP = float(os.environ.get("TRACK_WARMUP", "5"))
|
||||
CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8"))
|
||||
|
||||
|
||||
def compose(*args):
|
||||
cmd = ["docker", "compose"]
|
||||
if (ROOT / ".env").is_file():
|
||||
cmd.extend(["--env-file", str(ROOT / ".env")])
|
||||
cmd.extend(args)
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy())
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit((proc.stderr or proc.stdout)[-1000:])
|
||||
return proc
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
import urllib.request
|
||||
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
raise SystemExit("ready timeout")
|
||||
|
||||
|
||||
def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool):
|
||||
os.environ["API_CPUS"] = api_cpus
|
||||
os.environ["API_MEM"] = api_mem
|
||||
os.environ["POSTGRES_CPUS"] = postgres_cpus
|
||||
os.environ["DB_POOL_MAX"] = str(pool)
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "postgres")
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
probe = subprocess.run(
|
||||
["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if probe.returncode == 0:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise SystemExit("postgres did not become ready")
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"--scale",
|
||||
"api=%s" % replicas,
|
||||
"api",
|
||||
)
|
||||
wait_ready()
|
||||
running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()]
|
||||
if len(running) != replicas:
|
||||
raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running)))
|
||||
|
||||
|
||||
def run_load(tag, warmup=False):
|
||||
args = [
|
||||
sys.executable,
|
||||
"-X",
|
||||
"utf8",
|
||||
str(ROOT / "tests" / "load.py"),
|
||||
"--base-url",
|
||||
BASE,
|
||||
"--duration",
|
||||
str(WARMUP if warmup else DURATION),
|
||||
"--concurrency",
|
||||
str(CONCURRENCY),
|
||||
]
|
||||
dest = None
|
||||
if not warmup:
|
||||
dest = OUT / ("%s.json" % tag)
|
||||
args.extend(["--output", str(dest)])
|
||||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:]))
|
||||
if dest is None:
|
||||
return None
|
||||
return json.loads(dest.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def sample_resources():
|
||||
stats = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
activity = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-tAc",
|
||||
"SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'",
|
||||
)
|
||||
return {
|
||||
"docker_stats": [line for line in stats.stdout.splitlines() if line.strip()],
|
||||
"postgres_sessions": activity.stdout.strip(),
|
||||
}
|
||||
|
||||
|
||||
def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool):
|
||||
print("profile %s" % tag, flush=True)
|
||||
apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool)
|
||||
run_load(tag, warmup=True)
|
||||
box = {}
|
||||
|
||||
def snapshot():
|
||||
time.sleep(DURATION / 2)
|
||||
box["resources"] = sample_resources()
|
||||
|
||||
thread = threading.Thread(target=snapshot)
|
||||
thread.start()
|
||||
metrics = run_load(tag)
|
||||
thread.join()
|
||||
point = {
|
||||
"tag": tag,
|
||||
"replicas": replicas,
|
||||
"api_cpus_each": api_cpus,
|
||||
"api_mem_each": api_mem,
|
||||
"api_cpu_total": round(replicas * float(api_cpus), 3),
|
||||
"postgres_cpus": postgres_cpus,
|
||||
"db_pool_max_each": pool,
|
||||
"db_pool_max_total": replicas * pool,
|
||||
"load": metrics,
|
||||
"resources": box.get("resources"),
|
||||
}
|
||||
(OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True)
|
||||
return point
|
||||
|
||||
|
||||
def svg(points, path):
|
||||
width, height = 720, 320
|
||||
labels = [p["tag"] for p in points]
|
||||
values = [p["load"]["successful_rps"] for p in points]
|
||||
peak = max(values) or 1
|
||||
left, top, plot_w, plot_h = 48, 20, width - 70, height - 70
|
||||
bar_w = plot_w / len(points) * 0.62
|
||||
gap = plot_w / len(points)
|
||||
parts = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d">' % (width, height),
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
'<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency %d, %ss</text>'
|
||||
% (CONCURRENCY, int(DURATION)),
|
||||
]
|
||||
for index, (label, value) in enumerate(zip(labels, values)):
|
||||
h = plot_h * value / peak
|
||||
x = left + index * gap + (gap - bar_w) / 2
|
||||
y = top + 12 + (plot_h - h)
|
||||
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#2f6f4e"/>' % (x, y, bar_w, h))
|
||||
parts.append(
|
||||
'<text x="%.1f" y="%.1f" font-family="sans-serif" font-size="12" text-anchor="middle">%.1f</text>'
|
||||
% (x + bar_w / 2, y - 4, value)
|
||||
)
|
||||
parts.append(
|
||||
'<text x="%.1f" y="%d" font-family="sans-serif" font-size="11" text-anchor="middle">%s</text>'
|
||||
% (x + bar_w / 2, height - 18, label)
|
||||
)
|
||||
parts.append("</svg>")
|
||||
path.write_text("\n".join(parts) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
points = []
|
||||
# Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8.
|
||||
points.append(measure("A1", 1, "1.0", "512m", "1.0", 8))
|
||||
points.append(measure("A2", 2, "0.50", "256m", "1.0", 4))
|
||||
points.append(measure("A4", 4, "0.25", "128m", "1.0", 2))
|
||||
# Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU.
|
||||
points.append(measure("B1", 1, "0.50", "256m", "1.0", 4))
|
||||
points.append(measure("B2", 2, "0.50", "256m", "1.0", 4))
|
||||
points.append(measure("B4", 4, "0.50", "256m", "1.0", 4))
|
||||
# Candidate improvement: give the database the extra CPU the API side already has.
|
||||
points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4))
|
||||
# Candidate improvement: stop the 4 pools from opening 16 server connections.
|
||||
points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2))
|
||||
(OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
svg(points, OUT / "rps.svg")
|
||||
# Return the stand to the documented 2 x 0.50 CPU layout.
|
||||
apply_profile(2, "0.50", "256m", "1.0", 4)
|
||||
print("wrote %s" % (OUT / "summary.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user