From 32442d06db778650dadcf50be961704431679931 Mon Sep 17 00:00:00 2001 From: Sona Date: Tue, 10 Nov 2026 04:17:24 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=203=20=D1=83=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D0=BD=D1=8C=204:=20drain=20=D0=BF=D1=80=D0=B8=20SIGTERM,?= =?UTF-8?q?=20=D0=BB=D0=B8=D0=BC=D0=B8=D1=82=D1=8B=20=D1=80=D0=B5=D1=81?= =?UTF-8?q?=D1=83=D1=80=D1=81=D0=BE=D0=B2,=20=D0=BD=D0=B0=D0=B3=D1=80?= =?UTF-8?q?=D1=83=D0=B7=D0=BA=D0=B0=20=D1=81=20=D0=BE=D1=82=D0=BA=D0=B0?= =?UTF-8?q?=D0=B7=D0=B0=D0=BC=D0=B8,=20rolling=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 + compose.yaml | 34 +- evidence/docker-stats.txt | 6 + evidence/failover-level4.txt | 4 + evidence/idempotency-parallel.txt | 2 + evidence/load-baseline.json | 144 ++++++ evidence/load-rolling.json | 526 +++++++++++++++++++++ evidence/load-stop-0-ps.txt | 7 + evidence/load-stop-0.json | 195 ++++++++ evidence/load-stop-1-ps.txt | 7 + evidence/load-stop-1.json | 195 ++++++++ evidence/persistence-level4.txt | 3 + evidence/rolling-ps.txt | 6 + evidence/sigterm-level4.txt | 6 + evidence/smoke-level4.txt | 2 + infra/haproxy/haproxy.cfg | 4 + scripts/rolling-update.sh | 31 ++ scripts/run-failover-series.sh | 25 + src/config.py | 6 +- src/main.py | 26 +- student_tests/common.py | 34 ++ student_tests/loadgen.py | 106 +++++ student_tests/test_idempotency_parallel.py | 43 ++ student_tests/test_sigterm.py | 27 +- 24 files changed, 1429 insertions(+), 13 deletions(-) create mode 100644 evidence/docker-stats.txt create mode 100644 evidence/failover-level4.txt create mode 100644 evidence/idempotency-parallel.txt create mode 100644 evidence/load-baseline.json create mode 100644 evidence/load-rolling.json create mode 100644 evidence/load-stop-0-ps.txt create mode 100644 evidence/load-stop-0.json create mode 100644 evidence/load-stop-1-ps.txt create mode 100644 evidence/load-stop-1.json create mode 100644 evidence/persistence-level4.txt create mode 100644 evidence/rolling-ps.txt create mode 100644 evidence/sigterm-level4.txt create mode 100644 evidence/smoke-level4.txt create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/run-failover-series.sh create mode 100644 student_tests/common.py create mode 100644 student_tests/loadgen.py create mode 100644 student_tests/test_idempotency_parallel.py diff --git a/Dockerfile b/Dockerfile index 52a4ede3..570e3c9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ RUN pip install --no-cache-dir --timeout 60 -r requirements.txt COPY --chown=app:app src/ /app/src/ COPY --chown=app:app migrations/ /app/migrations/ +ARG APP_VERSION=v1 +ENV APP_VERSION=${APP_VERSION} + USER app EXPOSE 8080 diff --git a/compose.yaml b/compose.yaml index 6f6756c8..715e5ed2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,6 +17,11 @@ services: interval: 5s timeout: 3s retries: 10 + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEM:-512M} valkey: image: valkey/valkey:8.0.1-alpine @@ -27,10 +32,18 @@ services: interval: 5s timeout: 3s retries: 10 + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M # Одноразовая задача: применяет миграции и завершается. Реплики API миграции не запускают migrate: - build: . + build: + context: . + args: + APP_VERSION: ${APP_VERSION:-v1} image: mtusi-messenger:lab3 restart: "no" command: ["python", "-m", "src.migrate"] @@ -41,14 +54,24 @@ services: DATABASE_URL_FILE: /run/secrets/database_url secrets: - database_url + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M - # Реплики: число задаётся API_REPLICAS (по умолчанию 2). Без container_name и без published-портов + # Реплики: число API_REPLICAS, ресурсы каждой API_CPUS / API_MEM. Без container_name и published-портов api: image: mtusi-messenger:lab3 pull_policy: never restart: unless-stopped + stop_grace_period: 20s deploy: replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPUS:-0.5}" + memory: ${API_MEM:-256M} depends_on: postgres: condition: service_healthy @@ -60,6 +83,8 @@ services: APP_ENV: ${APP_ENV:-lab} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DRAIN_SECONDS: ${DRAIN_SECONDS:-5} + DB_POOL_MAX: ${DB_POOL_MAX:-10} DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL: redis://valkey:6379/0 secrets: @@ -77,6 +102,11 @@ services: - ./infra/haproxy:/usr/local/etc/haproxy:ro ports: - "127.0.0.1:8080:8080" + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M secrets: postgres_password: diff --git a/evidence/docker-stats.txt b/evidence/docker-stats.txt new file mode 100644 index 00000000..28e6d2fb --- /dev/null +++ b/evidence/docker-stats.txt @@ -0,0 +1,6 @@ +CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS +a66867a76907 mtusi-lab3-lb-1 0.28% 101.8MiB / 128MiB 79.52% 2.66MB / 1.7MB 0B / 0B 11 +f67725f7d715 mtusi-lab3-api-2 0.67% 40.29MiB / 256MiB 15.74% 1.69MB / 2.47MB 0B / 0B 5 +8c67bad5eac5 mtusi-lab3-api-1 0.58% 39.65MiB / 256MiB 15.49% 1.59MB / 2.33MB 0B / 0B 8 +c7d5c8d7b77a mtusi-lab3-postgres-1 2.26% 26.25MiB / 512MiB 5.13% 2.36MB / 1.77MB 0B / 614kB 8 +c36affad0631 mtusi-lab3-valkey-1 2.22% 10.35MiB / 128MiB 8.08% 57.3kB / 50.8kB 0B / 0B 5 diff --git a/evidence/failover-level4.txt b/evidence/failover-level4.txt new file mode 100644 index 00000000..ca7ca55c --- /dev/null +++ b/evidence/failover-level4.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 2.223} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/idempotency-parallel.txt b/evidence/idempotency-parallel.txt new file mode 100644 index 00000000..028bd497 --- /dev/null +++ b/evidence/idempotency-parallel.txt @@ -0,0 +1,2 @@ +{"rounds": 5, "parallel_requests_per_round": 30, "messages_in_history": 5, "replicas_that_answered": ["8c67bad5eac5", "f67725f7d715"]} +PASS: same key in parallel -> exactly one message; different body -> 409 diff --git a/evidence/load-baseline.json b/evidence/load-baseline.json new file mode 100644 index 00000000..a3e55d3b --- /dev/null +++ b/evidence/load-baseline.json @@ -0,0 +1,144 @@ +{ + "summary": { + "duration_s": 30.0, + "concurrency": 8, + "replicas_seen": [ + "8c67bad5eac5", + "f67725f7d715" + ], + "requests": 25691, + "ok": 25691, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 856.4, + "latency_ms": { + "p50": 6.4, + "p95": 28.7, + "p99": 61.7 + }, + "error_statuses": {}, + "action": {} + }, + "per_second": { + "0": { + "ok": 833, + "err": 0 + }, + "1": { + "ok": 1006, + "err": 0 + }, + "2": { + "ok": 1001, + "err": 0 + }, + "3": { + "ok": 656, + "err": 0 + }, + "4": { + "ok": 959, + "err": 0 + }, + "5": { + "ok": 577, + "err": 0 + }, + "6": { + "ok": 1010, + "err": 0 + }, + "7": { + "ok": 934, + "err": 0 + }, + "8": { + "ok": 973, + "err": 0 + }, + "9": { + "ok": 966, + "err": 0 + }, + "10": { + "ok": 976, + "err": 0 + }, + "11": { + "ok": 1011, + "err": 0 + }, + "12": { + "ok": 939, + "err": 0 + }, + "13": { + "ok": 660, + "err": 0 + }, + "14": { + "ok": 865, + "err": 0 + }, + "15": { + "ok": 686, + "err": 0 + }, + "16": { + "ok": 740, + "err": 0 + }, + "17": { + "ok": 632, + "err": 0 + }, + "18": { + "ok": 748, + "err": 0 + }, + "19": { + "ok": 919, + "err": 0 + }, + "20": { + "ok": 909, + "err": 0 + }, + "21": { + "ok": 932, + "err": 0 + }, + "22": { + "ok": 957, + "err": 0 + }, + "23": { + "ok": 953, + "err": 0 + }, + "24": { + "ok": 666, + "err": 0 + }, + "25": { + "ok": 993, + "err": 0 + }, + "26": { + "ok": 650, + "err": 0 + }, + "27": { + "ok": 859, + "err": 0 + }, + "28": { + "ok": 688, + "err": 0 + }, + "29": { + "ok": 993, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-rolling.json b/evidence/load-rolling.json new file mode 100644 index 00000000..ae6fcfb8 --- /dev/null +++ b/evidence/load-rolling.json @@ -0,0 +1,526 @@ +{ + "summary": { + "duration_s": 120.0, + "concurrency": 8, + "replicas_seen": [ + "4d6691ac2fdf", + "71148d9265a0" + ], + "requests": 95152, + "ok": 95152, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 792.9, + "latency_ms": { + "p50": 6.6, + "p95": 34.3, + "p99": 72.1 + }, + "error_statuses": {}, + "action": { + "target": "4d6691ac2fdf", + "started_at": 10.0, + "finished_at": 47.43, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.18, + "instances_in_last_5s": [ + "3ca50eabc36a", + "722fb646fbb5" + ], + "versions_first_last_seen_s": { + "v1": [ + 0.0, + 31.4 + ], + "v2": [ + 21.2, + 120.0 + ] + } + }, + "per_second": { + "0": { + "ok": 865, + "err": 0 + }, + "1": { + "ok": 709, + "err": 0 + }, + "2": { + "ok": 778, + "err": 0 + }, + "3": { + "ok": 657, + "err": 0 + }, + "4": { + "ok": 988, + "err": 0 + }, + "5": { + "ok": 854, + "err": 0 + }, + "6": { + "ok": 874, + "err": 0 + }, + "7": { + "ok": 823, + "err": 0 + }, + "8": { + "ok": 912, + "err": 0 + }, + "9": { + "ok": 909, + "err": 0 + }, + "10": { + "ok": 699, + "err": 0 + }, + "11": { + "ok": 680, + "err": 0 + }, + "12": { + "ok": 659, + "err": 0 + }, + "13": { + "ok": 514, + "err": 0 + }, + "14": { + "ok": 601, + "err": 0 + }, + "15": { + "ok": 563, + "err": 0 + }, + "16": { + "ok": 574, + "err": 0 + }, + "17": { + "ok": 580, + "err": 0 + }, + "18": { + "ok": 404, + "err": 0 + }, + "19": { + "ok": 590, + "err": 0 + }, + "20": { + "ok": 587, + "err": 0 + }, + "21": { + "ok": 683, + "err": 0 + }, + "22": { + "ok": 831, + "err": 0 + }, + "23": { + "ok": 582, + "err": 0 + }, + "24": { + "ok": 605, + "err": 0 + }, + "25": { + "ok": 842, + "err": 0 + }, + "26": { + "ok": 847, + "err": 0 + }, + "27": { + "ok": 881, + "err": 0 + }, + "28": { + "ok": 813, + "err": 0 + }, + "29": { + "ok": 831, + "err": 0 + }, + "30": { + "ok": 849, + "err": 0 + }, + "31": { + "ok": 588, + "err": 0 + }, + "32": { + "ok": 414, + "err": 0 + }, + "33": { + "ok": 549, + "err": 0 + }, + "34": { + "ok": 311, + "err": 0 + }, + "35": { + "ok": 419, + "err": 0 + }, + "36": { + "ok": 458, + "err": 0 + }, + "37": { + "ok": 583, + "err": 0 + }, + "38": { + "ok": 809, + "err": 0 + }, + "39": { + "ok": 972, + "err": 0 + }, + "40": { + "ok": 798, + "err": 0 + }, + "41": { + "ok": 594, + "err": 0 + }, + "42": { + "ok": 909, + "err": 0 + }, + "43": { + "ok": 902, + "err": 0 + }, + "44": { + "ok": 765, + "err": 0 + }, + "45": { + "ok": 699, + "err": 0 + }, + "46": { + "ok": 650, + "err": 0 + }, + "47": { + "ok": 809, + "err": 0 + }, + "48": { + "ok": 972, + "err": 0 + }, + "49": { + "ok": 800, + "err": 0 + }, + "50": { + "ok": 914, + "err": 0 + }, + "51": { + "ok": 655, + "err": 0 + }, + "52": { + "ok": 928, + "err": 0 + }, + "53": { + "ok": 900, + "err": 0 + }, + "54": { + "ok": 897, + "err": 0 + }, + "55": { + "ok": 702, + "err": 0 + }, + "56": { + "ok": 921, + "err": 0 + }, + "57": { + "ok": 891, + "err": 0 + }, + "58": { + "ok": 843, + "err": 0 + }, + "59": { + "ok": 921, + "err": 0 + }, + "60": { + "ok": 920, + "err": 0 + }, + "61": { + "ok": 856, + "err": 0 + }, + "62": { + "ok": 651, + "err": 0 + }, + "63": { + "ok": 889, + "err": 0 + }, + "64": { + "ok": 843, + "err": 0 + }, + "65": { + "ok": 690, + "err": 0 + }, + "66": { + "ok": 823, + "err": 0 + }, + "67": { + "ok": 978, + "err": 0 + }, + "68": { + "ok": 901, + "err": 0 + }, + "69": { + "ok": 911, + "err": 0 + }, + "70": { + "ok": 654, + "err": 0 + }, + "71": { + "ok": 758, + "err": 0 + }, + "72": { + "ok": 451, + "err": 0 + }, + "73": { + "ok": 612, + "err": 0 + }, + "74": { + "ok": 887, + "err": 0 + }, + "75": { + "ok": 915, + "err": 0 + }, + "76": { + "ok": 618, + "err": 0 + }, + "77": { + "ok": 983, + "err": 0 + }, + "78": { + "ok": 933, + "err": 0 + }, + "79": { + "ok": 944, + "err": 0 + }, + "80": { + "ok": 927, + "err": 0 + }, + "81": { + "ok": 960, + "err": 0 + }, + "82": { + "ok": 901, + "err": 0 + }, + "83": { + "ok": 593, + "err": 0 + }, + "84": { + "ok": 969, + "err": 0 + }, + "85": { + "ok": 911, + "err": 0 + }, + "86": { + "ok": 727, + "err": 0 + }, + "87": { + "ok": 836, + "err": 0 + }, + "88": { + "ok": 926, + "err": 0 + }, + "89": { + "ok": 920, + "err": 0 + }, + "90": { + "ok": 962, + "err": 0 + }, + "91": { + "ok": 940, + "err": 0 + }, + "92": { + "ok": 979, + "err": 0 + }, + "93": { + "ok": 691, + "err": 0 + }, + "94": { + "ok": 817, + "err": 0 + }, + "95": { + "ok": 967, + "err": 0 + }, + "96": { + "ok": 887, + "err": 0 + }, + "97": { + "ok": 570, + "err": 0 + }, + "98": { + "ok": 967, + "err": 0 + }, + "99": { + "ok": 934, + "err": 0 + }, + "100": { + "ok": 927, + "err": 0 + }, + "101": { + "ok": 947, + "err": 0 + }, + "102": { + "ok": 916, + "err": 0 + }, + "103": { + "ok": 997, + "err": 0 + }, + "104": { + "ok": 682, + "err": 0 + }, + "105": { + "ok": 902, + "err": 0 + }, + "106": { + "ok": 964, + "err": 0 + }, + "107": { + "ok": 691, + "err": 0 + }, + "108": { + "ok": 849, + "err": 0 + }, + "109": { + "ok": 924, + "err": 0 + }, + "110": { + "ok": 987, + "err": 0 + }, + "111": { + "ok": 935, + "err": 0 + }, + "112": { + "ok": 958, + "err": 0 + }, + "113": { + "ok": 804, + "err": 0 + }, + "114": { + "ok": 669, + "err": 0 + }, + "115": { + "ok": 806, + "err": 0 + }, + "116": { + "ok": 917, + "err": 0 + }, + "117": { + "ok": 969, + "err": 0 + }, + "118": { + "ok": 632, + "err": 0 + }, + "119": { + "ok": 918, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-stop-0-ps.txt b/evidence/load-stop-0-ps.txt new file mode 100644 index 00000000..47ba02e5 --- /dev/null +++ b/evidence/load-stop-0-ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 2 hours (healthy) 8080/tcp +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 2 hours ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp diff --git a/evidence/load-stop-0.json b/evidence/load-stop-0.json new file mode 100644 index 00000000..8cf9433d --- /dev/null +++ b/evidence/load-stop-0.json @@ -0,0 +1,195 @@ +{ + "summary": { + "duration_s": 40.0, + "concurrency": 8, + "replicas_seen": [ + "8c67bad5eac5", + "f67725f7d715" + ], + "requests": 25211, + "ok": 25211, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 630.3, + "latency_ms": { + "p50": 6.9, + "p95": 54.8, + "p99": 85.4 + }, + "error_statuses": {}, + "action": { + "target": "8c67bad5eac5", + "started_at": 10.0, + "finished_at": 12.29, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.17, + "instances_in_last_5s": [ + "f67725f7d715" + ] + }, + "per_second": { + "0": { + "ok": 957, + "err": 0 + }, + "1": { + "ok": 947, + "err": 0 + }, + "2": { + "ok": 918, + "err": 0 + }, + "3": { + "ok": 986, + "err": 0 + }, + "4": { + "ok": 608, + "err": 0 + }, + "5": { + "ok": 928, + "err": 0 + }, + "6": { + "ok": 680, + "err": 0 + }, + "7": { + "ok": 918, + "err": 0 + }, + "8": { + "ok": 913, + "err": 0 + }, + "9": { + "ok": 926, + "err": 0 + }, + "10": { + "ok": 878, + "err": 0 + }, + "11": { + "ok": 897, + "err": 0 + }, + "12": { + "ok": 503, + "err": 0 + }, + "13": { + "ok": 582, + "err": 0 + }, + "14": { + "ok": 488, + "err": 0 + }, + "15": { + "ok": 299, + "err": 0 + }, + "16": { + "ok": 575, + "err": 0 + }, + "17": { + "ok": 581, + "err": 0 + }, + "18": { + "ok": 601, + "err": 0 + }, + "19": { + "ok": 586, + "err": 0 + }, + "20": { + "ok": 585, + "err": 0 + }, + "21": { + "ok": 550, + "err": 0 + }, + "22": { + "ok": 579, + "err": 0 + }, + "23": { + "ok": 579, + "err": 0 + }, + "24": { + "ok": 579, + "err": 0 + }, + "25": { + "ok": 308, + "err": 0 + }, + "26": { + "ok": 378, + "err": 0 + }, + "27": { + "ok": 487, + "err": 0 + }, + "28": { + "ok": 561, + "err": 0 + }, + "29": { + "ok": 595, + "err": 0 + }, + "30": { + "ok": 602, + "err": 0 + }, + "31": { + "ok": 599, + "err": 0 + }, + "32": { + "ok": 548, + "err": 0 + }, + "33": { + "ok": 548, + "err": 0 + }, + "34": { + "ok": 578, + "err": 0 + }, + "35": { + "ok": 570, + "err": 0 + }, + "36": { + "ok": 330, + "err": 0 + }, + "37": { + "ok": 498, + "err": 0 + }, + "38": { + "ok": 484, + "err": 0 + }, + "39": { + "ok": 482, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-stop-1-ps.txt b/evidence/load-stop-1-ps.txt new file mode 100644 index 00000000..bbcccbfc --- /dev/null +++ b/evidence/load-stop-1-ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 56 seconds (healthy) 8080/tcp +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 56 seconds ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp diff --git a/evidence/load-stop-1.json b/evidence/load-stop-1.json new file mode 100644 index 00000000..a050253f --- /dev/null +++ b/evidence/load-stop-1.json @@ -0,0 +1,195 @@ +{ + "summary": { + "duration_s": 40.0, + "concurrency": 8, + "replicas_seen": [ + "8c67bad5eac5", + "f67725f7d715" + ], + "requests": 25380, + "ok": 25380, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 634.5, + "latency_ms": { + "p50": 6.9, + "p95": 54.6, + "p99": 81.2 + }, + "error_statuses": {}, + "action": { + "target": "f67725f7d715", + "started_at": 10.0, + "finished_at": 12.29, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.22, + "instances_in_last_5s": [ + "8c67bad5eac5" + ] + }, + "per_second": { + "0": { + "ok": 866, + "err": 0 + }, + "1": { + "ok": 945, + "err": 0 + }, + "2": { + "ok": 991, + "err": 0 + }, + "3": { + "ok": 987, + "err": 0 + }, + "4": { + "ok": 781, + "err": 0 + }, + "5": { + "ok": 967, + "err": 0 + }, + "6": { + "ok": 911, + "err": 0 + }, + "7": { + "ok": 959, + "err": 0 + }, + "8": { + "ok": 996, + "err": 0 + }, + "9": { + "ok": 611, + "err": 0 + }, + "10": { + "ok": 569, + "err": 0 + }, + "11": { + "ok": 743, + "err": 0 + }, + "12": { + "ok": 546, + "err": 0 + }, + "13": { + "ok": 501, + "err": 0 + }, + "14": { + "ok": 570, + "err": 0 + }, + "15": { + "ok": 600, + "err": 0 + }, + "16": { + "ok": 589, + "err": 0 + }, + "17": { + "ok": 557, + "err": 0 + }, + "18": { + "ok": 485, + "err": 0 + }, + "19": { + "ok": 491, + "err": 0 + }, + "20": { + "ok": 333, + "err": 0 + }, + "21": { + "ok": 571, + "err": 0 + }, + "22": { + "ok": 559, + "err": 0 + }, + "23": { + "ok": 565, + "err": 0 + }, + "24": { + "ok": 567, + "err": 0 + }, + "25": { + "ok": 551, + "err": 0 + }, + "26": { + "ok": 570, + "err": 0 + }, + "27": { + "ok": 536, + "err": 0 + }, + "28": { + "ok": 572, + "err": 0 + }, + "29": { + "ok": 519, + "err": 0 + }, + "30": { + "ok": 424, + "err": 0 + }, + "31": { + "ok": 415, + "err": 0 + }, + "32": { + "ok": 587, + "err": 0 + }, + "33": { + "ok": 541, + "err": 0 + }, + "34": { + "ok": 569, + "err": 0 + }, + "35": { + "ok": 584, + "err": 0 + }, + "36": { + "ok": 591, + "err": 0 + }, + "37": { + "ok": 587, + "err": 0 + }, + "38": { + "ok": 551, + "err": 0 + }, + "39": { + "ok": 523, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/persistence-level4.txt b/evidence/persistence-level4.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence-level4.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/rolling-ps.txt b/evidence/rolling-ps.txt new file mode 100644 index 00000000..f7318c27 --- /dev/null +++ b/evidence/rolling-ps.txt @@ -0,0 +1,6 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-3 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi-lab3-api-4 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 3 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp diff --git a/evidence/sigterm-level4.txt b/evidence/sigterm-level4.txt new file mode 100644 index 00000000..29e91c01 --- /dev/null +++ b/evidence/sigterm-level4.txt @@ -0,0 +1,6 @@ +Image: mtusi-messenger:lab3 +Scenario: idle; docker stop --timeout 20 mtusi-messenger-sigterm-99ecee4b +PASS idle: exit code 0; shutdown 5.353s; new connections rejected after 0.064s +Scenario: incomplete POST; docker stop --timeout 20 mtusi-messenger-sigterm-7da6202d +PASS incomplete POST: exit code 0; shutdown 13.414s; new connections rejected after 0.063s +PASS SIGTERM: both scenarios completed within 10 seconds diff --git a/evidence/smoke-level4.txt b/evidence/smoke-level4.txt new file mode 100644 index 00000000..44e41deb --- /dev/null +++ b/evidence/smoke-level4.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 58bbf906..a422f48f 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -8,6 +8,8 @@ defaults timeout connect 1s timeout client 30s timeout server 10s + timeout check 2s + timeout queue 5s retries 3 option redispatch @@ -28,6 +30,8 @@ frontend public backend api balance roundrobin + # Проверка /health/ready раз в 1 с; backend исключается после 2 неудач подряд, + # возвращается после 2 успехов. Худшее время обнаружения: 2 x 1 с + timeout check 2 с = 4 с. option httpchk http-check send meth GET uri /health/ready http-check expect status 200 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..cce3bf45 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Поочерёдная замена API-реплик новой версией образа (APP_VERSION=v2 scripts/rolling-update.sh). +# Миграции запускаются ОДИН раз до замены; migrations/002 только добавляет таблицу, поэтому старая версия с ней совместима. +set -eu +cd "$(dirname "$0")/.." +: "${APP_VERSION:?set new version, e.g. APP_VERSION=v2}" +export APP_VERSION +COMPOSE="docker compose --env-file .env" + +$COMPOSE build migrate +$COMPOSE run --rm migrate +N=$($COMPOSE ps -q api | wc -l | tr -d ' ') + +wait_healthy() { + i=0 + while [ "$i" -lt 90 ]; do + c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true) + [ "$c" -ge "$N" ] && return 0 + i=$((i + 1)); sleep 1 + done + echo "replicas did not become healthy" >&2; return 1 +} + +for cid in $($COMPOSE ps -q api); do + echo "replacing $cid" >&2 + docker stop -t 20 "$cid" >/dev/null # SIGTERM: readiness 503, drain 5 с, затем завершение + docker rm "$cid" >/dev/null + $COMPOSE up -d --no-deps --no-recreate --scale api="$N" api # создаётся реплика из нового образа + wait_healthy + sleep 5 # LB добавляет новую реплику после 2 успешных проверок +done \ No newline at end of file diff --git a/scripts/run-failover-series.sh b/scripts/run-failover-series.sh new file mode 100755 index 00000000..7b9841ed --- /dev/null +++ b/scripts/run-failover-series.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Базовая нагрузка, затем по очереди отказ каждой API-реплики под нагрузкой с восстановлением между ними. +set -eu +cd "$(dirname "$0")/.." +COMPOSE="docker compose --env-file .env" + +wait_healthy() { + i=0 + while [ "$i" -lt 90 ]; do + c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true) + [ "$c" -ge 2 ] && return 0 + i=$((i + 1)); sleep 1 + done + echo "replicas did not become healthy" >&2; return 1 +} + +python3 student_tests/loadgen.py --duration 30 --output evidence/load-baseline.json +for idx in 0 1; do + python3 student_tests/loadgen.py --duration 40 --action-script scripts/stop-one.sh \ + --action-at 10 --target-index "$idx" --output "evidence/load-stop-$idx.json" + $COMPOSE ps -a > "evidence/load-stop-$idx-ps.txt" + $COMPOSE start api + wait_healthy + sleep 10 # LB возвращает реплику после 2 успешных проверок +done \ No newline at end of file diff --git a/src/config.py b/src/config.py index 521c0342..0d5cec5b 100644 --- a/src/config.py +++ b/src/config.py @@ -26,4 +26,8 @@ CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.5")) # Идентификатор реплики: hostname контейнера = короткий ID контейнера (по нему stop-one.sh находит контейнер) INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() -IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) \ No newline at end of file +IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + +# SIGTERM: сколько секунд реплика отвечает 503 на /health/ready, пока LB её исключает, и продолжает обслуживать запросы. +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "5")) +APP_VERSION = os.environ.get("APP_VERSION", "dev") \ No newline at end of file diff --git a/src/main.py b/src/main.py index deb01747..766ed492 100644 --- a/src/main.py +++ b/src/main.py @@ -6,6 +6,7 @@ import uuid from contextlib import asynccontextmanager import asyncpg +import signal from fastapi import Depends, FastAPI, Request, Response from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, ConfigDict @@ -39,11 +40,31 @@ logger.addFilter(RequestIdFilter()) is_shutting_down = False USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +def _install_drain_handler(app: FastAPI) -> None: + """SIGTERM: сначала /health/ready -> 503, и только через DRAIN_SECONDS обычное завершение uvicorn.""" + loop = asyncio.get_running_loop() + original = signal.getsignal(signal.SIGTERM) + if not callable(original): + return + + def on_sigterm(signum, frame): + if app.state.draining: + return + app.state.draining = True + logger.info("SIGTERM received: draining for %s s", config.DRAIN_SECONDS) + loop.call_soon_threadsafe(loop.call_later, config.DRAIN_SECONDS, original, signum, frame) + + try: + signal.signal(signal.SIGTERM, on_sigterm) + except ValueError: # не главный поток (например, в тестах) + pass @asynccontextmanager async def lifespan(app: FastAPI): + app.state.draining = False app.state.pool = await create_pool_with_retry() app.state.cache = SessionCache(config.CACHE_URL) + _install_drain_handler(app) yield global is_shutting_down is_shutting_down = True @@ -66,6 +87,7 @@ async def middleware_pipeline(request: Request, call_next): response: Response = await call_next(request) response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = config.INSTANCE_ID + response.headers["X-App-Version"] = config.APP_VERSION logger.info(f"Completed {request.method} {request.url.path} with status {response.status_code}", extra={"request_id": req_id}) return response @@ -122,7 +144,9 @@ async def health_live(): @app.get("/health/ready") async def health_ready(request: Request): - # Критичная зависимость для готовности - PostgreSQL. + # Критичная зависимость для готовности - PostgreSQL. При SIGTERM реплика сразу «не готова». + if request.app.state.draining: + return make_error_response(503, "unavailable", "Draining", _req_id(request)) await request.app.state.pool.fetchval("SELECT 1", timeout=2) return {"status": "ok"} diff --git a/student_tests/common.py b/student_tests/common.py new file mode 100644 index 00000000..b649bc94 --- /dev/null +++ b/student_tests/common.py @@ -0,0 +1,34 @@ +import base64 +import json +import urllib.error +import urllib.request +import uuid + + +def call(base, method, path, body=None, headers=None, timeout=3.0): + """Возвращает (status, headers с ключами в нижнем регистре, bytes). Сетевые ошибки поднимаются наружу.""" + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json"} if data else {} + h.update(headers or {}) + req = urllib.request.Request(base + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, {k.lower(): v for k, v in r.headers.items()}, r.read() + except urllib.error.HTTPError as e: + return e.code, {k.lower(): v for k, v in e.headers.items()}, e.read() + + +def setup(base): + """Синтетический пользователь, сессия и чат из одного участника. Возвращает (auth_headers, chat_id).""" + name = "load_" + uuid.uuid4().hex[:12] + password = "load-password-" + uuid.uuid4().hex[:8] + s, _, b = call(base, "POST", "/api/v1/users", {"username": name, "display_name": "Load", "password": password}) + assert s == 201, f"register failed: {s}" + uid = json.loads(b)["id"] + cred = base64.b64encode(f"{name}:{password}".encode()).decode() + s, _, b = call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert s == 201, f"login failed: {s}" + auth = {"Authorization": "Bearer " + json.loads(b)["token"]} + s, _, b = call(base, "POST", "/api/v1/chats", {"title": "load", "member_ids": [uid]}, auth) + assert s == 201, f"chat failed: {s}" + return auth, json.loads(b)["id"] \ No newline at end of file diff --git a/student_tests/loadgen.py b/student_tests/loadgen.py new file mode 100644 index 00000000..aca9941d --- /dev/null +++ b/student_tests/loadgen.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Closed-loop нагрузка POST /messages через LB; во время неё выполняется action-скрипт (остановка реплики). +Без прозрачных retry: каждая неудача = ошибка. Токены в вывод не попадают.""" +import argparse +import json +import os +import subprocess +import threading +import time +import uuid + +from common import call, setup + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--duration", type=float, default=40) + ap.add_argument("--concurrency", type=int, default=8) + ap.add_argument("--action-script") + ap.add_argument("--action-at", type=float, default=10) + ap.add_argument("--target-index", type=int, default=0) + ap.add_argument("--output") + a = ap.parse_args() + + auth, chat = setup(a.base_url) + path = f"/api/v1/chats/{chat}/messages" + ids = set() + for _ in range(40): # какие реплики видны через LB + _, h, _ = call(a.base_url, "GET", "/api/v1/users/me", headers=auth) + ids.add(h.get("x-instance-id")) + ids = sorted(i for i in ids if i) + + records, lock = [], threading.Lock() # (t, status, latency, instance) + t_start = time.monotonic() + t_end = t_start + a.duration + + def worker(): + while time.monotonic() < t_end: + t0 = time.monotonic() + try: + s, h, _ = call(a.base_url, "POST", path, {"text": "load"}, + {**auth, "Idempotency-Key": uuid.uuid4().hex}, timeout=3) + inst, ver = h.get("x-instance-id"), h.get("x-app-version") + except Exception: + s, inst, ver = 0, None, None + with lock: + records.append((t0 - t_start, s, time.monotonic() - t0, inst, ver)) + + threads = [threading.Thread(target=worker) for _ in range(a.concurrency)] + for t in threads: + t.start() + + action = {} + if a.action_script: + time.sleep(a.action_at) + target = ids[a.target_index % len(ids)] + t0 = time.monotonic() - t_start + rc = subprocess.run([a.action_script], env={**os.environ, "TARGET_INSTANCE_ID": target}, + capture_output=True, timeout=300).returncode + action = {"target": target, "started_at": round(t0, 2), + "finished_at": round(time.monotonic() - t_start, 2), "exit_code": rc} + for t in threads: + t.join() + + ok = [r for r in records if r[1] == 201] + bad = [r for r in records if r[1] != 201] + lat = sorted(r[2] for r in ok) + pct = lambda p: round(lat[min(len(lat) - 1, int(len(lat) * p))] * 1000, 1) if lat else None + summary = { + "duration_s": a.duration, "concurrency": a.concurrency, "replicas_seen": ids, + "requests": len(records), "ok": len(ok), "errors": len(bad), + "error_rate": round(len(bad) / max(1, len(records)), 4), + "rps_ok": round(len(ok) / a.duration, 1), + "latency_ms": {"p50": pct(0.50), "p95": pct(0.95), "p99": pct(0.99)}, + "error_statuses": {str(s): sum(1 for r in bad if r[1] == s) for s in {r[1] for r in bad}}, + "action": action, + } + if action: + errs_after = [r[0] for r in bad if r[0] >= action["started_at"]] + times = sorted(r[0] for r in ok if r[0] >= action["started_at"]) + gaps = [b - a_ for a_, b in zip([action["started_at"]] + times, times)] + summary["errors_after_action"] = len(errs_after) + summary["recovery_s"] = round(max(errs_after) - action["started_at"], 2) if errs_after else 0.0 + summary["max_stall_s"] = round(max(gaps), 2) if gaps else None + summary["instances_in_last_5s"] = sorted({r[3] for r in ok if r[0] >= a.duration - 5 and r[3]}) + + vers = {} + for r in ok: + if r[4]: + v = vers.setdefault(r[4], [r[0], r[0]]) + v[0], v[1] = min(v[0], r[0]), max(v[1], r[0]) + summary["versions_first_last_seen_s"] = {k: [round(x, 1) for x in v] for k, v in vers.items()} + + timeline = {} + for t, s, *_ in records: + b = timeline.setdefault(int(t), {"ok": 0, "err": 0}) + b["ok" if s == 201 else "err"] += 1 + print(json.dumps(summary, ensure_ascii=False, indent=2)) + if a.output: + with open(a.output, "w") as f: + json.dump({"summary": summary, "per_second": timeline}, f, ensure_ascii=False, indent=1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_idempotency_parallel.py b/student_tests/test_idempotency_parallel.py new file mode 100644 index 00000000..56706014 --- /dev/null +++ b/student_tests/test_idempotency_parallel.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""30 параллельных POST с одним Idempotency-Key через LB: ровно одно сообщение, везде 201 и один id.""" +import json +import sys +import threading +import uuid + +from common import call, setup + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +THREADS, ROUNDS = 30, 5 + +auth, chat = setup(BASE) +path = f"/api/v1/chats/{chat}/messages" +replicas = set() + +for rnd in range(ROUNDS): + key = uuid.uuid4().hex + barrier = threading.Barrier(THREADS) + results = [None] * THREADS + + def work(i, rnd=rnd, key=key, barrier=barrier, results=results): + barrier.wait() + s, h, b = call(BASE, "POST", path, {"text": f"parallel {rnd}"}, {**auth, "Idempotency-Key": key}, timeout=10) + results[i] = (s, h.get("x-instance-id"), json.loads(b) if s == 201 else None) + + ts = [threading.Thread(target=work, args=(i,)) for i in range(THREADS)] + [t.start() for t in ts] + [t.join() for t in ts] + codes = {r[0] for r in results} + msg_ids = {r[2]["id"] for r in results if r[0] == 201} + replicas |= {r[1] for r in results} + assert codes == {201}, f"round {rnd}: codes {codes}" + assert len(msg_ids) == 1, f"round {rnd}: {len(msg_ids)} different messages" + s, _, _ = call(BASE, "POST", path, {"text": "other body"}, {**auth, "Idempotency-Key": key}) + assert s == 409, f"round {rnd}: other body gave {s}" + +s, _, b = call(BASE, "GET", path + "?limit=100", headers=auth) +count = len(json.loads(b)["items"]) +assert count == ROUNDS, f"history has {count} messages, expected {ROUNDS}" +print(json.dumps({"rounds": ROUNDS, "parallel_requests_per_round": THREADS, + "messages_in_history": count, "replicas_that_answered": sorted(r for r in replicas if r)})) +print("PASS: same key in parallel -> exactly one message; different body -> 409") \ No newline at end of file diff --git a/student_tests/test_sigterm.py b/student_tests/test_sigterm.py index 384cfc36..4651f00a 100644 --- a/student_tests/test_sigterm.py +++ b/student_tests/test_sigterm.py @@ -12,8 +12,8 @@ import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab2") -NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab2_default") +IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab3") +NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab3_default") PORT = int(os.environ.get("SIGTERM_PORT", "18080")) @@ -55,6 +55,15 @@ def connection_rejected(base_url): except (urllib.error.URLError, ConnectionError, TimeoutError): return True +def not_ready(base_url): + # Во время drain /health/ready отвечает 503 (или соединение уже закрыто) + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5): + return False + except urllib.error.HTTPError as exc: + return exc.code == 503 + except (urllib.error.URLError, ConnectionError, TimeoutError): + return True def test_shutdown(incomplete_request): name = "mtusi-messenger-sigterm-" + uuid.uuid4().hex[:8] @@ -82,24 +91,24 @@ def test_shutdown(incomplete_request): raise AssertionError(f"Expected 100 Continue, received {response!r}") pending.sendall(b"{") - print(f"Scenario: {scenario}; docker stop --timeout 10 {name}", flush=True) + print(f"Scenario: {scenario}; docker stop --timeout 20 {name}", flush=True) start = time.monotonic() with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: - stopped = executor.submit(run, "docker", "stop", "--timeout", "10", name) + stopped = executor.submit(run, "docker", "stop", "--timeout", "20", name) deadline = start + 2 - while not connection_rejected(base_url): + while not not_ready(base_url): if time.monotonic() >= deadline: - raise AssertionError("New connections are still accepted during shutdown") + raise AssertionError("Readiness still OK after SIGTERM") time.sleep(0.05) rejected_after = time.monotonic() - start - stopped.result(timeout=12) + stopped.result(timeout=25) elapsed = time.monotonic() - start exit_code = int(run("docker", "inspect", name, "--format", "{{.State.ExitCode}}").stdout) if exit_code != 0: raise AssertionError(f"Container exited with code {exit_code}, expected 0") - if elapsed > 10: - raise AssertionError(f"Shutdown exceeded 10 seconds: {elapsed:.3f}s") + if elapsed > 15: + raise AssertionError(f"Shutdown exceeded 15 seconds: {elapsed:.3f}s") if not connection_rejected(base_url): raise AssertionError("API still accepts connections after shutdown")