ЛР 3 уровень 4: drain при SIGTERM, лимиты ресурсов, нагрузка с отказами, rolling update

This commit is contained in:
2026-11-10 04:17:24 +03:00
parent de55dd1d8a
commit 32442d06db
24 changed files with 1429 additions and 13 deletions
+3
View File
@@ -16,6 +16,9 @@ RUN pip install --no-cache-dir --timeout 60 -r requirements.txt
COPY --chown=app:app src/ /app/src/
COPY --chown=app:app migrations/ /app/migrations/
ARG APP_VERSION=v1
ENV APP_VERSION=${APP_VERSION}
USER app
EXPOSE 8080
+32 -2
View File
@@ -17,6 +17,11 @@ services:
interval: 5s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "${PG_CPUS:-1.0}"
memory: ${PG_MEM:-512M}
valkey:
image: valkey/valkey:8.0.1-alpine
@@ -27,10 +32,18 @@ services:
interval: 5s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
# Одноразовая задача: применяет миграции и завершается. Реплики API миграции не запускают
migrate:
build: .
build:
context: .
args:
APP_VERSION: ${APP_VERSION:-v1}
image: mtusi-messenger:lab3
restart: "no"
command: ["python", "-m", "src.migrate"]
@@ -41,14 +54,24 @@ services:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
# Реплики: число задаётся API_REPLICAS (по умолчанию 2). Без container_name и без published-портов
# Реплики: число API_REPLICAS, ресурсы каждой API_CPUS / API_MEM. Без container_name и published-портов
api:
image: mtusi-messenger:lab3
pull_policy: never
restart: unless-stopped
stop_grace_period: 20s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPUS:-0.5}"
memory: ${API_MEM:-256M}
depends_on:
postgres:
condition: service_healthy
@@ -60,6 +83,8 @@ services:
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_SECONDS: ${DRAIN_SECONDS:-5}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://valkey:6379/0
secrets:
@@ -77,6 +102,11 @@ services:
- ./infra/haproxy:/usr/local/etc/haproxy:ro
ports:
- "127.0.0.1:8080:8080"
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
secrets:
postgres_password:
+6
View File
@@ -0,0 +1,6 @@
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
a66867a76907 mtusi-lab3-lb-1 0.28% 101.8MiB / 128MiB 79.52% 2.66MB / 1.7MB 0B / 0B 11
f67725f7d715 mtusi-lab3-api-2 0.67% 40.29MiB / 256MiB 15.74% 1.69MB / 2.47MB 0B / 0B 5
8c67bad5eac5 mtusi-lab3-api-1 0.58% 39.65MiB / 256MiB 15.49% 1.59MB / 2.33MB 0B / 0B 8
c7d5c8d7b77a mtusi-lab3-postgres-1 2.26% 26.25MiB / 512MiB 5.13% 2.36MB / 1.77MB 0B / 614kB 8
c36affad0631 mtusi-lab3-valkey-1 2.22% 10.35MiB / 128MiB 8.08% 57.3kB / 50.8kB 0B / 0B 5
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 2.223}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
{"rounds": 5, "parallel_requests_per_round": 30, "messages_in_history": 5, "replicas_that_answered": ["8c67bad5eac5", "f67725f7d715"]}
PASS: same key in parallel -> exactly one message; different body -> 409
+144
View File
@@ -0,0 +1,144 @@
{
"summary": {
"duration_s": 30.0,
"concurrency": 8,
"replicas_seen": [
"8c67bad5eac5",
"f67725f7d715"
],
"requests": 25691,
"ok": 25691,
"errors": 0,
"error_rate": 0.0,
"rps_ok": 856.4,
"latency_ms": {
"p50": 6.4,
"p95": 28.7,
"p99": 61.7
},
"error_statuses": {},
"action": {}
},
"per_second": {
"0": {
"ok": 833,
"err": 0
},
"1": {
"ok": 1006,
"err": 0
},
"2": {
"ok": 1001,
"err": 0
},
"3": {
"ok": 656,
"err": 0
},
"4": {
"ok": 959,
"err": 0
},
"5": {
"ok": 577,
"err": 0
},
"6": {
"ok": 1010,
"err": 0
},
"7": {
"ok": 934,
"err": 0
},
"8": {
"ok": 973,
"err": 0
},
"9": {
"ok": 966,
"err": 0
},
"10": {
"ok": 976,
"err": 0
},
"11": {
"ok": 1011,
"err": 0
},
"12": {
"ok": 939,
"err": 0
},
"13": {
"ok": 660,
"err": 0
},
"14": {
"ok": 865,
"err": 0
},
"15": {
"ok": 686,
"err": 0
},
"16": {
"ok": 740,
"err": 0
},
"17": {
"ok": 632,
"err": 0
},
"18": {
"ok": 748,
"err": 0
},
"19": {
"ok": 919,
"err": 0
},
"20": {
"ok": 909,
"err": 0
},
"21": {
"ok": 932,
"err": 0
},
"22": {
"ok": 957,
"err": 0
},
"23": {
"ok": 953,
"err": 0
},
"24": {
"ok": 666,
"err": 0
},
"25": {
"ok": 993,
"err": 0
},
"26": {
"ok": 650,
"err": 0
},
"27": {
"ok": 859,
"err": 0
},
"28": {
"ok": 688,
"err": 0
},
"29": {
"ok": 993,
"err": 0
}
}
}
+526
View File
@@ -0,0 +1,526 @@
{
"summary": {
"duration_s": 120.0,
"concurrency": 8,
"replicas_seen": [
"4d6691ac2fdf",
"71148d9265a0"
],
"requests": 95152,
"ok": 95152,
"errors": 0,
"error_rate": 0.0,
"rps_ok": 792.9,
"latency_ms": {
"p50": 6.6,
"p95": 34.3,
"p99": 72.1
},
"error_statuses": {},
"action": {
"target": "4d6691ac2fdf",
"started_at": 10.0,
"finished_at": 47.43,
"exit_code": 0
},
"errors_after_action": 0,
"recovery_s": 0.0,
"max_stall_s": 0.18,
"instances_in_last_5s": [
"3ca50eabc36a",
"722fb646fbb5"
],
"versions_first_last_seen_s": {
"v1": [
0.0,
31.4
],
"v2": [
21.2,
120.0
]
}
},
"per_second": {
"0": {
"ok": 865,
"err": 0
},
"1": {
"ok": 709,
"err": 0
},
"2": {
"ok": 778,
"err": 0
},
"3": {
"ok": 657,
"err": 0
},
"4": {
"ok": 988,
"err": 0
},
"5": {
"ok": 854,
"err": 0
},
"6": {
"ok": 874,
"err": 0
},
"7": {
"ok": 823,
"err": 0
},
"8": {
"ok": 912,
"err": 0
},
"9": {
"ok": 909,
"err": 0
},
"10": {
"ok": 699,
"err": 0
},
"11": {
"ok": 680,
"err": 0
},
"12": {
"ok": 659,
"err": 0
},
"13": {
"ok": 514,
"err": 0
},
"14": {
"ok": 601,
"err": 0
},
"15": {
"ok": 563,
"err": 0
},
"16": {
"ok": 574,
"err": 0
},
"17": {
"ok": 580,
"err": 0
},
"18": {
"ok": 404,
"err": 0
},
"19": {
"ok": 590,
"err": 0
},
"20": {
"ok": 587,
"err": 0
},
"21": {
"ok": 683,
"err": 0
},
"22": {
"ok": 831,
"err": 0
},
"23": {
"ok": 582,
"err": 0
},
"24": {
"ok": 605,
"err": 0
},
"25": {
"ok": 842,
"err": 0
},
"26": {
"ok": 847,
"err": 0
},
"27": {
"ok": 881,
"err": 0
},
"28": {
"ok": 813,
"err": 0
},
"29": {
"ok": 831,
"err": 0
},
"30": {
"ok": 849,
"err": 0
},
"31": {
"ok": 588,
"err": 0
},
"32": {
"ok": 414,
"err": 0
},
"33": {
"ok": 549,
"err": 0
},
"34": {
"ok": 311,
"err": 0
},
"35": {
"ok": 419,
"err": 0
},
"36": {
"ok": 458,
"err": 0
},
"37": {
"ok": 583,
"err": 0
},
"38": {
"ok": 809,
"err": 0
},
"39": {
"ok": 972,
"err": 0
},
"40": {
"ok": 798,
"err": 0
},
"41": {
"ok": 594,
"err": 0
},
"42": {
"ok": 909,
"err": 0
},
"43": {
"ok": 902,
"err": 0
},
"44": {
"ok": 765,
"err": 0
},
"45": {
"ok": 699,
"err": 0
},
"46": {
"ok": 650,
"err": 0
},
"47": {
"ok": 809,
"err": 0
},
"48": {
"ok": 972,
"err": 0
},
"49": {
"ok": 800,
"err": 0
},
"50": {
"ok": 914,
"err": 0
},
"51": {
"ok": 655,
"err": 0
},
"52": {
"ok": 928,
"err": 0
},
"53": {
"ok": 900,
"err": 0
},
"54": {
"ok": 897,
"err": 0
},
"55": {
"ok": 702,
"err": 0
},
"56": {
"ok": 921,
"err": 0
},
"57": {
"ok": 891,
"err": 0
},
"58": {
"ok": 843,
"err": 0
},
"59": {
"ok": 921,
"err": 0
},
"60": {
"ok": 920,
"err": 0
},
"61": {
"ok": 856,
"err": 0
},
"62": {
"ok": 651,
"err": 0
},
"63": {
"ok": 889,
"err": 0
},
"64": {
"ok": 843,
"err": 0
},
"65": {
"ok": 690,
"err": 0
},
"66": {
"ok": 823,
"err": 0
},
"67": {
"ok": 978,
"err": 0
},
"68": {
"ok": 901,
"err": 0
},
"69": {
"ok": 911,
"err": 0
},
"70": {
"ok": 654,
"err": 0
},
"71": {
"ok": 758,
"err": 0
},
"72": {
"ok": 451,
"err": 0
},
"73": {
"ok": 612,
"err": 0
},
"74": {
"ok": 887,
"err": 0
},
"75": {
"ok": 915,
"err": 0
},
"76": {
"ok": 618,
"err": 0
},
"77": {
"ok": 983,
"err": 0
},
"78": {
"ok": 933,
"err": 0
},
"79": {
"ok": 944,
"err": 0
},
"80": {
"ok": 927,
"err": 0
},
"81": {
"ok": 960,
"err": 0
},
"82": {
"ok": 901,
"err": 0
},
"83": {
"ok": 593,
"err": 0
},
"84": {
"ok": 969,
"err": 0
},
"85": {
"ok": 911,
"err": 0
},
"86": {
"ok": 727,
"err": 0
},
"87": {
"ok": 836,
"err": 0
},
"88": {
"ok": 926,
"err": 0
},
"89": {
"ok": 920,
"err": 0
},
"90": {
"ok": 962,
"err": 0
},
"91": {
"ok": 940,
"err": 0
},
"92": {
"ok": 979,
"err": 0
},
"93": {
"ok": 691,
"err": 0
},
"94": {
"ok": 817,
"err": 0
},
"95": {
"ok": 967,
"err": 0
},
"96": {
"ok": 887,
"err": 0
},
"97": {
"ok": 570,
"err": 0
},
"98": {
"ok": 967,
"err": 0
},
"99": {
"ok": 934,
"err": 0
},
"100": {
"ok": 927,
"err": 0
},
"101": {
"ok": 947,
"err": 0
},
"102": {
"ok": 916,
"err": 0
},
"103": {
"ok": 997,
"err": 0
},
"104": {
"ok": 682,
"err": 0
},
"105": {
"ok": 902,
"err": 0
},
"106": {
"ok": 964,
"err": 0
},
"107": {
"ok": 691,
"err": 0
},
"108": {
"ok": 849,
"err": 0
},
"109": {
"ok": 924,
"err": 0
},
"110": {
"ok": 987,
"err": 0
},
"111": {
"ok": 935,
"err": 0
},
"112": {
"ok": 958,
"err": 0
},
"113": {
"ok": 804,
"err": 0
},
"114": {
"ok": 669,
"err": 0
},
"115": {
"ok": 806,
"err": 0
},
"116": {
"ok": 917,
"err": 0
},
"117": {
"ok": 969,
"err": 0
},
"118": {
"ok": 632,
"err": 0
},
"119": {
"ok": 918,
"err": 0
}
}
}
+7
View File
@@ -0,0 +1,7 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago
mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 2 hours (healthy) 8080/tcp
mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp
mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 2 hours ago
mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp
mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp
+195
View File
@@ -0,0 +1,195 @@
{
"summary": {
"duration_s": 40.0,
"concurrency": 8,
"replicas_seen": [
"8c67bad5eac5",
"f67725f7d715"
],
"requests": 25211,
"ok": 25211,
"errors": 0,
"error_rate": 0.0,
"rps_ok": 630.3,
"latency_ms": {
"p50": 6.9,
"p95": 54.8,
"p99": 85.4
},
"error_statuses": {},
"action": {
"target": "8c67bad5eac5",
"started_at": 10.0,
"finished_at": 12.29,
"exit_code": 0
},
"errors_after_action": 0,
"recovery_s": 0.0,
"max_stall_s": 0.17,
"instances_in_last_5s": [
"f67725f7d715"
]
},
"per_second": {
"0": {
"ok": 957,
"err": 0
},
"1": {
"ok": 947,
"err": 0
},
"2": {
"ok": 918,
"err": 0
},
"3": {
"ok": 986,
"err": 0
},
"4": {
"ok": 608,
"err": 0
},
"5": {
"ok": 928,
"err": 0
},
"6": {
"ok": 680,
"err": 0
},
"7": {
"ok": 918,
"err": 0
},
"8": {
"ok": 913,
"err": 0
},
"9": {
"ok": 926,
"err": 0
},
"10": {
"ok": 878,
"err": 0
},
"11": {
"ok": 897,
"err": 0
},
"12": {
"ok": 503,
"err": 0
},
"13": {
"ok": 582,
"err": 0
},
"14": {
"ok": 488,
"err": 0
},
"15": {
"ok": 299,
"err": 0
},
"16": {
"ok": 575,
"err": 0
},
"17": {
"ok": 581,
"err": 0
},
"18": {
"ok": 601,
"err": 0
},
"19": {
"ok": 586,
"err": 0
},
"20": {
"ok": 585,
"err": 0
},
"21": {
"ok": 550,
"err": 0
},
"22": {
"ok": 579,
"err": 0
},
"23": {
"ok": 579,
"err": 0
},
"24": {
"ok": 579,
"err": 0
},
"25": {
"ok": 308,
"err": 0
},
"26": {
"ok": 378,
"err": 0
},
"27": {
"ok": 487,
"err": 0
},
"28": {
"ok": 561,
"err": 0
},
"29": {
"ok": 595,
"err": 0
},
"30": {
"ok": 602,
"err": 0
},
"31": {
"ok": 599,
"err": 0
},
"32": {
"ok": 548,
"err": 0
},
"33": {
"ok": 548,
"err": 0
},
"34": {
"ok": 578,
"err": 0
},
"35": {
"ok": 570,
"err": 0
},
"36": {
"ok": 330,
"err": 0
},
"37": {
"ok": 498,
"err": 0
},
"38": {
"ok": 484,
"err": 0
},
"39": {
"ok": 482,
"err": 0
}
}
}
+7
View File
@@ -0,0 +1,7 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 56 seconds (healthy) 8080/tcp
mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago
mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp
mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 56 seconds ago
mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp
mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp
+195
View File
@@ -0,0 +1,195 @@
{
"summary": {
"duration_s": 40.0,
"concurrency": 8,
"replicas_seen": [
"8c67bad5eac5",
"f67725f7d715"
],
"requests": 25380,
"ok": 25380,
"errors": 0,
"error_rate": 0.0,
"rps_ok": 634.5,
"latency_ms": {
"p50": 6.9,
"p95": 54.6,
"p99": 81.2
},
"error_statuses": {},
"action": {
"target": "f67725f7d715",
"started_at": 10.0,
"finished_at": 12.29,
"exit_code": 0
},
"errors_after_action": 0,
"recovery_s": 0.0,
"max_stall_s": 0.22,
"instances_in_last_5s": [
"8c67bad5eac5"
]
},
"per_second": {
"0": {
"ok": 866,
"err": 0
},
"1": {
"ok": 945,
"err": 0
},
"2": {
"ok": 991,
"err": 0
},
"3": {
"ok": 987,
"err": 0
},
"4": {
"ok": 781,
"err": 0
},
"5": {
"ok": 967,
"err": 0
},
"6": {
"ok": 911,
"err": 0
},
"7": {
"ok": 959,
"err": 0
},
"8": {
"ok": 996,
"err": 0
},
"9": {
"ok": 611,
"err": 0
},
"10": {
"ok": 569,
"err": 0
},
"11": {
"ok": 743,
"err": 0
},
"12": {
"ok": 546,
"err": 0
},
"13": {
"ok": 501,
"err": 0
},
"14": {
"ok": 570,
"err": 0
},
"15": {
"ok": 600,
"err": 0
},
"16": {
"ok": 589,
"err": 0
},
"17": {
"ok": 557,
"err": 0
},
"18": {
"ok": 485,
"err": 0
},
"19": {
"ok": 491,
"err": 0
},
"20": {
"ok": 333,
"err": 0
},
"21": {
"ok": 571,
"err": 0
},
"22": {
"ok": 559,
"err": 0
},
"23": {
"ok": 565,
"err": 0
},
"24": {
"ok": 567,
"err": 0
},
"25": {
"ok": 551,
"err": 0
},
"26": {
"ok": 570,
"err": 0
},
"27": {
"ok": 536,
"err": 0
},
"28": {
"ok": 572,
"err": 0
},
"29": {
"ok": 519,
"err": 0
},
"30": {
"ok": 424,
"err": 0
},
"31": {
"ok": 415,
"err": 0
},
"32": {
"ok": 587,
"err": 0
},
"33": {
"ok": 541,
"err": 0
},
"34": {
"ok": 569,
"err": 0
},
"35": {
"ok": 584,
"err": 0
},
"36": {
"ok": 591,
"err": 0
},
"37": {
"ok": 587,
"err": 0
},
"38": {
"ok": 551,
"err": 0
},
"39": {
"ok": 523,
"err": 0
}
}
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+6
View File
@@ -0,0 +1,6 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab3-api-3 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi-lab3-api-4 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 3 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp
mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp
mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp
+6
View File
@@ -0,0 +1,6 @@
Image: mtusi-messenger:lab3
Scenario: idle; docker stop --timeout 20 mtusi-messenger-sigterm-99ecee4b
PASS idle: exit code 0; shutdown 5.353s; new connections rejected after 0.064s
Scenario: incomplete POST; docker stop --timeout 20 mtusi-messenger-sigterm-7da6202d
PASS incomplete POST: exit code 0; shutdown 13.414s; new connections rejected after 0.063s
PASS SIGTERM: both scenarios completed within 10 seconds
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
+4
View File
@@ -8,6 +8,8 @@ defaults
timeout connect 1s
timeout client 30s
timeout server 10s
timeout check 2s
timeout queue 5s
retries 3
option redispatch
@@ -28,6 +30,8 @@ frontend public
backend api
balance roundrobin
# Проверка /health/ready раз в 1 с; backend исключается после 2 неудач подряд,
# возвращается после 2 успехов. Худшее время обнаружения: 2 x 1 с + timeout check 2 с = 4 с.
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
# Поочерёдная замена API-реплик новой версией образа (APP_VERSION=v2 scripts/rolling-update.sh).
# Миграции запускаются ОДИН раз до замены; migrations/002 только добавляет таблицу, поэтому старая версия с ней совместима.
set -eu
cd "$(dirname "$0")/.."
: "${APP_VERSION:?set new version, e.g. APP_VERSION=v2}"
export APP_VERSION
COMPOSE="docker compose --env-file .env"
$COMPOSE build migrate
$COMPOSE run --rm migrate
N=$($COMPOSE ps -q api | wc -l | tr -d ' ')
wait_healthy() {
i=0
while [ "$i" -lt 90 ]; do
c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true)
[ "$c" -ge "$N" ] && return 0
i=$((i + 1)); sleep 1
done
echo "replicas did not become healthy" >&2; return 1
}
for cid in $($COMPOSE ps -q api); do
echo "replacing $cid" >&2
docker stop -t 20 "$cid" >/dev/null # SIGTERM: readiness 503, drain 5 с, затем завершение
docker rm "$cid" >/dev/null
$COMPOSE up -d --no-deps --no-recreate --scale api="$N" api # создаётся реплика из нового образа
wait_healthy
sleep 5 # LB добавляет новую реплику после 2 успешных проверок
done
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
# Базовая нагрузка, затем по очереди отказ каждой API-реплики под нагрузкой с восстановлением между ними.
set -eu
cd "$(dirname "$0")/.."
COMPOSE="docker compose --env-file .env"
wait_healthy() {
i=0
while [ "$i" -lt 90 ]; do
c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true)
[ "$c" -ge 2 ] && return 0
i=$((i + 1)); sleep 1
done
echo "replicas did not become healthy" >&2; return 1
}
python3 student_tests/loadgen.py --duration 30 --output evidence/load-baseline.json
for idx in 0 1; do
python3 student_tests/loadgen.py --duration 40 --action-script scripts/stop-one.sh \
--action-at 10 --target-index "$idx" --output "evidence/load-stop-$idx.json"
$COMPOSE ps -a > "evidence/load-stop-$idx-ps.txt"
$COMPOSE start api
wait_healthy
sleep 10 # LB возвращает реплику после 2 успешных проверок
done
+5 -1
View File
@@ -26,4 +26,8 @@ CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.5"))
# Идентификатор реплики: hostname контейнера = короткий ID контейнера (по нему stop-one.sh находит контейнер)
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
# SIGTERM: сколько секунд реплика отвечает 503 на /health/ready, пока LB её исключает, и продолжает обслуживать запросы.
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "5"))
APP_VERSION = os.environ.get("APP_VERSION", "dev")
+25 -1
View File
@@ -6,6 +6,7 @@ import uuid
from contextlib import asynccontextmanager
import asyncpg
import signal
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from pydantic import BaseModel, ConfigDict
@@ -39,11 +40,31 @@ logger.addFilter(RequestIdFilter())
is_shutting_down = False
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
def _install_drain_handler(app: FastAPI) -> None:
"""SIGTERM: сначала /health/ready -> 503, и только через DRAIN_SECONDS обычное завершение uvicorn."""
loop = asyncio.get_running_loop()
original = signal.getsignal(signal.SIGTERM)
if not callable(original):
return
def on_sigterm(signum, frame):
if app.state.draining:
return
app.state.draining = True
logger.info("SIGTERM received: draining for %s s", config.DRAIN_SECONDS)
loop.call_soon_threadsafe(loop.call_later, config.DRAIN_SECONDS, original, signum, frame)
try:
signal.signal(signal.SIGTERM, on_sigterm)
except ValueError: # не главный поток (например, в тестах)
pass
@asynccontextmanager
async def lifespan(app: FastAPI):
app.state.draining = False
app.state.pool = await create_pool_with_retry()
app.state.cache = SessionCache(config.CACHE_URL)
_install_drain_handler(app)
yield
global is_shutting_down
is_shutting_down = True
@@ -66,6 +87,7 @@ async def middleware_pipeline(request: Request, call_next):
response: Response = await call_next(request)
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = config.INSTANCE_ID
response.headers["X-App-Version"] = config.APP_VERSION
logger.info(f"Completed {request.method} {request.url.path} with status {response.status_code}",
extra={"request_id": req_id})
return response
@@ -122,7 +144,9 @@ async def health_live():
@app.get("/health/ready")
async def health_ready(request: Request):
# Критичная зависимость для готовности - PostgreSQL.
# Критичная зависимость для готовности - PostgreSQL. При SIGTERM реплика сразу «не готова».
if request.app.state.draining:
return make_error_response(503, "unavailable", "Draining", _req_id(request))
await request.app.state.pool.fetchval("SELECT 1", timeout=2)
return {"status": "ok"}
+34
View File
@@ -0,0 +1,34 @@
import base64
import json
import urllib.error
import urllib.request
import uuid
def call(base, method, path, body=None, headers=None, timeout=3.0):
"""Возвращает (status, headers с ключами в нижнем регистре, bytes). Сетевые ошибки поднимаются наружу."""
data = json.dumps(body).encode() if body is not None else None
h = {"Content-Type": "application/json"} if data else {}
h.update(headers or {})
req = urllib.request.Request(base + path, data=data, method=method, headers=h)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, {k.lower(): v for k, v in r.headers.items()}, r.read()
except urllib.error.HTTPError as e:
return e.code, {k.lower(): v for k, v in e.headers.items()}, e.read()
def setup(base):
"""Синтетический пользователь, сессия и чат из одного участника. Возвращает (auth_headers, chat_id)."""
name = "load_" + uuid.uuid4().hex[:12]
password = "load-password-" + uuid.uuid4().hex[:8]
s, _, b = call(base, "POST", "/api/v1/users", {"username": name, "display_name": "Load", "password": password})
assert s == 201, f"register failed: {s}"
uid = json.loads(b)["id"]
cred = base64.b64encode(f"{name}:{password}".encode()).decode()
s, _, b = call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert s == 201, f"login failed: {s}"
auth = {"Authorization": "Bearer " + json.loads(b)["token"]}
s, _, b = call(base, "POST", "/api/v1/chats", {"title": "load", "member_ids": [uid]}, auth)
assert s == 201, f"chat failed: {s}"
return auth, json.loads(b)["id"]
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Closed-loop нагрузка POST /messages через LB; во время неё выполняется action-скрипт (остановка реплики).
Без прозрачных retry: каждая неудача = ошибка. Токены в вывод не попадают."""
import argparse
import json
import os
import subprocess
import threading
import time
import uuid
from common import call, setup
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--duration", type=float, default=40)
ap.add_argument("--concurrency", type=int, default=8)
ap.add_argument("--action-script")
ap.add_argument("--action-at", type=float, default=10)
ap.add_argument("--target-index", type=int, default=0)
ap.add_argument("--output")
a = ap.parse_args()
auth, chat = setup(a.base_url)
path = f"/api/v1/chats/{chat}/messages"
ids = set()
for _ in range(40): # какие реплики видны через LB
_, h, _ = call(a.base_url, "GET", "/api/v1/users/me", headers=auth)
ids.add(h.get("x-instance-id"))
ids = sorted(i for i in ids if i)
records, lock = [], threading.Lock() # (t, status, latency, instance)
t_start = time.monotonic()
t_end = t_start + a.duration
def worker():
while time.monotonic() < t_end:
t0 = time.monotonic()
try:
s, h, _ = call(a.base_url, "POST", path, {"text": "load"},
{**auth, "Idempotency-Key": uuid.uuid4().hex}, timeout=3)
inst, ver = h.get("x-instance-id"), h.get("x-app-version")
except Exception:
s, inst, ver = 0, None, None
with lock:
records.append((t0 - t_start, s, time.monotonic() - t0, inst, ver))
threads = [threading.Thread(target=worker) for _ in range(a.concurrency)]
for t in threads:
t.start()
action = {}
if a.action_script:
time.sleep(a.action_at)
target = ids[a.target_index % len(ids)]
t0 = time.monotonic() - t_start
rc = subprocess.run([a.action_script], env={**os.environ, "TARGET_INSTANCE_ID": target},
capture_output=True, timeout=300).returncode
action = {"target": target, "started_at": round(t0, 2),
"finished_at": round(time.monotonic() - t_start, 2), "exit_code": rc}
for t in threads:
t.join()
ok = [r for r in records if r[1] == 201]
bad = [r for r in records if r[1] != 201]
lat = sorted(r[2] for r in ok)
pct = lambda p: round(lat[min(len(lat) - 1, int(len(lat) * p))] * 1000, 1) if lat else None
summary = {
"duration_s": a.duration, "concurrency": a.concurrency, "replicas_seen": ids,
"requests": len(records), "ok": len(ok), "errors": len(bad),
"error_rate": round(len(bad) / max(1, len(records)), 4),
"rps_ok": round(len(ok) / a.duration, 1),
"latency_ms": {"p50": pct(0.50), "p95": pct(0.95), "p99": pct(0.99)},
"error_statuses": {str(s): sum(1 for r in bad if r[1] == s) for s in {r[1] for r in bad}},
"action": action,
}
if action:
errs_after = [r[0] for r in bad if r[0] >= action["started_at"]]
times = sorted(r[0] for r in ok if r[0] >= action["started_at"])
gaps = [b - a_ for a_, b in zip([action["started_at"]] + times, times)]
summary["errors_after_action"] = len(errs_after)
summary["recovery_s"] = round(max(errs_after) - action["started_at"], 2) if errs_after else 0.0
summary["max_stall_s"] = round(max(gaps), 2) if gaps else None
summary["instances_in_last_5s"] = sorted({r[3] for r in ok if r[0] >= a.duration - 5 and r[3]})
vers = {}
for r in ok:
if r[4]:
v = vers.setdefault(r[4], [r[0], r[0]])
v[0], v[1] = min(v[0], r[0]), max(v[1], r[0])
summary["versions_first_last_seen_s"] = {k: [round(x, 1) for x in v] for k, v in vers.items()}
timeline = {}
for t, s, *_ in records:
b = timeline.setdefault(int(t), {"ok": 0, "err": 0})
b["ok" if s == 201 else "err"] += 1
print(json.dumps(summary, ensure_ascii=False, indent=2))
if a.output:
with open(a.output, "w") as f:
json.dump({"summary": summary, "per_second": timeline}, f, ensure_ascii=False, indent=1)
if __name__ == "__main__":
main()
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""30 параллельных POST с одним Idempotency-Key через LB: ровно одно сообщение, везде 201 и один id."""
import json
import sys
import threading
import uuid
from common import call, setup
BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080"
THREADS, ROUNDS = 30, 5
auth, chat = setup(BASE)
path = f"/api/v1/chats/{chat}/messages"
replicas = set()
for rnd in range(ROUNDS):
key = uuid.uuid4().hex
barrier = threading.Barrier(THREADS)
results = [None] * THREADS
def work(i, rnd=rnd, key=key, barrier=barrier, results=results):
barrier.wait()
s, h, b = call(BASE, "POST", path, {"text": f"parallel {rnd}"}, {**auth, "Idempotency-Key": key}, timeout=10)
results[i] = (s, h.get("x-instance-id"), json.loads(b) if s == 201 else None)
ts = [threading.Thread(target=work, args=(i,)) for i in range(THREADS)]
[t.start() for t in ts]
[t.join() for t in ts]
codes = {r[0] for r in results}
msg_ids = {r[2]["id"] for r in results if r[0] == 201}
replicas |= {r[1] for r in results}
assert codes == {201}, f"round {rnd}: codes {codes}"
assert len(msg_ids) == 1, f"round {rnd}: {len(msg_ids)} different messages"
s, _, _ = call(BASE, "POST", path, {"text": "other body"}, {**auth, "Idempotency-Key": key})
assert s == 409, f"round {rnd}: other body gave {s}"
s, _, b = call(BASE, "GET", path + "?limit=100", headers=auth)
count = len(json.loads(b)["items"])
assert count == ROUNDS, f"history has {count} messages, expected {ROUNDS}"
print(json.dumps({"rounds": ROUNDS, "parallel_requests_per_round": THREADS,
"messages_in_history": count, "replicas_that_answered": sorted(r for r in replicas if r)}))
print("PASS: same key in parallel -> exactly one message; different body -> 409")
+18 -9
View File
@@ -12,8 +12,8 @@ import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab2")
NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab2_default")
IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab3")
NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab3_default")
PORT = int(os.environ.get("SIGTERM_PORT", "18080"))
@@ -55,6 +55,15 @@ def connection_rejected(base_url):
except (urllib.error.URLError, ConnectionError, TimeoutError):
return True
def not_ready(base_url):
# Во время drain /health/ready отвечает 503 (или соединение уже закрыто)
try:
with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5):
return False
except urllib.error.HTTPError as exc:
return exc.code == 503
except (urllib.error.URLError, ConnectionError, TimeoutError):
return True
def test_shutdown(incomplete_request):
name = "mtusi-messenger-sigterm-" + uuid.uuid4().hex[:8]
@@ -82,24 +91,24 @@ def test_shutdown(incomplete_request):
raise AssertionError(f"Expected 100 Continue, received {response!r}")
pending.sendall(b"{")
print(f"Scenario: {scenario}; docker stop --timeout 10 {name}", flush=True)
print(f"Scenario: {scenario}; docker stop --timeout 20 {name}", flush=True)
start = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
stopped = executor.submit(run, "docker", "stop", "--timeout", "10", name)
stopped = executor.submit(run, "docker", "stop", "--timeout", "20", name)
deadline = start + 2
while not connection_rejected(base_url):
while not not_ready(base_url):
if time.monotonic() >= deadline:
raise AssertionError("New connections are still accepted during shutdown")
raise AssertionError("Readiness still OK after SIGTERM")
time.sleep(0.05)
rejected_after = time.monotonic() - start
stopped.result(timeout=12)
stopped.result(timeout=25)
elapsed = time.monotonic() - start
exit_code = int(run("docker", "inspect", name, "--format", "{{.State.ExitCode}}").stdout)
if exit_code != 0:
raise AssertionError(f"Container exited with code {exit_code}, expected 0")
if elapsed > 10:
raise AssertionError(f"Shutdown exceeded 10 seconds: {elapsed:.3f}s")
if elapsed > 15:
raise AssertionError(f"Shutdown exceeded 15 seconds: {elapsed:.3f}s")
if not connection_rejected(base_url):
raise AssertionError("API still accepts connections after shutdown")