forked from mtusi2026/mtusi2026lab3
ЛР 3 уровень 4: drain при SIGTERM, лимиты ресурсов, нагрузка с отказами, rolling update
This commit is contained in:
@@ -16,6 +16,9 @@ RUN pip install --no-cache-dir --timeout 60 -r requirements.txt
|
||||
COPY --chown=app:app src/ /app/src/
|
||||
COPY --chown=app:app migrations/ /app/migrations/
|
||||
|
||||
ARG APP_VERSION=v1
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
+32
-2
@@ -17,6 +17,11 @@ services:
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${PG_CPUS:-1.0}"
|
||||
memory: ${PG_MEM:-512M}
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.1-alpine
|
||||
@@ -27,10 +32,18 @@ services:
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
# Одноразовая задача: применяет миграции и завершается. Реплики API миграции не запускают
|
||||
migrate:
|
||||
build: .
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-v1}
|
||||
image: mtusi-messenger:lab3
|
||||
restart: "no"
|
||||
command: ["python", "-m", "src.migrate"]
|
||||
@@ -41,14 +54,24 @@ services:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
# Реплики: число задаётся API_REPLICAS (по умолчанию 2). Без container_name и без published-портов
|
||||
# Реплики: число API_REPLICAS, ресурсы каждой API_CPUS / API_MEM. Без container_name и published-портов
|
||||
api:
|
||||
image: mtusi-messenger:lab3
|
||||
pull_policy: never
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-0.5}"
|
||||
memory: ${API_MEM:-256M}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -60,6 +83,8 @@ services:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-5}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: redis://valkey:6379/0
|
||||
secrets:
|
||||
@@ -77,6 +102,11 @@ services:
|
||||
- ./infra/haproxy:/usr/local/etc/haproxy:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
|
||||
a66867a76907 mtusi-lab3-lb-1 0.28% 101.8MiB / 128MiB 79.52% 2.66MB / 1.7MB 0B / 0B 11
|
||||
f67725f7d715 mtusi-lab3-api-2 0.67% 40.29MiB / 256MiB 15.74% 1.69MB / 2.47MB 0B / 0B 5
|
||||
8c67bad5eac5 mtusi-lab3-api-1 0.58% 39.65MiB / 256MiB 15.49% 1.59MB / 2.33MB 0B / 0B 8
|
||||
c7d5c8d7b77a mtusi-lab3-postgres-1 2.26% 26.25MiB / 512MiB 5.13% 2.36MB / 1.77MB 0B / 614kB 8
|
||||
c36affad0631 mtusi-lab3-valkey-1 2.22% 10.35MiB / 128MiB 8.08% 57.3kB / 50.8kB 0B / 0B 5
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 2.223}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
{"rounds": 5, "parallel_requests_per_round": 30, "messages_in_history": 5, "replicas_that_answered": ["8c67bad5eac5", "f67725f7d715"]}
|
||||
PASS: same key in parallel -> exactly one message; different body -> 409
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"summary": {
|
||||
"duration_s": 30.0,
|
||||
"concurrency": 8,
|
||||
"replicas_seen": [
|
||||
"8c67bad5eac5",
|
||||
"f67725f7d715"
|
||||
],
|
||||
"requests": 25691,
|
||||
"ok": 25691,
|
||||
"errors": 0,
|
||||
"error_rate": 0.0,
|
||||
"rps_ok": 856.4,
|
||||
"latency_ms": {
|
||||
"p50": 6.4,
|
||||
"p95": 28.7,
|
||||
"p99": 61.7
|
||||
},
|
||||
"error_statuses": {},
|
||||
"action": {}
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"ok": 833,
|
||||
"err": 0
|
||||
},
|
||||
"1": {
|
||||
"ok": 1006,
|
||||
"err": 0
|
||||
},
|
||||
"2": {
|
||||
"ok": 1001,
|
||||
"err": 0
|
||||
},
|
||||
"3": {
|
||||
"ok": 656,
|
||||
"err": 0
|
||||
},
|
||||
"4": {
|
||||
"ok": 959,
|
||||
"err": 0
|
||||
},
|
||||
"5": {
|
||||
"ok": 577,
|
||||
"err": 0
|
||||
},
|
||||
"6": {
|
||||
"ok": 1010,
|
||||
"err": 0
|
||||
},
|
||||
"7": {
|
||||
"ok": 934,
|
||||
"err": 0
|
||||
},
|
||||
"8": {
|
||||
"ok": 973,
|
||||
"err": 0
|
||||
},
|
||||
"9": {
|
||||
"ok": 966,
|
||||
"err": 0
|
||||
},
|
||||
"10": {
|
||||
"ok": 976,
|
||||
"err": 0
|
||||
},
|
||||
"11": {
|
||||
"ok": 1011,
|
||||
"err": 0
|
||||
},
|
||||
"12": {
|
||||
"ok": 939,
|
||||
"err": 0
|
||||
},
|
||||
"13": {
|
||||
"ok": 660,
|
||||
"err": 0
|
||||
},
|
||||
"14": {
|
||||
"ok": 865,
|
||||
"err": 0
|
||||
},
|
||||
"15": {
|
||||
"ok": 686,
|
||||
"err": 0
|
||||
},
|
||||
"16": {
|
||||
"ok": 740,
|
||||
"err": 0
|
||||
},
|
||||
"17": {
|
||||
"ok": 632,
|
||||
"err": 0
|
||||
},
|
||||
"18": {
|
||||
"ok": 748,
|
||||
"err": 0
|
||||
},
|
||||
"19": {
|
||||
"ok": 919,
|
||||
"err": 0
|
||||
},
|
||||
"20": {
|
||||
"ok": 909,
|
||||
"err": 0
|
||||
},
|
||||
"21": {
|
||||
"ok": 932,
|
||||
"err": 0
|
||||
},
|
||||
"22": {
|
||||
"ok": 957,
|
||||
"err": 0
|
||||
},
|
||||
"23": {
|
||||
"ok": 953,
|
||||
"err": 0
|
||||
},
|
||||
"24": {
|
||||
"ok": 666,
|
||||
"err": 0
|
||||
},
|
||||
"25": {
|
||||
"ok": 993,
|
||||
"err": 0
|
||||
},
|
||||
"26": {
|
||||
"ok": 650,
|
||||
"err": 0
|
||||
},
|
||||
"27": {
|
||||
"ok": 859,
|
||||
"err": 0
|
||||
},
|
||||
"28": {
|
||||
"ok": 688,
|
||||
"err": 0
|
||||
},
|
||||
"29": {
|
||||
"ok": 993,
|
||||
"err": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,526 @@
|
||||
{
|
||||
"summary": {
|
||||
"duration_s": 120.0,
|
||||
"concurrency": 8,
|
||||
"replicas_seen": [
|
||||
"4d6691ac2fdf",
|
||||
"71148d9265a0"
|
||||
],
|
||||
"requests": 95152,
|
||||
"ok": 95152,
|
||||
"errors": 0,
|
||||
"error_rate": 0.0,
|
||||
"rps_ok": 792.9,
|
||||
"latency_ms": {
|
||||
"p50": 6.6,
|
||||
"p95": 34.3,
|
||||
"p99": 72.1
|
||||
},
|
||||
"error_statuses": {},
|
||||
"action": {
|
||||
"target": "4d6691ac2fdf",
|
||||
"started_at": 10.0,
|
||||
"finished_at": 47.43,
|
||||
"exit_code": 0
|
||||
},
|
||||
"errors_after_action": 0,
|
||||
"recovery_s": 0.0,
|
||||
"max_stall_s": 0.18,
|
||||
"instances_in_last_5s": [
|
||||
"3ca50eabc36a",
|
||||
"722fb646fbb5"
|
||||
],
|
||||
"versions_first_last_seen_s": {
|
||||
"v1": [
|
||||
0.0,
|
||||
31.4
|
||||
],
|
||||
"v2": [
|
||||
21.2,
|
||||
120.0
|
||||
]
|
||||
}
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"ok": 865,
|
||||
"err": 0
|
||||
},
|
||||
"1": {
|
||||
"ok": 709,
|
||||
"err": 0
|
||||
},
|
||||
"2": {
|
||||
"ok": 778,
|
||||
"err": 0
|
||||
},
|
||||
"3": {
|
||||
"ok": 657,
|
||||
"err": 0
|
||||
},
|
||||
"4": {
|
||||
"ok": 988,
|
||||
"err": 0
|
||||
},
|
||||
"5": {
|
||||
"ok": 854,
|
||||
"err": 0
|
||||
},
|
||||
"6": {
|
||||
"ok": 874,
|
||||
"err": 0
|
||||
},
|
||||
"7": {
|
||||
"ok": 823,
|
||||
"err": 0
|
||||
},
|
||||
"8": {
|
||||
"ok": 912,
|
||||
"err": 0
|
||||
},
|
||||
"9": {
|
||||
"ok": 909,
|
||||
"err": 0
|
||||
},
|
||||
"10": {
|
||||
"ok": 699,
|
||||
"err": 0
|
||||
},
|
||||
"11": {
|
||||
"ok": 680,
|
||||
"err": 0
|
||||
},
|
||||
"12": {
|
||||
"ok": 659,
|
||||
"err": 0
|
||||
},
|
||||
"13": {
|
||||
"ok": 514,
|
||||
"err": 0
|
||||
},
|
||||
"14": {
|
||||
"ok": 601,
|
||||
"err": 0
|
||||
},
|
||||
"15": {
|
||||
"ok": 563,
|
||||
"err": 0
|
||||
},
|
||||
"16": {
|
||||
"ok": 574,
|
||||
"err": 0
|
||||
},
|
||||
"17": {
|
||||
"ok": 580,
|
||||
"err": 0
|
||||
},
|
||||
"18": {
|
||||
"ok": 404,
|
||||
"err": 0
|
||||
},
|
||||
"19": {
|
||||
"ok": 590,
|
||||
"err": 0
|
||||
},
|
||||
"20": {
|
||||
"ok": 587,
|
||||
"err": 0
|
||||
},
|
||||
"21": {
|
||||
"ok": 683,
|
||||
"err": 0
|
||||
},
|
||||
"22": {
|
||||
"ok": 831,
|
||||
"err": 0
|
||||
},
|
||||
"23": {
|
||||
"ok": 582,
|
||||
"err": 0
|
||||
},
|
||||
"24": {
|
||||
"ok": 605,
|
||||
"err": 0
|
||||
},
|
||||
"25": {
|
||||
"ok": 842,
|
||||
"err": 0
|
||||
},
|
||||
"26": {
|
||||
"ok": 847,
|
||||
"err": 0
|
||||
},
|
||||
"27": {
|
||||
"ok": 881,
|
||||
"err": 0
|
||||
},
|
||||
"28": {
|
||||
"ok": 813,
|
||||
"err": 0
|
||||
},
|
||||
"29": {
|
||||
"ok": 831,
|
||||
"err": 0
|
||||
},
|
||||
"30": {
|
||||
"ok": 849,
|
||||
"err": 0
|
||||
},
|
||||
"31": {
|
||||
"ok": 588,
|
||||
"err": 0
|
||||
},
|
||||
"32": {
|
||||
"ok": 414,
|
||||
"err": 0
|
||||
},
|
||||
"33": {
|
||||
"ok": 549,
|
||||
"err": 0
|
||||
},
|
||||
"34": {
|
||||
"ok": 311,
|
||||
"err": 0
|
||||
},
|
||||
"35": {
|
||||
"ok": 419,
|
||||
"err": 0
|
||||
},
|
||||
"36": {
|
||||
"ok": 458,
|
||||
"err": 0
|
||||
},
|
||||
"37": {
|
||||
"ok": 583,
|
||||
"err": 0
|
||||
},
|
||||
"38": {
|
||||
"ok": 809,
|
||||
"err": 0
|
||||
},
|
||||
"39": {
|
||||
"ok": 972,
|
||||
"err": 0
|
||||
},
|
||||
"40": {
|
||||
"ok": 798,
|
||||
"err": 0
|
||||
},
|
||||
"41": {
|
||||
"ok": 594,
|
||||
"err": 0
|
||||
},
|
||||
"42": {
|
||||
"ok": 909,
|
||||
"err": 0
|
||||
},
|
||||
"43": {
|
||||
"ok": 902,
|
||||
"err": 0
|
||||
},
|
||||
"44": {
|
||||
"ok": 765,
|
||||
"err": 0
|
||||
},
|
||||
"45": {
|
||||
"ok": 699,
|
||||
"err": 0
|
||||
},
|
||||
"46": {
|
||||
"ok": 650,
|
||||
"err": 0
|
||||
},
|
||||
"47": {
|
||||
"ok": 809,
|
||||
"err": 0
|
||||
},
|
||||
"48": {
|
||||
"ok": 972,
|
||||
"err": 0
|
||||
},
|
||||
"49": {
|
||||
"ok": 800,
|
||||
"err": 0
|
||||
},
|
||||
"50": {
|
||||
"ok": 914,
|
||||
"err": 0
|
||||
},
|
||||
"51": {
|
||||
"ok": 655,
|
||||
"err": 0
|
||||
},
|
||||
"52": {
|
||||
"ok": 928,
|
||||
"err": 0
|
||||
},
|
||||
"53": {
|
||||
"ok": 900,
|
||||
"err": 0
|
||||
},
|
||||
"54": {
|
||||
"ok": 897,
|
||||
"err": 0
|
||||
},
|
||||
"55": {
|
||||
"ok": 702,
|
||||
"err": 0
|
||||
},
|
||||
"56": {
|
||||
"ok": 921,
|
||||
"err": 0
|
||||
},
|
||||
"57": {
|
||||
"ok": 891,
|
||||
"err": 0
|
||||
},
|
||||
"58": {
|
||||
"ok": 843,
|
||||
"err": 0
|
||||
},
|
||||
"59": {
|
||||
"ok": 921,
|
||||
"err": 0
|
||||
},
|
||||
"60": {
|
||||
"ok": 920,
|
||||
"err": 0
|
||||
},
|
||||
"61": {
|
||||
"ok": 856,
|
||||
"err": 0
|
||||
},
|
||||
"62": {
|
||||
"ok": 651,
|
||||
"err": 0
|
||||
},
|
||||
"63": {
|
||||
"ok": 889,
|
||||
"err": 0
|
||||
},
|
||||
"64": {
|
||||
"ok": 843,
|
||||
"err": 0
|
||||
},
|
||||
"65": {
|
||||
"ok": 690,
|
||||
"err": 0
|
||||
},
|
||||
"66": {
|
||||
"ok": 823,
|
||||
"err": 0
|
||||
},
|
||||
"67": {
|
||||
"ok": 978,
|
||||
"err": 0
|
||||
},
|
||||
"68": {
|
||||
"ok": 901,
|
||||
"err": 0
|
||||
},
|
||||
"69": {
|
||||
"ok": 911,
|
||||
"err": 0
|
||||
},
|
||||
"70": {
|
||||
"ok": 654,
|
||||
"err": 0
|
||||
},
|
||||
"71": {
|
||||
"ok": 758,
|
||||
"err": 0
|
||||
},
|
||||
"72": {
|
||||
"ok": 451,
|
||||
"err": 0
|
||||
},
|
||||
"73": {
|
||||
"ok": 612,
|
||||
"err": 0
|
||||
},
|
||||
"74": {
|
||||
"ok": 887,
|
||||
"err": 0
|
||||
},
|
||||
"75": {
|
||||
"ok": 915,
|
||||
"err": 0
|
||||
},
|
||||
"76": {
|
||||
"ok": 618,
|
||||
"err": 0
|
||||
},
|
||||
"77": {
|
||||
"ok": 983,
|
||||
"err": 0
|
||||
},
|
||||
"78": {
|
||||
"ok": 933,
|
||||
"err": 0
|
||||
},
|
||||
"79": {
|
||||
"ok": 944,
|
||||
"err": 0
|
||||
},
|
||||
"80": {
|
||||
"ok": 927,
|
||||
"err": 0
|
||||
},
|
||||
"81": {
|
||||
"ok": 960,
|
||||
"err": 0
|
||||
},
|
||||
"82": {
|
||||
"ok": 901,
|
||||
"err": 0
|
||||
},
|
||||
"83": {
|
||||
"ok": 593,
|
||||
"err": 0
|
||||
},
|
||||
"84": {
|
||||
"ok": 969,
|
||||
"err": 0
|
||||
},
|
||||
"85": {
|
||||
"ok": 911,
|
||||
"err": 0
|
||||
},
|
||||
"86": {
|
||||
"ok": 727,
|
||||
"err": 0
|
||||
},
|
||||
"87": {
|
||||
"ok": 836,
|
||||
"err": 0
|
||||
},
|
||||
"88": {
|
||||
"ok": 926,
|
||||
"err": 0
|
||||
},
|
||||
"89": {
|
||||
"ok": 920,
|
||||
"err": 0
|
||||
},
|
||||
"90": {
|
||||
"ok": 962,
|
||||
"err": 0
|
||||
},
|
||||
"91": {
|
||||
"ok": 940,
|
||||
"err": 0
|
||||
},
|
||||
"92": {
|
||||
"ok": 979,
|
||||
"err": 0
|
||||
},
|
||||
"93": {
|
||||
"ok": 691,
|
||||
"err": 0
|
||||
},
|
||||
"94": {
|
||||
"ok": 817,
|
||||
"err": 0
|
||||
},
|
||||
"95": {
|
||||
"ok": 967,
|
||||
"err": 0
|
||||
},
|
||||
"96": {
|
||||
"ok": 887,
|
||||
"err": 0
|
||||
},
|
||||
"97": {
|
||||
"ok": 570,
|
||||
"err": 0
|
||||
},
|
||||
"98": {
|
||||
"ok": 967,
|
||||
"err": 0
|
||||
},
|
||||
"99": {
|
||||
"ok": 934,
|
||||
"err": 0
|
||||
},
|
||||
"100": {
|
||||
"ok": 927,
|
||||
"err": 0
|
||||
},
|
||||
"101": {
|
||||
"ok": 947,
|
||||
"err": 0
|
||||
},
|
||||
"102": {
|
||||
"ok": 916,
|
||||
"err": 0
|
||||
},
|
||||
"103": {
|
||||
"ok": 997,
|
||||
"err": 0
|
||||
},
|
||||
"104": {
|
||||
"ok": 682,
|
||||
"err": 0
|
||||
},
|
||||
"105": {
|
||||
"ok": 902,
|
||||
"err": 0
|
||||
},
|
||||
"106": {
|
||||
"ok": 964,
|
||||
"err": 0
|
||||
},
|
||||
"107": {
|
||||
"ok": 691,
|
||||
"err": 0
|
||||
},
|
||||
"108": {
|
||||
"ok": 849,
|
||||
"err": 0
|
||||
},
|
||||
"109": {
|
||||
"ok": 924,
|
||||
"err": 0
|
||||
},
|
||||
"110": {
|
||||
"ok": 987,
|
||||
"err": 0
|
||||
},
|
||||
"111": {
|
||||
"ok": 935,
|
||||
"err": 0
|
||||
},
|
||||
"112": {
|
||||
"ok": 958,
|
||||
"err": 0
|
||||
},
|
||||
"113": {
|
||||
"ok": 804,
|
||||
"err": 0
|
||||
},
|
||||
"114": {
|
||||
"ok": 669,
|
||||
"err": 0
|
||||
},
|
||||
"115": {
|
||||
"ok": 806,
|
||||
"err": 0
|
||||
},
|
||||
"116": {
|
||||
"ok": 917,
|
||||
"err": 0
|
||||
},
|
||||
"117": {
|
||||
"ok": 969,
|
||||
"err": 0
|
||||
},
|
||||
"118": {
|
||||
"ok": 632,
|
||||
"err": 0
|
||||
},
|
||||
"119": {
|
||||
"ok": 918,
|
||||
"err": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago
|
||||
mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 2 hours (healthy) 8080/tcp
|
||||
mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 2 hours ago
|
||||
mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp
|
||||
mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp
|
||||
@@ -0,0 +1,195 @@
|
||||
{
|
||||
"summary": {
|
||||
"duration_s": 40.0,
|
||||
"concurrency": 8,
|
||||
"replicas_seen": [
|
||||
"8c67bad5eac5",
|
||||
"f67725f7d715"
|
||||
],
|
||||
"requests": 25211,
|
||||
"ok": 25211,
|
||||
"errors": 0,
|
||||
"error_rate": 0.0,
|
||||
"rps_ok": 630.3,
|
||||
"latency_ms": {
|
||||
"p50": 6.9,
|
||||
"p95": 54.8,
|
||||
"p99": 85.4
|
||||
},
|
||||
"error_statuses": {},
|
||||
"action": {
|
||||
"target": "8c67bad5eac5",
|
||||
"started_at": 10.0,
|
||||
"finished_at": 12.29,
|
||||
"exit_code": 0
|
||||
},
|
||||
"errors_after_action": 0,
|
||||
"recovery_s": 0.0,
|
||||
"max_stall_s": 0.17,
|
||||
"instances_in_last_5s": [
|
||||
"f67725f7d715"
|
||||
]
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"ok": 957,
|
||||
"err": 0
|
||||
},
|
||||
"1": {
|
||||
"ok": 947,
|
||||
"err": 0
|
||||
},
|
||||
"2": {
|
||||
"ok": 918,
|
||||
"err": 0
|
||||
},
|
||||
"3": {
|
||||
"ok": 986,
|
||||
"err": 0
|
||||
},
|
||||
"4": {
|
||||
"ok": 608,
|
||||
"err": 0
|
||||
},
|
||||
"5": {
|
||||
"ok": 928,
|
||||
"err": 0
|
||||
},
|
||||
"6": {
|
||||
"ok": 680,
|
||||
"err": 0
|
||||
},
|
||||
"7": {
|
||||
"ok": 918,
|
||||
"err": 0
|
||||
},
|
||||
"8": {
|
||||
"ok": 913,
|
||||
"err": 0
|
||||
},
|
||||
"9": {
|
||||
"ok": 926,
|
||||
"err": 0
|
||||
},
|
||||
"10": {
|
||||
"ok": 878,
|
||||
"err": 0
|
||||
},
|
||||
"11": {
|
||||
"ok": 897,
|
||||
"err": 0
|
||||
},
|
||||
"12": {
|
||||
"ok": 503,
|
||||
"err": 0
|
||||
},
|
||||
"13": {
|
||||
"ok": 582,
|
||||
"err": 0
|
||||
},
|
||||
"14": {
|
||||
"ok": 488,
|
||||
"err": 0
|
||||
},
|
||||
"15": {
|
||||
"ok": 299,
|
||||
"err": 0
|
||||
},
|
||||
"16": {
|
||||
"ok": 575,
|
||||
"err": 0
|
||||
},
|
||||
"17": {
|
||||
"ok": 581,
|
||||
"err": 0
|
||||
},
|
||||
"18": {
|
||||
"ok": 601,
|
||||
"err": 0
|
||||
},
|
||||
"19": {
|
||||
"ok": 586,
|
||||
"err": 0
|
||||
},
|
||||
"20": {
|
||||
"ok": 585,
|
||||
"err": 0
|
||||
},
|
||||
"21": {
|
||||
"ok": 550,
|
||||
"err": 0
|
||||
},
|
||||
"22": {
|
||||
"ok": 579,
|
||||
"err": 0
|
||||
},
|
||||
"23": {
|
||||
"ok": 579,
|
||||
"err": 0
|
||||
},
|
||||
"24": {
|
||||
"ok": 579,
|
||||
"err": 0
|
||||
},
|
||||
"25": {
|
||||
"ok": 308,
|
||||
"err": 0
|
||||
},
|
||||
"26": {
|
||||
"ok": 378,
|
||||
"err": 0
|
||||
},
|
||||
"27": {
|
||||
"ok": 487,
|
||||
"err": 0
|
||||
},
|
||||
"28": {
|
||||
"ok": 561,
|
||||
"err": 0
|
||||
},
|
||||
"29": {
|
||||
"ok": 595,
|
||||
"err": 0
|
||||
},
|
||||
"30": {
|
||||
"ok": 602,
|
||||
"err": 0
|
||||
},
|
||||
"31": {
|
||||
"ok": 599,
|
||||
"err": 0
|
||||
},
|
||||
"32": {
|
||||
"ok": 548,
|
||||
"err": 0
|
||||
},
|
||||
"33": {
|
||||
"ok": 548,
|
||||
"err": 0
|
||||
},
|
||||
"34": {
|
||||
"ok": 578,
|
||||
"err": 0
|
||||
},
|
||||
"35": {
|
||||
"ok": 570,
|
||||
"err": 0
|
||||
},
|
||||
"36": {
|
||||
"ok": 330,
|
||||
"err": 0
|
||||
},
|
||||
"37": {
|
||||
"ok": 498,
|
||||
"err": 0
|
||||
},
|
||||
"38": {
|
||||
"ok": 484,
|
||||
"err": 0
|
||||
},
|
||||
"39": {
|
||||
"ok": 482,
|
||||
"err": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 56 seconds (healthy) 8080/tcp
|
||||
mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago
|
||||
mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 56 seconds ago
|
||||
mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp
|
||||
mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp
|
||||
@@ -0,0 +1,195 @@
|
||||
{
|
||||
"summary": {
|
||||
"duration_s": 40.0,
|
||||
"concurrency": 8,
|
||||
"replicas_seen": [
|
||||
"8c67bad5eac5",
|
||||
"f67725f7d715"
|
||||
],
|
||||
"requests": 25380,
|
||||
"ok": 25380,
|
||||
"errors": 0,
|
||||
"error_rate": 0.0,
|
||||
"rps_ok": 634.5,
|
||||
"latency_ms": {
|
||||
"p50": 6.9,
|
||||
"p95": 54.6,
|
||||
"p99": 81.2
|
||||
},
|
||||
"error_statuses": {},
|
||||
"action": {
|
||||
"target": "f67725f7d715",
|
||||
"started_at": 10.0,
|
||||
"finished_at": 12.29,
|
||||
"exit_code": 0
|
||||
},
|
||||
"errors_after_action": 0,
|
||||
"recovery_s": 0.0,
|
||||
"max_stall_s": 0.22,
|
||||
"instances_in_last_5s": [
|
||||
"8c67bad5eac5"
|
||||
]
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"ok": 866,
|
||||
"err": 0
|
||||
},
|
||||
"1": {
|
||||
"ok": 945,
|
||||
"err": 0
|
||||
},
|
||||
"2": {
|
||||
"ok": 991,
|
||||
"err": 0
|
||||
},
|
||||
"3": {
|
||||
"ok": 987,
|
||||
"err": 0
|
||||
},
|
||||
"4": {
|
||||
"ok": 781,
|
||||
"err": 0
|
||||
},
|
||||
"5": {
|
||||
"ok": 967,
|
||||
"err": 0
|
||||
},
|
||||
"6": {
|
||||
"ok": 911,
|
||||
"err": 0
|
||||
},
|
||||
"7": {
|
||||
"ok": 959,
|
||||
"err": 0
|
||||
},
|
||||
"8": {
|
||||
"ok": 996,
|
||||
"err": 0
|
||||
},
|
||||
"9": {
|
||||
"ok": 611,
|
||||
"err": 0
|
||||
},
|
||||
"10": {
|
||||
"ok": 569,
|
||||
"err": 0
|
||||
},
|
||||
"11": {
|
||||
"ok": 743,
|
||||
"err": 0
|
||||
},
|
||||
"12": {
|
||||
"ok": 546,
|
||||
"err": 0
|
||||
},
|
||||
"13": {
|
||||
"ok": 501,
|
||||
"err": 0
|
||||
},
|
||||
"14": {
|
||||
"ok": 570,
|
||||
"err": 0
|
||||
},
|
||||
"15": {
|
||||
"ok": 600,
|
||||
"err": 0
|
||||
},
|
||||
"16": {
|
||||
"ok": 589,
|
||||
"err": 0
|
||||
},
|
||||
"17": {
|
||||
"ok": 557,
|
||||
"err": 0
|
||||
},
|
||||
"18": {
|
||||
"ok": 485,
|
||||
"err": 0
|
||||
},
|
||||
"19": {
|
||||
"ok": 491,
|
||||
"err": 0
|
||||
},
|
||||
"20": {
|
||||
"ok": 333,
|
||||
"err": 0
|
||||
},
|
||||
"21": {
|
||||
"ok": 571,
|
||||
"err": 0
|
||||
},
|
||||
"22": {
|
||||
"ok": 559,
|
||||
"err": 0
|
||||
},
|
||||
"23": {
|
||||
"ok": 565,
|
||||
"err": 0
|
||||
},
|
||||
"24": {
|
||||
"ok": 567,
|
||||
"err": 0
|
||||
},
|
||||
"25": {
|
||||
"ok": 551,
|
||||
"err": 0
|
||||
},
|
||||
"26": {
|
||||
"ok": 570,
|
||||
"err": 0
|
||||
},
|
||||
"27": {
|
||||
"ok": 536,
|
||||
"err": 0
|
||||
},
|
||||
"28": {
|
||||
"ok": 572,
|
||||
"err": 0
|
||||
},
|
||||
"29": {
|
||||
"ok": 519,
|
||||
"err": 0
|
||||
},
|
||||
"30": {
|
||||
"ok": 424,
|
||||
"err": 0
|
||||
},
|
||||
"31": {
|
||||
"ok": 415,
|
||||
"err": 0
|
||||
},
|
||||
"32": {
|
||||
"ok": 587,
|
||||
"err": 0
|
||||
},
|
||||
"33": {
|
||||
"ok": 541,
|
||||
"err": 0
|
||||
},
|
||||
"34": {
|
||||
"ok": 569,
|
||||
"err": 0
|
||||
},
|
||||
"35": {
|
||||
"ok": 584,
|
||||
"err": 0
|
||||
},
|
||||
"36": {
|
||||
"ok": 591,
|
||||
"err": 0
|
||||
},
|
||||
"37": {
|
||||
"ok": 587,
|
||||
"err": 0
|
||||
},
|
||||
"38": {
|
||||
"ok": 551,
|
||||
"err": 0
|
||||
},
|
||||
"39": {
|
||||
"ok": 523,
|
||||
"err": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,6 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab3-api-3 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi-lab3-api-4 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 3 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp
|
||||
mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp
|
||||
@@ -0,0 +1,6 @@
|
||||
Image: mtusi-messenger:lab3
|
||||
Scenario: idle; docker stop --timeout 20 mtusi-messenger-sigterm-99ecee4b
|
||||
PASS idle: exit code 0; shutdown 5.353s; new connections rejected after 0.064s
|
||||
Scenario: incomplete POST; docker stop --timeout 20 mtusi-messenger-sigterm-7da6202d
|
||||
PASS incomplete POST: exit code 0; shutdown 13.414s; new connections rejected after 0.063s
|
||||
PASS SIGTERM: both scenarios completed within 10 seconds
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -8,6 +8,8 @@ defaults
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout check 2s
|
||||
timeout queue 5s
|
||||
retries 3
|
||||
option redispatch
|
||||
|
||||
@@ -28,6 +30,8 @@ frontend public
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Проверка /health/ready раз в 1 с; backend исключается после 2 неудач подряд,
|
||||
# возвращается после 2 успехов. Худшее время обнаружения: 2 x 1 с + timeout check 2 с = 4 с.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
# Поочерёдная замена API-реплик новой версией образа (APP_VERSION=v2 scripts/rolling-update.sh).
|
||||
# Миграции запускаются ОДИН раз до замены; migrations/002 только добавляет таблицу, поэтому старая версия с ней совместима.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${APP_VERSION:?set new version, e.g. APP_VERSION=v2}"
|
||||
export APP_VERSION
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
|
||||
$COMPOSE build migrate
|
||||
$COMPOSE run --rm migrate
|
||||
N=$($COMPOSE ps -q api | wc -l | tr -d ' ')
|
||||
|
||||
wait_healthy() {
|
||||
i=0
|
||||
while [ "$i" -lt 90 ]; do
|
||||
c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true)
|
||||
[ "$c" -ge "$N" ] && return 0
|
||||
i=$((i + 1)); sleep 1
|
||||
done
|
||||
echo "replicas did not become healthy" >&2; return 1
|
||||
}
|
||||
|
||||
for cid in $($COMPOSE ps -q api); do
|
||||
echo "replacing $cid" >&2
|
||||
docker stop -t 20 "$cid" >/dev/null # SIGTERM: readiness 503, drain 5 с, затем завершение
|
||||
docker rm "$cid" >/dev/null
|
||||
$COMPOSE up -d --no-deps --no-recreate --scale api="$N" api # создаётся реплика из нового образа
|
||||
wait_healthy
|
||||
sleep 5 # LB добавляет новую реплику после 2 успешных проверок
|
||||
done
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Базовая нагрузка, затем по очереди отказ каждой API-реплики под нагрузкой с восстановлением между ними.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
|
||||
wait_healthy() {
|
||||
i=0
|
||||
while [ "$i" -lt 90 ]; do
|
||||
c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true)
|
||||
[ "$c" -ge 2 ] && return 0
|
||||
i=$((i + 1)); sleep 1
|
||||
done
|
||||
echo "replicas did not become healthy" >&2; return 1
|
||||
}
|
||||
|
||||
python3 student_tests/loadgen.py --duration 30 --output evidence/load-baseline.json
|
||||
for idx in 0 1; do
|
||||
python3 student_tests/loadgen.py --duration 40 --action-script scripts/stop-one.sh \
|
||||
--action-at 10 --target-index "$idx" --output "evidence/load-stop-$idx.json"
|
||||
$COMPOSE ps -a > "evidence/load-stop-$idx-ps.txt"
|
||||
$COMPOSE start api
|
||||
wait_healthy
|
||||
sleep 10 # LB возвращает реплику после 2 успешных проверок
|
||||
done
|
||||
+5
-1
@@ -26,4 +26,8 @@ CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.5"))
|
||||
|
||||
# Идентификатор реплики: hostname контейнера = короткий ID контейнера (по нему stop-one.sh находит контейнер)
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
||||
IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
|
||||
# SIGTERM: сколько секунд реплика отвечает 503 на /health/ready, пока LB её исключает, и продолжает обслуживать запросы.
|
||||
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "5"))
|
||||
APP_VERSION = os.environ.get("APP_VERSION", "dev")
|
||||
+25
-1
@@ -6,6 +6,7 @@ import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import asyncpg
|
||||
import signal
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
@@ -39,11 +40,31 @@ logger.addFilter(RequestIdFilter())
|
||||
is_shutting_down = False
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
|
||||
def _install_drain_handler(app: FastAPI) -> None:
|
||||
"""SIGTERM: сначала /health/ready -> 503, и только через DRAIN_SECONDS обычное завершение uvicorn."""
|
||||
loop = asyncio.get_running_loop()
|
||||
original = signal.getsignal(signal.SIGTERM)
|
||||
if not callable(original):
|
||||
return
|
||||
|
||||
def on_sigterm(signum, frame):
|
||||
if app.state.draining:
|
||||
return
|
||||
app.state.draining = True
|
||||
logger.info("SIGTERM received: draining for %s s", config.DRAIN_SECONDS)
|
||||
loop.call_soon_threadsafe(loop.call_later, config.DRAIN_SECONDS, original, signum, frame)
|
||||
|
||||
try:
|
||||
signal.signal(signal.SIGTERM, on_sigterm)
|
||||
except ValueError: # не главный поток (например, в тестах)
|
||||
pass
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
app.state.draining = False
|
||||
app.state.pool = await create_pool_with_retry()
|
||||
app.state.cache = SessionCache(config.CACHE_URL)
|
||||
_install_drain_handler(app)
|
||||
yield
|
||||
global is_shutting_down
|
||||
is_shutting_down = True
|
||||
@@ -66,6 +87,7 @@ async def middleware_pipeline(request: Request, call_next):
|
||||
response: Response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
response.headers["X-App-Version"] = config.APP_VERSION
|
||||
logger.info(f"Completed {request.method} {request.url.path} with status {response.status_code}",
|
||||
extra={"request_id": req_id})
|
||||
return response
|
||||
@@ -122,7 +144,9 @@ async def health_live():
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def health_ready(request: Request):
|
||||
# Критичная зависимость для готовности - PostgreSQL.
|
||||
# Критичная зависимость для готовности - PostgreSQL. При SIGTERM реплика сразу «не готова».
|
||||
if request.app.state.draining:
|
||||
return make_error_response(503, "unavailable", "Draining", _req_id(request))
|
||||
await request.app.state.pool.fetchval("SELECT 1", timeout=2)
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import base64
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
def call(base, method, path, body=None, headers=None, timeout=3.0):
|
||||
"""Возвращает (status, headers с ключами в нижнем регистре, bytes). Сетевые ошибки поднимаются наружу."""
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
h = {"Content-Type": "application/json"} if data else {}
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(base + path, data=data, method=method, headers=h)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, {k.lower(): v for k, v in r.headers.items()}, r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, {k.lower(): v for k, v in e.headers.items()}, e.read()
|
||||
|
||||
|
||||
def setup(base):
|
||||
"""Синтетический пользователь, сессия и чат из одного участника. Возвращает (auth_headers, chat_id)."""
|
||||
name = "load_" + uuid.uuid4().hex[:12]
|
||||
password = "load-password-" + uuid.uuid4().hex[:8]
|
||||
s, _, b = call(base, "POST", "/api/v1/users", {"username": name, "display_name": "Load", "password": password})
|
||||
assert s == 201, f"register failed: {s}"
|
||||
uid = json.loads(b)["id"]
|
||||
cred = base64.b64encode(f"{name}:{password}".encode()).decode()
|
||||
s, _, b = call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert s == 201, f"login failed: {s}"
|
||||
auth = {"Authorization": "Bearer " + json.loads(b)["token"]}
|
||||
s, _, b = call(base, "POST", "/api/v1/chats", {"title": "load", "member_ids": [uid]}, auth)
|
||||
assert s == 201, f"chat failed: {s}"
|
||||
return auth, json.loads(b)["id"]
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Closed-loop нагрузка POST /messages через LB; во время неё выполняется action-скрипт (остановка реплики).
|
||||
Без прозрачных retry: каждая неудача = ошибка. Токены в вывод не попадают."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from common import call, setup
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--duration", type=float, default=40)
|
||||
ap.add_argument("--concurrency", type=int, default=8)
|
||||
ap.add_argument("--action-script")
|
||||
ap.add_argument("--action-at", type=float, default=10)
|
||||
ap.add_argument("--target-index", type=int, default=0)
|
||||
ap.add_argument("--output")
|
||||
a = ap.parse_args()
|
||||
|
||||
auth, chat = setup(a.base_url)
|
||||
path = f"/api/v1/chats/{chat}/messages"
|
||||
ids = set()
|
||||
for _ in range(40): # какие реплики видны через LB
|
||||
_, h, _ = call(a.base_url, "GET", "/api/v1/users/me", headers=auth)
|
||||
ids.add(h.get("x-instance-id"))
|
||||
ids = sorted(i for i in ids if i)
|
||||
|
||||
records, lock = [], threading.Lock() # (t, status, latency, instance)
|
||||
t_start = time.monotonic()
|
||||
t_end = t_start + a.duration
|
||||
|
||||
def worker():
|
||||
while time.monotonic() < t_end:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
s, h, _ = call(a.base_url, "POST", path, {"text": "load"},
|
||||
{**auth, "Idempotency-Key": uuid.uuid4().hex}, timeout=3)
|
||||
inst, ver = h.get("x-instance-id"), h.get("x-app-version")
|
||||
except Exception:
|
||||
s, inst, ver = 0, None, None
|
||||
with lock:
|
||||
records.append((t0 - t_start, s, time.monotonic() - t0, inst, ver))
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(a.concurrency)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
action = {}
|
||||
if a.action_script:
|
||||
time.sleep(a.action_at)
|
||||
target = ids[a.target_index % len(ids)]
|
||||
t0 = time.monotonic() - t_start
|
||||
rc = subprocess.run([a.action_script], env={**os.environ, "TARGET_INSTANCE_ID": target},
|
||||
capture_output=True, timeout=300).returncode
|
||||
action = {"target": target, "started_at": round(t0, 2),
|
||||
"finished_at": round(time.monotonic() - t_start, 2), "exit_code": rc}
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
ok = [r for r in records if r[1] == 201]
|
||||
bad = [r for r in records if r[1] != 201]
|
||||
lat = sorted(r[2] for r in ok)
|
||||
pct = lambda p: round(lat[min(len(lat) - 1, int(len(lat) * p))] * 1000, 1) if lat else None
|
||||
summary = {
|
||||
"duration_s": a.duration, "concurrency": a.concurrency, "replicas_seen": ids,
|
||||
"requests": len(records), "ok": len(ok), "errors": len(bad),
|
||||
"error_rate": round(len(bad) / max(1, len(records)), 4),
|
||||
"rps_ok": round(len(ok) / a.duration, 1),
|
||||
"latency_ms": {"p50": pct(0.50), "p95": pct(0.95), "p99": pct(0.99)},
|
||||
"error_statuses": {str(s): sum(1 for r in bad if r[1] == s) for s in {r[1] for r in bad}},
|
||||
"action": action,
|
||||
}
|
||||
if action:
|
||||
errs_after = [r[0] for r in bad if r[0] >= action["started_at"]]
|
||||
times = sorted(r[0] for r in ok if r[0] >= action["started_at"])
|
||||
gaps = [b - a_ for a_, b in zip([action["started_at"]] + times, times)]
|
||||
summary["errors_after_action"] = len(errs_after)
|
||||
summary["recovery_s"] = round(max(errs_after) - action["started_at"], 2) if errs_after else 0.0
|
||||
summary["max_stall_s"] = round(max(gaps), 2) if gaps else None
|
||||
summary["instances_in_last_5s"] = sorted({r[3] for r in ok if r[0] >= a.duration - 5 and r[3]})
|
||||
|
||||
vers = {}
|
||||
for r in ok:
|
||||
if r[4]:
|
||||
v = vers.setdefault(r[4], [r[0], r[0]])
|
||||
v[0], v[1] = min(v[0], r[0]), max(v[1], r[0])
|
||||
summary["versions_first_last_seen_s"] = {k: [round(x, 1) for x in v] for k, v in vers.items()}
|
||||
|
||||
timeline = {}
|
||||
for t, s, *_ in records:
|
||||
b = timeline.setdefault(int(t), {"ok": 0, "err": 0})
|
||||
b["ok" if s == 201 else "err"] += 1
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
if a.output:
|
||||
with open(a.output, "w") as f:
|
||||
json.dump({"summary": summary, "per_second": timeline}, f, ensure_ascii=False, indent=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""30 параллельных POST с одним Idempotency-Key через LB: ровно одно сообщение, везде 201 и один id."""
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from common import call, setup
|
||||
|
||||
BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080"
|
||||
THREADS, ROUNDS = 30, 5
|
||||
|
||||
auth, chat = setup(BASE)
|
||||
path = f"/api/v1/chats/{chat}/messages"
|
||||
replicas = set()
|
||||
|
||||
for rnd in range(ROUNDS):
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(THREADS)
|
||||
results = [None] * THREADS
|
||||
|
||||
def work(i, rnd=rnd, key=key, barrier=barrier, results=results):
|
||||
barrier.wait()
|
||||
s, h, b = call(BASE, "POST", path, {"text": f"parallel {rnd}"}, {**auth, "Idempotency-Key": key}, timeout=10)
|
||||
results[i] = (s, h.get("x-instance-id"), json.loads(b) if s == 201 else None)
|
||||
|
||||
ts = [threading.Thread(target=work, args=(i,)) for i in range(THREADS)]
|
||||
[t.start() for t in ts]
|
||||
[t.join() for t in ts]
|
||||
codes = {r[0] for r in results}
|
||||
msg_ids = {r[2]["id"] for r in results if r[0] == 201}
|
||||
replicas |= {r[1] for r in results}
|
||||
assert codes == {201}, f"round {rnd}: codes {codes}"
|
||||
assert len(msg_ids) == 1, f"round {rnd}: {len(msg_ids)} different messages"
|
||||
s, _, _ = call(BASE, "POST", path, {"text": "other body"}, {**auth, "Idempotency-Key": key})
|
||||
assert s == 409, f"round {rnd}: other body gave {s}"
|
||||
|
||||
s, _, b = call(BASE, "GET", path + "?limit=100", headers=auth)
|
||||
count = len(json.loads(b)["items"])
|
||||
assert count == ROUNDS, f"history has {count} messages, expected {ROUNDS}"
|
||||
print(json.dumps({"rounds": ROUNDS, "parallel_requests_per_round": THREADS,
|
||||
"messages_in_history": count, "replicas_that_answered": sorted(r for r in replicas if r)}))
|
||||
print("PASS: same key in parallel -> exactly one message; different body -> 409")
|
||||
@@ -12,8 +12,8 @@ import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab2")
|
||||
NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab2_default")
|
||||
IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab3")
|
||||
NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab3_default")
|
||||
PORT = int(os.environ.get("SIGTERM_PORT", "18080"))
|
||||
|
||||
|
||||
@@ -55,6 +55,15 @@ def connection_rejected(base_url):
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError):
|
||||
return True
|
||||
|
||||
def not_ready(base_url):
|
||||
# Во время drain /health/ready отвечает 503 (или соединение уже закрыто)
|
||||
try:
|
||||
with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5):
|
||||
return False
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code == 503
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError):
|
||||
return True
|
||||
|
||||
def test_shutdown(incomplete_request):
|
||||
name = "mtusi-messenger-sigterm-" + uuid.uuid4().hex[:8]
|
||||
@@ -82,24 +91,24 @@ def test_shutdown(incomplete_request):
|
||||
raise AssertionError(f"Expected 100 Continue, received {response!r}")
|
||||
pending.sendall(b"{")
|
||||
|
||||
print(f"Scenario: {scenario}; docker stop --timeout 10 {name}", flush=True)
|
||||
print(f"Scenario: {scenario}; docker stop --timeout 20 {name}", flush=True)
|
||||
start = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
stopped = executor.submit(run, "docker", "stop", "--timeout", "10", name)
|
||||
stopped = executor.submit(run, "docker", "stop", "--timeout", "20", name)
|
||||
deadline = start + 2
|
||||
while not connection_rejected(base_url):
|
||||
while not not_ready(base_url):
|
||||
if time.monotonic() >= deadline:
|
||||
raise AssertionError("New connections are still accepted during shutdown")
|
||||
raise AssertionError("Readiness still OK after SIGTERM")
|
||||
time.sleep(0.05)
|
||||
rejected_after = time.monotonic() - start
|
||||
stopped.result(timeout=12)
|
||||
stopped.result(timeout=25)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
exit_code = int(run("docker", "inspect", name, "--format", "{{.State.ExitCode}}").stdout)
|
||||
if exit_code != 0:
|
||||
raise AssertionError(f"Container exited with code {exit_code}, expected 0")
|
||||
if elapsed > 10:
|
||||
raise AssertionError(f"Shutdown exceeded 10 seconds: {elapsed:.3f}s")
|
||||
if elapsed > 15:
|
||||
raise AssertionError(f"Shutdown exceeded 15 seconds: {elapsed:.3f}s")
|
||||
if not connection_rejected(base_url):
|
||||
raise AssertionError("API still accepts connections after shutdown")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user