feat: перенос логики с 4лабф

This commit is contained in:
Enterxx2
2026-11-09 22:33:50 +03:00
parent 276dbdad8a
commit 1d8dfb7ea4
166 changed files with 13171 additions and 6 deletions
+59
View File
@@ -0,0 +1,59 @@
#!/bin/sh
set -eu
# Миграции с нуля и повторно, отказ API на непримененной схеме: отдельный
# compose-проект с собственным пустым volume. Рабочий стенд и его данные
# не затрагиваются; временный volume этого проекта удаляется в конце.
cd "$(dirname "$0")/.."
PROJECT=messenger-lab5-migrations-check
# Свои порты, чтобы не конфликтовать с основным стендом
export LB_PORT="${MIGRATIONS_CHECK_LB_PORT:-58080}"
export LB_EXPIRY_PORT="${MIGRATIONS_CHECK_LB_EXPIRY_PORT:-58081}"
export DB_HOST_PORT="${MIGRATIONS_CHECK_DB_HOST_PORT:-55434}"
COMPOSE="docker compose --env-file ${ENV_FILE:-.env} -p $PROJECT"
trap '$COMPOSE down -v --remove-orphans >/dev/null 2>&1' EXIT
$COMPOSE up -d --wait postgres >/dev/null 2>&1
# Тот же образ и сеть, что у сервиса migrate; apply_migrations трижды подряд
$COMPOSE run --rm --no-deps -T migrate python - <<'EOF'
import asyncio
import sys
sys.path.insert(0, "src")
from infrastructure.dependencies.database import database_pool_manager as db
async def main() -> None:
await db.create_pool(30)
async with db.acquire() as conn:
tables = await conn.fetchval(
"SELECT count(*) FROM pg_tables WHERE schemaname = 'public';"
)
print(f"пустой volume: таблиц в public = {tables}")
# Так стартует API-реплика: без migrate она отказывается работать
try:
await db.check_migrations()
except RuntimeError as error:
print(f"API до migrate: {error}")
else:
raise SystemExit("FAIL: API стартовал бы на пустой схеме")
for attempt in (1, 2, 3):
await db.apply_migrations()
async with db.acquire() as conn:
versions = await conn.fetch(
"SELECT version, name FROM schema_migrations ORDER BY version;"
)
tables = await conn.fetchval(
"SELECT count(*) FROM pg_tables WHERE schemaname = 'public';"
)
print(
f"запуск {attempt}: schema_migrations = "
f"{[(r['version'], r['name']) for r in versions]}, таблиц = {tables}"
)
await db.check_migrations()
print("API после migrate: схема совместима")
await db.close()
asyncio.run(main())
EOF
+91
View File
@@ -0,0 +1,91 @@
#!/bin/sh
set -eu
# Совместимость текущей схемы с предыдущей версией API — условие rolling update:
# миграции применяются до замены реплик, и старые реплики продолжают работать.
# Образ собирается из коммита PREVIOUS_REVISION (по умолчанию lab2-код aa971fb),
# запускается одним процессом в сети backend рабочего стенда (наружу порт
# не публикуется) и выполняет вход, создание чата, запись и чтение сообщения.
# Создаются только синтетические данные; volumes не затрагиваются.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
REVISION="${PREVIOUS_REVISION:-aa971fb}"
IMAGE="mtusi-messenger:compat-$REVISION"
PROJECT=$(docker compose --env-file "$ENV_FILE" config --format json | python3 -c \
'import json, sys; print(json.load(sys.stdin)["name"])')
NETWORK="${PROJECT}_backend"
git archive "$REVISION" | docker build --quiet --tag "$IMAGE" - >/dev/null
echo "previous version image: $IMAGE ($(git log -1 --format='%h %s' "$REVISION"))"
echo "schema on stand: $(docker compose --env-file "$ENV_FILE" exec -T postgres sh -c \
'psql -X -q -tA -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT string_agg(name, \$\$, \$\$ ORDER BY version) FROM schema_migrations;"')"
docker run --rm -i --network "$NETWORK" --env-file "$ENV_FILE" \
-e DB_HOST=postgres -e DB_PORT=5432 -e VALKEY_HOST=valkey -e VALKEY_PORT=6379 \
"$IMAGE" sh -c 'python src/gateway.py >/tmp/api.log 2>&1 & exec python -' <<'EOF'
import base64
import json
import time
import urllib.error
import urllib.request
import uuid
BASE = "http://127.0.0.1:8080"
def call(method, path, body=None, headers=None):
data = None if body is None else json.dumps(body).encode()
request = urllib.request.Request(BASE + path, data=data, method=method)
request.add_header("Content-Type", "application/json")
for name, value in (headers or {}).items():
request.add_header(name, value)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status, json.loads(response.read() or b"null")
except urllib.error.HTTPError as error:
return error.code, None
for _ in range(60):
try:
if call("GET", "/health/ready")[0] == 200:
break
except OSError:
pass
time.sleep(0.5)
else:
print(open("/tmp/api.log").read()[-2000:])
raise SystemExit("FAIL: previous version did not become ready on current schema")
username = "compat_" + uuid.uuid4().hex[:12]
password = "Compat-" + uuid.uuid4().hex
status_user, user = call(
"POST", "/api/v1/users",
{"username": username, "display_name": "Compat", "password": password},
)
credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
status_login, session = call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credentials}
)
bearer = {"Authorization": "Bearer " + session["token"]}
status_chat, chat = call(
"POST", "/api/v1/chats", {"title": "compat", "member_ids": [user["id"]]}, bearer
)
path = f"/api/v1/chats/{chat['id']}/messages"
status_send, message = call("POST", path, {"text": "старая версия пишет"}, bearer)
status_read, page = call("GET", path, headers=bearer)
result = {
"ready": True,
"register": status_user,
"login": status_login,
"create_chat": status_chat,
"send_message_without_key": status_send,
"read_history": status_read,
"history_matches": page["items"] == [message],
}
print(json.dumps(result, ensure_ascii=False))
log = open("/tmp/api.log").read()
print("startup log:", [line.split(" - ", 1)[-1] for line in log.splitlines() if "миграц" in line.lower() or "Схема" in line])
if any(code >= 400 for code in result.values() if isinstance(code, int)) or not result["history_matches"]:
raise SystemExit("FAIL: previous version is not compatible with current schema")
print("PASS: previous API version works on the current schema (expand-only migration)")
EOF
+482
View File
@@ -0,0 +1,482 @@
#!/usr/bin/env python3
"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных.
failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики
(docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление
(scripts/restore.sh) → нагрузка после восстановления.
rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик).
Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же
Idempotency-Key и телом. После прогона вся история читается заново: каждое
подтверждённое сообщение на месте и не изменено, дублей нет.
Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
REQUEST_TIMEOUT_SEC = 5
RETRY_PAUSE_SEC = 0.05
STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
@dataclass
class Attempt:
"""Одна HTTP-попытка клиента; время — секунды от начала прогона"""
op: str
started: float
finished: float
status: int | None
error: str | None
instance: str | None
retry: bool
@property
def ok(self) -> bool:
return self.status in (200, 201)
class Recorder:
def __init__(self) -> None:
self.started = time.monotonic()
self.lock = threading.Lock()
self.attempts: list[Attempt] = []
self.confirmed: list[dict] = []
self.events: dict[str, float] = {}
def now(self) -> float:
return time.monotonic() - self.started
def mark(self, name: str) -> None:
self.events[name] = round(self.now(), 3)
def add(self, attempt: Attempt, message: dict | None = None) -> None:
with self.lock:
self.attempts.append(attempt)
if message is not None:
self.confirmed.append(message)
def request(
base: str, method: str, path: str, token: str, body: dict | None, key: str | None
) -> tuple[int | None, str | None, str | None, dict | None]:
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
data = None
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
headers["Content-Type"] = "application/json"
if key is not None:
headers["Idempotency-Key"] = key
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response:
payload = json.loads(response.read())
return response.status, None, response.headers.get("X-Instance-Id"), payload
except urllib.error.HTTPError as error:
error.read()
return error.code, None, error.headers.get("X-Instance-Id"), None
except OSError as error:
return None, type(error).__name__, None, None
def worker(
base: str,
token: str,
chat_id: str,
worker_id: int,
recorder: Recorder,
stop: threading.Event,
) -> None:
"""Запись с повтором по тому же ключу до подтверждения, затем чтение"""
path = f"/api/v1/chats/{chat_id}/messages"
index = 0
while not stop.is_set():
key = uuid.uuid4().hex
body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"}
retry = False
while not stop.is_set():
started = recorder.now()
status, error, instance, payload = request(
base, "POST", path, token, body, key
)
attempt = Attempt(
"write", started, recorder.now(), status, error, instance, retry
)
recorder.add(attempt, payload if status == 201 else None)
if status == 201:
break
retry = True
time.sleep(RETRY_PAUSE_SEC)
started = recorder.now()
status, error, instance, _ = request(
base, "GET", path + "?limit=1", token, None, None
)
recorder.add(
Attempt("read", started, recorder.now(), status, error, instance, False)
)
index += 1
def percentile(values: list[float], p: float) -> float | None:
if not values:
return None
ordered = sorted(values)
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1)
def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict:
inside = [a for a in attempts if start <= a.finished < end]
duration = max(end - start, 1e-9)
result: dict = {"seconds": round(duration, 3)}
for op in ("write", "read"):
ops = [a for a in inside if a.op == op]
ok = [a for a in ops if a.ok]
result[op] = {
"ok": len(ok),
"errors": len(ops) - len(ok),
"ok_rps": round(len(ok) / duration, 1),
"latency_ms": {
"p50": percentile([a.finished - a.started for a in ok], 0.50),
"p95": percentile([a.finished - a.started for a in ok], 0.95),
"p99": percentile([a.finished - a.started for a in ok], 0.99),
},
}
return result
def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict:
"""
Время восстановления записи: от первой ошибки после воздействия до первой
успешной записи после последней ошибки; без ошибок — 0. Отдельно —
наибольший разрыв между подтверждениями записи (задержка без ошибок).
"""
window = sorted(
(a for a in attempts if a.op == "write" and a.finished >= action_start - 1),
key=lambda a: a.finished,
)
errors = [a for a in window if not a.ok and a.finished >= action_start]
kinds: dict[str, int] = {}
for a in errors:
kind = a.error or f"HTTP {a.status}"
kinds[kind] = kinds.get(kind, 0) + 1
if errors:
last_error = errors[-1].finished
first_ok_after = next(
(a.finished for a in window if a.ok and a.finished > last_error), end
)
recovery = first_ok_after - errors[0].finished
else:
recovery = 0.0
oks = [a.finished for a in window if a.ok and a.finished <= end]
gap = max((b - a for a, b in zip(oks, oks[1:])), default=None)
return {
"write_errors": len(errors),
"error_kinds": kinds,
"write_recovery_seconds": round(recovery, 3),
"max_write_gap_seconds": None if gap is None else round(gap, 3),
}
def timeline(attempts: list[Attempt], end: float) -> list[dict]:
rows = []
for second in range(int(end) + 1):
inside = [a for a in attempts if second <= a.finished < second + 1]
instances: dict[str, int] = {}
for a in inside:
if a.ok and a.instance:
instances[a.instance] = instances.get(a.instance, 0) + 1
rows.append(
{
"t": second,
"ok": sum(a.ok for a in inside),
"errors": sum(not a.ok for a in inside),
"instances": instances,
}
)
return rows
def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]:
path = f"/api/v1/chats/{chat_id}/messages?limit=100"
items: list[dict] = []
while True:
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
if page["next_cursor"] is None:
return items
path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote(
page["next_cursor"], safe=""
)
def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict:
history = read_history(client, actor, chat_id)
by_id = {m["id"]: m for m in history}
texts = [m["text"] for m in history]
confirmed = {m["id"]: m for m in recorder.confirmed}
return {
"messages_in_history": len(history),
"confirmed_unique": len(confirmed),
"confirmed_missing_or_changed": sum(
by_id.get(i) != m for i, m in confirmed.items()
),
"duplicate_ids": len(history) - len(by_id),
"duplicate_texts": len(texts) - len(set(texts)),
"retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed),
}
def run(command: list[str], env: dict | None = None) -> str:
result = subprocess.run(
command, cwd=ROOT, env=env, check=True, capture_output=True, text=True
)
return result.stdout.strip()
def api_containers() -> list[str]:
ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split()
names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids]
return sorted(names)
def instance_of(container: str) -> str:
return run(["docker", "exec", container, "cat", "/tmp/instance-id"])
def container_stats() -> list[str]:
names = run(
["docker", "compose", "--env-file", ".env", "ps", "-q", "nginx", "postgres", "api"]
).split()
output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names])
return output.splitlines()
def scenario(
args: argparse.Namespace,
label: str,
action: list[str],
restore: list[str] | None,
target: str | None = None,
env: dict | None = None,
) -> dict:
client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC)
alice, _, _, chat = client.bootstrap()
base = client.base
recorder = Recorder()
stop = threading.Event()
threads = [
threading.Thread(
target=worker,
args=(base, alice["token"], chat["id"], n, recorder, stop),
daemon=True,
)
for n in range(args.workers)
]
for thread in threads:
thread.start()
time.sleep(args.before / 2)
stats = container_stats()
time.sleep(args.before / 2)
instances_before = sorted(
{a.instance for a in recorder.attempts if a.ok and a.instance}
)
recorder.mark("action_start")
action_output = run(action, env)
recorder.mark("action_end")
restore_output = ""
if restore is not None:
time.sleep(args.outage)
recorder.mark("restore_start")
restore_output = run(restore)
recorder.mark("restore_end")
time.sleep(args.after)
stop.set()
for thread in threads:
thread.join(timeout=REQUEST_TIMEOUT_SEC + 1)
recorder.mark("end")
events = recorder.events
end = events["end"]
after_start = events.get("restore_end", events["action_end"])
attempts = recorder.attempts
seen_after = [
a for a in attempts if a.ok and a.finished >= after_start and a.instance
]
back_in_rotation = next(
(
round(a.finished - after_start, 3)
for a in sorted(seen_after, key=lambda a: a.finished)
if a.instance not in instances_before
),
None,
)
served = sorted(
(a for a in attempts if a.ok and a.instance), key=lambda a: a.finished
)
instances_seen: dict[str, list[float]] = {}
for a in served:
offset = round(a.finished - events["action_start"], 3)
instances_seen.setdefault(a.instance, [offset, offset])[1] = offset
phases = {
"before": phase_stats(attempts, 0, events["action_start"]),
"during": phase_stats(attempts, events["action_start"], after_start),
"after": phase_stats(attempts, after_start, end),
}
target_after_stop = None
if target is not None:
target_after_stop = sum(
1
for a in attempts
if a.ok
and a.instance == target
and a.finished > events["action_end"]
and a.finished < events.get("restore_start", end)
)
return {
"scenario": label,
"target_instance": target,
"workers": args.workers,
"events_seconds": events,
"instances_before": instances_before,
"instances_after": sorted({a.instance for a in seen_after}),
"target_responses_while_stopped": target_after_stop,
"restored_replica_in_rotation_seconds": back_in_rotation
if restore is not None
else None,
# alias → [первый, последний] успешный ответ, секунды от начала воздействия
"instances_seen_from_action": instances_seen,
"phases": phases,
"recovery": recovery_stats(attempts, events["action_start"], end),
"integrity": integrity(client, alice, chat["id"], recorder),
"container_stats_before_action": stats,
"action_output": action_output.splitlines(),
"restore_output_tail": restore_output.splitlines()[-2:],
"timeline": timeline(attempts, end),
}
def failover(args: argparse.Namespace) -> list[dict]:
results = []
for signal in args.signals:
for container in api_containers():
target = instance_of(container)
if signal == "SIGTERM":
action = ["docker", "stop", "--timeout", "10", container]
else:
action = ["docker", "kill", "--signal", "SIGKILL", container]
label = f"{signal} {container}"
print(f"== {label} (instance {target})", flush=True)
result = scenario(
args, label, action, ["./scripts/restore.sh"], target=target
)
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
results.append(result)
return results
def rolling(args: argparse.Namespace) -> list[dict]:
env = dict(os.environ)
env.setdefault("APP_REVISION", f"exp-{int(time.time())}")
print(f"== rolling update to revision {env['APP_REVISION']}", flush=True)
result = scenario(
args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env
)
result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines()
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
return [result]
def brief(result: dict) -> dict:
"""Краткая строка прогона для терминала; полная версия — в --output"""
return {
"scenario": result["scenario"],
"rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()},
"rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()},
"errors": {
k: v["write"]["errors"] + v["read"]["errors"]
for k, v in result["phases"].items()
},
"recovery": result["recovery"],
"target_responses_while_stopped": result["target_responses_while_stopped"],
"restored_replica_in_rotation_seconds": result[
"restored_replica_in_rotation_seconds"
],
"instances_seen_from_action": result["instances_seen_from_action"],
"integrity": result["integrity"],
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=["failover", "rolling"])
parser.add_argument("--base-url", default=None)
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа")
parser.add_argument("--outage", type=float, default=10, help="реплика остановлена")
parser.add_argument("--after", type=float, default=8, help="после восстановления")
parser.add_argument(
"--signals",
nargs="+",
default=["SIGTERM", "SIGKILL"],
choices=["SIGTERM", "SIGKILL"],
)
parser.add_argument("--output", default=None, help="JSON с полными результатами")
args = parser.parse_args()
started = time.strftime("%Y-%m-%dT%H:%M:%S%z")
results = failover(args) if args.mode == "failover" else rolling(args)
report = {
"mode": args.mode,
"started": started,
"profile": {
"workers": args.workers,
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
"client_timeout_seconds": REQUEST_TIMEOUT_SEC,
"phases_seconds": {
"before": args.before,
"outage": args.outage,
"after": args.after,
},
},
"runs": results,
}
if args.output:
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(f"saved: {args.output}")
failed = [
r["scenario"]
for r in results
if r["integrity"]["confirmed_missing_or_changed"]
or r["integrity"]["duplicate_ids"]
or r["integrity"]["duplicate_texts"]
or r["target_responses_while_stopped"]
]
if failed:
raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}")
print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет")
if __name__ == "__main__":
main()
+111
View File
@@ -0,0 +1,111 @@
-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на ≥10 000 сообщений.
-- Запуск: make explain → evidence/explain_history.txt
-- Сид синтетический и идемпотентный: повторный запуск данных не дублирует.
-- Пользователь explain_seed не может войти: password_hash не является hash Argon2.
\pset pager off
\timing off
DO $$
DECLARE
seed_user uuid;
big_chat uuid;
noise uuid;
BEGIN
IF EXISTS (SELECT 1 FROM chats WHERE title = 'explain-big') THEN
RETURN;
END IF;
INSERT INTO users (username, display_name, password_hash)
VALUES ('explain_seed', 'EXPLAIN seed', 'not-an-argon2-hash')
ON CONFLICT (username) DO UPDATE SET display_name = EXCLUDED.display_name
RETURNING id INTO seed_user;
-- Большой чат: 20 000 сообщений
INSERT INTO chats (title, created_by) VALUES ('explain-big', seed_user)
RETURNING id INTO big_chat;
INSERT INTO chat_members (chat_id, user_id, position) VALUES (big_chat, seed_user, 0);
INSERT INTO messages (chat_id, sender_id, text, created_at)
SELECT big_chat, seed_user, 'synthetic message #' || n,
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second'
FROM generate_series(1, 20000) AS n;
-- Шум: 50 чатов по 1 000 сообщений, чтобы таблица была больше одного чата
FOR i IN 1..50 LOOP
INSERT INTO chats (title, created_by) VALUES ('explain-noise-' || i, seed_user)
RETURNING id INTO noise;
INSERT INTO chat_members (chat_id, user_id, position) VALUES (noise, seed_user, 0);
INSERT INTO messages (chat_id, sender_id, text, created_at)
SELECT noise, seed_user, 'noise #' || n,
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second'
FROM generate_series(1, 1000) AS n;
END LOOP;
END $$;
ANALYZE users, chats, chat_members, messages;
SELECT id AS big_chat FROM chats WHERE title = 'explain-big' \gset
-- Cursor из середины истории: ключ 10 000-го сообщения
SELECT created_at AS cursor_created_at, id AS cursor_id
FROM messages WHERE chat_id = :'big_chat'
ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
\echo '== Объём данных'
SELECT
(SELECT count(*) FROM messages) AS messages_total,
(SELECT count(*) FROM messages WHERE chat_id = :'big_chat') AS messages_in_big_chat,
pg_size_pretty(pg_relation_size('messages')) AS messages_heap,
pg_size_pretty(pg_relation_size('messages_chat_id_created_at_id_idx')) AS history_index;
\echo ''
\echo '== 1. Первая страница истории (limit 50 + 1), как list_for_chat'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :'big_chat'
ORDER BY created_at, id
LIMIT 51;
\echo ''
\echo '== 2. Страница из середины по cursor (после 10 000-го), как list_for_chat_after'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :'big_chat'
AND (created_at, id) > (:'cursor_created_at'::timestamptz, :'cursor_id'::uuid)
ORDER BY created_at, id
LIMIT 51;
\echo ''
\echo '== 3. Тот же запрос как generic plan prepared statement (так его исполняет asyncpg)'
SET plan_cache_mode = force_generic_plan;
PREPARE history_page(uuid, timestamptz, uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
LIMIT $4;
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
EXECUTE history_page(:'big_chat', :'cursor_created_at', :'cursor_id', 51);
DEALLOCATE history_page;
RESET plan_cache_mode;
\echo ''
\echo '== 4. MAX(created_at) при вставке сообщения (монотонное время чата)'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT max(created_at) FROM messages WHERE chat_id = :'big_chat';
\echo ''
\echo '== 5. Для сравнения: тот же запрос из середины БЕЗ индекса истории'
\echo ' (DROP INDEX внутри транзакции с ROLLBACK — индекс не теряется)'
BEGIN;
DROP INDEX messages_chat_id_created_at_id_idx;
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :'big_chat'
AND (created_at, id) > (:'cursor_created_at'::timestamptz, :'cursor_id'::uuid)
ORDER BY created_at, id
LIMIT 51;
ROLLBACK;
+73
View File
@@ -0,0 +1,73 @@
#!/bin/sh
set -eu
# Локальные синтетические секреты стенда в secrets/ (каталог в .gitignore)
# и .env из .env.example, если их ещё нет. Повторный запуск ничего не меняет:
# существующие ключи не перезаписываются, иначе данные bucket станут недоступны.
#
# s3_admin_* — ключи s3-bootstrap: создание bucket (действие Admin)
# s3_access_* — ключи API и janitor: Read/Write/List только своего bucket
# s3_identities — конфигурация identities SeaweedFS из этих ключей
#
# Значения не печатаются. Файлы 0644 внутри каталога 0700: процесс API в
# контейнере работает не от root и должен прочитать смонтированный секрет.
cd "$(dirname "$0")/.."
umask 077
if [ ! -f .env ]; then
cp .env.example .env
echo "created .env from .env.example"
fi
BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env)
: "${BUCKET:?S3_BUCKET is not set in .env}"
mkdir -p secrets
chmod 700 secrets
random() {
LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c "$1"
}
secret() {
file="secrets/$1"
if [ ! -s "$file" ]; then
random "$2" >"$file"
echo "created $file"
fi
chmod 644 "$file"
}
secret s3_admin_access_key_id 20
secret s3_admin_secret_access_key 40
secret s3_access_key_id 20
secret s3_secret_access_key 40
# Без identity "anonymous" SeaweedFS отклоняет любой неподписанный запрос
cat >secrets/s3_identities.json <<EOF
{
"identities": [
{
"name": "bootstrap",
"credentials": [
{
"accessKey": "$(cat secrets/s3_admin_access_key_id)",
"secretKey": "$(cat secrets/s3_admin_secret_access_key)"
}
],
"actions": ["Admin"]
},
{
"name": "messenger-api",
"credentials": [
{
"accessKey": "$(cat secrets/s3_access_key_id)",
"secretKey": "$(cat secrets/s3_secret_access_key)"
}
],
"actions": ["Read:$BUCKET", "Write:$BUCKET", "List:$BUCKET"]
}
]
}
EOF
chmod 644 secrets/s3_identities.json
echo "secrets ready in secrets/ (values are not printed)"
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
# Операторский вид API-реплик: контейнер ↔ alias X-Instance-Id ↔ ревизия образа.
# Нужен, чтобы показать, какой реальный контейнер обслужил запрос или остановлен:
# заголовок сам по себе не доказывает разные процессы.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
printf '%-24s %-28s %-14s %-8s %s\n' CONTAINER STATE INSTANCE PID REVISION
for container in $(docker compose --env-file "$ENV_FILE" ps -a -q api); do
name=$(docker inspect -f '{{.Name}}' "$container")
state=$(docker inspect -f '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$container")
pid=$(docker inspect -f '{{.State.Pid}}' "$container")
revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container")
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || echo -)
printf '%-24s %-28s %-14s %-8s %s\n' "${name#/}" "$state" "$alias" "$pid" "$revision"
done
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey,
# S3), затем все API-реплики, janitor и балансировщик. Никаких down -v, flush
# или повторного seed; migrate и s3-bootstrap не запускаются — схема и bucket
# уже в volumes. Готовность ждёт тест.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey s3
docker compose --env-file "$ENV_FILE" restart --no-deps api janitor
docker compose --env-file "$ENV_FILE" restart --no-deps nginx
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
# Возврат API-реплик к числу API_REPLICAS после stop-one.sh: запуск остановленных
# контейнеров и ожидание healthy. Работающие реплики не пересоздаются
# (--no-recreate), хранилища, migrate и LB не перезапускаются.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
docker compose --env-file "$ENV_FILE" up -d --no-deps --no-recreate --wait --wait-timeout 60 api
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
# Поочерёдная замена API-реплик новой сборкой без остановки обслуживания.
# 1. build нового образа (ревизия видна в метке образа и журнале старта);
# 2. migrate однократно: миграции только расширяют схему, старые реплики
# продолжают работать на новой схеме;
# 3. для каждой старой реплики: +1 новая → ждать healthy → пауза на обновление
# DNS в nginx → graceful stop старой (SIGTERM, LB уводит трафик) → удалить.
# Во время замены реплик на одну больше, чем API_REPLICAS: ресурс заложен заранее.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
COMPOSE="docker compose --env-file $ENV_FILE"
APP_REVISION="${APP_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)}"
export APP_REVISION
# ID образа зависит только от содержимого (без provenance-attestation)
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
# Больше resolver valid=2s в nginx: новая реплика уже в upstream до остановки старой
LB_DNS_REFRESH_SEC=3
STOP_TIMEOUT_SEC=10
log() {
echo "[$(date +%H:%M:%S)] $*"
}
log "build revision $APP_REVISION"
$COMPOSE build --quiet api
log "migrate"
$COMPOSE run --rm --no-deps migrate >/dev/null
old_containers=$($COMPOSE ps -q api)
for container in $old_containers; do
name=$(docker inspect -f '{{.Name}}' "$container")
running=$($COMPOSE ps -q api | wc -l | tr -d ' ')
log "scale api to $((running + 1)), wait healthy"
$COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 60 \
--scale "api=$((running + 1))" api
sleep "$LB_DNS_REFRESH_SEC"
log "graceful stop ${name#/}"
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
docker rm "$container" >/dev/null
done
log "done: all API replicas run revision $APP_REVISION"
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""Временный read-only доступ к Filer UI SeaweedFS (просмотр бакетов) с хоста.
Filer UI слушает только 127.0.0.1 внутри контейнера s3 и в сеть не открыт
(требование: административная консоль не доступна в общей сети). Этот скрипт
— операторский инструмент на время демонстрации:
* слушает только 127.0.0.1 хоста и живёт, пока запущен (Ctrl+C — закрыть);
* пропускает только GET/HEAD: удаление/загрузка через UI (POST/PUT/DELETE)
отклоняются 405, данные bucket через него не изменить;
* каждый запрос выполняется в контейнере s3 (docker exec … curl) к
127.0.0.1:8888 — порты контейнера и сети стенда не меняются.
make s3-ui # http://127.0.0.1:8888/buckets/
python3 scripts/s3_ui.py --port 18888
Python 3.9+, стандартная библиотека.
"""
from __future__ import annotations
import argparse
import subprocess
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FILER_URL = "http://127.0.0.1:8888"
CURL_TIMEOUT_SEC = 30
# Заголовки клиента, передаваемые filer: от Accept зависит HTML или JSON
FORWARDED_HEADERS = ("Accept", "Range")
# Ответ пересобирается целиком: длина и кодирование задаются заново
DROPPED_HEADERS = {"transfer-encoding", "connection", "content-length", "keep-alive"}
def s3_container() -> str:
container = subprocess.run(
["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
).stdout.strip()
if not container:
raise SystemExit("контейнер s3 не запущен: сначала make up")
return container
class ReadOnlyFilerProxy(BaseHTTPRequestHandler):
container = ""
def do_GET(self) -> None:
self._proxy(head=False)
def do_HEAD(self) -> None:
self._proxy(head=True)
def _reject(self) -> None:
self.send_error(405, "Read-only: изменение bucket через UI запрещено")
do_POST = do_PUT = do_DELETE = do_PATCH = _reject
def _proxy(self, head: bool) -> None:
command = ["docker", "exec", self.container, "curl", "-s", "-i"]
if head:
command.append("-I")
for name in FORWARDED_HEADERS:
if self.headers.get(name):
command += ["-H", f"{name}: {self.headers[name]}"]
command.append(FILER_URL + self.path)
try:
raw = subprocess.run(
command, capture_output=True, timeout=CURL_TIMEOUT_SEC, check=True
).stdout
except (subprocess.SubprocessError, OSError):
self.send_error(502, "Filer недоступен")
return
head_part, _, body = raw.partition(b"\r\n\r\n")
lines = head_part.decode("latin-1").split("\r\n")
status = int(lines[0].split()[1])
self.send_response(status)
for line in lines[1:]:
name, _, value = line.partition(":")
if name.strip().lower() not in DROPPED_HEADERS:
self.send_header(name.strip(), value.strip())
self.send_header("Content-Length", str(len(body)))
self.send_header("Connection", "close")
self.end_headers()
if not head:
self.wfile.write(body)
def log_message(self, format: str, *args: object) -> None:
print(f"{self.command} {self.path.split('?')[0]} -> {args[1] if len(args) > 1 else ''}")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
parser.add_argument("--port", type=int, default=8888)
args = parser.parse_args()
ReadOnlyFilerProxy.container = s3_container()
server = ThreadingHTTPServer(("127.0.0.1", args.port), ReadOnlyFilerProxy)
print(f"Filer UI (read-only): http://127.0.0.1:{args.port}/buckets/ — Ctrl+C, чтобы закрыть")
try:
server.serve_forever()
except KeyboardInterrupt:
pass
finally:
server.server_close()
print("проброс закрыт")
if __name__ == "__main__":
main()
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""SIGTERM одной API-реплики за балансировщиком.
Реплика перестаёт принимать соединения, дожидается текущих запросов и
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
в ответах не появляется.
"""
import argparse
import json
import os
import subprocess
import threading
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
SHUTDOWN_LIMIT_SEC = 10
PROBE_INTERVAL_SEC = 0.02
PROBERS = 4
class Prober(threading.Thread):
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
def __init__(self, url: str) -> None:
super().__init__(daemon=True)
self.url = url
self.stop = threading.Event()
self.ok: list[tuple[float, str]] = []
self.errors: list[tuple[float, str]] = []
def run(self) -> None:
while not self.stop.is_set():
try:
with urllib.request.urlopen(self.url, timeout=2) as response:
self.ok.append(
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
)
except urllib.error.HTTPError as error:
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
except OSError as error:
self.errors.append((time.monotonic(), type(error).__name__))
time.sleep(PROBE_INTERVAL_SEC)
def docker(*args: str) -> str:
result = subprocess.run(
["docker", *args], check=True, capture_output=True, text=True
)
return result.stdout.strip()
def first_api_container() -> str:
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
if not ids:
raise SystemExit("FAIL: нет запущенных API-реплик")
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
)
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
args = parser.parse_args()
os.chdir(ROOT)
container = args.container or first_api_container()
instance = docker("exec", container, "cat", "/tmp/instance-id")
logs = subprocess.Popen(
["docker", "logs", "--follow", "--since", "0s", container],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
waiter = subprocess.Popen(
["docker", "wait", container], stdout=subprocess.PIPE, text=True
)
probers = [
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
]
for prober in probers:
prober.start()
time.sleep(1)
before = {alias for prober in probers for _, alias in prober.ok}
if instance not in before:
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
sent_at = time.monotonic()
docker("kill", "--signal", "SIGTERM", container)
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
exited_at = time.monotonic()
time.sleep(1)
for prober in probers:
prober.stop.set()
prober.join()
logs.terminate()
shutdown_log = logs.communicate(timeout=5)[0]
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
last_target = max((t for t, a in ok_after if a == instance), default=None)
print("Журнал реплики после SIGTERM:")
print(
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
)
summary = {
"container": container,
"instance": instance,
"exit_code": exit_code,
"shutdown_seconds": round(exited_at - sent_at, 3),
"limit_seconds": SHUTDOWN_LIMIT_SEC,
"instances_before": sorted(before),
"probes_ok_after_sigterm": len(ok_after),
"instances_after_sigterm": sorted({a for _, a in ok_after}),
"last_target_response_after_sigterm_seconds": None
if last_target is None
else round(last_target - sent_at, 3),
"target_responses_after_exit": len(target_after_exit),
"errors_after_sigterm": len(errors_after),
"error_kinds": sorted({e for _, e in errors_after}),
}
print(json.dumps(summary, indent=2, ensure_ascii=False))
if (
exit_code != 0
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
or errors_after
or target_after_exit
or not ok_after
):
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
print(
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
% (container, exited_at - sent_at, len(ok_after))
)
if __name__ == "__main__":
main()
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
# Остановка ОДНОЙ API-реплики, обслужившей тест: TARGET_INSTANCE_ID → контейнер.
# Каждый процесс API пишет свой alias (X-Instance-Id) в /tmp/instance-id.
# docker stop: SIGTERM → graceful shutdown; restart-политика unless-stopped
# не поднимает вручную остановленный контейнер, он остаётся остановленным.
# БД, LB и volumes не затрагиваются. Восстановление — scripts/restore.sh.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
# Меньше 10 с лимита hook; больше drain + graceful shutdown (1 + 5 с)
STOP_TIMEOUT_SEC=8
for container in $(docker compose --env-file "$ENV_FILE" ps -q api); do
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || true)
if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then
name=$(docker inspect -f '{{.Name}}' "$container")
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
echo "stopped ${name#/} (instance $alias)" >&2
exit 0
fi
done
echo "no running API replica with instance $TARGET_INSTANCE_ID" >&2
exit 1
+317
View File
@@ -0,0 +1,317 @@
#!/usr/bin/env python3
"""Эксперимент: N одновременных загрузок вложений и пик памяти API-реплик.
Гипотеза: память реплики на загрузку ограничена (spool ≤ 1 MiB RAM, потоковый
PUT в S3, проверка изображения в пуле из IMAGE_INSPECT_WORKERS потоков),
поэтому пик RSS растёт не пропорционально N × размер файла.
Профили:
png — PNG ~9 MiB из шума (почти не сжимается): нагрузка на приём тела,
spool, SHA-256 и PUT в S3. Генерируется стандартной библиотекой.
jpeg — progressive JPEG 8000×5000 = 40 Мпикс (предел): худший случай
декодера libjpeg. Нужен Pillow (poetry run python ...).
Перед каждым прогоном API-реплики перезапускаются: VmHWM (пиковый RSS
процесса) начинается заново. Память опрашивается внутри контейнеров каждые
0.2 с (cgroup memory.current и VmRSS процесса). Все N запросов стартуют по
одному барьеру. Tokens не печатаются и не сохраняются.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import os
import platform
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
REQUEST_TIMEOUT_SEC = 120
READY_TIMEOUT_SEC = 90
SAMPLE_INTERVAL_SEC = 0.2
COMPOSE = ["docker", "compose", "--env-file", ".env"]
SAMPLER = (
"while :; do "
"echo $(cat /sys/fs/cgroup/memory.current) "
"$(awk '/VmRSS/ {print $2}' /proc/1/status); "
f"sleep {SAMPLE_INTERVAL_SEC}; done"
)
def png_noise(width: int, height: int) -> bytes:
"""Настоящий RGB PNG из случайных байт: размер ≈ width × height × 3"""
def chunk(kind: bytes, data: bytes) -> bytes:
return (
struct.pack(">I", len(data))
+ kind
+ data
+ struct.pack(">I", zlib.crc32(kind + data))
)
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw, 1))
+ chunk(b"IEND", b"")
)
def jpeg_worst_case() -> bytes:
from PIL import Image
Image.MAX_IMAGE_PIXELS = None
image = Image.effect_noise((8000, 5000), 64).convert("RGB")
buffer = io.BytesIO()
image.save(buffer, "JPEG", quality=10, progressive=True, subsampling=0)
return buffer.getvalue()
def compose(*args: str) -> str:
return subprocess.run(
[*COMPOSE, *args], cwd=ROOT, capture_output=True, text=True, check=True
).stdout
def api_containers() -> list[str]:
return compose("ps", "-q", "api").split()
def container_name(container: str) -> str:
return subprocess.run(
["docker", "inspect", "-f", "{{.Name}}", container],
capture_output=True, text=True, check=True,
).stdout.strip().lstrip("/")
def process_status(container: str) -> dict:
"""VmRSS/VmHWM процесса API (PID 1), байты, и его X-Instance-Id"""
output = subprocess.run(
["docker", "exec", container, "sh", "-c",
"grep -E 'VmRSS|VmHWM' /proc/1/status; cat /tmp/instance-id"],
capture_output=True, text=True, check=True,
).stdout.split("\n")
values: dict = {line.split(":")[0]: int(line.split()[1]) * 1024 for line in output[:2]}
values["instance"] = output[2].strip()
return values
def restart_api(base_url: str) -> None:
compose("restart", "api")
deadline = time.monotonic() + READY_TIMEOUT_SEC
while time.monotonic() < deadline:
states = [
subprocess.run(
["docker", "inspect", "-f", "{{.State.Health.Status}}", c],
capture_output=True, text=True,
).stdout.strip()
for c in api_containers()
]
if states and all(state == "healthy" for state in states):
return
time.sleep(1)
raise SystemExit("API replicas are not healthy after restart")
class Sampler(threading.Thread):
"""Опрос памяти одного контейнера изнутри одним долгим docker exec"""
def __init__(self, container: str) -> None:
super().__init__(daemon=True)
self.container = container
self.cgroup_max = 0
self.rss_max = 0
self.samples = 0
self._process = subprocess.Popen(
["docker", "exec", container, "sh", "-c", SAMPLER],
stdout=subprocess.PIPE, text=True,
)
def run(self) -> None:
assert self._process.stdout is not None
for line in self._process.stdout:
parts = line.split()
if len(parts) == 2:
self.cgroup_max = max(self.cgroup_max, int(parts[0]))
self.rss_max = max(self.rss_max, int(parts[1]) * 1024)
self.samples += 1
def stop(self) -> None:
self._process.terminate()
self._process.wait()
def upload_once(base: str, token: str, chat_id: str, content: bytes, content_type: str) -> dict:
request = urllib.request.Request(
f"{base}/api/v1/chats/{chat_id}/attachments?filename=experiment",
data=content,
method="POST",
headers={"Authorization": "Bearer " + token, "Content-Type": content_type},
)
started = time.monotonic()
try:
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SEC) as response:
body, status, instance = response.read(), response.status, response.headers.get("X-Instance-Id")
except urllib.error.HTTPError as error:
body, status, instance = error.read(), error.code, error.headers.get("X-Instance-Id")
except OSError as error:
return {"status": None, "error": type(error).__name__, "seconds": time.monotonic() - started}
result = {"status": status, "instance": instance, "seconds": time.monotonic() - started}
if status == 201:
result["id"] = json.loads(body)["id"]
return result
def percentile(values: list[float], share: float) -> float:
ordered = sorted(values)
return ordered[min(len(ordered) - 1, round(share * (len(ordered) - 1)))]
def run_round(client: Client, actor: dict, chat: dict, profile: str, content: bytes,
content_type: str, concurrency: int, restart: bool) -> dict:
if restart:
restart_api(client.base)
containers = api_containers()
# Прогрев: первый запрос каждой реплики открывает пулы БД/S3
warmup = png_noise(4, 4)
for _ in range(2 * len(containers)):
upload_once(client.base, actor["token"], chat["id"], warmup, "image/png")
before = {container_name(c): process_status(c) for c in containers}
samplers = [Sampler(c) for c in containers]
for sampler in samplers:
sampler.start()
time.sleep(1)
barrier = threading.Barrier(concurrency)
results: list[dict] = [{} for _ in range(concurrency)]
def worker(index: int) -> None:
barrier.wait()
results[index] = upload_once(client.base, actor["token"], chat["id"], content, content_type)
started = time.monotonic()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
wall = time.monotonic() - started
time.sleep(1)
for sampler in samplers:
sampler.stop()
after = {container_name(c): process_status(c) for c in containers}
ok = [r for r in results if r.get("status") == 201]
sample = ok[0]["id"] if ok else None
checksum_ok = None
if sample:
downloaded = client.call("GET", f"/api/v1/attachments/{sample}/content", actor=actor)["bytes"]
checksum_ok = hashlib.sha256(downloaded).digest() == hashlib.sha256(content).digest()
seconds = [r["seconds"] for r in ok]
mib = 1024 * 1024
replicas = {}
for sampler in samplers:
name = container_name(sampler.container)
replicas[name] = {
"uploads_served": sum(1 for r in ok if r.get("instance") == after[name].get("instance")),
"rss_before_mib": round(before[name]["VmRSS"] / mib, 1),
"rss_peak_process_vmhwm_mib": round(after[name]["VmHWM"] / mib, 1),
"rss_peak_sampled_mib": round(sampler.rss_max / mib, 1),
"cgroup_peak_sampled_mib": round(sampler.cgroup_max / mib, 1),
"samples": sampler.samples,
}
by_instance: dict[str, int] = {}
for r in ok:
by_instance[r["instance"]] = by_instance.get(r["instance"], 0) + 1
return {
"profile": profile,
"file_bytes": len(content),
"concurrency": concurrency,
"replicas_restarted_before": restart,
"status_counts": {str(s): sum(1 for r in results if r.get("status") == s)
for s in sorted({r.get("status") for r in results}, key=str)},
"uploads_by_instance": by_instance,
"wall_seconds": round(wall, 2),
"latency_seconds": {
"p50": round(percentile(seconds, 0.5), 3),
"p95": round(percentile(seconds, 0.95), 3),
"max": round(max(seconds), 3),
} if seconds else None,
"throughput_mib_per_second": round(len(ok) * len(content) / mib / wall, 1),
"download_sha256_matches": checksum_ok,
"replicas": replicas,
}
def environment() -> dict:
info = json.loads(subprocess.run(
["docker", "info", "--format", "{{json .}}"], capture_output=True, text=True, check=True
).stdout)
settings = {}
for line in (ROOT / ".env").read_text().splitlines():
key = line.split("=", 1)[0]
if key in {"IMAGE_INSPECT_WORKERS", "UPLOAD_SPOOL_MEMORY_BYTES", "MAX_UPLOAD_BYTES",
"MAX_IMAGE_PIXELS", "API_REPLICAS", "S3_POOL_MAX_SIZE"}:
settings[key] = line.split("=", 1)[1]
return {
"host": f"{platform.system()} {platform.machine()}",
"docker": info.get("ServerVersion"),
"docker_vm_cpus": info.get("NCPU"),
"docker_vm_memory_gib": round(info.get("MemTotal", 0) / 2**30, 1),
"api_limits": "cpus 1.0, memory 512M per replica (compose.yaml)",
"s3": "chrislusf/seaweedfs:4.48",
"settings": settings,
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
parser.add_argument("--base-url", default=None)
parser.add_argument("--output", default="evidence/experiment-uploads.json")
parser.add_argument("--concurrency", default="10,20")
parser.add_argument("--profiles", default="png,jpeg")
parser.add_argument("--no-restart", action="store_true")
args = parser.parse_args()
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
payloads = {}
if "png" in args.profiles:
payloads["png"] = (png_noise(1750, 1750), "image/png")
if "jpeg" in args.profiles:
try:
payloads["jpeg"] = (jpeg_worst_case(), "image/jpeg")
except ImportError:
print("jpeg profile skipped: Pillow is not installed (use poetry run)")
report = {"environment": environment(), "rounds": []}
for profile, (content, content_type) in payloads.items():
for concurrency in (int(x) for x in args.concurrency.split(",")):
result = run_round(client, alice, chat, profile, content, content_type,
concurrency, not args.no_restart)
report["rounds"].append(result)
print(json.dumps(result, ensure_ascii=False))
output = ROOT / args.output
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print("saved:", args.output)
if __name__ == "__main__":
main()