Добавить балансировку API и атомарные повторы сообщений

This commit is contained in:
Горбанюк
2026-10-17 18:41:13 +03:00
parent 0ab3a341aa
commit 534ddd1371
39 changed files with 4113 additions and 1 deletions
+6
View File
@@ -17,4 +17,10 @@ backups
__pycache__
node_modules
evidence
student-tests
tests
research
REPORT.md
SCHEMA.md
requirements-dev.txt
.DS_Store
+2
View File
@@ -3,6 +3,7 @@
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
PUBLIC_PORT=8083
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
@@ -11,3 +12,4 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
DATABASE_READ_AUTOCOMMIT=1
+22
View File
@@ -0,0 +1,22 @@
name: Application integration
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.14.7'
- run: python -m pip install uv==0.12.13
- run: uv sync --locked
- run: make bootstrap lint check
- run: VCS_REF=$(git rev-parse HEAD) docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
- run: PYTEST_ADDOPTS=--disable-warnings make test-student
- run: make test
- run: uv run --locked python scripts/verify_cluster.py --output /tmp/cluster-ci.json
- if: always()
run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml down
+1
View File
@@ -0,0 +1 @@
3.14.7
+28
View File
@@ -0,0 +1,28 @@
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
WORKDIR /app
RUN python -m pip install --no-cache-dir uv==0.12.13
ENV UV_PYTHON_DOWNLOADS=never
COPY pyproject.toml uv.lock .python-version ./
COPY src ./src
RUN uv sync --frozen --no-dev --no-editable --compile-bytecode
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY migrations /app/migrations
ARG VCS_REF=uncommitted
LABEL org.opencontainers.image.revision=$VCS_REF
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
APP_LOG_LEVEL=INFO
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
CMD ["python", "-m", "messenger"]
+22 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8083
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,24 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
RESEARCH_OUTPUT ?= evidence/scaling-new
.PHONY: bootstrap up down image lint test-student verify research
bootstrap:
$(PYTHON) scripts/bootstrap.py
up:
VCS_REF=$$(git rev-parse HEAD) docker compose --env-file .env up -d --build --wait
down:
docker compose --env-file .env down
image:
VCS_REF=$$(git rev-parse HEAD) docker compose --env-file .env build
lint:
uv run --locked ruff check src student-tests scripts/bootstrap.py scripts/cluster.py scripts/stop-one.py scripts/rolling-update.py scripts/verify_cluster.py research
test-student:
uv run --locked pytest
verify:
uv run --locked python scripts/verify_cluster.py
research:
uv run --locked python research/run.py --output "$(RESEARCH_OUTPUT)"
uv run --locked --group research python research/analyze.py "$(RESEARCH_OUTPUT)"
+38
View File
@@ -0,0 +1,38 @@
# Lab3: общая идемпотентность и транзакции
Основа — собственная lab2. PostgreSQL остаётся источником истины пользователей,
чатов, членства, сообщений и сессий. Valkey — восстанавливаемый кэш User,
а не доказательство действительности Bearer. У API нет обязательного локального
состояния. Каждому процессу при старте назначается случайный X-Instance-Id.
Новая миграция добавляет message_requests. Область ключа:
user_id + HTTP method POST + chat_id (путь коллекции сообщений) + ASCII key.
PK обеспечивает точную уникальность области; payload_hash содержит SHA-256
канонического валидированного тела. response_message_id ссылается на неизменяемое
Message. Срок хранения результата не менее 86400 секунд, часы — PostgreSQL.
Отправка под READ COMMITTED:
1. Аутентификация и проверка членства выполняются заново, включая повтор.
2. Transaction-level advisory lock сериализует одну область между процессами.
64-битный hash используется только как lock: коллизия вызывает лишнее ожидание,
но не подменяет точный PK и сравнение payload_hash.
3. Неистёкший результат с тем же body возвращает прежнее Message/201; другой body
даёт 409. Истёкшую запись ключа можно заменить, старое Message остаётся.
4. INSERT Message и INSERT результата идут в одной транзакции. HTTP 201 выдаётся
после COMMIT. Ошибка между вставками откатывает обе; 5xx не запоминаются.
Канонизация сохраняет точный text, сортирует JSON-ключи и учитывает пустой список
вложений для последующего расширения. Внешняя схема lab3 по OpenAPI пока принимает
только text; поля будущих лабораторных не включаются в обязательный контракт.
Миграции исполняет отдельный migrate service под advisory lock; API не запускают
DDL. Миграция 002 только добавляет таблицу и индексы: она совместима с lab2, но
старый API не умеет обязательную идемпотентность, поэтому его нельзя смешивать с
lab3 за одним LB. Внутри lab3 поочерёдная замена процесса не меняет схему.
Исследуемая оптимизация: одиночные SELECT выполняются в autocommit, без
отдельных сетевых команд BEGIN/COMMIT. PostgreSQL по-прежнему исполняет каждый
SELECT в транзакции. Вставки, отзыв сессии, многокомандное создание чата и
идемпотентная отправка всегда используют явную транзакцию и COMMIT до ответа.
DATABASE_READ_AUTOCOMMIT=0 включает базовый вариант с явными транзакциями и для
чтения; =1 — оптимизацию. Уровень изоляции, SQL, пул и обработчики одинаковы.
+6
View File
@@ -0,0 +1,6 @@
# Only DB/cache test ports; API replicas always stay behind the LB.
services:
postgres:
ports: ["127.0.0.1:${TEST_PG_PORT:-55433}:5432"]
valkey:
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56380}:6379"]
+108
View File
@@ -0,0 +1,108 @@
name: mtusi-lab3
x-api: &api
image: ${API_IMAGE:-mtusi-messenger:lab3}
build:
context: .
args:
VCS_REF: ${VCS_REF:-uncommitted}
user: "10001:10001"
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
pids_limit: 96
cpus: ${API_CPU:-1}
mem_limit: ${API_MEMORY:-512m}
memswap_limit: ${API_MEMORY:-512m}
stop_grace_period: 7s
secrets: [database_url, cache_url]
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
APP_ENV: lab3
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
networks: [backend]
services:
postgres:
image: postgres:18.3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql]
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
-c, shared_preload_libraries=pg_stat_statements]
healthcheck:
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
cpus: ${POSTGRES_CPU:-1}
mem_limit: 768m
memswap_limit: 768m
networks: [backend]
valkey:
image: valkey/valkey:8.1.3-alpine
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
--maxmemory-policy, allkeys-lru]
healthcheck:
test: [CMD, valkey-cli, ping]
interval: 2s
timeout: 1s
retries: 20
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
networks: [backend]
migrate:
<<: *api
command: [python, -m, messenger.migrate]
healthcheck: {disable: true}
depends_on:
postgres: {condition: service_healthy}
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
valkey: {condition: service_healthy}
lb:
image: haproxy:3.2.1-alpine
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports: ["127.0.0.1:${PUBLIC_PORT:-8083}:8080"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 2s
retries: 30
depends_on:
api: {condition: service_healthy}
networks: [backend]
networks:
backend: {}
volumes:
postgres_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
+44
View File
@@ -0,0 +1,44 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
timeout connect 500ms
timeout check 500ms
timeout client 10s
timeout server 8s
timeout http-request 3s
retries 0
resolvers docker
nameserver embedded 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 0s
hold nx 1s
hold refused 1s
hold timeout 1s
hold other 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
listen metrics
bind :8404
stats enable
stats uri /stats
stats refresh 1s
+37
View File
@@ -0,0 +1,37 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id,user_id),
UNIQUE (chat_id,position)
);
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry ON sessions (expires_at);
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
chat_id uuid NOT NULL REFERENCES chats(id),
idempotency_key varchar(128) COLLATE "C" NOT NULL
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
response_message_id uuid NOT NULL REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
CHECK (expires_at > created_at)
);
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
+43
View File
@@ -0,0 +1,43 @@
[project]
name = "mtusi-messenger"
version = "0.3.0"
description = "MTUSI lab 3: replicated messenger with atomic request idempotency"
requires-python = ">=3.14,<3.15"
dependencies = [
"argon2-cffi==25.1.0",
"fastapi==0.143.3",
"psycopg[binary,pool]==3.3.6",
"pydantic==2.14.1",
"redis==8.1.0",
"uvicorn==0.54.0",
]
[build-system]
requires = ["hatchling==1.27.0"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/messenger"]
[dependency-groups]
dev = [
"httpx==0.28.1",
"openapi-spec-validator==0.7.2",
"pytest==9.1.1",
"ruff==0.17.1",
]
research = ["matplotlib==3.11.2"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra --tb=short"
[tool.ruff]
target-version = "py314"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
+181
View File
@@ -0,0 +1,181 @@
#!/usr/bin/env python3
"""Summarize independent repeats without treating them as one latency sample."""
import argparse
import csv
import json
import re
from collections import defaultdict
from pathlib import Path
from statistics import mean, median
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
def mib(value):
number, unit = re.fullmatch(r"([0-9.]+)(\w+)", value.strip()).groups()
return float(number) * {"B": 1 / 1048576, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit]
def resources(trial):
cpu, memory = defaultdict(list), defaultdict(list)
for sample in trial["resource_samples"]:
per_role = defaultdict(float)
for container in sample.get("containers", []):
role = next(
(r for r in ("api", "postgres", "valkey", "lb") if f"-{r}-" in container["Name"]),
"generator",
)
per_role[role] += float(container["CPUPerc"].rstrip("%"))
memory[role].append(mib(container["MemUsage"].split("/")[0]))
for role, value in per_role.items():
cpu[role].append(value)
return {
role: {"cpu_percent": mean(cpu[role]), "max_container_mib": max(memory[role])}
for role in cpu
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
args = parser.parse_args()
manifest = json.loads((args.directory / "manifest.json").read_text())
groups = defaultdict(list)
for path in sorted(args.directory.glob("trial-*.json")):
d = json.loads(path.read_text())
c = d["configuration"]
groups[
(c["series"], c["replicas"], c["concurrency"], c["optimized"], c.get("postgres_cpu", 1))
].append(d)
assert sum(map(len, groups.values())) == len(manifest["trials"])
rows = []
for (series, n, vus, optimized, pgcpu), trials in sorted(groups.items()):
assert len(trials) == 3, "A series needs three complete independent repeats"
resource = [resources(t) for t in trials]
c = trials[0]["configuration"]
row = {
"series": series,
"replicas": n,
"concurrency": vus,
"optimized": optimized,
"postgres_cpu": pgcpu,
"total_api_cpu": c["cpu_each"] * n,
"total_api_memory_mib": c["memory_mib_each"] * n,
"repeats": len(trials),
"requests": sum(t["requests"] for t in trials),
"errors": sum(t["errors"] for t in trials),
}
for metric in ("successful_rps", "p50_ms", "p95_ms", "p99_ms"):
values = [t[metric] for t in trials]
row[metric] = median(values)
row[metric + "_min"] = min(values)
row[metric + "_max"] = max(values)
row["begin_per_http_request"] = median(
sum(q["calls"] for q in t["sql_stats"] if q["query"].strip() == "BEGIN")
/ t["raw_k6"]["metrics"]["http_reqs"]["values"]["count"]
for t in trials
)
for role in ("api", "postgres", "lb", "valkey", "generator"):
row[role + "_cpu_percent"] = median(r[role]["cpu_percent"] for r in resource)
row[role + "_max_container_mib"] = max(r[role]["max_container_mib"] for r in resource)
row["api_quota_utilization_percent"] = row["api_cpu_percent"] / row["total_api_cpu"]
row["postgres_quota_utilization_percent"] = row["postgres_cpu_percent"] / pgcpu
rows.append(row)
(args.directory / "analysis.json").write_text(json.dumps(rows, indent=2) + "\n")
with (args.directory / "analysis.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
lines = [
"| Series | API | VU | PG CPU | RPS median [min; max] | p50/p95/p99 ms | "
"Errors/requests | API CPU / PG CPU % | Max API MiB |",
"| --- | ---: | ---: | ---: | --- | --- | --- | --- | ---: |",
]
for r in rows:
lines.append(
f"| {r['series']} | {r['replicas']} | {r['concurrency']} | {r['postgres_cpu']} | "
f"{r['successful_rps']:.1f} [{r['successful_rps_min']:.1f}; "
f"{r['successful_rps_max']:.1f}] | "
f"{r['p50_ms']:.1f}/{r['p95_ms']:.1f}/{r['p99_ms']:.1f} | "
f"{r['errors']}/{r['requests']} | "
f"{r['api_cpu_percent']:.1f}/{r['postgres_cpu_percent']:.1f} | "
f"{r['api_max_container_mib']:.1f} |"
)
(args.directory / "results.md").write_text("\n".join(lines) + "\n")
plt.rcParams.update(
{
"font.size": 10,
"figure.dpi": 140,
"axes.grid": True,
"grid.alpha": 0.2,
"axes.spines.top": False,
"axes.spines.right": False,
}
)
fig, axes = plt.subplots(1, 3, figsize=(13, 4), constrained_layout=True)
for series, label in [
("fixed", "Fixed total: 2 CPU / 1 GiB"),
("growing", "Per replica: 1 CPU / 512 MiB"),
]:
values = [r for r in rows if r["series"] == series]
xs = [r["replicas"] for r in values]
for ax, metric, title in zip(
axes,
["successful_rps", "p95_ms", "api_quota_utilization_percent"],
["Successful requests/s", "p95 latency (ms)", "API CPU / quota (%)"],
strict=True,
):
ax.plot(xs, [r[metric] for r in values], marker="o", label=label)
if metric != "api_quota_utilization_percent":
ax.fill_between(
xs,
[r[metric + "_min"] for r in values],
[r[metric + "_max"] for r in values],
alpha=0.12,
)
ax.set(title=title, xlabel="API replicas", xticks=[1, 2, 4])
axes[0].legend(fontsize=8)
fig.suptitle("Scaling: same 90% read / 10% write workload, 32 VUs; median of 3 repeats")
fig.savefig(args.directory / "scaling.png")
plt.close(fig)
baseline = next(r for r in rows if r["series"] == "fixed" and r["replicas"] == 2)
optimized = next(r for r in rows if r["series"] == "improvement")
values = sorted(
[baseline] + [r for r in rows if r["series"] == "saturation"],
key=lambda r: r["concurrency"],
)
fig, axes = plt.subplots(1, 3, figsize=(13, 4), constrained_layout=True)
xs = [r["concurrency"] for r in values]
axes[0].plot(xs, [r["successful_rps"] for r in values], "o-", label="RPS")
tail = axes[0].twinx()
tail.plot(xs, [r["p95_ms"] for r in values], "s--", color="tab:orange", label="p95 ms")
tail.set_ylabel("p95 ms", color="tab:orange")
axes[0].set(title="Concurrency, baseline", xlabel="VUs", ylabel="Successful RPS")
db = sorted(
[optimized] + [r for r in rows if r["series"] == "database"],
key=lambda r: r["postgres_cpu"],
)
axes[1].plot([r["postgres_cpu"] for r in db], [r["successful_rps"] for r in db], "o-")
axes[1].set(
title="Controlled PostgreSQL saturation", xlabel="PostgreSQL CPU quota", ylabel="RPS"
)
axes[2].bar(
["Baseline", "Autocommit\nsingle SELECT"],
[baseline["successful_rps"], optimized["successful_rps"]],
color=["tab:blue", "tab:green"],
)
axes[2].set(title="Same 2 API / 2 CPU / 32 VUs", ylabel="Successful RPS")
gain = (optimized["successful_rps"] / baseline["successful_rps"] - 1) * 100
fig.suptitle(f"Bottlenecks and one change: median RPS gain {gain:+.1f}%")
fig.savefig(args.directory / "bottlenecks.png")
plt.close(fig)
print("\n".join(lines))
print(f"Optimization median gain: {gain:+.2f}%")
if __name__ == "__main__":
main()
+61
View File
@@ -0,0 +1,61 @@
import http from 'k6/http';
import { Counter, Trend } from 'k6/metrics';
const fixture = JSON.parse(open('/input/session.json'));
const requests = new Counter('measured_requests');
const successes = new Counter('measured_successes');
const errors = new Counter('measured_errors');
const latency = new Trend('successful_latency', true);
const allLatency = new Trend('all_latency', true);
const allWrites = new Counter('all_confirmed_writes');
const served = {};
for (const alias of fixture.instances) served[alias] = new Counter('served_' + alias);
const warmup = Number(__ENV.WARMUP || '5');
const duration = Number(__ENV.DURATION || '15');
const concurrency = Number(__ENV.CONCURRENCY || '32');
export const options = {
scenarios: {
warmup: { executor: 'constant-vus', vus: concurrency, duration: `${warmup}s`,
exec: 'warm', gracefulStop: '1s' },
measure: { executor: 'constant-vus', vus: concurrency, duration: `${duration}s`,
startTime: `${warmup + 1}s`, exec: 'measure', gracefulStop: '5s' },
},
summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)'],
discardResponseBodies: true,
};
function request(measured) {
const path = `http://lb:8080/api/v1/chats/${fixture.chat}/messages`;
const key = `${fixture.run}:${measured ? 'm' : 'w'}:${__VU}:${__ITER}`;
const params = { headers: { 'Authorization': 'Bearer ' + fixture.token,
'Content-Type': 'application/json', 'Idempotency-Key': key },
timeout: '10s', tags: { name: 'messages' } };
const write = __ITER % 10 === 9;
const cursor = __ITER % 2 === 0 ? `&cursor=${encodeURIComponent(fixture.cursor)}` : '';
const response = write
? http.post(path, JSON.stringify({ text: key }), params)
: http.get(`${path}?limit=50${cursor}`, params);
if (write && response.status === 201) allWrites.add(1);
if (measured) {
requests.add(1);
allLatency.add(response.timings.duration);
const alias = response.headers['X-Instance-Id'];
const ok = response.status === (write ? 201 : 200) && alias in served
&& Boolean(response.headers['X-Request-Id']);
(ok ? successes : errors).add(1);
if (ok) {
latency.add(response.timings.duration);
served[alias].add(1);
}
}
}
export function warm() { request(false); }
export function measure() { request(true); }
export function handleSummary(data) {
data.profile = { warmup_seconds: warmup, duration_seconds: duration,
concurrency: concurrency, measurement_wall_seconds:
data.state.testRunDurationMs / 1000 - warmup - 1 };
return { '/output/k6.json': JSON.stringify(data, null, 2) };
}
+414
View File
@@ -0,0 +1,414 @@
#!/usr/bin/env python3
"""Repeatable scaling matrix on an isolated Compose project and fresh data."""
import argparse
import hashlib
import json
import os
import platform
import random
import subprocess
import sys
import tempfile
import threading
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import psycopg
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
from messenger.cursor import encode_cursor
from messenger.repository import stamp
ROOT = Path(__file__).resolve().parents[1]
PROJECT = "mtusi-lab3-research"
K6 = "grafana/k6@sha256:3ddc8b1a33a2c3d8edc6e99b6a762ae36cba08788463458f5e6a7703e14eb77d"
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def dc(*args, env):
return subprocess.run(
[
"docker",
"compose",
"-p",
PROJECT,
"--env-file",
".env",
"-f",
"compose.yaml",
"-f",
"compose.test.yaml",
*args,
],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=120,
)
def count(metrics, key):
return metrics.get(key, {}).get("values", {}).get("count", 0)
def wait_api(client, expected):
seen, deadline = set(), time.monotonic() + 60
while time.monotonic() < deadline:
try:
response = client.call("GET", "/health/ready", schema="Health")
seen.add(response["headers"]["X-Instance-Id"])
if set(expected) <= seen:
return sorted(expected)
except Failure:
pass
time.sleep(0.1)
raise RuntimeError("Research APIs failed to become ready")
def samples(stop, container_ids, result):
while not stop.is_set():
try:
r = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *container_ids],
capture_output=True,
text=True,
check=True,
timeout=5,
)
result.append(
{
"at": datetime.now(UTC).isoformat(),
"containers": [
{k: value[k] for k in ("ID", "Name", "CPUPerc", "MemUsage", "MemPerc")}
for value in map(json.loads, r.stdout.splitlines())
],
}
)
except OSError, ValueError, KeyError, subprocess.SubprocessError:
result.append({"error": "resource_sample_unavailable"})
stop.wait(0.25)
def fixture(conn, client):
# This connection only belongs to PROJECT's separate PostgreSQL container.
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions,message_requests CASCADE")
alice, _, _, chat = client.bootstrap()
conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
"SELECT md5(%s || g::text)::uuid,%s,%s,repeat('m',128),"
"'2026-10-17T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
"FROM generate_series(1,10000) g",
(chat["id"], chat["id"], alice["id"]),
)
conn.execute("VACUUM ANALYZE messages")
row = conn.execute(
"SELECT created_at,id FROM messages WHERE chat_id=%s "
"ORDER BY created_at,id OFFSET 9000 LIMIT 1",
(chat["id"],),
).fetchone()
cursor = encode_cursor("messages:" + chat["id"], (stamp(row["created_at"]), str(row["id"])))
return {"token": alice["token"], "chat": chat["id"], "cursor": cursor, "run": uuid4().hex}
def run_trial(config, env, output, dsn, duration, warmup):
env = {
**env,
"API_REPLICAS": str(config["replicas"]),
"POSTGRES_CPU": str(config.get("postgres_cpu", 1)),
"API_CPU": str(config["cpu_each"]),
"API_MEMORY": str(config["memory_mib_each"]) + "m",
"DATABASE_READ_AUTOCOMMIT": str(int(config["optimized"])),
}
dc("up", "-d", "--no-build", "--wait", env=env)
# PostgreSQL CPU changes recreate its container; never reuse that old socket.
with psycopg.connect(dsn, autocommit=True, row_factory=dict_row) as conn:
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
measure(config, env, output, conn, duration, warmup)
def measure(config, env, output, conn, duration, warmup):
client = Client("http://127.0.0.1:18083", timeout=5)
ids = dc("ps", "-q", "api", env=env).stdout.split()
assert len(ids) == config["replicas"]
aliases = []
for container in ids:
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"
)
aliases.append(
subprocess.check_output(
["docker", "exec", container, "python", "-c", code], text=True
).strip()
)
aliases = wait_api(client, aliases)
dc("exec", "-T", "valkey", "valkey-cli", "FLUSHDB", env=env)
data = fixture(conn, client)
data["instances"] = aliases
conn.execute("SELECT pg_stat_statements_reset()")
resources, stop, generator = [], threading.Event(), None
with tempfile.TemporaryDirectory(prefix="lab3-load-") as temporary:
temporary = Path(temporary)
session = temporary / "session.json"
session.write_text(json.dumps(data))
os.chmod(session, 0o600)
logs = temporary / "out"
logs.mkdir()
try:
r = subprocess.run(
[
"docker",
"run",
"-d",
"--network",
PROJECT + "_backend",
"--cpus",
"2",
"--memory",
"384m",
"--memory-swap",
"384m",
"--user",
str(os.getuid()) + ":" + str(os.getgid()),
"--read-only",
"--tmpfs",
"/tmp:rw,size=16m",
"-v",
str(ROOT / "research/load.js") + ":/script/load.js:ro",
"-v",
str(session) + ":/input/session.json:ro",
"-v",
str(logs) + ":/output",
"-e",
"K6_NO_USAGE_REPORT=true",
"-e",
"DURATION=" + str(duration),
"-e",
"WARMUP=" + str(warmup),
"-e",
"CONCURRENCY=" + str(config["concurrency"]),
K6,
"run",
"--quiet",
"/script/load.js",
],
check=True,
capture_output=True,
text=True,
)
generator = r.stdout.strip()
ids = dc("ps", "-q", env=env).stdout.split() + [generator]
sampler = threading.Thread(target=samples, args=(stop, ids, resources), daemon=True)
sampler.start()
result = subprocess.run(
["docker", "wait", generator],
check=True,
capture_output=True,
text=True,
timeout=duration + warmup + 35,
)
if result.stdout.strip() != "0":
raise RuntimeError("Load generator failed")
raw = json.loads((logs / "k6.json").read_text())
finally:
stop.set()
if "sampler" in locals():
sampler.join(timeout=6)
if generator:
subprocess.run(["docker", "rm", "-f", generator], capture_output=True, check=True)
sql_stats = [
dict(row)
for row in conn.execute(
"SELECT query,calls,total_exec_time,mean_exec_time,rows FROM pg_stat_statements "
"WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) "
"ORDER BY calls DESC LIMIT 30"
).fetchall()
]
metrics = raw["metrics"]
requests, successful = count(metrics, "measured_requests"), count(metrics, "measured_successes")
latency = metrics.get("successful_latency", {}).get("values", {})
final_rows = conn.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]
key_rows = conn.execute("SELECT count(*) AS n FROM message_requests").fetchone()["n"]
assert final_rows - 10000 == key_rows, "Message/result transaction invariant failed"
assert key_rows >= count(metrics, "all_confirmed_writes"), "Confirmed writes missing"
summary = {
"configuration": config,
"requests": requests,
"successes": successful,
"errors": count(metrics, "measured_errors"),
"error_fraction": (requests - successful) / requests,
"successful_rps": successful / raw["profile"]["measurement_wall_seconds"],
"p50_ms": latency.get("med"),
"p95_ms": latency.get("p(95)"),
"p99_ms": latency.get("p(99)"),
"final_messages": final_rows,
"idempotency_results": key_rows,
"resource_samples": resources,
"sql_stats": sql_stats,
"raw_k6": raw,
}
output.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
print(
json.dumps(
{
"trial": output.name,
"rps": round(summary["successful_rps"], 2),
"p95_ms": round(summary["p95_ms"], 2),
"errors": summary["errors"],
}
),
flush=True,
)
def matrix(pilot):
if pilot:
return [
{
"series": "pilot",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": 32,
"optimized": mode,
"repeat": 1,
}
for mode in (False, True)
]
values = []
for repeat in range(1, 4):
for series in ("fixed", "growing"):
for replicas in (1, 2, 4):
values.append(
{
"series": series,
"replicas": replicas,
"cpu_each": 2 / replicas if series == "fixed" else 1,
"memory_mib_each": 1024 // replicas if series == "fixed" else 512,
"concurrency": 32,
"optimized": False,
"repeat": repeat,
}
)
for vus in (8, 96):
values.append(
{
"series": "saturation",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": vus,
"optimized": False,
"repeat": repeat,
}
)
values.append(
{
"series": "improvement",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": 32,
"optimized": True,
"repeat": repeat,
}
)
for quota in (0.1, 0.25):
values.append(
{
"series": "database",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": 32,
"optimized": True,
"postgres_cpu": quota,
"repeat": repeat,
}
)
random.Random(2301).shuffle(values)
return values
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, default=ROOT / "evidence/scaling")
parser.add_argument("--duration", type=int, default=15)
parser.add_argument("--warmup", type=int, default=5)
parser.add_argument("--pilot", action="store_true")
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Choose a new output directory")
args.output.mkdir(parents=True)
env = {
**os.environ,
"PUBLIC_PORT": "18083",
"TEST_PG_PORT": "55434",
"TEST_CACHE_PORT": "56381",
"API_REPLICAS": "2",
"SESSION_TTL_SECONDS": "3600",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
# Separate project name and published DB port prevent touching the main lab3 data.
dc("up", "-d", "--no-build", "--wait", env=env)
password = (ROOT / "secrets/postgres_password").read_text().strip()
dsn = make_conninfo(
host="127.0.0.1",
port=55434,
dbname="messenger",
user="messenger",
password=password,
options="-c timezone=UTC",
connect_timeout=3,
)
metadata = {
"started_at": datetime.now(UTC).isoformat(),
"project": PROJECT,
"source_sha256": {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "src/messenger").glob("*.py")
},
"k6_image": K6,
"docker_host": json.loads(
subprocess.check_output(
["docker", "info", "--format", '{"cpu":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'],
text=True,
)
),
"api_image": subprocess.check_output(
["docker", "image", "inspect", "mtusi-messenger:lab3", "--format", "{{.Id}}"], text=True
).strip(),
"client_architecture": platform.machine(),
"trials": matrix(args.pilot),
}
try:
for index, config in enumerate(metadata["trials"], 1):
run_trial(
config,
env,
args.output / f"trial-{index:02d}.json",
dsn,
args.duration,
args.warmup,
)
assert metadata["source_sha256"] == {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "src/messenger").glob("*.py")
}, "Runtime source changed mid-run"
metadata["finished_at"] = datetime.now(UTC).isoformat()
(args.output / "manifest.json").write_text(json.dumps(metadata, indent=2) + "\n")
print("PASS: isolated scaling matrix completed", flush=True)
finally:
dc("down", env=env) # Keeps this project's volume; never touches the main project.
if __name__ == "__main__":
main()
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once, without printing their values."""
import os
import secrets
from pathlib import Path
root = Path(__file__).resolve().parents[1]
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_urlsafe(32) + "\n")
password = password_file.read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
"cache_url": "redis://valkey:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n")
# Compose bind-mount secrets must be readable by the non-root container process.
# The parent directory remains 0700 on the host.
for name in ("postgres_password", "database_url", "cache_url"):
os.chmod(directory / name, 0o644)
env = root / ".env"
if not env.exists():
env.write_text((root / ".env.example").read_text())
print("Local secret files and environment are ready; existing values preserved.")
+82
View File
@@ -0,0 +1,82 @@
"""Read-only discovery and scoped Compose operations for the local lab3 stack."""
import json
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="http://127.0.0.1:8083", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# All running application/state services; no migration, down -v, flush or seed.
docker compose --env-file .env restart --timeout 7 api postgres valkey lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8083"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8083")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8083")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""One-process implementation of the lab 1 messenger contract."""
+24
View File
@@ -0,0 +1,24 @@
import logging
import os
import uvicorn
def main():
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
logging.basicConfig(level=level, format="%(message)s")
uvicorn.run(
"messenger.app:create_app",
factory=True,
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
access_log=False,
log_level=level.lower(),
timeout_keep_alive=2,
timeout_graceful_shutdown=5,
)
if __name__ == "__main__":
main()
+286
View File
@@ -0,0 +1,286 @@
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
import json
import logging
import os
import re
from contextlib import asynccontextmanager
from time import perf_counter
from typing import Annotated
from uuid import uuid4
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from starlette.exceptions import HTTPException
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
from messenger.cache import SessionCache
from messenger.domain import DomainError, User, invalid
from messenger.repository import PostgresRepository
from messenger.schemas import (
ChatPage,
ChatView,
CreateChat,
CreateMessage,
CreateUser,
MessagePage,
MessageView,
SessionView,
UserView,
canonical_uuid,
)
from messenger.service import Messenger
from messenger.settings import Settings
logger = logging.getLogger("messenger")
class RequestContext:
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id
async def __call__(self, scope: Scope, receive: Receive, send: Send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
request_id = str(uuid4())
scope.setdefault("state", {})["request_id"] = request_id
start, status, response_started = perf_counter(), 500, False
async def with_id(message: Message):
nonlocal status, response_started
if message["type"] == "http.response.start":
response_started = True
status = message["status"]
message["headers"] = [
pair
for pair in message.get("headers", [])
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
] + [
(b"x-request-id", request_id.encode()),
(b"x-instance-id", self.instance_id.encode()),
]
await send(message)
try:
await self.app(scope, receive, with_id)
except Exception as exc:
# Do not let Uvicorn print exception values that could contain user data.
if response_started:
raise
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
response = error_response(
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
)
await response(scope, receive, with_id)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request_id,
"method": scope["method"],
"route": route,
"status": status,
"duration_ms": round((perf_counter() - start) * 1000, 3),
}
)
)
def error_response(
request: Request, status: int, code: str, message: str, headers: dict | None = None
) -> JSONResponse:
identity = getattr(request.state, "request_id", str(uuid4()))
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": identity}},
headers={"X-Request-Id": identity, **(headers or {})},
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings.from_env()
repository = PostgresRepository(
settings.database_url,
settings.pool_size,
settings.idempotency_ttl,
settings.read_autocommit,
)
cache = SessionCache(settings.cache_url)
authentication = AuthService(repository, cache, settings.session_ttl)
@asynccontextmanager
async def lifespan(app: FastAPI):
repository.open()
app.state.ready = True
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
try:
yield
finally:
app.state.ready = False
repository.close()
cache.close()
logger.info(json.dumps({"event": "stopped"}))
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
app.state.messenger = Messenger(repository)
app.state.repository = repository
app.state.authentication = authentication
app.state.cache = cache
app.state.ready = False
@app.exception_handler(DomainError)
async def domain_error(request: Request, exc: DomainError):
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
@app.exception_handler(HTTPException)
async def http_error(request: Request, exc: HTTPException):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
@app.exception_handler(Exception)
async def unexpected_error(request: Request, exc: Exception):
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
def service(request: Request) -> Messenger:
return request.app.state.messenger
def token_digest(request: Request) -> bytes:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized()
return bearer_digest(request.headers.get("authorization"))
def identity(
digest: Annotated[bytes, Depends(token_digest)],
) -> Identity:
return authentication.identify(digest)
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
return current.user
async def credentials(request: Request) -> tuple[str, str]:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized("Basic")
result = basic_credentials(request.headers.get("authorization"))
if await request.body():
raise invalid("Вход не принимает тело запроса")
return result
def resource_id(chat_id: str) -> str:
try:
return canonical_uuid(chat_id)
except ValueError:
raise invalid("Некорректный UUID чата") from None
def pagination(request: Request) -> tuple[int, str | None]:
for key in ("limit", "cursor"):
if len(request.query_params.getlist(key)) > 1:
raise invalid("Параметр пагинации указан несколько раз")
raw = request.query_params.get("limit", "50")
try:
if not re.fullmatch(r"[0-9]+", raw):
raise ValueError
limit = int(raw)
if not 1 <= limit <= 100:
raise ValueError
except ValueError:
raise invalid("limit должен быть целым от 1 до 100") from None
return limit, request.query_params.get("cursor")
def json_input(request: Request):
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media != "application/json":
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
def message_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
raise invalid("Требуется корректный Idempotency-Key")
return values[0]
CurrentUser = Annotated[User, Depends(actor)]
Service = Annotated[Messenger, Depends(service)]
ChatId = Annotated[str, Depends(resource_id)]
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready(request: Request):
if not request.app.state.ready:
raise DomainError(503, "unavailable", "Приложение не готово")
repository.ready()
return {"status": "ok"}
@app.post(
"/api/v1/users",
status_code=201,
response_model=UserView,
dependencies=[Depends(json_input)],
)
def register(body: CreateUser):
return authentication.register(body.username, body.display_name, body.password)
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
return authentication.login(*value)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(current: Annotated[Identity, Depends(identity)]):
authentication.logout(current)
return Response(status_code=204)
@app.get("/api/v1/users/me", response_model=UserView)
def me(current: CurrentUser):
return current
@app.post(
"/api/v1/chats",
status_code=201,
response_model=ChatView,
dependencies=[Depends(json_input)],
)
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
return messenger.create_chat(current, body.title, body.member_ids)
@app.get("/api/v1/chats", response_model=ChatPage)
def chats(current: CurrentUser, messenger: Service, page: Pagination):
return messenger.chats(current, *page)
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
return messenger.chat(current, chat)
@app.post(
"/api/v1/chats/{chat_id}/messages",
status_code=201,
response_model=MessageView,
dependencies=[Depends(json_input)],
)
def send_message(
body: CreateMessage,
current: CurrentUser,
chat: ChatId,
messenger: Service,
key: Annotated[str, Depends(message_key)],
):
return messenger.send(current, chat, body.text, key)
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
return messenger.messages(current, chat, *page)
return app
+127
View File
@@ -0,0 +1,127 @@
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
import base64
import binascii
import hashlib
import re
import secrets
from contextlib import contextmanager
from dataclasses import dataclass
from threading import BoundedSemaphore
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.profiles import RFC_9106_LOW_MEMORY
from messenger.cache import SessionCache
from messenger.domain import DomainError, User
from messenger.repository import PostgresRepository
def unauthorized(scheme: str = "Bearer") -> DomainError:
challenge = scheme + ' realm="messenger"'
if scheme == "Basic":
challenge += ', charset="UTF-8"'
return DomainError(
401,
"unauthorized",
"Требуется действительная идентификация",
{"WWW-Authenticate": challenge},
)
def basic_credentials(header: str | None) -> tuple[str, str]:
try:
if header is None or len(header) > 2048:
raise ValueError
scheme, encoded = header.split(" ", 1)
if scheme.lower() != "basic":
raise ValueError
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = raw.split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
raise ValueError
if not 12 <= len(password) <= 128 or not password.strip():
raise ValueError
return username, password
except ValueError, UnicodeError, binascii.Error:
raise unauthorized("Basic") from None
def bearer_digest(header: str | None) -> bytes:
if header:
parts = header.split(" ")
if len(parts) == 2 and parts[0].lower() == "bearer":
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
return hashlib.sha256(parts[1].encode()).digest()
raise unauthorized()
@dataclass(frozen=True)
class Identity:
user: User
token_hash: bytes
@dataclass(frozen=True)
class Session:
token: str
token_type: str
expires_at: str
user: User
class AuthService:
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
self.repository, self.cache, self.ttl = repository, cache, ttl
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
self.kdf_slots = BoundedSemaphore(2)
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
@contextmanager
def kdf(self):
if not self.kdf_slots.acquire(timeout=0.5):
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
try:
yield
finally:
self.kdf_slots.release()
def register(self, username: str, display_name: str, password: str) -> User:
with self.kdf():
encoded = self.hasher.hash(password)
return self.repository.add_user(username, display_name, encoded)
def login(self, username: str, password: str) -> Session:
credentials = self.repository.credentials(username)
encoded = credentials[1] if credentials else self.dummy_hash
try:
with self.kdf():
self.hasher.verify(encoded, password)
except VerificationError, InvalidHashError:
raise unauthorized("Basic") from None
if credentials is None:
raise unauthorized("Basic")
token = secrets.token_urlsafe(32)
digest = hashlib.sha256(token.encode()).digest()
user = credentials[0]
expires = self.repository.create_session(digest, user.id, self.ttl)
return Session(token, "Bearer", expires, user)
def identify(self, digest: bytes) -> Identity:
session = self.repository.active_session(digest)
if session is None:
raise unauthorized()
user_id = str(session["user_id"])
user = self.cache.get(digest, user_id)
if user is None:
user = self.repository.get_user(user_id)
if user is None:
raise unauthorized()
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
self.cache.put(digest, user, ttl)
return Identity(user, digest)
def logout(self, identity: Identity):
self.repository.revoke_session(identity.token_hash)
self.cache.delete(identity.token_hash)
+57
View File
@@ -0,0 +1,57 @@
"""Disposable session-user cache; PostgreSQL always decides token validity."""
import json
import logging
from dataclasses import asdict
from redis import Redis
from redis.backoff import NoBackoff
from redis.exceptions import RedisError
from redis.retry import Retry
from messenger.domain import User
logger = logging.getLogger("messenger")
class SessionCache:
def __init__(self, url: str):
self.client = Redis.from_url(
url,
socket_connect_timeout=0.2,
socket_timeout=0.2,
max_connections=8,
decode_responses=True,
retry_on_timeout=False,
retry=Retry(NoBackoff(), 0),
)
@staticmethod
def key(digest: bytes) -> str:
return "session-user:" + digest.hex()
def get(self, digest: bytes, user_id: str) -> User | None:
try:
raw = self.client.get(self.key(digest))
if raw:
value = User(**json.loads(raw))
if value.id == user_id:
return value
except RedisError, ValueError, TypeError:
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
return None
def put(self, digest: bytes, user: User, ttl: int):
try:
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
def delete(self, digest: bytes):
try:
self.client.delete(self.key(digest))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
def close(self):
self.client.close()
+41
View File
@@ -0,0 +1,41 @@
"""Versioned keyset position, bound to a list and its resource, never an offset."""
import base64
import binascii
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.domain import HistoryKey, invalid
def encode_cursor(scope: str, position: HistoryKey) -> str:
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
return (
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
.decode()
.rstrip("=")
)
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
if value is None:
return None
try:
if not value or len(value) > 1024:
raise ValueError
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
data = json.loads(raw)
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
raise ValueError
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
raise ValueError
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
raise ValueError
stamp = datetime.fromisoformat(data["created_at"])
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
raise ValueError
identity = str(UUID(data["id"]))
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
raise invalid("Некорректный cursor или другая область пагинации") from None
+62
View File
@@ -0,0 +1,62 @@
"""Immutable domain values; no dependency on HTTP or a storage technology."""
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import uuid4
def utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
def new_id() -> str:
return str(uuid4())
class DomainError(Exception):
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid(message: str = "Некорректный запрос") -> DomainError:
return DomainError(400, "invalid_request", message)
def not_found() -> DomainError:
return DomainError(404, "not_found", "Объект не найден")
@dataclass(frozen=True, slots=True)
class User:
id: str
username: str
display_name: str
created_at: str
@dataclass(frozen=True, slots=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: str
@dataclass(frozen=True, slots=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: str
type HistoryKey = tuple[str, str]
def history_key(item: Chat | Message) -> HistoryKey:
return item.created_at, item.id
+46
View File
@@ -0,0 +1,46 @@
"""Transactional, checksummed migrations, serialized across concurrent starts."""
import hashlib
import os
from pathlib import Path
import psycopg
from messenger.settings import secret
def migrate(conninfo: str, directory: Path):
with psycopg.connect(conninfo, connect_timeout=5) as conn:
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
)
for path in sorted(directory.glob("*.sql")):
content = path.read_text()
checksum = hashlib.sha256(content.encode()).hexdigest()
current = conn.execute(
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
).fetchone()
if current:
if current[0] != checksum:
raise RuntimeError("Applied migration checksum changed: " + path.stem)
continue
conn.execute(content)
conn.execute(
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
(path.stem, checksum),
)
print("Applied migration " + path.stem)
def main():
migrate(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
)
if __name__ == "__main__":
main()
+263
View File
@@ -0,0 +1,263 @@
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
import hashlib
import json
from contextlib import contextmanager, nullcontext
from datetime import datetime
from typing import Protocol
import psycopg
from psycopg import errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
from messenger.domain import Chat, DomainError, HistoryKey, Message, User, invalid, new_id
def stamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
def user_value(row) -> User:
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
def chat_value(row) -> Chat:
return Chat(
str(row["id"]),
row["title"],
tuple(str(x) for x in row["member_ids"]),
stamp(row["created_at"]),
)
def message_value(row) -> Message:
return Message(
str(row["id"]),
str(row["chat_id"]),
str(row["sender_id"]),
row["text"],
stamp(row["created_at"]),
)
class Repository(Protocol):
def get_user(self, user_id: str) -> User | None: ...
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
def get_chat(self, chat_id: str) -> Chat | None: ...
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
def idempotent_message(self, chat_id: str, sender_id: str, text: str, key: str) -> Message: ...
def page_messages(
self, chat_id: str, after: HistoryKey | None, limit: int
) -> list[Message]: ...
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
class PostgresRepository:
def __init__(
self,
conninfo: str,
pool_size: int = 8,
idempotency_ttl: int = 86400,
read_autocommit: bool = True,
):
self.read_autocommit = read_autocommit
self.idempotency_ttl = idempotency_ttl
self.pool = ConnectionPool(
conninfo,
min_size=1,
max_size=pool_size,
timeout=1,
max_waiting=32,
open=False,
check=ConnectionPool.check_connection,
kwargs={
"row_factory": dict_row,
"autocommit": True,
"connect_timeout": 2,
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
},
)
def open(self):
self.pool.open(wait=True, timeout=15)
def close(self):
self.pool.close(timeout=5)
@contextmanager
def connection(self, *, write: bool = False):
try:
with self.pool.connection() as conn:
transaction = (
conn.transaction() if write or not self.read_autocommit else nullcontext()
)
with transaction:
yield conn
except (
psycopg.OperationalError,
psycopg.InterfaceError,
PoolTimeout,
TooManyRequests,
errors.QueryCanceled,
errors.LockNotAvailable,
):
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
def ready(self):
with self.connection() as conn:
row = conn.execute(
"SELECT count(*) AS n FROM schema_migrations "
"WHERE version IN ('001_initial','002_idempotency')"
).fetchone()
if row["n"] != 2:
raise DomainError(503, "unavailable", "Схема ещё не готова")
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO users (id,username,display_name,password_hash) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), username, display_name, password_hash),
).fetchone()
return user_value(row)
except errors.UniqueViolation:
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
def credentials(self, username: str):
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
return (user_value(row), row["password_hash"]) if row else None
def get_user(self, user_id: str) -> User | None:
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
return user_value(row) if row else None
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
[(row["id"], member, i) for i, member in enumerate(member_ids)],
)
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
except errors.ForeignKeyViolation:
raise invalid("Указан неизвестный участник") from None
def get_chat(self, chat_id: str) -> Chat | None:
with self.connection() as conn:
row = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
).fetchone()
return chat_value(row) if row else None
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (user_id, *after, limit) if after else (user_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
params,
).fetchall()
return [chat_value(row) for row in rows]
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
return message_value(row)
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
condition = " AND (created_at,id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (chat_id, *after, limit) if after else (chat_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT * FROM messages WHERE chat_id=%s"
+ condition
+ " ORDER BY created_at,id LIMIT %s",
params,
).fetchall()
return [message_value(row) for row in rows]
def idempotent_message(self, chat_id: str, sender_id: str, text: str, key: str) -> Message:
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
payload = json.dumps(
{"text": text, "attachment_ids": []},
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
)
digest = hashlib.sha256(payload.encode()).digest()
with self.connection(write=True) as conn:
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
previous = conn.execute(
"SELECT r.payload_hash,m.* FROM message_requests r "
"JOIN messages m ON m.id=r.response_message_id "
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
(sender_id, chat_id, key),
).fetchone()
if previous:
if previous["payload_hash"] != digest:
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
return message_value(previous)
conn.execute(
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
"AND chat_id=%s AND idempotency_key=%s",
(sender_id, chat_id, key),
)
message = message_value(
conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
)
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
return message
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
conn.execute(
"INSERT INTO message_requests "
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
)
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO sessions (token_hash,user_id,expires_at) "
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
(token_hash, user_id, ttl),
).fetchone()
return stamp(row["expires_at"])
def active_session(self, token_hash: bytes):
with self.connection() as conn:
return conn.execute(
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
(token_hash,),
).fetchone()
def revoke_session(self, token_hash: bytes):
with self.connection(write=True) as conn:
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
+92
View File
@@ -0,0 +1,92 @@
"""Strict lab 2 wire types; password never belongs to a response User."""
import re
from typing import Annotated
from uuid import UUID
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator
def canonical_uuid(value: str) -> str:
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
raise ValueError("Ожидается UUID")
return str(UUID(value))
def nonblank(value: str) -> str:
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
raise ValueError("Ожидается непустая строка Unicode")
return value
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
class Input(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
class CreateUser(Input):
username: str = Field(min_length=3, max_length=32)
display_name: ShortText
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
@field_validator("username")
@classmethod
def valid_username(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Недопустимое имя пользователя")
return value
class CreateChat(Input):
title: ShortText
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
class CreateMessage(Input):
text: Annotated[str, Field(min_length=1, max_length=4000), AfterValidator(nonblank)]
class View(BaseModel):
model_config = ConfigDict(from_attributes=True, extra="forbid")
class UserView(View):
id: str
username: str
display_name: str
created_at: str
class SessionView(View):
token: str
token_type: str
expires_at: str
user: UserView
class ChatView(View):
id: str
title: str
member_ids: list[str]
created_at: str
class MessageView(View):
id: str
chat_id: str
sender_id: str
text: str
created_at: str
class ChatPage(View):
items: list[ChatView]
next_cursor: str | None
class MessagePage(View):
items: list[MessageView]
next_cursor: str | None
+50
View File
@@ -0,0 +1,50 @@
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
from dataclasses import dataclass
from messenger.cursor import decode_cursor, encode_cursor
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
from messenger.repository import Repository
@dataclass(frozen=True)
class Page[T]:
items: list[T]
next_cursor: str | None
class Messenger:
def __init__(self, repository: Repository):
self.repository = repository
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
return self.repository.add_chat(title, tuple(member_ids))
def chat(self, actor: User, chat_id: str) -> Chat:
chat = self.repository.get_chat(chat_id)
if chat is None or actor.id not in chat.member_ids:
raise not_found()
return chat
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
scope = "chats:" + actor.id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
def send(self, actor: User, chat_id: str, text: str, key: str) -> Message:
self.chat(actor, chat_id)
return self.repository.idempotent_message(chat_id, actor.id, text, key)
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
self.chat(actor, chat_id)
scope = "messages:" + chat_id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
@staticmethod
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
visible = items[:limit]
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
return Page(visible, cursor)
+37
View File
@@ -0,0 +1,37 @@
"""Explicit configuration; credentials are read from untracked secret files."""
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str, default: str) -> str:
return Path(os.getenv(name, default)).read_text().strip()
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 8
idempotency_ttl: int = 86400
read_autocommit: bool = True
@classmethod
def from_env(cls):
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
if idempotency_ttl < 86400:
raise ValueError("Idempotency retention must be at least 24 hours")
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
raise ValueError("Invalid session lifetime or connection pool size")
return cls(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
ttl,
size,
idempotency_ttl,
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
)
+72
View File
@@ -0,0 +1,72 @@
import base64
from pathlib import Path
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from psycopg import sql
from psycopg.conninfo import make_conninfo
from messenger.app import create_app
from messenger.migrate import migrate
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
PASSWORD = "Учебный:пароль😀-12345"
@pytest.fixture(scope="session")
def settings():
password = (ROOT / "secrets/postgres_password").read_text().strip()
admin = make_conninfo(
host="127.0.0.1", port=55433, user="messenger", password=password, dbname="messenger"
)
database = "student_tests_" + uuid4().hex
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
config = Settings(make_conninfo(admin, dbname=database), "redis://127.0.0.1:56380/1")
migrate(config.database_url, ROOT / "migrations")
yield config
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
@pytest.fixture
def client(settings):
with psycopg.connect(settings.database_url) as conn:
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
client.app.state.cache.client.flushdb()
yield client
def basic(username, password=PASSWORD):
return {
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
}
@pytest.fixture
def actors(client):
result = []
for name in ("alice", "bob", "eve"):
r = client.post(
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
)
assert r.status_code == 201
session = client.post("/api/v1/auth/sessions", headers=basic(name))
assert session.status_code == 201
result.append({**r.json(), "token": session.json()["token"]})
return result
@pytest.fixture
def room(client, actors):
response = client.post(
"/api/v1/chats",
headers={"Authorization": "Bearer " + actors[0]["token"]},
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
)
assert response.status_code == 201
return response.json()
+299
View File
@@ -0,0 +1,299 @@
import hashlib
import json
import re
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import psycopg
import pytest
from conftest import PASSWORD, ROOT, basic
from fastapi.testclient import TestClient
from messenger.app import create_app
from messenger.auth import basic_credentials
from messenger.migrate import migrate
def headers(actor):
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
def error(response, status, code):
assert response.status_code == status, response.text
body = response.json()["error"]
assert set(body) == {"code", "message", "request_id"}
assert body["code"] == code
assert body["request_id"] == response.headers["X-Request-Id"]
assert len(response.headers.get_list("X-Request-Id")) == 1
if status == 401:
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
def test_malformed_json(client, body):
error(
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
400,
"invalid_request",
)
@pytest.mark.parametrize(
"change",
[
{"username": "ab"},
{"username": "a" * 33},
{"username": "Upper"},
{"username": "кириллица"},
{"username": "valid\n"},
{"username": 123},
{"username": True},
{"display_name": 123},
{"display_name": None},
{"display_name": " \n\t"},
{"display_name": "😀" * 101},
{"display_name": "\ud800"},
{"password": "short"},
{"password": "😀" * 129},
{"password": 123},
{"password": " " * 12},
{"password": "\ud800" * 12},
{"extra": True},
],
)
def test_strict_registration(client, change):
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
response = client.post(
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
)
error(response, 400, "invalid_request")
def test_unicode_password_hash_and_authentication(client, settings, actors):
for u in actors:
assert "password" not in u and "password_hash" not in u
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
with psycopg.connect(settings.database_url) as conn:
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
0
]
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
assert (
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
== 32
)
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
error(bad, 401, "unauthorized")
error(absent, 401, "unauthorized")
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
for value in (
{},
basic("alice"),
{"X-User-Id": actors[0]["id"]},
{"Authorization": "Bearer unknown"},
):
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
actual = client.get(
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
).json()
assert actual["id"] == actors[0]["id"]
error(
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
400,
"invalid_request",
)
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
def test_invalid_basic(value):
with pytest.raises(Exception) as caught:
basic_credentials(value)
assert caught.value.status == 401
def test_registration_race(client, settings):
def register(_):
return client.post(
"/api/v1/users",
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
).status_code
with ThreadPoolExecutor(max_workers=2) as pool:
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
path = "/api/v1/chats/" + room["id"]
for suffix in ("", "/messages"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
error(
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
)
with psycopg.connect(settings.database_url) as conn:
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
error(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "rollback", "member_ids": members},
),
400,
"invalid_request",
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
for body in (
{"text": ""},
{"text": " \t"},
{"text": "😀" * 4001},
{"text": 123},
{"text": None},
{"text": "ok", "sender_id": actors[1]["id"]},
):
error(
client.post(path + "/messages", headers=headers(actors[0]), json=body),
400,
"invalid_request",
)
for query in (
"limit=0",
"limit=101",
"limit=true",
"limit=1.0",
"limit=1&limit=2",
"cursor=",
"cursor=bad!",
"cursor=e30",
"cursor=x&cursor=y",
):
error(
client.get(path + "/messages?" + query, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
def send(i):
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
assert r.status_code == 201
return r.json()
with ThreadPoolExecutor(max_workers=10) as pool:
sent = list(pool.map(send, range(100)))
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
)
for m in sent:
m["created_at"] = "2026-10-17T12:00:00.000000Z"
found, cursor = [], None
while True:
query = {"limit": 17 if cursor else 1}
if cursor:
query["cursor"] = cursor
response = client.get(path, params=query, headers=headers(actors[1]))
assert response.status_code == 200
page = response.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
assert len(found) == len({x["id"] for x in found}) == 100
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
error(
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
user = actors[0]
url = "/api/v1/users/me"
assert client.get(url, headers=headers(user)).status_code == 200
repository = client.app.state.repository
original = repository.get_user
def must_be_cached(*args):
raise AssertionError("cache hit should avoid fetching immutable user again")
monkeypatch.setattr(repository, "get_user", must_be_cached)
assert client.get(url, headers=headers(user)).status_code == 200
monkeypatch.setattr(repository, "get_user", original)
cache = client.app.state.cache.client
digest = hashlib.sha256(user["token"].encode()).digest()
key = client.app.state.cache.key(digest)
assert 0 < cache.ttl(key) <= 3600
stale = cache.get(key)
cache.flushdb()
assert client.get(url, headers=headers(user)).status_code == 200
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
error(client.get(url, headers=headers(user)), 401, "unauthorized")
error(
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
)
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
database_file.write_text(settings.database_url)
cache_file.write_text(settings.cache_url)
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
with TestClient(create_app()) as short:
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
assert response.status_code == 201
token = {"Authorization": "Bearer " + response.json()["token"]}
assert short.get("/api/v1/users/me", headers=token).status_code == 200
with TestClient(create_app(settings)) as restarted:
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
time.sleep(2.1)
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
migrate(settings.database_url, ROOT / "migrations")
modified = tmp_path / "001_initial.sql"
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
with pytest.raises(RuntimeError, match="checksum changed"):
migrate(settings.database_url, tmp_path)
def test_logs_and_safe_errors(client, monkeypatch, caplog):
canary = "PRIVATE_BODY_CANARY_123"
with caplog.at_level("INFO", logger="messenger"):
r = client.post(
"/api/v1/users",
json={"username": "logging", "display_name": canary, "password": PASSWORD},
)
assert r.status_code == 201
assert canary not in caplog.text and PASSWORD not in caplog.text
def broken(*args):
raise RuntimeError(canary)
monkeypatch.setattr(client.app.state.authentication, "register", broken)
response = client.post(
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
)
error(response, 500, "internal_error")
assert canary not in response.text and canary not in caplog.text
error(
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
415,
"unsupported_media_type",
)
+137
View File
@@ -0,0 +1,137 @@
import json
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from test_api import error, headers
from messenger.app import create_app
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
def test_message_key_required(client, actors, room, key):
h = {"Authorization": "Bearer " + actors[0]["token"]}
if key is not None:
h["Idempotency-Key"] = key
error(
client.post(
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
),
400,
"invalid_request",
)
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
path = f"/api/v1/chats/{room['id']}/messages"
key = "Scope.Key:123"
h = {**headers(actors[0]), "Idempotency-Key": key}
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
assert first.status_code == 201
repeat = client.post(
path,
headers={**h, "Content-Type": "application/json"},
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
)
assert repeat.status_code == 201 and repeat.json() == first.json()
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
outsider = {**headers(actors[2]), "Idempotency-Key": key}
error(
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
)
other_user = {**headers(actors[1]), "Idempotency-Key": key}
own = client.post(path, headers=other_user, json={"text": "another author"})
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
chat = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "another", "member_ids": [actors[0]["id"]]},
).json()
other_chat = client.post(
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
)
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
with TestClient(create_app(settings)) as second:
assert second.app.state.instance_id != client.app.state.instance_id
def send(i):
c = client if i % 2 else second
r = c.post(path, headers=h, json={"text": "same concurrent body"})
assert r.status_code == 201
return r.json(), r.headers["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({item[0]["id"] for item in results}) == 1
assert len({item[1] for item in results}) == 2
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
repository = client.app.state.repository
original = repository._save_request
def fail_after_message_insert(*args):
raise RuntimeError("injected between message and result")
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
error(
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
monkeypatch.setattr(repository, "_save_request", original)
result = client.post(path, headers=h, json={"text": "retry after rollback"})
assert result.status_code == 201
assert (
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
)
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
first = client.post(path, headers=h, json={"text": "old"}).json()
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
"expires_at=clock_timestamp()-interval '1 day'"
)
second = client.post(path, headers=h, json={"text": "new"})
assert second.status_code == 201 and second.json()["id"] != first["id"]
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
@pytest.mark.parametrize("optimized", [True, False])
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
c.get("/api/v1/users/me", headers=headers(actors[0]))
pool = c.app.state.repository.pool
before = pool.get_stats().get("requests_num", 0)
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
assert response.status_code == 200 and response.json()["items"] == []
assert pool.get_stats()["requests_num"] - before == 3
def test_instance_header_covers_health_errors_and_success(client):
values = [
client.get("/health/live"),
client.get("/health/ready"),
client.get("/api/v1/users/me"),
client.get("/not-a-route"),
]
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
Generated
+1022
View File
File diff suppressed because it is too large Load Diff