forked from mtusi2026/mtusi2026lab3
Реализовать лабораторную 3
This commit is contained in:
@@ -18,3 +18,4 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
**/__pycache__
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
## Лабораторная 3
|
||||
## Лабораторная 3. Балансировка и идемпотентность
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор и группа — Samodurov Nikita, bpi2303
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Целевая оценка — 4
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Две реплики API принимают запросы через HAProxy и используют общие сессии в PostgreSQL. Параллельные повторы отправки с одним ключом создают одно сообщение.
|
||||
|
||||
Команды запуска и ограничения описаны в [REPORT.md](REPORT.md). Результаты smoke, перезапуска и дополнительных сценариев сохранены в [evidence/local-checks.txt](evidence/local-checks.txt).
|
||||
|
||||
Источники — материалы и контракт курса.
|
||||
|
||||
Использование ИИ — консультации и проверки ИИ.
|
||||
|
||||
@@ -12,5 +12,30 @@ jobs:
|
||||
python-version: '3.12'
|
||||
- run: python -m pip install -r requirements-dev.txt
|
||||
- run: make check validate PYTHON=python
|
||||
# This job validates the skeleton, not a student solution.
|
||||
# Add build, readiness and make test in a separate application job.
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Build and start stack
|
||||
run: |
|
||||
python scripts/setup.py
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
docker compose build migrate
|
||||
docker compose up -d --wait
|
||||
- name: Run HTTP and resilience checks
|
||||
run: |
|
||||
make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose start api-a api-b
|
||||
python student-tests/check_replicas.py
|
||||
- name: Show service status
|
||||
if: failure()
|
||||
run: docker compose ps -a
|
||||
- name: Stop stack
|
||||
if: always()
|
||||
run: docker compose down
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
FROM python:3.13.5-slim-bookworm@sha256:4c2cf9917bd1cbacc5e9b07320025bdb7cdf2df7b0ceaccb55e9dd7e30987419
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY migrations/ ./migrations/
|
||||
COPY src/ ./src/
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=3s --timeout=3s --start-period=5s --retries=4 \
|
||||
CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=2).close()"]
|
||||
CMD ["python", "src/server.py"]
|
||||
@@ -1,37 +1,53 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор и группа — Samodurov Nikita, bpi2303
|
||||
- Целевая оценка — 4
|
||||
- Источники — материалы и контракт курса.
|
||||
- Использование ИИ — консультации и проверки ИИ.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск из чистого клона
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine с Compose v2 и Python 3.9+ на хосте. Контейнер использует Python 3.13.5. По умолчанию API доступен на http://127.0.0.1:8080. При занятом порте задайте `PUBLIC_PORT` и соответствующий `BASE_URL`. Команда setup создаёт локальные `.env` и `secrets/`.
|
||||
|
||||
## Архитектура и решения
|
||||
```bash
|
||||
python scripts/setup.py
|
||||
make check
|
||||
docker compose config --quiet
|
||||
docker compose build migrate
|
||||
docker compose up -d --wait
|
||||
make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose start api-a api-b
|
||||
python student-tests/check_replicas.py
|
||||
docker compose down
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
В Windows PowerShell используйте следующие проверки после запуска стенда. Перед проверками установите `$env:PYTHONUTF8 = '1'`.
|
||||
|
||||
## Доказательства критериев
|
||||
```powershell
|
||||
python scripts/check_contract.py
|
||||
python tests/smoke.py
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
docker compose start api-a api-b
|
||||
python student-tests/check_replicas.py
|
||||
```
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Для полной валидации контракта установите `requirements-dev.txt` в отдельное окружение и выполните `python scripts/validate_openapi.py`. Команда `docker compose down` оставляет именованные тома. Для намеренного сброса учебных данных выполните `docker compose down -v`.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Архитектура
|
||||
|
||||
## Эксперименты и отказы
|
||||
Из ЛР 2 перенесены PostgreSQL, сессии и кэш профилей. HAProxy распределяет запросы между api-a и api-b по round-robin и проверяет /health/ready раз в секунду. После двух неуспешных проверок реплика исключается. Таймаут каждой проверки — одна секунда. Штатный выход по SIGTERM закрывает слушающий сокет и оставляет обработчикам до пяти секунд. На хост опубликован только HAProxy. Алиас X-Instance-Id соответствует имени сервиса. Разрешение имён использует Docker DNS.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
PostgreSQL хранит результат отправки с ключом в области «пользователь + чат + POST». Транзакционная advisory-блокировка сериализует запросы с одинаковым ключом. Сообщение и запись результата фиксируются одним COMMIT. Тот же текст в течение 24 часов возвращает исходный ответ, другой текст даёт 409. После истечения TTL запись того же ключа заменяется при следующем использовании. Сессия и членство проверяются на каждом повторе.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Версионируемые миграции выполняет сервис migrate под блокировкой. Проверка замены реплик использует ту же версию схемы. Миграции с несовместимыми изменениями требуют отдельного плана обновления.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Проверки
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Результаты локального Docker-стенда и команды приведены в [evidence/local-checks.txt](evidence/local-checks.txt).
|
||||
|
||||
## Ограничения
|
||||
|
||||
Реплики защищают от отказа одного процесса API. HAProxy и PostgreSQL работают по одному экземпляру. Во время исключения остановленного backend возможны короткие ошибки, POST сообщения можно повторить с исходным ключом.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
x-app: &app
|
||||
image: mtusi-messenger:lab3
|
||||
cpus: 1.0
|
||||
mem_limit: 320m
|
||||
env_file: .env
|
||||
environment:
|
||||
CURSOR_KEY_FILE: /run/secrets/cursor_key
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
secrets: [database_url, cache_url, cursor_key]
|
||||
|
||||
services:
|
||||
db:
|
||||
cpus: 1.0
|
||||
mem_limit: 384m
|
||||
image: postgres:17.5-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
secrets: [db_password]
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
|
||||
cache:
|
||||
cpus: 0.25
|
||||
mem_limit: 96m
|
||||
image: redis:7.4.3-alpine
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "32mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
|
||||
migrate:
|
||||
<<: *app
|
||||
build: .
|
||||
command: ["python", "src/migrate.py"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
||||
api-a: &api
|
||||
<<: *app
|
||||
environment:
|
||||
CURSOR_KEY_FILE: /run/secrets/cursor_key
|
||||
INSTANCE_ID: api-a
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
cache:
|
||||
condition: service_started
|
||||
stop_grace_period: 8s
|
||||
|
||||
api-b:
|
||||
<<: *api
|
||||
environment:
|
||||
CURSOR_KEY_FILE: /run/secrets/cursor_key
|
||||
INSTANCE_ID: api-b
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.11-alpine
|
||||
cpus: 0.5
|
||||
mem_limit: 96m
|
||||
ports: ["127.0.0.1:${PUBLIC_PORT:-8080}:8080"]
|
||||
volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
|
||||
depends_on:
|
||||
api-a:
|
||||
condition: service_healthy
|
||||
api-b:
|
||||
condition: service_healthy
|
||||
|
||||
secrets:
|
||||
db_password:
|
||||
file: ./secrets/db_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
cursor_key:
|
||||
file: ./secrets/cursor_key
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
@@ -0,0 +1,30 @@
|
||||
2026-11-12. Windows, Python 3.13.5, Docker Desktop с Linux-контейнерами.
|
||||
PUBLIC_PORT=18083, BASE_URL=http://127.0.0.1:18083
|
||||
|
||||
> python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
> python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
|
||||
> python tests/smoke.py --base-url http://127.0.0.1:18083
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
> python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
Hook: docker compose restart db cache api-a api-b lb.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
> python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
{"observed_instances_before": 2, "successful_writes": 106, "failed_attempts": 4, "first_recovery_seconds": 1.658}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
> python student-tests/check_replicas.py
|
||||
PASS: 8 concurrent replays through both replicas, 1 message
|
||||
PASS: stop api-a, 5.8 write/s, errors=3, survivor=0.085s
|
||||
PASS: stop api-b, 3.4 write/s, errors=5, survivor=0.048s
|
||||
PASS: recreate api-a, 8.2 write/s, errors=2, survivor=0.130s
|
||||
PASS: recreate api-b, 8.0 write/s, errors=3, survivor=0.094s
|
||||
|
||||
write/s считается за весь короткий сценарий. survivor — время до первого
|
||||
успешного ответа оставшейся реплики после начала действия. Публичный failover
|
||||
отдельно измеряет первый ответ после завершения hook.
|
||||
@@ -0,0 +1,34 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option http-server-close
|
||||
timeout connect 1s
|
||||
timeout client 15s
|
||||
timeout server 12s
|
||||
timeout http-request 8s
|
||||
timeout check 1s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 2
|
||||
timeout resolve 500ms
|
||||
timeout retry 500ms
|
||||
hold valid 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend messenger
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
server api-a api-a:8080
|
||||
server api-b api-b:8080
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash bytea NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE members (
|
||||
chat_id uuid REFERENCES chats(id),
|
||||
user_id uuid REFERENCES users(id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX members_user_idx ON members (user_id, chat_id);
|
||||
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES members (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE message_requests (
|
||||
actor_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL,
|
||||
fingerprint bytea NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (actor_id, chat_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX message_requests_expiry_idx ON message_requests (expires_at);
|
||||
@@ -0,0 +1,5 @@
|
||||
psycopg==3.2.9
|
||||
psycopg-binary==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.2.0
|
||||
typing-extensions==4.14.0
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart db cache api-a api-b lb
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart db cache api-a api-b lb
|
||||
@@ -0,0 +1,28 @@
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
|
||||
|
||||
def ensure(name, value):
|
||||
path = directory / name
|
||||
try:
|
||||
with path.open("x", encoding="utf-8") as file:
|
||||
file.write(value + "\n")
|
||||
except FileExistsError:
|
||||
pass
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
password = ensure("db_password", secrets.token_hex(24))
|
||||
ensure("database_url", f"postgresql://messenger:{password}@db:5432/messenger")
|
||||
ensure("cache_url", "redis://cache:6379/0")
|
||||
ensure("cursor_key", secrets.token_hex(32))
|
||||
|
||||
try:
|
||||
with (root / ".env").open("xb") as file:
|
||||
file.write((root / ".env.example").read_bytes())
|
||||
except FileExistsError:
|
||||
pass
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="api-a" goto stop
|
||||
if "%TARGET_INSTANCE_ID%"=="api-b" goto stop
|
||||
exit /b 2
|
||||
:stop
|
||||
docker compose stop --timeout 5 %TARGET_INSTANCE_ID%
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${TARGET_INSTANCE_ID:-}" in
|
||||
api-a|api-b) docker compose stop --timeout 5 "$TARGET_INSTANCE_ID" ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,13 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name):
|
||||
return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def positive_int(name, default):
|
||||
value = int(os.getenv(name, default))
|
||||
if value < 1:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
return value
|
||||
@@ -0,0 +1,19 @@
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from config import secret
|
||||
|
||||
|
||||
def main():
|
||||
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as connection:
|
||||
connection.execute("SELECT pg_advisory_xact_lock(230301)")
|
||||
connection.execute("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY)")
|
||||
for path in sorted(Path("migrations").glob("*.sql")):
|
||||
if connection.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone():
|
||||
continue
|
||||
connection.execute(path.read_text(encoding="utf-8"))
|
||||
connection.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,))
|
||||
print(f"Applied {path.name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
from concurrent.futures import Future, TimeoutError
|
||||
from queue import Empty, Full, Queue
|
||||
from threading import Event, Thread
|
||||
|
||||
import redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
|
||||
class ProfileCache:
|
||||
def __init__(self, url):
|
||||
self.client = redis.Redis.from_url(
|
||||
url, socket_timeout=0.25, socket_connect_timeout=0.25,
|
||||
max_connections=1, retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
self.work = Queue(maxsize=4)
|
||||
self.stopping = Event()
|
||||
Thread(target=self.run, daemon=True, name="profile-cache").start()
|
||||
|
||||
def close(self):
|
||||
self.stopping.set()
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
while not self.stopping.is_set():
|
||||
try:
|
||||
result, operation, args = self.work.get(timeout=0.25)
|
||||
except Empty:
|
||||
continue
|
||||
if not result.set_running_or_notify_cancel():
|
||||
continue
|
||||
try:
|
||||
result.set_result(operation(*args))
|
||||
except redis.RedisError:
|
||||
result.set_result(None)
|
||||
finally:
|
||||
self.client.close()
|
||||
|
||||
def submit(self, operation, *args):
|
||||
result = Future()
|
||||
try:
|
||||
self.work.put_nowait((result, operation, args))
|
||||
except Full:
|
||||
result.set_result(None)
|
||||
return result
|
||||
|
||||
def get(self, key):
|
||||
result = self.submit(self.client.get, key)
|
||||
try:
|
||||
return result.result(timeout=0.35)
|
||||
except TimeoutError:
|
||||
result.cancel()
|
||||
return None
|
||||
|
||||
def set(self, key, value):
|
||||
self.submit(self.client.setex, key, 300, value)
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
from uuid import uuid4
|
||||
|
||||
from storage import DB_UNAVAILABLE, Store, json_value, unauthorized, unavailable
|
||||
from validation import ApiError, canonical_id, invalid, not_found, require_fields, require_text
|
||||
|
||||
CHAT_PATH = re.compile(r"/api/v1/chats/([^/]+)(/messages)?")
|
||||
MAX_JSON_BYTES = 64 * 1024
|
||||
LOG = logging.getLogger("messenger")
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = dict(pairs)
|
||||
if len(result) != len(pairs):
|
||||
raise ValueError
|
||||
return result
|
||||
|
||||
|
||||
def page_query(query):
|
||||
try:
|
||||
params = parse_qs(query, keep_blank_values=True, max_num_fields=16)
|
||||
except ValueError:
|
||||
raise invalid("Invalid query") from None
|
||||
limits = params.get("limit", ["50"])
|
||||
cursors = params.get("cursor", [None])
|
||||
if len(limits) != 1 or not re.fullmatch(r"[0-9]{1,3}", limits[0]):
|
||||
raise invalid("Invalid limit")
|
||||
limit = int(limits[0])
|
||||
if not 1 <= limit <= 100:
|
||||
raise invalid("Invalid limit")
|
||||
if len(cursors) != 1:
|
||||
raise invalid("Invalid cursor")
|
||||
return limit, cursors[0]
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.handle_api()
|
||||
|
||||
def do_POST(self):
|
||||
self.handle_api()
|
||||
|
||||
def do_DELETE(self):
|
||||
self.handle_api()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
def log_request(self, code="-", size="-"):
|
||||
LOG.info("request_id=%s status=%s", self.request_id, code)
|
||||
|
||||
def send_error(self, code, message=None, explain=None):
|
||||
self.request_id = uuid4().hex
|
||||
self.response_headers = {}
|
||||
self.send_json(code, {"error": {
|
||||
"code": "invalid_request",
|
||||
"message": "Invalid HTTP request",
|
||||
"request_id": self.request_id,
|
||||
}})
|
||||
|
||||
def handle_api(self):
|
||||
self.request_id = uuid4().hex
|
||||
self.response_headers = {}
|
||||
try:
|
||||
try:
|
||||
status, payload = self.route()
|
||||
except ApiError as error:
|
||||
status, payload = self.error_payload(error)
|
||||
except DB_UNAVAILABLE:
|
||||
status, payload = self.error_payload(unavailable())
|
||||
except (TimeoutError, ConnectionError):
|
||||
return
|
||||
except Exception:
|
||||
LOG.error("request_id=%s unexpected error", self.request_id)
|
||||
status, payload = self.error_payload(
|
||||
ApiError(500, "internal_error", "Internal server error")
|
||||
)
|
||||
self.send_json(status, payload)
|
||||
except (TimeoutError, ConnectionError):
|
||||
return
|
||||
|
||||
def error_payload(self, error):
|
||||
self.response_headers = error.headers
|
||||
return error.status, {"error": {
|
||||
"code": error.code,
|
||||
"message": str(error),
|
||||
"request_id": self.request_id,
|
||||
}}
|
||||
|
||||
def send_json(self, status, payload):
|
||||
data = b"" if status == 204 else json.dumps(payload, ensure_ascii=False, default=json_value).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("X-Request-Id", self.request_id)
|
||||
self.send_header("X-Instance-Id", self.server.instance_id)
|
||||
if status != 204:
|
||||
self.send_header("Content-Type", "application/json")
|
||||
for name, value in self.response_headers.items():
|
||||
self.send_header(name, value)
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(data)
|
||||
|
||||
def read_json(self):
|
||||
lengths = self.headers.get_all("Content-Length", [])
|
||||
if (
|
||||
self.headers.get("Transfer-Encoding")
|
||||
or len(lengths) != 1
|
||||
or not re.fullmatch(r"[0-9]{1,10}", lengths[0])
|
||||
or self.headers.get_content_type() != "application/json"
|
||||
):
|
||||
raise invalid("Expected a JSON body")
|
||||
length = int(lengths[0])
|
||||
if length > MAX_JSON_BYTES:
|
||||
raise invalid("JSON body is too large")
|
||||
try:
|
||||
data = self.rfile.read(length)
|
||||
if len(data) != length:
|
||||
raise ValueError
|
||||
return json.loads(data.decode("utf-8"), object_pairs_hook=unique_object)
|
||||
except (ValueError, RecursionError):
|
||||
raise invalid("Invalid JSON") from None
|
||||
|
||||
def route(self):
|
||||
try:
|
||||
target = urlsplit(self.path)
|
||||
except ValueError:
|
||||
raise invalid("Invalid request path") from None
|
||||
path, method = target.path, self.command
|
||||
app = self.server.app
|
||||
|
||||
if method == "GET" and path == "/health/live":
|
||||
return 200, {"status": "ok"}
|
||||
if method == "GET" and path == "/health/ready":
|
||||
app.ready()
|
||||
return 200, {"status": "ok"}
|
||||
if method == "POST" and path == "/api/v1/users":
|
||||
body = self.read_json()
|
||||
require_fields(body, "username", "display_name", "password")
|
||||
username = body["username"]
|
||||
if not isinstance(username, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
raise invalid("Invalid username")
|
||||
password = body["password"]
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||
raise invalid("Invalid password")
|
||||
try:
|
||||
password.encode("utf-8")
|
||||
except UnicodeError:
|
||||
raise invalid("Invalid password") from None
|
||||
return 201, app.register(username, require_text(body["display_name"], 100), password)
|
||||
if method == "POST" and path == "/api/v1/auth/sessions":
|
||||
return 201, app.login(*self.basic_credentials())
|
||||
|
||||
match = CHAT_PATH.fullmatch(path)
|
||||
if path not in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions/current") and not match:
|
||||
raise not_found()
|
||||
header = self.headers.get("Authorization", "")
|
||||
bearer = re.fullmatch(r"(?i:Bearer) ([A-Za-z0-9_-]{43})", header)
|
||||
if not bearer:
|
||||
raise unauthorized("Bearer")
|
||||
digest = app.token_hash(bearer[1])
|
||||
actor = app.actor(digest)
|
||||
if path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||
app.logout(digest)
|
||||
return 204, None
|
||||
if path == "/api/v1/users/me" and method == "GET":
|
||||
return 200, app.user(actor)
|
||||
if path == "/api/v1/chats":
|
||||
if method == "POST":
|
||||
body = self.read_json()
|
||||
require_fields(body, "title", "member_ids")
|
||||
title = require_text(body["title"], 100)
|
||||
members = body["member_ids"]
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise invalid("Invalid member list")
|
||||
members = [canonical_id(member) for member in members]
|
||||
if actor not in members or len(set(members)) != len(members):
|
||||
raise invalid("Invalid member list")
|
||||
return 201, app.create_chat(title, members)
|
||||
if method == "GET":
|
||||
return 200, app.chat_page(actor, *page_query(target.query))
|
||||
if match:
|
||||
chat_id = canonical_id(unquote(match[1]))
|
||||
chat = app.chat(actor, chat_id)
|
||||
if match[2]:
|
||||
if method == "POST":
|
||||
body = self.read_json()
|
||||
require_fields(body, "text")
|
||||
key = self.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise invalid("Invalid Idempotency-Key")
|
||||
return 201, app.send(actor, chat_id, require_text(body["text"], 4000), key)
|
||||
if method == "GET":
|
||||
return 200, app.message_page(chat_id, *page_query(target.query))
|
||||
elif method == "GET":
|
||||
return 200, chat
|
||||
raise not_found()
|
||||
|
||||
def basic_credentials(self):
|
||||
if self.headers.get("Content-Length", "0") != "0" or self.headers.get("Transfer-Encoding"):
|
||||
raise invalid("Login does not accept a body")
|
||||
match = re.fullmatch(r"(?i:Basic) ([A-Za-z0-9+/=]{1,800})",
|
||||
self.headers.get("Authorization", ""))
|
||||
if not match:
|
||||
raise unauthorized("Basic")
|
||||
try:
|
||||
value = base64.b64decode(match[1], validate=True).decode("utf-8")
|
||||
except ValueError:
|
||||
raise unauthorized("Basic") from None
|
||||
username, separator, password = value.partition(":")
|
||||
if not separator or not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||
raise unauthorized("Basic")
|
||||
return username, password
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
timeout = 0.25
|
||||
request_queue_size = 64
|
||||
|
||||
def __init__(self, address):
|
||||
self.app = Store()
|
||||
self.instance_id = os.getenv("INSTANCE_ID") or uuid4().hex
|
||||
self.stopping = False
|
||||
self.active = 0
|
||||
self.workers = threading.Condition()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def get_request(self):
|
||||
connection, address = super().get_request()
|
||||
connection.settimeout(4)
|
||||
return connection, address
|
||||
|
||||
def verify_request(self, request, client_address):
|
||||
return not self.stopping
|
||||
|
||||
def process_request(self, request, client_address):
|
||||
with self.workers:
|
||||
self.active += 1
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except Exception:
|
||||
self.worker_done()
|
||||
raise
|
||||
|
||||
def process_request_thread(self, request, client_address):
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
self.worker_done()
|
||||
|
||||
def worker_done(self):
|
||||
with self.workers:
|
||||
self.active -= 1
|
||||
self.workers.notify_all()
|
||||
|
||||
def handle_error(self, request, client_address):
|
||||
LOG.error("HTTP connection handler failed")
|
||||
|
||||
def stop(self, signum, frame):
|
||||
self.stopping = True
|
||||
|
||||
def run(self):
|
||||
while not self.stopping:
|
||||
self.handle_request()
|
||||
self.server_close()
|
||||
with self.workers:
|
||||
self.workers.wait_for(lambda: self.active == 0, timeout=5)
|
||||
self.app.close()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
host = os.getenv("APP_HOST", "0.0.0.0")
|
||||
port = int(os.getenv("APP_PORT", "8080"))
|
||||
environment = os.getenv("APP_ENV", "lab")
|
||||
with Server((host, port)) as server:
|
||||
signal.signal(signal.SIGTERM, server.stop)
|
||||
signal.signal(signal.SIGINT, server.stop)
|
||||
LOG.info("started host=%s port=%s env=%s", host, server.server_port, environment)
|
||||
server.run()
|
||||
LOG.info("stopped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
from config import positive_int, secret
|
||||
from profile_cache import ProfileCache
|
||||
from validation import ApiError, invalid, not_found
|
||||
|
||||
USER = "id, username, display_name, created_at"
|
||||
MESSAGE = "id, chat_id, sender_id, text, created_at"
|
||||
CHAT = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
|
||||
DB_UNAVAILABLE = (psycopg.OperationalError, PoolTimeout, psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable)
|
||||
LOG = logging.getLogger("messenger")
|
||||
|
||||
|
||||
def json_value(value):
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(value, UUID):
|
||||
return str(value)
|
||||
raise TypeError(type(value).__name__)
|
||||
|
||||
|
||||
def unavailable():
|
||||
return ApiError(503, "unavailable", "Dependency temporarily unavailable")
|
||||
|
||||
|
||||
def unauthorized(scheme):
|
||||
challenge = f'{scheme} realm="messenger"'
|
||||
if scheme == "Basic":
|
||||
challenge += ', charset="UTF-8"'
|
||||
return ApiError(401, "unauthorized", "Invalid credentials", {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
secret("DATABASE_URL"), min_size=1, max_size=8, max_waiting=24, timeout=1.5,
|
||||
kwargs={"row_factory": dict_row, "connect_timeout": 2,
|
||||
"options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"},
|
||||
open=True, check=ConnectionPool.check_connection,
|
||||
)
|
||||
self.cache = ProfileCache(secret("CACHE_URL"))
|
||||
self.cursor_key = bytes.fromhex(secret("CURSOR_KEY"))
|
||||
self.ttl = positive_int("SESSION_TTL_SECONDS", 3600)
|
||||
self.idempotency_ttl = positive_int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
self.hash_slots = BoundedSemaphore(4)
|
||||
self.dummy_hash = bytes(16 + 32)
|
||||
|
||||
def close(self):
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self):
|
||||
with self.pool.connection() as db:
|
||||
db.execute("SELECT 1 FROM users LIMIT 1")
|
||||
|
||||
def password_hash(self, password, salt):
|
||||
if not self.hash_slots.acquire(timeout=0.5):
|
||||
raise unavailable()
|
||||
try:
|
||||
return salt + hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=2**15, r=8, p=1,
|
||||
maxmem=64 * 1024**2, dklen=32,
|
||||
)
|
||||
finally:
|
||||
self.hash_slots.release()
|
||||
|
||||
def register(self, username, name, password):
|
||||
password_hash = self.password_hash(password, secrets.token_bytes(16))
|
||||
try:
|
||||
with self.pool.connection() as db:
|
||||
return db.execute(f"""
|
||||
INSERT INTO users (username,display_name,password_hash) VALUES (%s,%s,%s)
|
||||
RETURNING {USER}
|
||||
""", (username, name, password_hash)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, "conflict", "Username already exists") from None
|
||||
|
||||
def login(self, username, password):
|
||||
with self.pool.connection() as db:
|
||||
row = db.execute(f"SELECT {USER},password_hash FROM users WHERE username=%s",
|
||||
(username,)).fetchone()
|
||||
stored = row.pop("password_hash") if row else self.dummy_hash
|
||||
matches = hmac.compare_digest(self.password_hash(password, stored[:16]), stored)
|
||||
if not matches or row is None:
|
||||
raise unauthorized("Basic")
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = self.token_hash(token)
|
||||
with self.pool.connection() as db:
|
||||
db.execute("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
session = db.execute("""
|
||||
INSERT INTO sessions (token_hash,user_id,expires_at)
|
||||
VALUES (%s,%s,clock_timestamp() + %s * interval '1 second')
|
||||
RETURNING expires_at
|
||||
""", (digest, row["id"], self.ttl)).fetchone()
|
||||
return {"token": token, "token_type": "Bearer", **session, "user": row}
|
||||
|
||||
@staticmethod
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
def actor(self, digest):
|
||||
with self.pool.connection() as db:
|
||||
row = db.execute("""
|
||||
SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()
|
||||
""", (digest,)).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized("Bearer")
|
||||
return row["user_id"]
|
||||
|
||||
def user(self, user_id):
|
||||
key = f"user:v1:{user_id}"
|
||||
value = self.cache.get(key)
|
||||
if value is not None:
|
||||
try:
|
||||
return json.loads(value)
|
||||
except ValueError:
|
||||
LOG.info("invalid profile cache value")
|
||||
with self.pool.connection() as db:
|
||||
user = db.execute(f"SELECT {USER} FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||
self.cache.set(key, json.dumps(user, default=json_value))
|
||||
return user
|
||||
|
||||
def logout(self, digest):
|
||||
with self.pool.connection() as db:
|
||||
removed = db.execute("""
|
||||
DELETE FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()
|
||||
""", (digest,)).rowcount
|
||||
if not removed:
|
||||
raise unauthorized("Bearer")
|
||||
|
||||
def create_chat(self, title, members):
|
||||
try:
|
||||
with self.pool.connection() as db:
|
||||
chat = db.execute("INSERT INTO chats (title) VALUES (%s) RETURNING *",
|
||||
(title,)).fetchone()
|
||||
with db.cursor() as cursor:
|
||||
cursor.executemany("INSERT INTO members (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||
[(chat["id"], member, index) for index, member in enumerate(members)])
|
||||
return {**chat, "member_ids": members}
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid("Unknown member") from None
|
||||
|
||||
def chat(self, actor, chat_id):
|
||||
with self.pool.connection() as db:
|
||||
row = db.execute(f"""
|
||||
SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id
|
||||
WHERE m.user_id=%s AND c.id=%s
|
||||
""", (actor, chat_id)).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
def send(self, actor, chat_id, text, key):
|
||||
fingerprint = hashlib.sha256(text.encode("utf-8")).digest()
|
||||
identity = (actor, chat_id, key)
|
||||
with self.pool.connection() as db:
|
||||
db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))",
|
||||
(f"{actor}:{chat_id}:{key}",))
|
||||
request = db.execute("""
|
||||
SELECT fingerprint,message_id FROM message_requests
|
||||
WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at > clock_timestamp()
|
||||
""", identity).fetchone()
|
||||
if request is not None:
|
||||
if not hmac.compare_digest(request["fingerprint"], fingerprint):
|
||||
raise ApiError(409, "conflict", "Idempotency key already used for another body")
|
||||
return db.execute(f"SELECT {MESSAGE} FROM messages WHERE id=%s",
|
||||
(request["message_id"],)).fetchone()
|
||||
message = db.execute(f"""
|
||||
INSERT INTO messages (chat_id,sender_id,text) VALUES (%s,%s,%s) RETURNING {MESSAGE}
|
||||
""", (chat_id, actor, text)).fetchone()
|
||||
db.execute("""
|
||||
INSERT INTO message_requests (actor_id,chat_id,key,fingerprint,message_id,expires_at)
|
||||
VALUES (%s,%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second')
|
||||
ON CONFLICT (actor_id,chat_id,key) DO UPDATE
|
||||
SET fingerprint=excluded.fingerprint,message_id=excluded.message_id,expires_at=excluded.expires_at
|
||||
""", (*identity, fingerprint, message["id"], self.idempotency_ttl))
|
||||
return message
|
||||
|
||||
def chat_page(self, actor, limit, cursor):
|
||||
return self.page(f"""SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id
|
||||
WHERE m.user_id=%s""", "c.", ("chats", actor), limit, cursor)
|
||||
|
||||
def message_page(self, chat_id, limit, cursor):
|
||||
return self.page(f"SELECT {MESSAGE} FROM messages WHERE chat_id=%s",
|
||||
"", ("messages", chat_id), limit, cursor)
|
||||
|
||||
def page(self, query, prefix, scope, limit, cursor):
|
||||
columns = f"{prefix}created_at, {prefix}id"
|
||||
params = [scope[1]]
|
||||
if cursor is not None:
|
||||
params.extend(self.decode_cursor(cursor, scope))
|
||||
query += f" AND ({columns}) > (%s,%s)"
|
||||
with self.pool.connection() as db:
|
||||
rows = db.execute(f"{query} ORDER BY {columns} LIMIT %s", (*params, limit + 1)).fetchall()
|
||||
items = rows[:limit]
|
||||
next_cursor = self.encode_cursor(scope, items[-1]) if len(rows) > limit else None
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
def encode_cursor(self, scope, row):
|
||||
payload = json.dumps([scope, row["created_at"], row["id"]],
|
||||
separators=(",", ":"), default=json_value).encode("ascii")
|
||||
signature = hmac.digest(self.cursor_key, payload, "sha256")
|
||||
return base64.urlsafe_b64encode(signature + payload).decode("ascii").rstrip("=")
|
||||
|
||||
def decode_cursor(self, cursor, scope):
|
||||
try:
|
||||
if not 1 <= len(cursor) <= 256 or not cursor.isascii():
|
||||
raise ValueError
|
||||
data = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)
|
||||
signature, payload = data[:32], data[32:]
|
||||
if not hmac.compare_digest(signature, hmac.digest(self.cursor_key, payload, "sha256")):
|
||||
raise ValueError
|
||||
saved_scope, timestamp, record = json.loads(payload)
|
||||
if saved_scope != [scope[0], str(scope[1])]:
|
||||
raise ValueError
|
||||
return datetime.fromisoformat(timestamp), UUID(record)
|
||||
except ValueError:
|
||||
raise invalid("Invalid cursor") from None
|
||||
@@ -0,0 +1,43 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message):
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found():
|
||||
return ApiError(404, "not_found", "Resource not found")
|
||||
|
||||
|
||||
def canonical_id(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
|
||||
):
|
||||
raise invalid("Invalid UUID")
|
||||
return UUID(value)
|
||||
|
||||
|
||||
def require_fields(body, *fields):
|
||||
if not isinstance(body, dict) or body.keys() != set(fields):
|
||||
raise invalid("Invalid request fields")
|
||||
|
||||
|
||||
def require_text(value, maximum):
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or not value.strip()
|
||||
or len(value) > maximum
|
||||
or "\x00" in value
|
||||
or re.search(r"[\ud800-\udfff]", value)
|
||||
):
|
||||
raise invalid("Invalid text field")
|
||||
return value
|
||||
@@ -0,0 +1,118 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from threading import Barrier, Event
|
||||
import time
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, check
|
||||
|
||||
REPLICAS = ("api-a", "api-b")
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(["docker", "compose", *args], text=True, timeout=45).strip()
|
||||
|
||||
|
||||
def wait_instances(expected):
|
||||
client = Client(timeout=1)
|
||||
deadline = time.monotonic() + 25
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
instance = client.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
|
||||
seen.add(instance)
|
||||
if seen == expected:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"Expected {expected}, observed {seen}")
|
||||
|
||||
|
||||
def parallel_replay(actor, chat):
|
||||
start = Barrier(8)
|
||||
key = uuid.uuid4().hex
|
||||
|
||||
def write(_):
|
||||
start.wait(timeout=10)
|
||||
return Client().message(actor, chat, "Concurrent replay", key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as workers:
|
||||
results = list(workers.map(write, range(8)))
|
||||
ids = {item["json"]["id"] for item in results}
|
||||
instances = {item["headers"]["X-Instance-Id"] for item in results}
|
||||
check(len(ids) == 1 and instances == set(REPLICAS), "parallel replay across replicas failed")
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
page = Client().call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
check(sum(item["id"] in ids for item in page["items"]) == 1, "duplicate in history")
|
||||
print("PASS: 8 concurrent replays through both replicas, 1 message", flush=True)
|
||||
|
||||
|
||||
def exercise(actor, chat, target, recreate):
|
||||
stop = Event()
|
||||
results = []
|
||||
client = Client(timeout=2)
|
||||
started = time.monotonic()
|
||||
|
||||
def traffic():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
try:
|
||||
response = client.message(actor, chat, "Write " + key, key=key)
|
||||
results.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
|
||||
key = uuid.uuid4().hex
|
||||
except Failure:
|
||||
results.append((time.monotonic(), None, None))
|
||||
stop.wait(0.05)
|
||||
|
||||
old_id = compose("ps", "-q", target)
|
||||
with ThreadPoolExecutor(max_workers=1) as workers:
|
||||
worker = workers.submit(traffic)
|
||||
try:
|
||||
time.sleep(0.5)
|
||||
changed = time.monotonic()
|
||||
if recreate:
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", target)
|
||||
check(old_id != compose("ps", "-q", target), "container was not replaced")
|
||||
wait_instances(set(REPLICAS))
|
||||
else:
|
||||
compose("stop", "--timeout", "5", target)
|
||||
check(not compose("ps", "-q", "--status", "running", target), "replica is still running")
|
||||
time.sleep(2.5)
|
||||
finally:
|
||||
stop.set()
|
||||
worker.result(timeout=5)
|
||||
if not recreate:
|
||||
compose("start", target)
|
||||
wait_instances(set(REPLICAS))
|
||||
|
||||
survivor = next(name for name in REPLICAS if name != target)
|
||||
successful = [row for row in results if row[1] is not None]
|
||||
first = next(row[0] for row in successful if row[0] >= changed and row[1] == survivor)
|
||||
check(first - changed <= 10, "surviving replica took too long")
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
|
||||
observed = set()
|
||||
while path is not None:
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
observed.update(item["id"] for item in page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={cursor}" if cursor else None
|
||||
check({row[2] for row in successful} <= observed, "confirmed writes lost")
|
||||
action = "recreate" if recreate else "stop"
|
||||
elapsed = results[-1][0] - started
|
||||
print(f"PASS: {action} {target}, {len(successful) / elapsed:.1f} write/s, "
|
||||
f"errors={len(results) - len(successful)}, survivor={first - changed:.3f}s", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
compose("start", *REPLICAS)
|
||||
wait_instances(set(REPLICAS))
|
||||
owner, _, _, chat = Client().bootstrap()
|
||||
parallel_replay(owner, chat)
|
||||
for name in REPLICAS:
|
||||
exercise(owner, chat, name, recreate=False)
|
||||
for name in REPLICAS:
|
||||
exercise(owner, chat, name, recreate=True)
|
||||
Reference in New Issue
Block a user